From 14f7b5711c58a258246d2ae0f322cb7a7bbfcb9c Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 23 Feb 2024 11:49:12 -0500 Subject: [PATCH 01/16] MIGRATIONS-1519: Add support for providing CPU arch Signed-off-by: Tanner Lewis --- .../default-values.json | 1 + .../lib/common-utilities.ts | 13 +++++++++ .../lib/fetch-migration-stack.ts | 11 +++++--- .../service-stacks/capture-proxy-es-stack.ts | 4 ++- .../lib/service-stacks/capture-proxy-stack.ts | 4 ++- .../lib/service-stacks/elasticsearch-stack.ts | 4 ++- .../lib/service-stacks/kafka-broker-stack.ts | 5 +++- .../service-stacks/kafka-zookeeper-stack.ts | 5 +++- .../migration-analytics-stack.ts | 4 ++- .../service-stacks/migration-console-stack.ts | 4 ++- .../service-stacks/migration-service-core.ts | 9 +++++-- .../opensearch-container-stack.ts | 6 +++-- .../service-stacks/traffic-replayer-stack.ts | 4 ++- .../lib/stack-composer.ts | 14 ++++++++++ .../opensearch-service-migration/options.md | 1 + .../test/common-utilities.test.ts | 27 +++++++++++++++++++ .../test/default-values-test.json | 3 ++- .../test/fetch-migration-stack.test.ts | 4 ++- 18 files changed, 106 insertions(+), 17 deletions(-) create mode 100644 deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts diff --git a/deployment/cdk/opensearch-service-migration/default-values.json b/deployment/cdk/opensearch-service-migration/default-values.json index f7c2caeb9..069676cf7 100644 --- a/deployment/cdk/opensearch-service-migration/default-values.json +++ b/deployment/cdk/opensearch-service-migration/default-values.json @@ -6,6 +6,7 @@ "nodeToNodeEncryptionEnabled": true, "encryptionAtRestEnabled": true, "vpcEnabled": true, + "defaultFargateCpuArch": "X86_64", "migrationAssistanceEnabled": true, "migrationConsoleServiceEnabled": true, "trafficReplayerServiceEnabled": true, diff --git a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts index a28fa2b61..45864b7ce 100644 --- a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts +++ b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts @@ -1,6 +1,7 @@ import {Effect, PolicyStatement, Role, ServicePrincipal} from "aws-cdk-lib/aws-iam"; import {Construct} from "constructs"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; +import {CpuArchitecture} from "aws-cdk-lib/aws-ecs"; export function createOpenSearchIAMAccessPolicy(region: string, accountId: string): PolicyStatement { return new PolicyStatement({ @@ -120,4 +121,16 @@ export function createDefaultECSTaskRole(scope: Construct, serviceName: string): ] })) return serviceTaskRole +} + +export function validateFargateCpuArch(cpuArch: string): CpuArchitecture { + if (cpuArch.toUpperCase() === "X86_64") { + return CpuArchitecture.X86_64 + } + else if (cpuArch.toUpperCase() === "ARM64") { + return CpuArchitecture.ARM64 + } + else { + throw new Error(`Unknown Fargate cpu architecture provided: ${cpuArch}`) + } } \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts b/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts index dd712543d..dff7e65fb 100644 --- a/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts @@ -3,9 +3,9 @@ import {IVpc} from "aws-cdk-lib/aws-ec2"; import {Construct} from "constructs"; import { Cluster, - ContainerImage, + ContainerImage, CpuArchitecture, FargateTaskDefinition, - LogDrivers, + LogDrivers, OperatingSystemFamily, Secret as ECSSecret } from "aws-cdk-lib/aws-ecs"; import {Secret as SMSecret} from "aws-cdk-lib/aws-secretsmanager"; @@ -22,7 +22,8 @@ import { export interface FetchMigrationProps extends StackPropsExt { readonly vpc: IVpc, readonly dpPipelineTemplatePath: string, - readonly sourceEndpoint: string + readonly sourceEndpoint: string, + readonly fargateCpuArch: CpuArchitecture } export class FetchMigrationStack extends Stack { @@ -49,6 +50,10 @@ export class FetchMigrationStack extends Stack { ecsTaskRole.addToPolicy(openSearchServerlessPolicy) // ECS Task Definition const fetchMigrationFargateTask = new FargateTaskDefinition(this, "fetchMigrationFargateTask", { + runtimePlatform: { + operatingSystemFamily: OperatingSystemFamily.LINUX, + cpuArchitecture: props.fargateCpuArch + }, family: `migration-${props.stage}-${serviceName}`, memoryLimitMiB: 8192, cpu: 2048, diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts index 38fcee01e..37204c403 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts @@ -1,6 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; @@ -14,6 +14,7 @@ export interface CaptureProxyESProps extends StackPropsExt { readonly vpc: IVpc, readonly streamingSourceType: StreamingSourceType, readonly analyticsServiceEnabled: boolean, + readonly fargateCpuArch: CpuArchitecture, readonly extraArgs?: string, } @@ -75,6 +76,7 @@ export class CaptureProxyESStack extends MigrationServiceCore { serviceConnectServices: [serviceConnectService, esServiceConnectService], serviceDiscoveryEnabled: true, serviceDiscoveryPort: 19200, + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 1024, taskMemoryLimitMiB: 4096, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts index 91f40e44b..66842ca84 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts @@ -1,6 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; @@ -13,6 +13,7 @@ import {createMSKProducerIAMPolicies} from "../common-utilities"; export interface CaptureProxyProps extends StackPropsExt { readonly vpc: IVpc, readonly streamingSourceType: StreamingSourceType, + readonly fargateCpuArch: CpuArchitecture, readonly customSourceClusterEndpoint?: string, readonly analyticsServiceEnabled?: boolean, readonly extraArgs?: string, @@ -60,6 +61,7 @@ export class CaptureProxyStack extends MigrationServiceCore { taskRolePolicies: servicePolicies, portMappings: [servicePort], serviceConnectServices: [serviceConnectService], + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 512, taskMemoryLimitMiB: 2048, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts index ca1bbca88..a14513c64 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts @@ -1,6 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; @@ -10,6 +10,7 @@ import {StringParameter} from "aws-cdk-lib/aws-ssm"; export interface ElasticsearchProps extends StackPropsExt { readonly vpc: IVpc, + readonly fargateCpuArch: CpuArchitecture } /** @@ -45,6 +46,7 @@ export class ElasticsearchStack extends MigrationServiceCore { serviceConnectServices: [serviceConnectService], serviceDiscoveryEnabled: true, serviceDiscoveryPort: 9200, + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 512, taskMemoryLimitMiB: 2048, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts index 7276d41ac..d8bf72b95 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts @@ -1,6 +1,7 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; import { + CpuArchitecture, PortMapping, Protocol } from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; @@ -9,7 +10,8 @@ import {StringParameter} from "aws-cdk-lib/aws-ssm"; import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; export interface KafkaBrokerProps extends StackPropsExt { - readonly vpc: IVpc + readonly vpc: IVpc, + readonly fargateCpuArch: CpuArchitecture } /** @@ -62,6 +64,7 @@ export class KafkaBrokerStack extends MigrationServiceCore { }, portMappings: [servicePort], serviceConnectServices: [serviceConnectService], + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 256, taskMemoryLimitMiB: 2048, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts index e204b226e..405af42eb 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts @@ -1,6 +1,7 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; import { + CpuArchitecture, PortMapping, Protocol, } from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; @@ -9,7 +10,8 @@ import {StringParameter} from "aws-cdk-lib/aws-ssm"; import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; export interface KafkaZookeeperProps extends StackPropsExt { - readonly vpc: IVpc + readonly vpc: IVpc, + readonly fargateCpuArch: CpuArchitecture } /** @@ -46,6 +48,7 @@ export class KafkaZookeeperStack extends MigrationServiceCore { }, portMappings: [servicePort], serviceConnectServices: [serviceConnectService], + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 256, taskMemoryLimitMiB: 512, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-analytics-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-analytics-stack.ts index ca5cd62b0..22075a0ca 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-analytics-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-analytics-stack.ts @@ -6,7 +6,7 @@ import { SecurityGroup, IVpc, } from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; @@ -15,6 +15,7 @@ import {createAwsDistroForOtelPushInstrumentationPolicy} from "../common-utiliti export interface MigrationAnalyticsProps extends StackPropsExt { readonly vpc: IVpc, + readonly fargateCpuArch: CpuArchitecture, readonly bastionHostEnabled?: boolean } @@ -82,6 +83,7 @@ export class MigrationAnalyticsStack extends MigrationServiceCore { dockerDirectoryPath: join(__dirname, "../../../../../", "TrafficCapture/dockerSolution/src/main/docker/otelCollector"), dockerImageCommand: ["--config=/etc/otel-config-aws.yaml"], securityGroups: securityGroups, + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 1024, taskMemoryLimitMiB: 4096, portMappings: [otelCollectorPort, otelHealthCheckPort], diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts index 16013cb40..e1d931e0a 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts @@ -1,6 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {MountPoint, Volume} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, MountPoint, Volume} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; @@ -17,6 +17,7 @@ export interface MigrationConsoleProps extends StackPropsExt { readonly vpc: IVpc, readonly streamingSourceType: StreamingSourceType, readonly fetchMigrationEnabled: boolean, + readonly fargateCpuArch: CpuArchitecture, readonly migrationAnalyticsEnabled: boolean } @@ -154,6 +155,7 @@ export class MigrationConsoleStack extends MigrationServiceCore { mountPoints: [replayerOutputMountPoint], environment: environment, taskRolePolicies: servicePolicies, + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 512, taskMemoryLimitMiB: 1024, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts index c419609dc..d030179e2 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts @@ -3,11 +3,11 @@ import {ISecurityGroup, IVpc, SubnetType} from "aws-cdk-lib/aws-ec2"; import { CfnService as FargateCfnService, Cluster, - ContainerImage, + ContainerImage, CpuArchitecture, FargateService, FargateTaskDefinition, LogDrivers, - MountPoint, + MountPoint, OperatingSystemFamily, PortMapping, Ulimit, Volume } from "aws-cdk-lib/aws-ecs"; @@ -25,6 +25,7 @@ export interface MigrationServiceCoreProps extends StackPropsExt { readonly serviceName: string, readonly vpc: IVpc, readonly securityGroups: ISecurityGroup[], + readonly cpuArchitecture: CpuArchitecture, readonly dockerFilePath?: string, readonly dockerDirectoryPath?: string, readonly dockerImageRegistryName?: string, @@ -87,6 +88,10 @@ export class MigrationServiceCore extends Stack { const serviceTaskDef = new FargateTaskDefinition(this, "ServiceTaskDef", { ephemeralStorageGiB: 75, + runtimePlatform: { + operatingSystemFamily: OperatingSystemFamily.LINUX, + cpuArchitecture: props.cpuArchitecture + }, family: `migration-${props.stage}-${props.serviceName}`, memoryLimitMiB: props.taskMemoryLimitMiB ? props.taskMemoryLimitMiB : 1024, cpu: props.taskCpuUnits ? props.taskCpuUnits : 256, diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts index 4bfc13b2b..420f94bbe 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts @@ -1,13 +1,14 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {PortMapping, Protocol, Ulimit, UlimitName} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, PortMapping, Protocol, Ulimit, UlimitName} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {MigrationServiceCore} from "./migration-service-core"; import {StringParameter} from "aws-cdk-lib/aws-ssm"; import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service"; export interface OpenSearchContainerProps extends StackPropsExt { - readonly vpc: IVpc + readonly vpc: IVpc, + readonly fargateCpuArch: CpuArchitecture } /** @@ -61,6 +62,7 @@ export class OpenSearchContainerStack extends MigrationServiceCore { serviceConnectServices: [serviceConnectService], taskCpuUnits: 1024, taskMemoryLimitMiB: 4096, + cpuArchitecture: props.fargateCpuArch, ulimits: ulimits, ...props }); diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts index a290b86c7..b6a183367 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts @@ -1,6 +1,6 @@ import {StackPropsExt} from "../stack-composer"; import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2"; -import {MountPoint, Volume} from "aws-cdk-lib/aws-ecs"; +import {CpuArchitecture, MountPoint, Volume} from "aws-cdk-lib/aws-ecs"; import {Construct} from "constructs"; import {join} from "path"; import {MigrationServiceCore} from "./migration-service-core"; @@ -18,6 +18,7 @@ export interface TrafficReplayerProps extends StackPropsExt { readonly vpc: IVpc, readonly enableClusterFGACAuth: boolean, readonly streamingSourceType: StreamingSourceType, + readonly fargateCpuArch: CpuArchitecture, readonly addOnMigrationId?: string, readonly customKafkaGroupId?: string, readonly userAgentSuffix?: string, @@ -102,6 +103,7 @@ export class TrafficReplayerStack extends MigrationServiceCore { environment: { "TUPLE_DIR_PATH": `/shared-replayer-output/traffic-replayer-${deployId}` }, + cpuArchitecture: props.fargateCpuArch, taskCpuUnits: 1024, taskMemoryLimitMiB: 4096, ...props diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts index 87d432714..9147caab4 100644 --- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts +++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts @@ -18,6 +18,7 @@ import {KafkaZookeeperStack} from "./service-stacks/kafka-zookeeper-stack"; import {Application} from "@aws-cdk/aws-servicecatalogappregistry-alpha"; import {OpenSearchContainerStack} from "./service-stacks/opensearch-container-stack"; import {determineStreamingSourceType, StreamingSourceType} from "./streaming-source-type"; +import {validateFargateCpuArch} from "./common-utilities"; export interface StackPropsExt extends StackProps { readonly stage: string, @@ -159,6 +160,7 @@ export class StackComposer { const mskBrokerNodeCount = this.getContextForType('mskBrokerNodeCount', 'number', defaultValues, contextJSON) const mskSubnetIds = this.getContextForType('mskSubnetIds', 'object', defaultValues, contextJSON) const addOnMigrationDeployId = this.getContextForType('addOnMigrationDeployId', 'string', defaultValues, contextJSON) + const defaultFargateCpuArch = this.getContextForType('defaultFargateCpuArch', 'string', defaultValues, contextJSON) const captureProxyESServiceEnabled = this.getContextForType('captureProxyESServiceEnabled', 'boolean', defaultValues, contextJSON) const captureProxyESExtraArgs = this.getContextForType('captureProxyESExtraArgs', 'string', defaultValues, contextJSON) const migrationConsoleServiceEnabled = this.getContextForType('migrationConsoleServiceEnabled', 'boolean', defaultValues, contextJSON) @@ -211,6 +213,8 @@ export class StackComposer { } else if (targetClusterEndpoint || osContainerServiceEnabled) { targetEndpoint = targetClusterEndpoint ? targetClusterEndpoint : "https://opensearch:9200" } + + const fargateCpuArch = validateFargateCpuArch(defaultFargateCpuArch) const streamingSourceType = determineStreamingSourceType(kafkaBrokerServiceEnabled) const engineVersion = this.getContextForType('engineVersion', 'string', defaultValues, contextJSON) @@ -384,6 +388,7 @@ export class StackComposer { migrationAnalyticsStack = new MigrationAnalyticsStack(scope, "migration-analytics", { stackName: `OSMigrations-${stage}-${region}-MigrationAnalytics`, description: "This stack contains the OpenTelemetry Collector and Bastion Host", + fargateCpuArch: fargateCpuArch, bastionHostEnabled: migrationAnalyticsBastionHostEnabled, vpc:networkStack.vpc, stage: stage, @@ -403,6 +408,7 @@ export class StackComposer { description: "This stack contains resources for the OpenSearch Container ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) this.addDependentStacks(osContainerStack, [migrationStack]) @@ -417,6 +423,7 @@ export class StackComposer { description: "This stack contains resources for the Kafka Zookeeper ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) this.addDependentStacks(kafkaZookeeperStack, [migrationStack]) @@ -431,6 +438,7 @@ export class StackComposer { description: "This stack contains resources for the Kafka Broker ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) this.addDependentStacks(kafkaBrokerStack, [migrationStack, kafkaZookeeperStack]) @@ -448,6 +456,7 @@ export class StackComposer { description: "This stack contains resources to assist migrating historical data to an OpenSearch Service domain", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) this.addDependentStacks(fetchMigrationStack, [migrationStack, openSearchStack, osContainerStack]) @@ -465,6 +474,7 @@ export class StackComposer { description: "This stack contains resources for the Capture Proxy/Elasticsearch ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) // The analytics stack dependency is necessary to ensure the otel collector is available (and can be found via service connect) @@ -487,6 +497,7 @@ export class StackComposer { description: "This stack contains resources for the Traffic Replayer ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) // The analytics stack dependency is necessary to ensure the otel collector is available (and can be found via service connect) @@ -503,6 +514,7 @@ export class StackComposer { description: "This stack contains resources for a testing mock Elasticsearch single node cluster ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) this.addDependentStacks(elasticsearchStack, [migrationStack]) @@ -521,6 +533,7 @@ export class StackComposer { description: "This stack contains resources for the Capture Proxy ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) // The analytics stack dependency is necessary to ensure the otel collector is available (and can be found via service connect) @@ -540,6 +553,7 @@ export class StackComposer { description: "This stack contains resources for the Migration Console ECS service", stage: stage, defaultDeployId: defaultDeployId, + fargateCpuArch: fargateCpuArch, ...props, }) // To enable the Migration Console to make requests to other service endpoints with Service Connect, diff --git a/deployment/cdk/opensearch-service-migration/options.md b/deployment/cdk/opensearch-service-migration/options.md index 68005b7ba..7f9ecdc2c 100644 --- a/deployment/cdk/opensearch-service-migration/options.md +++ b/deployment/cdk/opensearch-service-migration/options.md @@ -7,6 +7,7 @@ These tables list all CDK context configuration values a user can specify for th |--------------------------------------|---------|------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | migrationAssistanceEnabled | boolean | true | Flag which controls deploying common Migration Service resources such as MSK, EFS, and an ECS cluster. **Note**: This option must be enabled to use any Migration service | | addOnMigrationDeployId | string | "cit2-replay" | Specify an ID string to use for an additional replay [scenario](./README.md#how-to-run-multiple-traffic-replayer-scenarios). **Note**: This option should not be used for initial deployments | +| defaultFargateCpuArch | string | "X86_64", "ARM64" | Provide a default CPU architecture that should be used for all containers. Defaults to `X86_64` | | captureProxyESServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack | | captureProxyESExtraArgs | string | `"--suppressCaptureForHeaderMatch user-agent .*elastic-java/7.17.0.*"` | Extra arguments to provide to the Capture Proxy command. This includes available arguments specified by the [Capture Proxy](../../../TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java). | | migrationConsoleServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack | diff --git a/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts b/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts new file mode 100644 index 000000000..ae2455f1a --- /dev/null +++ b/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts @@ -0,0 +1,27 @@ +import {CpuArchitecture} from "aws-cdk-lib/aws-ecs"; +import {validateFargateCpuArch} from "../lib/common-utilities"; + +test('Test valid fargate cpu arch strings can be parsed', () => { + const cpuArch1 = "arm64" + const detectedArch1 = validateFargateCpuArch(cpuArch1) + expect(detectedArch1).toEqual(CpuArchitecture.ARM64) + + const cpuArch2 = "ARM64" + const detectedArch2 = validateFargateCpuArch(cpuArch2) + expect(detectedArch2).toEqual(CpuArchitecture.ARM64) + + const cpuArch3 = "x86_64" + const detectedArch3 = validateFargateCpuArch(cpuArch3) + expect(detectedArch3).toEqual(CpuArchitecture.X86_64) + + const cpuArch4 = "X86_64" + const detectedArch4 = validateFargateCpuArch(cpuArch4) + expect(detectedArch4).toEqual(CpuArchitecture.X86_64) +}) + +test('Test invalid fargate cpu arch strings throws error', () => { + const cpuArch = "arm32" + const getArchFunction = () => validateFargateCpuArch(cpuArch) + expect(getArchFunction).toThrowError() +}) + diff --git a/deployment/cdk/opensearch-service-migration/test/default-values-test.json b/deployment/cdk/opensearch-service-migration/test/default-values-test.json index d3ddac369..53c4c8ec9 100644 --- a/deployment/cdk/opensearch-service-migration/test/default-values-test.json +++ b/deployment/cdk/opensearch-service-migration/test/default-values-test.json @@ -1,5 +1,6 @@ { "engineVersion": "OS_1.0", "domainName": "sample-cdk-unit-test-domain", - "dpPipelineTemplatePath": "./dp_pipeline_template.yaml" + "dpPipelineTemplatePath": "./dp_pipeline_template.yaml", + "defaultFargateCpuArch": "X86_64" } \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/test/fetch-migration-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/fetch-migration-stack.test.ts index aa2bba94e..911f420ff 100644 --- a/deployment/cdk/opensearch-service-migration/test/fetch-migration-stack.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/fetch-migration-stack.test.ts @@ -2,6 +2,7 @@ import {App} from 'aws-cdk-lib'; import {FetchMigrationStack} from "../lib/fetch-migration-stack"; import {Template} from "aws-cdk-lib/assertions"; import {NetworkStack} from "../lib/network-stack"; +import {CpuArchitecture} from "aws-cdk-lib/aws-ecs"; test('Test default fetch migration stack creates required resources', () => { const app = new App(); @@ -16,7 +17,8 @@ test('Test default fetch migration stack creates required resources', () => { dpPipelineTemplatePath: "./dp_pipeline_template.yaml", sourceEndpoint: "https://test-cluster", defaultDeployId: "default", - stage: "unit-test" + stage: "unit-test", + fargateCpuArch: CpuArchitecture.X86_64, }) From 42180aaa59d373bf5323a4408ef5cb5cb1badc81 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 23 Feb 2024 13:59:36 -0500 Subject: [PATCH 02/16] Github e2e action test Signed-off-by: Tanner Lewis --- TrafficCapture/dockerSolution/build.gradle | 2 +- .../src/main/docker/docker-compose.yml | 70 +++++++++---------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle index 672ecc89c..0e8838d9f 100644 --- a/TrafficCapture/dockerSolution/build.gradle +++ b/TrafficCapture/dockerSolution/build.gradle @@ -66,7 +66,7 @@ dockerCompose { ['src/main/docker/docker-compose.yml', "${extensionsDir}/" + (project.hasProperty("otel-collector") ? "${project.getProperty('otel-collector')}" : - "otel-prometheus-jaeger-opensearch.yml"), + "otel-image-default-config.yml"), "${extensionsDir}" + (project.hasProperty("multiProxy") ? "proxy-multi.yml" : "proxy-single.yml") ] } diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index bff5b3c42..41a0ff5d6 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -2,43 +2,43 @@ version: '3.7' services: - prometheus: - container_name: prometheus - image: prom/prometheus:latest - networks: - - migrations - volumes: - - ./prometheus.yaml:/etc/prometheus/prometheus.yml - ports: - - "9090:9090" - command: - - '--config.file=/etc/prometheus/prometheus.yml' - - '--enable-feature=exemplar-storage' +# prometheus: +# container_name: prometheus +# image: prom/prometheus:latest +# networks: +# - migrations +# volumes: +# - ./prometheus.yaml:/etc/prometheus/prometheus.yml +# ports: +# - "9090:9090" +# command: +# - '--config.file=/etc/prometheus/prometheus.yml' +# - '--enable-feature=exemplar-storage' - # Jaeger - jaeger: - image: jaegertracing/all-in-one:latest - networks: - - migrations - ports: - - "16686:16686" - - "4317" - - "4318" - environment: - - COLLECTOR_OTLP_ENABLED=true +# # Jaeger +# jaeger: +# image: jaegertracing/all-in-one:latest +# networks: +# - migrations +# ports: +# - "16686:16686" +# - "4317" +# - "4318" +# environment: +# - COLLECTOR_OTLP_ENABLED=true - grafana: - image: grafana/grafana:latest - networks: - - migrations - ports: - - "3000:3000" - volumes: - - ./grafana_data:/var/lib/grafana - environment: - - GF_SECURITY_ADMIN_PASSWORD=admin - depends_on: - - prometheus +# grafana: +# image: grafana/grafana:latest +# networks: +# - migrations +# ports: +# - "3000:3000" +# volumes: +# - ./grafana_data:/var/lib/grafana +# environment: +# - GF_SECURITY_ADMIN_PASSWORD=admin +# depends_on: +# - prometheus zookeeper: image: docker.io/bitnami/zookeeper:3.8 From 8336d4db38c705f5cbcd9d5b3b5e12c725e1a0b9 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 23 Feb 2024 14:07:35 -0500 Subject: [PATCH 03/16] OS version test Signed-off-by: Tanner Lewis --- .../dockerSolution/src/main/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 41a0ff5d6..9a7c85b3b 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -85,7 +85,7 @@ services: command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --otelCollectorEndpoint http://otel-collector:4317" opensearchtarget: - image: 'opensearchproject/opensearch:latest' + image: 'opensearchproject/opensearch:2.9.0' environment: - discovery.type=single-node networks: From b3c536d3ac644eb5b06392a3fdfa0bcb26d83488 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 23 Feb 2024 14:46:12 -0500 Subject: [PATCH 04/16] OS version test2 Signed-off-by: Tanner Lewis --- .../dockerSolution/src/main/docker/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 9a7c85b3b..c27e2e263 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -68,6 +68,7 @@ services: - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 depends_on: - zookeeper + - opensearchtarget replayer: image: 'migrations/traffic_replayer:latest' @@ -85,7 +86,7 @@ services: command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --otelCollectorEndpoint http://otel-collector:4317" opensearchtarget: - image: 'opensearchproject/opensearch:2.9.0' + image: 'opensearchproject/opensearch:1.3.12' environment: - discovery.type=single-node networks: From 274c710f7852474fb2ab76eb78e4cd8fd345e8d0 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 23 Feb 2024 14:54:59 -0500 Subject: [PATCH 05/16] OS version test3 Signed-off-by: Tanner Lewis --- .../dockerSolution/src/main/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index c27e2e263..153c9ba57 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -86,7 +86,7 @@ services: command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --otelCollectorEndpoint http://otel-collector:4317" opensearchtarget: - image: 'opensearchproject/opensearch:1.3.12' + image: 'opensearchproject/opensearch:latest' environment: - discovery.type=single-node networks: From aca0900e41fcb4d120bca62b35c4648edbb59fc1 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 23 Feb 2024 15:08:26 -0500 Subject: [PATCH 06/16] OS version test4 Signed-off-by: Tanner Lewis --- .../src/main/docker/docker-compose.yml | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 153c9ba57..654c63997 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -89,33 +89,34 @@ services: image: 'opensearchproject/opensearch:latest' environment: - discovery.type=single-node + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin networks: - migrations ports: - "29200:9200" - opensearchanalytics: - image: 'opensearchproject/opensearch:latest' - environment: - - discovery.type=single-node - networks: - - migrations - ports: - - "39200:9200" +# opensearchanalytics: +# image: 'opensearchproject/opensearch:latest' +# environment: +# - discovery.type=single-node +# networks: +# - migrations +# ports: +# - "39200:9200" - opensearch-dashboards: - image: migrations/opensearch_dashboards:latest # Make sure the version of opensearch-dashboards matches the version of opensearch installed on other nodes - container_name: opensearch-dashboards - ports: - - "5601:5601" # Map host port 5601 to container port 5601 - expose: - - "5601" # Expose port 5601 for web access to OpenSearch Dashboards - networks: - - migrations - volumes: - - ./opensearchDashboards/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml - depends_on: - - opensearchanalytics +# opensearch-dashboards: +# image: migrations/opensearch_dashboards:latest # Make sure the version of opensearch-dashboards matches the version of opensearch installed on other nodes +# container_name: opensearch-dashboards +# ports: +# - "5601:5601" # Map host port 5601 to container port 5601 +# expose: +# - "5601" # Expose port 5601 for web access to OpenSearch Dashboards +# networks: +# - migrations +# volumes: +# - ./opensearchDashboards/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml +# depends_on: +# - opensearchanalytics # otel-collector: # image: public.ecr.aws/a0w2c5q7/otelcol-with-opensearch:latest From f0703e9a0a1ab18d782f94e59d290e46f6d714d2 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 23 Feb 2024 15:48:18 -0500 Subject: [PATCH 07/16] OS version test5 Signed-off-by: Tanner Lewis --- TrafficCapture/dockerSolution/build.gradle | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle index 0e8838d9f..6e6e0c2e9 100644 --- a/TrafficCapture/dockerSolution/build.gradle +++ b/TrafficCapture/dockerSolution/build.gradle @@ -62,13 +62,9 @@ javaContainerServices.forEach { projectName, dockerImageName -> dockerCompose { def extensionsDir = "src/main/docker/composeExtensions/" - useComposeFiles = - ['src/main/docker/docker-compose.yml', - "${extensionsDir}/" + (project.hasProperty("otel-collector") ? - "${project.getProperty('otel-collector')}" : - "otel-image-default-config.yml"), - "${extensionsDir}" + (project.hasProperty("multiProxy") ? "proxy-multi.yml" : "proxy-single.yml") - ] + useComposeFiles = project.hasProperty('multiProxy') ? + ['src/main/docker/docker-compose.yml', 'src/main/docker/docker-compose-multi.yml'] : + ['src/main/docker/docker-compose.yml', "${extensionsDir}" + "proxy-single.yml"] } task buildDockerImages { From b5465a382b7deed1ccf142085665db14af1e3e32 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 23 Feb 2024 16:54:30 -0500 Subject: [PATCH 08/16] Change OS version Signed-off-by: Tanner Lewis --- .../dockerSolution/src/main/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 654c63997..f70bb2ac9 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -86,7 +86,7 @@ services: command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --otelCollectorEndpoint http://otel-collector:4317" opensearchtarget: - image: 'opensearchproject/opensearch:latest' + image: 'opensearchproject/opensearch:2.2.1' environment: - discovery.type=single-node - OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin From f39e1e41c32239b1826ed05abf5b6f1f59795f80 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 23 Feb 2024 17:13:49 -0500 Subject: [PATCH 09/16] Change OS version2 Signed-off-by: Tanner Lewis --- .../dockerSolution/src/main/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index f70bb2ac9..d5ad1c59b 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -86,7 +86,7 @@ services: command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --otelCollectorEndpoint http://otel-collector:4317" opensearchtarget: - image: 'opensearchproject/opensearch:2.2.1' + image: 'opensearchproject/opensearch:2.8.0' environment: - discovery.type=single-node - OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin From b3844739283d03bbb6bad4d2e4fa2a44a241fa45 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 23 Feb 2024 17:22:53 -0500 Subject: [PATCH 10/16] Change OS version3 Signed-off-by: Tanner Lewis --- .../dockerSolution/src/main/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index d5ad1c59b..5975d3a86 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -86,7 +86,7 @@ services: command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --otelCollectorEndpoint http://otel-collector:4317" opensearchtarget: - image: 'opensearchproject/opensearch:2.8.0' + image: 'opensearchproject/opensearch:2.11.0' environment: - discovery.type=single-node - OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin From b1d5ce5e1cfa65d32108bd7d2e7a17a99ff88908 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 23 Feb 2024 17:32:27 -0500 Subject: [PATCH 11/16] Change OS version4 Signed-off-by: Tanner Lewis --- .../dockerSolution/src/main/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 5975d3a86..08014b68f 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -86,7 +86,7 @@ services: command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --otelCollectorEndpoint http://otel-collector:4317" opensearchtarget: - image: 'opensearchproject/opensearch:2.11.0' + image: 'opensearchproject/opensearch:2.12.0' environment: - discovery.type=single-node - OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin From efe8379d8ae85a62107d3f277adfd627f6003192 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Fri, 23 Feb 2024 17:59:06 -0500 Subject: [PATCH 12/16] Update os target env variable pass Signed-off-by: Tanner Lewis --- .../dockerSolution/src/main/docker/docker-compose.yml | 4 ++-- test/conftest.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 08014b68f..e029ff9df 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -86,10 +86,10 @@ services: command: /bin/sh -c "/runJavaWithClasspath.sh org.opensearch.migrations.replay.TrafficReplayer https://opensearchtarget:9200 --auth-header-value Basic\\ YWRtaW46YWRtaW4= --insecure --kafka-traffic-brokers kafka:9092 --kafka-traffic-topic logging-traffic-topic --kafka-traffic-group-id default-logging-group --otelCollectorEndpoint http://otel-collector:4317" opensearchtarget: - image: 'opensearchproject/opensearch:2.12.0' + image: 'opensearchproject/opensearch:latest' environment: - discovery.type=single-node - - OPENSEARCH_INITIAL_ADMIN_PASSWORD=admin + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=DemoPass123! networks: - migrations ports: diff --git a/test/conftest.py b/test/conftest.py index 7a558b544..e2d98d053 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -26,7 +26,7 @@ def pytest_addoption(parser): parser.addoption("--source_username", action="store", default="admin") parser.addoption("--source_password", action="store", default="admin") parser.addoption("--target_username", action="store", default="admin") - parser.addoption("--target_password", action="store", default="admin") + parser.addoption("--target_password", action="store", default="DemoPass123!") parser.addoption("--unique_id", action="store", default=uuid.uuid4().hex) From 12a96e7a35f941addc9185f6f6b28383429ee602 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Mon, 26 Feb 2024 10:43:11 -0500 Subject: [PATCH 13/16] Reset docker testing values Signed-off-by: Tanner Lewis --- TrafficCapture/dockerSolution/build.gradle | 10 +- .../src/main/docker/docker-compose.yml | 114 +++++++++--------- 2 files changed, 63 insertions(+), 61 deletions(-) diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle index 6e6e0c2e9..d0885e887 100644 --- a/TrafficCapture/dockerSolution/build.gradle +++ b/TrafficCapture/dockerSolution/build.gradle @@ -62,9 +62,13 @@ javaContainerServices.forEach { projectName, dockerImageName -> dockerCompose { def extensionsDir = "src/main/docker/composeExtensions/" - useComposeFiles = project.hasProperty('multiProxy') ? - ['src/main/docker/docker-compose.yml', 'src/main/docker/docker-compose-multi.yml'] : - ['src/main/docker/docker-compose.yml', "${extensionsDir}" + "proxy-single.yml"] + useComposeFiles = + ['src/main/docker/docker-compose.yml', + "${extensionsDir}/" + (project.hasProperty("otel-collector") ? + "${project.getProperty('otel-collector')}" : + "otel-prometheus-jaeger-opensearch.yml"), + "${extensionsDir}" + (project.hasProperty("multiProxy") ? "proxy-multi.yml" : "proxy-single.yml") + ] } task buildDockerImages { diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index e029ff9df..bff5b3c42 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -2,43 +2,43 @@ version: '3.7' services: -# prometheus: -# container_name: prometheus -# image: prom/prometheus:latest -# networks: -# - migrations -# volumes: -# - ./prometheus.yaml:/etc/prometheus/prometheus.yml -# ports: -# - "9090:9090" -# command: -# - '--config.file=/etc/prometheus/prometheus.yml' -# - '--enable-feature=exemplar-storage' + prometheus: + container_name: prometheus + image: prom/prometheus:latest + networks: + - migrations + volumes: + - ./prometheus.yaml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--enable-feature=exemplar-storage' -# # Jaeger -# jaeger: -# image: jaegertracing/all-in-one:latest -# networks: -# - migrations -# ports: -# - "16686:16686" -# - "4317" -# - "4318" -# environment: -# - COLLECTOR_OTLP_ENABLED=true + # Jaeger + jaeger: + image: jaegertracing/all-in-one:latest + networks: + - migrations + ports: + - "16686:16686" + - "4317" + - "4318" + environment: + - COLLECTOR_OTLP_ENABLED=true -# grafana: -# image: grafana/grafana:latest -# networks: -# - migrations -# ports: -# - "3000:3000" -# volumes: -# - ./grafana_data:/var/lib/grafana -# environment: -# - GF_SECURITY_ADMIN_PASSWORD=admin -# depends_on: -# - prometheus + grafana: + image: grafana/grafana:latest + networks: + - migrations + ports: + - "3000:3000" + volumes: + - ./grafana_data:/var/lib/grafana + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin + depends_on: + - prometheus zookeeper: image: docker.io/bitnami/zookeeper:3.8 @@ -68,7 +68,6 @@ services: - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092 depends_on: - zookeeper - - opensearchtarget replayer: image: 'migrations/traffic_replayer:latest' @@ -89,34 +88,33 @@ services: image: 'opensearchproject/opensearch:latest' environment: - discovery.type=single-node - - OPENSEARCH_INITIAL_ADMIN_PASSWORD=DemoPass123! networks: - migrations ports: - "29200:9200" -# opensearchanalytics: -# image: 'opensearchproject/opensearch:latest' -# environment: -# - discovery.type=single-node -# networks: -# - migrations -# ports: -# - "39200:9200" + opensearchanalytics: + image: 'opensearchproject/opensearch:latest' + environment: + - discovery.type=single-node + networks: + - migrations + ports: + - "39200:9200" -# opensearch-dashboards: -# image: migrations/opensearch_dashboards:latest # Make sure the version of opensearch-dashboards matches the version of opensearch installed on other nodes -# container_name: opensearch-dashboards -# ports: -# - "5601:5601" # Map host port 5601 to container port 5601 -# expose: -# - "5601" # Expose port 5601 for web access to OpenSearch Dashboards -# networks: -# - migrations -# volumes: -# - ./opensearchDashboards/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml -# depends_on: -# - opensearchanalytics + opensearch-dashboards: + image: migrations/opensearch_dashboards:latest # Make sure the version of opensearch-dashboards matches the version of opensearch installed on other nodes + container_name: opensearch-dashboards + ports: + - "5601:5601" # Map host port 5601 to container port 5601 + expose: + - "5601" # Expose port 5601 for web access to OpenSearch Dashboards + networks: + - migrations + volumes: + - ./opensearchDashboards/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml + depends_on: + - opensearchanalytics # otel-collector: # image: public.ecr.aws/a0w2c5q7/otelcol-with-opensearch:latest From dd98c8bf5ae2b54c5c480b79f123804506b25202 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Mon, 26 Feb 2024 12:27:13 -0500 Subject: [PATCH 14/16] Allow detecting process arch as a default Signed-off-by: Tanner Lewis --- .../default-values.json | 1 - .../lib/common-utilities.ts | 20 ++++++++++++------- .../opensearch-service-migration/options.md | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/deployment/cdk/opensearch-service-migration/default-values.json b/deployment/cdk/opensearch-service-migration/default-values.json index 069676cf7..f7c2caeb9 100644 --- a/deployment/cdk/opensearch-service-migration/default-values.json +++ b/deployment/cdk/opensearch-service-migration/default-values.json @@ -6,7 +6,6 @@ "nodeToNodeEncryptionEnabled": true, "encryptionAtRestEnabled": true, "vpcEnabled": true, - "defaultFargateCpuArch": "X86_64", "migrationAssistanceEnabled": true, "migrationConsoleServiceEnabled": true, "trafficReplayerServiceEnabled": true, diff --git a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts index 45864b7ce..62512fcd8 100644 --- a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts +++ b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts @@ -123,14 +123,20 @@ export function createDefaultECSTaskRole(scope: Construct, serviceName: string): return serviceTaskRole } -export function validateFargateCpuArch(cpuArch: string): CpuArchitecture { - if (cpuArch.toUpperCase() === "X86_64") { +export function validateFargateCpuArch(cpuArch?: string): CpuArchitecture { + const desiredArch = cpuArch ? cpuArch : process.arch + const desiredArchUpper = desiredArch.toUpperCase() + + if (desiredArchUpper === "X86_64" || desiredArchUpper === "X64") { return CpuArchitecture.X86_64 - } - else if (cpuArch.toUpperCase() === "ARM64") { + } else if (desiredArchUpper === "ARM64") { return CpuArchitecture.ARM64 - } - else { - throw new Error(`Unknown Fargate cpu architecture provided: ${cpuArch}`) + } else { + if (cpuArch) { + throw new Error(`Unknown Fargate cpu architecture provided: ${desiredArch}`) + } + else { + throw new Error(`Unsupported process cpu architecture detected: ${desiredArch}, CDK requires X64 or ARM64 for Docker image compatability`) + } } } \ No newline at end of file diff --git a/deployment/cdk/opensearch-service-migration/options.md b/deployment/cdk/opensearch-service-migration/options.md index 7f9ecdc2c..68a8c847d 100644 --- a/deployment/cdk/opensearch-service-migration/options.md +++ b/deployment/cdk/opensearch-service-migration/options.md @@ -7,7 +7,7 @@ These tables list all CDK context configuration values a user can specify for th |--------------------------------------|---------|------------------------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | migrationAssistanceEnabled | boolean | true | Flag which controls deploying common Migration Service resources such as MSK, EFS, and an ECS cluster. **Note**: This option must be enabled to use any Migration service | | addOnMigrationDeployId | string | "cit2-replay" | Specify an ID string to use for an additional replay [scenario](./README.md#how-to-run-multiple-traffic-replayer-scenarios). **Note**: This option should not be used for initial deployments | -| defaultFargateCpuArch | string | "X86_64", "ARM64" | Provide a default CPU architecture that should be used for all containers. Defaults to `X86_64` | +| defaultFargateCpuArch | string | "X86_64", "ARM64" | Provide a default CPU architecture that should be used for all containers. Defaults to using `process.arch` to determine the proper architecture to use | | captureProxyESServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack | | captureProxyESExtraArgs | string | `"--suppressCaptureForHeaderMatch user-agent .*elastic-java/7.17.0.*"` | Extra arguments to provide to the Capture Proxy command. This includes available arguments specified by the [Capture Proxy](../../../TrafficCapture/trafficCaptureProxyServer/src/main/java/org/opensearch/migrations/trafficcapture/proxyserver/CaptureProxy.java). | | migrationConsoleServiceEnabled | boolean | true | Enable deploying the given service, via a new CloudFormation stack | From 42158fe5d1b0bd8f9f7308e050d8bb4696ff0bfc Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Mon, 26 Feb 2024 12:33:04 -0500 Subject: [PATCH 15/16] Remove testing target pass change Signed-off-by: Tanner Lewis --- test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index e2d98d053..7a558b544 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -26,7 +26,7 @@ def pytest_addoption(parser): parser.addoption("--source_username", action="store", default="admin") parser.addoption("--source_password", action="store", default="admin") parser.addoption("--target_username", action="store", default="admin") - parser.addoption("--target_password", action="store", default="DemoPass123!") + parser.addoption("--target_password", action="store", default="admin") parser.addoption("--unique_id", action="store", default=uuid.uuid4().hex) From 3c1e08f3c19587652dbe7f1e525fb653cf7925f2 Mon Sep 17 00:00:00 2001 From: Tanner Lewis Date: Mon, 26 Feb 2024 15:56:51 -0500 Subject: [PATCH 16/16] Add basic test for detecting cpu arch Signed-off-by: Tanner Lewis --- .../test/common-utilities.test.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts b/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts index ae2455f1a..cfbf2a8be 100644 --- a/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts @@ -25,3 +25,11 @@ test('Test invalid fargate cpu arch strings throws error', () => { expect(getArchFunction).toThrowError() }) +test('Test detected fargate cpu arch is valid', () => { + const detectedArch = process.arch + const detectedArchUpper = detectedArch.toUpperCase() + + const expectedCpuArch = detectedArchUpper === "X64" ? CpuArchitecture.X86_64 : CpuArchitecture.ARM64 + const cpuArch = validateFargateCpuArch() + expect(cpuArch).toEqual(expectedCpuArch) +}) \ No newline at end of file