MIGRATIONS-1519: Add support for providing Fargate CPU arch in CDK (#520

) * MIGRATIONS-1519: Add support for providing CPU arch Signed-off-by: Tanner Lewis <lewijacn@amazon.com>
opensearch-project · Feb 26, 2024 · d8763bb · d8763bb
1 parent f18c4bf
commit d8763bb
Show file tree

Hide file tree

Showing 18 changed files with 120 additions and 18 deletions.
diff --git a/TrafficCapture/dockerSolution/build.gradle b/TrafficCapture/dockerSolution/build.gradle
@@ -68,7 +68,7 @@ dockerCompose {
                      "${project.getProperty('otel-collector')}" :
                      "otel-prometheus-jaeger-opensearch.yml"),
              "${extensionsDir}" + (project.hasProperty("multiProxy") ? "proxy-multi.yml" : "proxy-single.yml")
-                     ]
+            ]
 }
 
 task buildDockerImages {

diff --git a/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts b/deployment/cdk/opensearch-service-migration/lib/common-utilities.ts
@@ -1,6 +1,7 @@
 import {Effect, PolicyStatement, Role, ServicePrincipal} from "aws-cdk-lib/aws-iam";
 import {Construct} from "constructs";
 import {StringParameter} from "aws-cdk-lib/aws-ssm";
+import {CpuArchitecture} from "aws-cdk-lib/aws-ecs";
 
 export function createOpenSearchIAMAccessPolicy(region: string, accountId: string): PolicyStatement {
     return new PolicyStatement({
@@ -120,4 +121,22 @@ export function createDefaultECSTaskRole(scope: Construct, serviceName: string):
         ]
     }))
     return serviceTaskRole
+}
+
+export function validateFargateCpuArch(cpuArch?: string): CpuArchitecture {
+    const desiredArch = cpuArch ? cpuArch : process.arch
+    const desiredArchUpper = desiredArch.toUpperCase()
+
+    if (desiredArchUpper === "X86_64" || desiredArchUpper === "X64") {
+        return CpuArchitecture.X86_64
+    } else if (desiredArchUpper === "ARM64") {
+        return CpuArchitecture.ARM64
+    } else {
+        if (cpuArch) {
+            throw new Error(`Unknown Fargate cpu architecture provided: ${desiredArch}`)
+        }
+        else {
+            throw new Error(`Unsupported process cpu architecture detected: ${desiredArch}, CDK requires X64 or ARM64 for Docker image compatability`)
+        }
+    }
 }
diff --git a/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts b/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts
@@ -3,9 +3,9 @@ import {IVpc} from "aws-cdk-lib/aws-ec2";
 import {Construct} from "constructs";
 import {
     Cluster,
-    ContainerImage,
+    ContainerImage, CpuArchitecture,
     FargateTaskDefinition,
-    LogDrivers,
+    LogDrivers, OperatingSystemFamily,
     Secret as ECSSecret
 } from "aws-cdk-lib/aws-ecs";
 import {Secret as SMSecret} from "aws-cdk-lib/aws-secretsmanager";
@@ -22,7 +22,8 @@ import {
 export interface FetchMigrationProps extends StackPropsExt {
     readonly vpc: IVpc,
     readonly dpPipelineTemplatePath: string,
-    readonly sourceEndpoint: string
+    readonly sourceEndpoint: string,
+    readonly fargateCpuArch: CpuArchitecture
 }
 
 export class FetchMigrationStack extends Stack {
@@ -49,6 +50,10 @@ export class FetchMigrationStack extends Stack {
         ecsTaskRole.addToPolicy(openSearchServerlessPolicy)
         // ECS Task Definition
         const fetchMigrationFargateTask = new FargateTaskDefinition(this, "fetchMigrationFargateTask", {
+            runtimePlatform: {
+                operatingSystemFamily: OperatingSystemFamily.LINUX,
+                cpuArchitecture: props.fargateCpuArch
+            },
             family: `migration-${props.stage}-${serviceName}`,
             memoryLimitMiB: 8192,
             cpu: 2048,

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-es-stack.ts
@@ -1,6 +1,6 @@
 import {StackPropsExt} from "../stack-composer";
 import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
-import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs";
+import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
 import {join} from "path";
 import {MigrationServiceCore} from "./migration-service-core";
@@ -14,6 +14,7 @@ export interface CaptureProxyESProps extends StackPropsExt {
     readonly vpc: IVpc,
     readonly streamingSourceType: StreamingSourceType,
     readonly analyticsServiceEnabled: boolean,
+    readonly fargateCpuArch: CpuArchitecture,
     readonly extraArgs?: string,
 }
 
@@ -75,6 +76,7 @@ export class CaptureProxyESStack extends MigrationServiceCore {
             serviceConnectServices: [serviceConnectService, esServiceConnectService],
             serviceDiscoveryEnabled: true,
             serviceDiscoveryPort: 19200,
+            cpuArchitecture: props.fargateCpuArch,
             taskCpuUnits: 1024,
             taskMemoryLimitMiB: 4096,
             ...props

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/capture-proxy-stack.ts
@@ -1,6 +1,6 @@
 import {StackPropsExt} from "../stack-composer";
 import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
-import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs";
+import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
 import {join} from "path";
 import {MigrationServiceCore} from "./migration-service-core";
@@ -13,6 +13,7 @@ import {createMSKProducerIAMPolicies} from "../common-utilities";
 export interface CaptureProxyProps extends StackPropsExt {
     readonly vpc: IVpc,
     readonly streamingSourceType: StreamingSourceType,
+    readonly fargateCpuArch: CpuArchitecture,
     readonly customSourceClusterEndpoint?: string,
     readonly analyticsServiceEnabled?: boolean,
     readonly extraArgs?: string,
@@ -60,6 +61,7 @@ export class CaptureProxyStack extends MigrationServiceCore {
             taskRolePolicies: servicePolicies,
             portMappings: [servicePort],
             serviceConnectServices: [serviceConnectService],
+            cpuArchitecture: props.fargateCpuArch,
             taskCpuUnits: 512,
             taskMemoryLimitMiB: 2048,
             ...props

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/elasticsearch-stack.ts
@@ -1,6 +1,6 @@
 import {StackPropsExt} from "../stack-composer";
 import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
-import {PortMapping, Protocol} from "aws-cdk-lib/aws-ecs";
+import {CpuArchitecture, PortMapping, Protocol} from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
 import {join} from "path";
 import {MigrationServiceCore} from "./migration-service-core";
@@ -10,6 +10,7 @@ import {StringParameter} from "aws-cdk-lib/aws-ssm";
 
 export interface ElasticsearchProps extends StackPropsExt {
     readonly vpc: IVpc,
+    readonly fargateCpuArch: CpuArchitecture
 }
 
 /**
@@ -45,6 +46,7 @@ export class ElasticsearchStack extends MigrationServiceCore {
             serviceConnectServices: [serviceConnectService],
             serviceDiscoveryEnabled: true,
             serviceDiscoveryPort: 9200,
+            cpuArchitecture: props.fargateCpuArch,
             taskCpuUnits: 512,
             taskMemoryLimitMiB: 2048,
             ...props

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-broker-stack.ts
@@ -1,6 +1,7 @@
 import {StackPropsExt} from "../stack-composer";
 import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
 import {
+    CpuArchitecture,
     PortMapping, Protocol
 } from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
@@ -9,7 +10,8 @@ import {StringParameter} from "aws-cdk-lib/aws-ssm";
 import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service";
 
 export interface KafkaBrokerProps extends StackPropsExt {
-    readonly vpc: IVpc
+    readonly vpc: IVpc,
+    readonly fargateCpuArch: CpuArchitecture
 }
 
 /**
@@ -62,6 +64,7 @@ export class KafkaBrokerStack extends MigrationServiceCore {
             },
             portMappings: [servicePort],
             serviceConnectServices: [serviceConnectService],
+            cpuArchitecture: props.fargateCpuArch,
             taskCpuUnits: 256,
             taskMemoryLimitMiB: 2048,
             ...props

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/kafka-zookeeper-stack.ts
@@ -1,6 +1,7 @@
 import {StackPropsExt} from "../stack-composer";
 import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
 import {
+    CpuArchitecture,
     PortMapping, Protocol,
 } from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
@@ -9,7 +10,8 @@ import {StringParameter} from "aws-cdk-lib/aws-ssm";
 import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service";
 
 export interface KafkaZookeeperProps extends StackPropsExt {
-    readonly vpc: IVpc
+    readonly vpc: IVpc,
+    readonly fargateCpuArch: CpuArchitecture
 }
 
 /**
@@ -46,6 +48,7 @@ export class KafkaZookeeperStack extends MigrationServiceCore {
             },
             portMappings: [servicePort],
             serviceConnectServices: [serviceConnectService],
+            cpuArchitecture: props.fargateCpuArch,
             taskCpuUnits: 256,
             taskMemoryLimitMiB: 512,
             ...props

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-analytics-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-analytics-stack.ts
@@ -6,7 +6,7 @@ import {
   SecurityGroup,
   IVpc,
 } from "aws-cdk-lib/aws-ec2";
-import {PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs";
+import {CpuArchitecture, PortMapping, Protocol, ServiceConnectService} from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
 import {join} from "path";
 import {MigrationServiceCore} from "./migration-service-core";
@@ -15,6 +15,7 @@ import {createAwsDistroForOtelPushInstrumentationPolicy} from "../common-utiliti
 
 export interface MigrationAnalyticsProps extends StackPropsExt {
     readonly vpc: IVpc,
+    readonly fargateCpuArch: CpuArchitecture,
     readonly bastionHostEnabled?: boolean
 }
 
@@ -82,6 +83,7 @@ export class MigrationAnalyticsStack extends MigrationServiceCore {
             dockerDirectoryPath: join(__dirname, "../../../../../", "TrafficCapture/dockerSolution/src/main/docker/otelCollector"),
             dockerImageCommand: ["--config=/etc/otel-config-aws.yaml"],
             securityGroups: securityGroups,
+            cpuArchitecture: props.fargateCpuArch,
             taskCpuUnits: 1024,
             taskMemoryLimitMiB: 4096,
             portMappings: [otelCollectorPort, otelHealthCheckPort],

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts
@@ -1,6 +1,6 @@
 import {StackPropsExt} from "../stack-composer";
 import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
-import {MountPoint, Volume} from "aws-cdk-lib/aws-ecs";
+import {CpuArchitecture, MountPoint, Volume} from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
 import {join} from "path";
 import {MigrationServiceCore} from "./migration-service-core";
@@ -17,6 +17,7 @@ export interface MigrationConsoleProps extends StackPropsExt {
     readonly vpc: IVpc,
     readonly streamingSourceType: StreamingSourceType,
     readonly fetchMigrationEnabled: boolean,
+    readonly fargateCpuArch: CpuArchitecture,
     readonly migrationAnalyticsEnabled: boolean
 }
 
@@ -154,6 +155,7 @@ export class MigrationConsoleStack extends MigrationServiceCore {
             mountPoints: [replayerOutputMountPoint],
             environment: environment,
             taskRolePolicies: servicePolicies,
+            cpuArchitecture: props.fargateCpuArch,
             taskCpuUnits: 512,
             taskMemoryLimitMiB: 1024,
             ...props

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-service-core.ts
@@ -3,11 +3,11 @@ import {ISecurityGroup, IVpc, SubnetType} from "aws-cdk-lib/aws-ec2";
 import {
     CfnService as FargateCfnService,
     Cluster,
-    ContainerImage,
+    ContainerImage, CpuArchitecture,
     FargateService,
     FargateTaskDefinition,
     LogDrivers,
-    MountPoint,
+    MountPoint, OperatingSystemFamily,
     PortMapping, Ulimit,
     Volume
 } from "aws-cdk-lib/aws-ecs";
@@ -25,6 +25,7 @@ export interface MigrationServiceCoreProps extends StackPropsExt {
     readonly serviceName: string,
     readonly vpc: IVpc,
     readonly securityGroups: ISecurityGroup[],
+    readonly cpuArchitecture: CpuArchitecture,
     readonly dockerFilePath?: string,
     readonly dockerDirectoryPath?: string,
     readonly dockerImageRegistryName?: string,
@@ -87,6 +88,10 @@ export class MigrationServiceCore extends Stack {
 
         const serviceTaskDef = new FargateTaskDefinition(this, "ServiceTaskDef", {
             ephemeralStorageGiB: 75,
+            runtimePlatform: {
+                operatingSystemFamily: OperatingSystemFamily.LINUX,
+                cpuArchitecture: props.cpuArchitecture
+            },
             family: `migration-${props.stage}-${props.serviceName}`,
             memoryLimitMiB: props.taskMemoryLimitMiB ? props.taskMemoryLimitMiB : 1024,
             cpu: props.taskCpuUnits ? props.taskCpuUnits : 256,

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/opensearch-container-stack.ts
@@ -1,13 +1,14 @@
 import {StackPropsExt} from "../stack-composer";
 import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
-import {PortMapping, Protocol, Ulimit, UlimitName} from "aws-cdk-lib/aws-ecs";
+import {CpuArchitecture, PortMapping, Protocol, Ulimit, UlimitName} from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
 import {MigrationServiceCore} from "./migration-service-core";
 import {StringParameter} from "aws-cdk-lib/aws-ssm";
 import {ServiceConnectService} from "aws-cdk-lib/aws-ecs/lib/base/base-service";
 
 export interface OpenSearchContainerProps extends StackPropsExt {
-    readonly vpc: IVpc
+    readonly vpc: IVpc,
+    readonly fargateCpuArch: CpuArchitecture
 }
 
 /**
@@ -61,6 +62,7 @@ export class OpenSearchContainerStack extends MigrationServiceCore {
             serviceConnectServices: [serviceConnectService],
             taskCpuUnits: 1024,
             taskMemoryLimitMiB: 4096,
+            cpuArchitecture: props.fargateCpuArch,
             ulimits: ulimits,
             ...props
         });

diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/traffic-replayer-stack.ts
@@ -1,6 +1,6 @@
 import {StackPropsExt} from "../stack-composer";
 import {IVpc, SecurityGroup} from "aws-cdk-lib/aws-ec2";
-import {MountPoint, Volume} from "aws-cdk-lib/aws-ecs";
+import {CpuArchitecture, MountPoint, Volume} from "aws-cdk-lib/aws-ecs";
 import {Construct} from "constructs";
 import {join} from "path";
 import {MigrationServiceCore} from "./migration-service-core";
@@ -18,6 +18,7 @@ export interface TrafficReplayerProps extends StackPropsExt {
     readonly vpc: IVpc,
     readonly enableClusterFGACAuth: boolean,
     readonly streamingSourceType: StreamingSourceType,
+    readonly fargateCpuArch: CpuArchitecture,
     readonly addOnMigrationId?: string,
     readonly customKafkaGroupId?: string,
     readonly userAgentSuffix?: string,
@@ -102,6 +103,7 @@ export class TrafficReplayerStack extends MigrationServiceCore {
             environment: {
                 "TUPLE_DIR_PATH": `/shared-replayer-output/traffic-replayer-${deployId}`
             },
+            cpuArchitecture: props.fargateCpuArch,
             taskCpuUnits: 1024,
             taskMemoryLimitMiB: 4096,
             ...props