From 90eb5853684e2e4c5a3d7a961b7ceeeb5a9ccef2 Mon Sep 17 00:00:00 2001
From: Tanner Lewis <lewijacn@amazon.com>
Date: Fri, 10 May 2024 09:44:21 -0400
Subject: [PATCH 1/5] Get basic working e2e script with context from file

Signed-off-by: Tanner Lewis <lewijacn@amazon.com>
---
 .gitignore                                    |   1 +
 .../lib/stack-composer.ts                     |  37 ++++--
 test/awsE2ESolutionSetup.sh                   | 116 ++++++++++++------
 test/defaultCDKContext.json                   |  37 ++++++
 4 files changed, 143 insertions(+), 48 deletions(-)
 create mode 100644 test/defaultCDKContext.json

diff --git a/.gitignore b/.gitignore
index 58319b627..92ee33c81 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@ __pycache__
 .python-version
 logs
 .vscode/
+tmp/
 
 # Build files
 plugins/opensearch/loggable-transport-netty4/.gradle/
diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts
index 09edc69d1..6033b5e8c 100644
--- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts
+++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts
@@ -1,5 +1,6 @@
 import {Construct} from "constructs";
 import {Duration, Stack, StackProps} from "aws-cdk-lib";
+import {readFileSync} from 'fs';
 import {OpenSearchDomainStack} from "./opensearch-domain-stack";
 import {EngineVersion, TLSSecurityPolicy} from "aws-cdk-lib/aws-opensearchservice";
 import * as defaultValuesJson from "../default-values.json"
@@ -98,6 +99,31 @@ export class StackComposer {
         }
     }
 
+    private parseContextBlock(scope: Construct, contextId: string) {
+        const contextFile = scope.node.tryGetContext("contextFile")
+        if (contextFile) {
+            const fileString = readFileSync(contextFile, 'utf-8');
+            const fileJSON = JSON.parse(fileString)
+            const contextBlock = fileJSON[contextId]
+            if (!contextBlock) {
+                throw new Error(`No CDK context block found for contextId '${contextId}' in file ${contextFile}`)
+            }
+            return contextBlock
+        }
+
+        let contextJSON = scope.node.tryGetContext(contextId)
+        if (!contextJSON) {
+            throw new Error(`No CDK context block found for contextId '${contextId}'`)
+        }
+        // For a context block to be provided as a string (as in the case of providing via command line) it will need to be properly escaped
+        // to be captured. This requires JSON to parse twice, 1. Returns a normal JSON string with no escaping 2. Returns a JSON object for use
+        if (typeof contextJSON === 'string') {
+            contextJSON = JSON.parse(JSON.parse(contextJSON))
+        }
+        return contextJSON
+
+    }
+
     constructor(scope: Construct, props: StackComposerProps) {
 
         const defaultValues: { [x: string]: (any); } = defaultValuesJson
@@ -108,18 +134,11 @@ export class StackComposer {
         if (!contextId) {
             throw new Error("Required context field 'contextId' not provided")
         }
-        let contextJSON = scope.node.tryGetContext(contextId)
-        if (!contextJSON) {
-            throw new Error(`No CDK context block found for contextId '${contextId}'`)
-        }
+        const contextJSON = this.parseContextBlock(scope, contextId)
         console.log('Received following context block for deployment: ')
         console.log(contextJSON)
         console.log('End of context block.')
-        // For a context block to be provided as a string (as in the case of providing via command line) it will need to be properly escaped
-        // to be captured. This requires JSON to parse twice, 1. Returns a normal JSON string with no escaping 2. Returns a JSON object for use
-        if (typeof contextJSON === 'string') {
-            contextJSON = JSON.parse(JSON.parse(contextJSON))
-        }
+
         const stage = this.getContextForType('stage', 'string', defaultValues, contextJSON)
 
         let version: EngineVersion
diff --git a/test/awsE2ESolutionSetup.sh b/test/awsE2ESolutionSetup.sh
index d48ab665f..247b8e786 100755
--- a/test/awsE2ESolutionSetup.sh
+++ b/test/awsE2ESolutionSetup.sh
@@ -3,6 +3,12 @@
 # Note: As this script will deploy an E2E solution in AWS, it assumes all the dependencies of the migration solution (e.g. aws cli, cdk cli,
 # configured aws credentials, git, java, docker) as well as 'jq'
 
+script_abs_path=$(readlink -f "$0")
+TEST_DIR_PATH=$(dirname "$script_abs_path")
+ROOT_REPO_PATH=$(dirname "$TEST_DIR_PATH")
+TMP_DIR_PATH="$TEST_DIR_PATH/tmp"
+EC2_SOURCE_CDK_PATH="$ROOT_REPO_PATH/test/opensearch-cluster-cdk"
+MIGRATION_CDK_PATH="$ROOT_REPO_PATH/deployment/cdk/opensearch-service-migration"
 
 # Note: This function is still in an experimental state
 # Modify EC2 nodes to add required Kafka security group if it doesn't exist, as well as call the ./startCaptureProxy.sh
@@ -48,7 +54,7 @@ prepare_source_nodes_for_capture () {
     sleep 10
     command_status=$(aws ssm get-command-invocation --command-id "$command_id" --instance-id "$id" --output text --query 'Status')
     # TODO for multi-node setups, we should collect all command ids and allow to run in parallel
-    while [ "$command_status" != "Success" ] && [ "$command_status" != "Failed" ]
+    while [ "$command_status" != "Success" ] && [ "$command_status" != "Failed" ] && [ "$command_status" != "TimedOut" ]
     do
       echo "Waiting for command to complete, current status is $command_status"
       sleep 60
@@ -56,9 +62,9 @@ prepare_source_nodes_for_capture () {
     done
     echo "Command has completed with status: $command_status, appending output"
     echo "Standard Output:"
-    aws ssm get-command-invocation --command-id "$command_id" --instance-id "$id" --output text --query 'StandardOutputContent'
+    aws --no-cli-pager ssm get-command-invocation --command-id "$command_id" --instance-id "$id" --output text --query 'StandardOutputContent'
     echo "Standard Error:"
-    aws ssm get-command-invocation --command-id "$command_id" --instance-id "$id" --output text --query 'StandardErrorContent'
+    aws --no-cli-pager ssm get-command-invocation --command-id "$command_id" --instance-id "$id" --output text --query 'StandardErrorContent'
   done
 }
 
@@ -83,10 +89,28 @@ create_service_linked_roles () {
   aws iam create-service-linked-role --aws-service-name ecs.amazonaws.com
 }
 
+clean_up_all () {
+  cdk_context_var=$1
+  vpc_id=$2
+  default_sg=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$vpc_id" "Name=group-name,Values=default" --query "SecurityGroups[*].GroupId" --output text)
+  instance_ids=($(aws ec2 describe-instances --filters 'Name=tag:Name,Values=opensearch-infra-stack*' 'Name=instance-state-name,Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text))
+  # Revert source cluster back to default SG to remove added SGs
+  for id in "${instance_ids[@]}"
+  do
+    echo "Removing security groups from node: $id"
+    aws ec2 modify-instance-attribute --instance-id $id --groups $default_sg
+  done
+
+  cd "$MIGRATION_CDK_PATH" || exit
+  cdk destroy "*" --force --c aws-existing-source=$cdk_context_var --c contextId=aws-existing-source
+  cd "$EC2_SOURCE_CDK_PATH" || exit
+  cdk destroy "*" --force --c suffix="Migration-Source" --c distVersion="7.10.2" --c distributionUrl="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.2-linux-x86_64.tar.gz" --c securityDisabled=true --c minDistribution=false --c cpuArch="x64" --c isInternal=true --c singleNodeCluster=false --c networkAvailabilityZones=2 --c dataNodeCount=2 --c managerNodeCount=0 --c serverAccessType="ipv4" --c restrictServerAccessTo="0.0.0.0/0" --c captureProxyEnabled=false
+}
+
 # One-time required CDK bootstrap setup for a given region. Only required if the 'CDKToolkit' CFN stack does not exist
 bootstrap_region () {
   # Picking arbitrary context values to satisfy required values for CDK synthesis. These should not need to be kept in sync with the actual deployment context values
-  cdk bootstrap "*" --require-approval never --c suffix="Migration-Source" --c distVersion="7.10.2" --c distributionUrl="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.2-linux-x86_64.tar.gz" --c securityDisabled=true --c minDistribution=false --c cpuArch="x64" --c isInternal=true --c singleNodeCluster=false --c networkAvailabilityZones=2 --c dataNodeCount=2 --c managerNodeCount=0 --c serverAccessType="ipv4" --c restrictServerAccessTo="0.0.0.0/0" --c captureProxyEnabled=false
+  cdk bootstrap --require-approval never --c suffix="Migration-Source" --c distVersion="7.10.2" --c distributionUrl="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.2-linux-x86_64.tar.gz" --c securityDisabled=true --c minDistribution=false --c cpuArch="x64" --c isInternal=true --c singleNodeCluster=false --c networkAvailabilityZones=2 --c dataNodeCount=2 --c managerNodeCount=0 --c serverAccessType="ipv4" --c restrictServerAccessTo="0.0.0.0/0" --c captureProxyEnabled=false
 }
 
 usage() {
@@ -98,6 +122,10 @@ usage() {
   echo "  ./awsE2ESolutionSetup.sh <>"
   echo ""
   echo "Options:"
+  echo "  --context-file                                   A file path for a given context file from which source and target context options will be used, default is './defaultCDKContext.json'."
+  echo "  --source-context-id                              The CDK context block identifier within the context-file to use, default is 'source-single-node-ec2'."
+  echo "  --migration-context-id                           The CDK context block identifier within the context-file to use, default is 'migration-default'."
+
   echo "  --migrations-git-url                             The Github http url used for building the capture proxy on setups with a dedicated source cluster, default is 'https://github.com/opensearch-project/opensearch-migrations.git'."
   echo "  --migrations-git-branch                          The Github branch associated with the 'git-url' to pull from, default is 'main'."
   echo "  --stage                                          The stage name to use for naming/grouping of AWS deployment resources, default is 'aws-integ'."
@@ -107,19 +135,24 @@ usage() {
   echo "  --skip-capture-proxy                             Flag to skip setting up the Capture Proxy on source cluster nodes"
   echo "  --skip-source-deploy                             Flag to skip deploying the EC2 source cluster"
   echo "  --skip-migration-deploy                          Flag to skip deploying the Migration solution"
+  echo "  --clean-up-all                                   Flag to remove all deployed CloudFormation resources"
   echo ""
   exit 1
 }
 
-STAGE='aws-integ'
+STAGE='script-test'
 RUN_POST_ACTIONS=false
 CREATE_SLR=false
 BOOTSTRAP_REGION=false
 SKIP_CAPTURE_PROXY=false
 SKIP_SOURCE_DEPLOY=false
 SKIP_MIGRATION_DEPLOY=false
+CONTEXT_FILE='./defaultCDKContext.json'
+SOURCE_CONTEXT_ID='source-single-node-ec2'
+MIGRATION_CONTEXT_ID='migration-default'
 MIGRATIONS_GIT_URL='https://github.com/opensearch-project/opensearch-migrations.git'
 MIGRATIONS_GIT_BRANCH='main'
+CLEAN_UP_ALL=false
 
 while [[ $# -gt 0 ]]; do
   case $1 in
@@ -147,6 +180,21 @@ while [[ $# -gt 0 ]]; do
       SKIP_MIGRATION_DEPLOY=true
       shift # past argument
       ;;
+    --context-file)
+      MIGRATIONS_GIT_URL="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --source-context-id)
+      MIGRATIONS_GIT_URL="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --migration-context-id)
+      MIGRATIONS_GIT_URL="$2"
+      shift # past argument
+      shift # past value
+      ;;
     --migrations-git-url)
       MIGRATIONS_GIT_URL="$2"
       shift # past argument
@@ -162,6 +210,10 @@ while [[ $# -gt 0 ]]; do
       shift # past argument
       shift # past value
       ;;
+    --clean-up-all)
+      CLEAN_UP_ALL=true
+      shift # past argument
+      ;;
     -h|--h|--help)
       usage
       ;;
@@ -184,68 +236,54 @@ if [ "$CREATE_SLR" = true ] ; then
   create_service_linked_roles
 fi
 
-# Store CDK context for migration solution deployment in variable
-read -r -d '' cdk_context << EOM
-{
-    "stage": "$STAGE",
-    "vpcId": "<VPC_ID>",
-    "engineVersion": "OS_2.11",
-    "domainName": "opensearch-cluster-aws-integ",
-    "dataNodeCount": 2,
-    "openAccessPolicyEnabled": true,
-    "domainRemovalPolicy": "DESTROY",
-    "artifactBucketRemovalPolicy": "DESTROY",
-    "trafficReplayerExtraArgs": "--speedup-factor 10.0",
-    "fetchMigrationEnabled": true,
-    "reindexFromSnapshotServiceEnabled": true,
-    "sourceClusterEndpoint": "<SOURCE_CLUSTER_ENDPOINT>",
-    "dpPipelineTemplatePath": "../../../test/dp_pipeline_aws_integ.yaml"
-}
-EOM
+mkdir -p "$TMP_DIR_PATH"
+cp $CONTEXT_FILE "$TMP_DIR_PATH/generatedCDKContext.json"
+sed -i -e "s/<STAGE>/$STAGE/g" "$TMP_DIR_PATH/generatedCDKContext.json"
 
 if [ ! -d "opensearch-cluster-cdk" ]; then
   git clone https://github.com/lewijacn/opensearch-cluster-cdk.git
 else
   echo "Repo already exists, skipping clone."
 fi
-cd opensearch-cluster-cdk && git checkout migration-es
-git pull
+cd opensearch-cluster-cdk && git pull && git checkout tanner-migration-testing
 npm install
 if [ "$BOOTSTRAP_REGION" = true ] ; then
   bootstrap_region
 fi
 
-if [ "$SKIP_SOURCE_DEPLOY" = false ] ; then
+if [ "$SKIP_SOURCE_DEPLOY" = false ] && [ "$CLEAN_UP_ALL" = false ] ; then
   # Deploy source cluster on EC2 instances
-  cdk deploy "*" --require-approval never --c suffix="Migration-Source" --c distVersion="7.10.2" --c distributionUrl="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.2-linux-x86_64.tar.gz" --c captureProxyEnabled=true --c securityDisabled=true --c minDistribution=false --c cpuArch="x64" --c isInternal=true --c singleNodeCluster=true --c networkAvailabilityZones=2 --c dataNodeCount=1 --c managerNodeCount=0 --c serverAccessType="ipv4" --c restrictServerAccessTo="0.0.0.0/0"
+  cdk deploy "*" --c contextFile="$TMP_DIR_PATH/generatedCDKContext.json" --c contextId="$SOURCE_CONTEXT_ID" --require-approval never
   if [ $? -ne 0 ]; then
     echo "Error: deploy source cluster failed, exiting."
     exit 1
   fi
 fi
 
-source_endpoint=$(aws cloudformation describe-stacks --stack-name opensearch-infra-stack-Migration-Source --query "Stacks[0].Outputs[?OutputKey==\`loadbalancerurl\`].OutputValue" --output text)
+source_endpoint=$(aws cloudformation describe-stacks --stack-name "opensearch-infra-stack-ec2-source-$STAGE" --query "Stacks[0].Outputs[?OutputKey==\`loadbalancerurl\`].OutputValue" --output text)
 echo "Source endpoint: $source_endpoint"
-vpc_id=$(aws cloudformation describe-stacks --stack-name opensearch-network-stack --query "Stacks[0].Outputs[?contains(OutputValue, 'vpc')].OutputValue" --output text)
+vpc_id=$(aws cloudformation describe-stacks --stack-name "opensearch-network-stack-ec2-source-$STAGE" --query "Stacks[0].Outputs[?contains(OutputValue, 'vpc')].OutputValue" --output text)
 echo "VPC ID: $vpc_id"
 
-if [ "$SKIP_MIGRATION_DEPLOY" = false ] ; then
-  cdk_context=$(echo "${cdk_context/<VPC_ID>/$vpc_id}")
-  cdk_context=$(echo "${cdk_context/<SOURCE_CLUSTER_ENDPOINT>/http://${source_endpoint}:9200}")
-  cdk_context=$(echo $cdk_context | jq '@json')
-  # TODO Further verify space escaping for JSON
-  # Escape spaces for CDK JSON parsing to handle
-  cdk_context=$(echo "${cdk_context/ /\\u0020}")
-  echo $cdk_context
+# Replace source specific placeholders
+sed -i -e "s/<VPC_ID>/$vpc_id/g" "$TMP_DIR_PATH/generatedCDKContext.json"
+sed -i -e "s/<SOURCE_CLUSTER_ENDPOINT>/$source_endpoint/g" "$TMP_DIR_PATH/generatedCDKContext.json"
 
-  cd ../../deployment/cdk/opensearch-service-migration || exit
+if [ "$CLEAN_UP_ALL" = true ] ; then
+  #TODO adjust for file
+  clean_up_all "$cdk_context" "$vpc_id"
+  exit 0
+fi
+
+if [ "$SKIP_MIGRATION_DEPLOY" = false ] ; then
+  cd "$MIGRATION_CDK_PATH" || exit
   ./buildDockerImages.sh
   if [ $? -ne 0 ]; then
     echo "Error: building docker images failed, exiting."
     exit 1
   fi
   npm install
-  cdk deploy "*" --c aws-existing-source=$cdk_context --c contextId=aws-existing-source --require-approval never --concurrency 3
+  cdk deploy "*" --c contextFile="$TMP_DIR_PATH/generatedCDKContext.json" --c contextId="$MIGRATION_CONTEXT_ID" --require-approval never --concurrency 3
   if [ $? -ne 0 ]; then
     echo "Error: deploying migration stacks failed, exiting."
     exit 1
diff --git a/test/defaultCDKContext.json b/test/defaultCDKContext.json
new file mode 100644
index 000000000..620b57d5f
--- /dev/null
+++ b/test/defaultCDKContext.json
@@ -0,0 +1,37 @@
+{
+  "migration-default": {
+    "stage": "<STAGE>",
+    "vpcId": "<VPC_ID>",
+    "engineVersion": "OS_2.11",
+    "domainName": "os-cluster-<STAGE>",
+    "dataNodeCount": 2,
+    "openAccessPolicyEnabled": true,
+    "domainRemovalPolicy": "DESTROY",
+    "artifactBucketRemovalPolicy": "DESTROY",
+    "trafficReplayerExtraArgs": "--speedup-factor 10.0",
+    "fetchMigrationEnabled": true,
+    "reindexFromSnapshotServiceEnabled": true,
+    "sourceClusterEndpoint": "<SOURCE_CLUSTER_ENDPOINT>",
+    "dpPipelineTemplatePath": "../../../test/dp_pipeline_aws_integ.yaml",
+    "migrationConsoleEnableOSI": true
+  },
+  "source-single-node-ec2": {
+    "stage": "<STAGE>",
+    "suffix": "ec2-source-<STAGE>",
+    "networkStackSuffix": "ec2-source-<STAGE>",
+    "distVersion": "7.10.2",
+    "cidr": "12.0.0.0/16",
+    "distributionUrl": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.2-linux-x86_64.tar.gz",
+    "captureProxyEnabled": true,
+    "securityDisabled": true,
+    "minDistribution": false,
+    "cpuArch": "x64",
+    "isInternal": true,
+    "singleNodeCluster": true,
+    "networkAvailabilityZones": 2,
+    "dataNodeCount": 1,
+    "managerNodeCount": 0,
+    "serverAccessType": "ipv4",
+    "restrictServerAccessTo": "0.0.0.0/0"
+  }
+}
\ No newline at end of file

From 4ed11b38dbf25cf825bd20d8ed69f6d3d4adc04f Mon Sep 17 00:00:00 2001
From: Tanner Lewis <lewijacn@amazon.com>
Date: Fri, 10 May 2024 18:42:09 -0400
Subject: [PATCH 2/5] Testing updates and minor fixes

Signed-off-by: Tanner Lewis <lewijacn@amazon.com>
---
 .../lambda/msk-ordered-endpoints-handler.ts   |  8 +++
 .../test/common-utilities.test.ts             |  2 +-
 .../test/network-stack.test.ts                |  6 +-
 .../test/resources/invalid-context-file.json  |  7 ++
 .../test/resources/sample-context-file.json   |  9 +++
 .../test/stack-composer.test.ts               | 70 ++++++++++++++++---
 .../test/test-utils.ts                        | 12 +++-
 test/awsE2ESolutionSetup.sh                   | 43 ++++++------
 8 files changed, 123 insertions(+), 34 deletions(-)
 create mode 100644 deployment/cdk/opensearch-service-migration/test/resources/invalid-context-file.json
 create mode 100644 deployment/cdk/opensearch-service-migration/test/resources/sample-context-file.json

diff --git a/deployment/cdk/opensearch-service-migration/lib/lambda/msk-ordered-endpoints-handler.ts b/deployment/cdk/opensearch-service-migration/lib/lambda/msk-ordered-endpoints-handler.ts
index 2759e435d..f857d9aa2 100644
--- a/deployment/cdk/opensearch-service-migration/lib/lambda/msk-ordered-endpoints-handler.ts
+++ b/deployment/cdk/opensearch-service-migration/lib/lambda/msk-ordered-endpoints-handler.ts
@@ -29,6 +29,14 @@ export const handler = async (event: any, context: Context) => {
     console.log('Lambda is invoked with event:' + JSON.stringify(event));
     console.log('Lambda is invoked with context:' + JSON.stringify(context));
 
+    if (event.RequestType === 'Delete') {
+        console.log('Skipping processing for Delete event...')
+        return {
+            statusCode: 200,
+            UniqueId: "getMSKBrokers"
+        }
+    }
+
     if (!process.env.MSK_ARN) {
         throw Error(`Missing at least one required environment variable [MSK_ARN: ${process.env.MSK_ARN}]`)
     }
diff --git a/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts b/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts
index cfbf2a8be..7cb30e3c2 100644
--- a/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts
+++ b/deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts
@@ -22,7 +22,7 @@ test('Test valid fargate cpu arch strings can be parsed', () => {
 test('Test invalid fargate cpu arch strings throws error', () => {
     const cpuArch = "arm32"
     const getArchFunction = () => validateFargateCpuArch(cpuArch)
-    expect(getArchFunction).toThrowError()
+    expect(getArchFunction).toThrow()
 })
 
 test('Test detected fargate cpu arch is valid', () => {
diff --git a/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts b/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts
index 2a1bc748f..e19d93c7d 100644
--- a/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts
+++ b/deployment/cdk/opensearch-service-migration/test/network-stack.test.ts
@@ -99,17 +99,17 @@ test('Test valid imported target cluster endpoint having port and ending in slas
 test('Test target cluster endpoint with no protocol throws error', () => {
     const inputTargetEndpoint = "vpc-domain-abcdef.us-east-1.es.amazonaws.com:443/"
     const validateAndFormatURL = () => NetworkStack.validateAndReturnFormattedHttpURL(inputTargetEndpoint)
-    expect(validateAndFormatURL).toThrowError()
+    expect(validateAndFormatURL).toThrow()
 })
 
 test('Test target cluster endpoint with path throws error', () => {
     const inputTargetEndpoint = "https://vpc-domain-abcdef.us-east-1.es.amazonaws.com:443/indexes"
     const validateAndFormatURL = () => NetworkStack.validateAndReturnFormattedHttpURL(inputTargetEndpoint)
-    expect(validateAndFormatURL).toThrowError()
+    expect(validateAndFormatURL).toThrow()
 })
 
 test('Test invalid target cluster endpoint throws error', () => {
     const inputTargetEndpoint = "vpc-domain-abcdef"
     const validateAndFormatURL = () => NetworkStack.validateAndReturnFormattedHttpURL(inputTargetEndpoint)
-    expect(validateAndFormatURL).toThrowError()
+    expect(validateAndFormatURL).toThrow()
 })
\ No newline at end of file
diff --git a/deployment/cdk/opensearch-service-migration/test/resources/invalid-context-file.json b/deployment/cdk/opensearch-service-migration/test/resources/invalid-context-file.json
new file mode 100644
index 000000000..6a393f0e2
--- /dev/null
+++ b/deployment/cdk/opensearch-service-migration/test/resources/invalid-context-file.json
@@ -0,0 +1,7 @@
+"unit-test-1": {
+"stage": "unittest",
+"vpcEnabled": true,
+"migrationAssistanceEnabled": true,
+"kafkaBrokerServiceEnabled": true,
+"otelCollectorEnabled": false
+}
\ No newline at end of file
diff --git a/deployment/cdk/opensearch-service-migration/test/resources/sample-context-file.json b/deployment/cdk/opensearch-service-migration/test/resources/sample-context-file.json
new file mode 100644
index 000000000..2b6456250
--- /dev/null
+++ b/deployment/cdk/opensearch-service-migration/test/resources/sample-context-file.json
@@ -0,0 +1,9 @@
+{
+  "unit-test-1": {
+    "stage": "unittest",
+    "vpcEnabled": true,
+    "migrationAssistanceEnabled": true,
+    "kafkaBrokerServiceEnabled": true,
+    "otelCollectorEnabled": false
+  }
+}
\ No newline at end of file
diff --git a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts
index 69e3a403b..2282508e8 100644
--- a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts
+++ b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts
@@ -1,8 +1,9 @@
 import {Template} from "aws-cdk-lib/assertions";
 import {OpenSearchDomainStack} from "../lib/opensearch-domain-stack";
-import {createStackComposer} from "./test-utils";
+import {createStackComposer, createStackComposerOnlyPassedContext} from "./test-utils";
 import {App} from "aws-cdk-lib";
 import {StackComposer} from "../lib/stack-composer";
+import {KafkaStack, OpenSearchContainerStack, OtelCollectorStack} from "../lib";
 
 test('Test empty string provided for a parameter which has a default value, uses the default value', () => {
 
@@ -27,7 +28,7 @@ test('Test invalid engine version format throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test ES 7.10 engine version format is parsed', () => {
@@ -117,7 +118,7 @@ test('Test access policy missing Statement throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test access policy with empty Statement array throws error', () => {
@@ -128,7 +129,7 @@ test('Test access policy with empty Statement array throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test access policy with empty Statement block throws error', () => {
@@ -139,7 +140,7 @@ test('Test access policy with empty Statement block throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test access policy with improper Statement throws error', () => {
@@ -151,7 +152,7 @@ test('Test access policy with improper Statement throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test invalid TLS security policy throws error', () => {
@@ -162,7 +163,7 @@ test('Test invalid TLS security policy throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test invalid EBS volume type throws error', () => {
@@ -173,7 +174,7 @@ test('Test invalid EBS volume type throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test invalid domain removal policy type throws error', () => {
@@ -184,7 +185,7 @@ test('Test invalid domain removal policy type throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test that app registry association is created when migrationsAppRegistryARN is provided', () => {
@@ -223,3 +224,54 @@ test('Test that with analytics and assistance stacks enabled, creates one opense
     const domainStacks = openSearchStacks.stacks.filter((s) => s instanceof OpenSearchDomainStack)
     expect(domainStacks.length).toEqual(1)
 })
+
+
+test('Test that loading context via a file is successful', () => {
+
+    const contextOptions = {
+        contextFile: './test/resources/sample-context-file.json',
+        contextId: 'unit-test-1'
+    }
+    const stacks =  createStackComposerOnlyPassedContext(contextOptions)
+    const kafkaContainerStack = stacks.stacks.filter((s) => s instanceof KafkaStack)
+    expect(kafkaContainerStack.length).toEqual(1)
+    const otelContainerStack = stacks.stacks.filter((s) => s instanceof OtelCollectorStack)
+    expect(otelContainerStack.length).toEqual(0)
+})
+
+test('Test that loading context via a file errors if file does not exist', () => {
+
+    const contextOptions = {
+        contextFile: './test/resources/missing-file.json',
+        contextId: 'unit-test-1'
+    }
+
+    const createStackFunc = () => createStackComposerOnlyPassedContext(contextOptions)
+
+    expect(createStackFunc).toThrow()
+})
+
+//TODO add better error here
+test('Test that loading context via a file errors if file is not proper json', () => {
+
+    const contextOptions = {
+        contextFile: './test/resources/invalid-context-file.json',
+        contextId: 'unit-test-1'
+    }
+
+    const createStackFunc = () => createStackComposerOnlyPassedContext(contextOptions)
+
+    expect(createStackFunc).toThrow()
+})
+
+test('Test that loading context via a file errors if contextId does not exist', () => {
+
+    const contextOptions = {
+        contextFile: './test/resources/sample-context-file.json',
+        contextId: 'unit-test-fake'
+    }
+
+    const createStackFunc = () => createStackComposerOnlyPassedContext(contextOptions)
+
+    expect(createStackFunc).toThrow()
+})
\ No newline at end of file
diff --git a/deployment/cdk/opensearch-service-migration/test/test-utils.ts b/deployment/cdk/opensearch-service-migration/test/test-utils.ts
index e28a25c9a..dea702f3d 100644
--- a/deployment/cdk/opensearch-service-migration/test/test-utils.ts
+++ b/deployment/cdk/opensearch-service-migration/test/test-utils.ts
@@ -14,4 +14,14 @@ export function createStackComposer(contextBlock: { [x: string]: (any); }) {
         env: {account: "test-account", region: "us-east-1"},
         migrationsSolutionVersion: "1.0.0"
     })
-}
\ No newline at end of file
+}
+
+export function createStackComposerOnlyPassedContext(contextBlock: { [x: string]: (any); }) {
+    const app = new App({
+        context: contextBlock
+    })
+    return new StackComposer(app, {
+        env: {account: "test-account", region: "us-east-1"},
+        migrationsSolutionVersion: "1.0.0"
+    })
+}
diff --git a/test/awsE2ESolutionSetup.sh b/test/awsE2ESolutionSetup.sh
index 247b8e786..0ec93d5bf 100755
--- a/test/awsE2ESolutionSetup.sh
+++ b/test/awsE2ESolutionSetup.sh
@@ -16,7 +16,7 @@ MIGRATION_CDK_PATH="$ROOT_REPO_PATH/deployment/cdk/opensearch-service-migration"
 # to mediate if this is not the case.
 prepare_source_nodes_for_capture () {
   deploy_stage=$1
-  instance_ids=($(aws ec2 describe-instances --filters 'Name=tag:Name,Values=opensearch-infra-stack*' 'Name=instance-state-name,Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text))
+  instance_ids=($(aws ec2 describe-instances --filters "Name=tag:Name,Values=$SOURCE_INFRA_STACK_NAME/*" 'Name=instance-state-name,Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text))
   kafka_brokers=$(aws ssm get-parameter --name "/migration/$deploy_stage/default/kafkaBrokers" --query 'Parameter.Value' --output text)
   # Substitute @ to be used instead of ',' for cases where ',' would disrupt formatting of arguments, i.e. AWS SSM commands
   kafka_brokers=$(echo "$kafka_brokers" | tr ',' '@')
@@ -70,7 +70,7 @@ prepare_source_nodes_for_capture () {
 
 restore_and_record () {
   deploy_stage=$1
-  source_lb_endpoint=$(aws cloudformation describe-stacks --stack-name opensearch-infra-stack-Migration-Source --query "Stacks[0].Outputs[?OutputKey==\`loadbalancerurl\`].OutputValue" --output text)
+  source_lb_endpoint=$(aws cloudformation describe-stacks --stack-name "$SOURCE_INFRA_STACK_NAME" --query "Stacks[0].Outputs[?OutputKey==\`loadbalancerurl\`].OutputValue" --output text)
   source_endpoint="http://${source_lb_endpoint}:19200"
   kafka_brokers=$(aws ssm get-parameter --name "/migration/$deploy_stage/default/kafkaBrokers" --query 'Parameter.Value' --output text)
   console_task_arn=$(aws ecs list-tasks --cluster migration-${deploy_stage}-ecs-cluster --family "migration-${deploy_stage}-migration-console" | jq --raw-output '.taskArns[0]')
@@ -93,7 +93,7 @@ clean_up_all () {
   cdk_context_var=$1
   vpc_id=$2
   default_sg=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$vpc_id" "Name=group-name,Values=default" --query "SecurityGroups[*].GroupId" --output text)
-  instance_ids=($(aws ec2 describe-instances --filters 'Name=tag:Name,Values=opensearch-infra-stack*' 'Name=instance-state-name,Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text))
+  instance_ids=($(aws ec2 describe-instances --filters "Name=tag:Name,Values=$SOURCE_INFRA_STACK_NAME/*" 'Name=instance-state-name,Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text))
   # Revert source cluster back to default SG to remove added SGs
   for id in "${instance_ids[@]}"
   do
@@ -102,15 +102,15 @@ clean_up_all () {
   done
 
   cd "$MIGRATION_CDK_PATH" || exit
-  cdk destroy "*" --force --c aws-existing-source=$cdk_context_var --c contextId=aws-existing-source
+  cdk destroy "*" --force --c contextFile="$GEN_CONTEXT_FILE" --c contextId="$MIGRATION_CONTEXT_ID"
   cd "$EC2_SOURCE_CDK_PATH" || exit
-  cdk destroy "*" --force --c suffix="Migration-Source" --c distVersion="7.10.2" --c distributionUrl="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.2-linux-x86_64.tar.gz" --c securityDisabled=true --c minDistribution=false --c cpuArch="x64" --c isInternal=true --c singleNodeCluster=false --c networkAvailabilityZones=2 --c dataNodeCount=2 --c managerNodeCount=0 --c serverAccessType="ipv4" --c restrictServerAccessTo="0.0.0.0/0" --c captureProxyEnabled=false
+  cdk destroy "*" --force --c contextFile="$GEN_CONTEXT_FILE" --c contextId="$SOURCE_CONTEXT_ID"
 }
 
 # One-time required CDK bootstrap setup for a given region. Only required if the 'CDKToolkit' CFN stack does not exist
 bootstrap_region () {
   # Picking arbitrary context values to satisfy required values for CDK synthesis. These should not need to be kept in sync with the actual deployment context values
-  cdk bootstrap --require-approval never --c suffix="Migration-Source" --c distVersion="7.10.2" --c distributionUrl="https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-7.10.2-linux-x86_64.tar.gz" --c securityDisabled=true --c minDistribution=false --c cpuArch="x64" --c isInternal=true --c singleNodeCluster=false --c networkAvailabilityZones=2 --c dataNodeCount=2 --c managerNodeCount=0 --c serverAccessType="ipv4" --c restrictServerAccessTo="0.0.0.0/0" --c captureProxyEnabled=false
+  cdk bootstrap --require-approval never --c contextFile="$GEN_CONTEXT_FILE" --c contextId="$SOURCE_CONTEXT_ID"
 }
 
 usage() {
@@ -125,7 +125,6 @@ usage() {
   echo "  --context-file                                   A file path for a given context file from which source and target context options will be used, default is './defaultCDKContext.json'."
   echo "  --source-context-id                              The CDK context block identifier within the context-file to use, default is 'source-single-node-ec2'."
   echo "  --migration-context-id                           The CDK context block identifier within the context-file to use, default is 'migration-default'."
-
   echo "  --migrations-git-url                             The Github http url used for building the capture proxy on setups with a dedicated source cluster, default is 'https://github.com/opensearch-project/opensearch-migrations.git'."
   echo "  --migrations-git-branch                          The Github branch associated with the 'git-url' to pull from, default is 'main'."
   echo "  --stage                                          The stage name to use for naming/grouping of AWS deployment resources, default is 'aws-integ'."
@@ -181,17 +180,17 @@ while [[ $# -gt 0 ]]; do
       shift # past argument
       ;;
     --context-file)
-      MIGRATIONS_GIT_URL="$2"
+      CONTEXT_FILE="$2"
       shift # past argument
       shift # past value
       ;;
     --source-context-id)
-      MIGRATIONS_GIT_URL="$2"
+      SOURCE_CONTEXT_ID="$2"
       shift # past argument
       shift # past value
       ;;
     --migration-context-id)
-      MIGRATIONS_GIT_URL="$2"
+      MIGRATION_CONTEXT_ID="$2"
       shift # past argument
       shift # past value
       ;;
@@ -236,9 +235,14 @@ if [ "$CREATE_SLR" = true ] ; then
   create_service_linked_roles
 fi
 
+SOURCE_NETWORK_STACK_NAME="opensearch-network-stack-ec2-source-$STAGE"
+SOURCE_INFRA_STACK_NAME="opensearch-infra-stack-ec2-source-$STAGE"
+GEN_CONTEXT_FILE="$TMP_DIR_PATH/generatedCDKContext.json"
+
+# Replace preliminary placeholders in CDK context into a generated context file
 mkdir -p "$TMP_DIR_PATH"
-cp $CONTEXT_FILE "$TMP_DIR_PATH/generatedCDKContext.json"
-sed -i -e "s/<STAGE>/$STAGE/g" "$TMP_DIR_PATH/generatedCDKContext.json"
+cp $CONTEXT_FILE "$GEN_CONTEXT_FILE"
+sed -i -e "s/<STAGE>/$STAGE/g" "$GEN_CONTEXT_FILE"
 
 if [ ! -d "opensearch-cluster-cdk" ]; then
   git clone https://github.com/lewijacn/opensearch-cluster-cdk.git
@@ -253,24 +257,23 @@ fi
 
 if [ "$SKIP_SOURCE_DEPLOY" = false ] && [ "$CLEAN_UP_ALL" = false ] ; then
   # Deploy source cluster on EC2 instances
-  cdk deploy "*" --c contextFile="$TMP_DIR_PATH/generatedCDKContext.json" --c contextId="$SOURCE_CONTEXT_ID" --require-approval never
+  cdk deploy "*" --c contextFile="$GEN_CONTEXT_FILE" --c contextId="$SOURCE_CONTEXT_ID" --require-approval never
   if [ $? -ne 0 ]; then
     echo "Error: deploy source cluster failed, exiting."
     exit 1
   fi
 fi
 
-source_endpoint=$(aws cloudformation describe-stacks --stack-name "opensearch-infra-stack-ec2-source-$STAGE" --query "Stacks[0].Outputs[?OutputKey==\`loadbalancerurl\`].OutputValue" --output text)
+source_endpoint=$(aws cloudformation describe-stacks --stack-name "$SOURCE_INFRA_STACK_NAME" --query "Stacks[0].Outputs[?OutputKey==\`loadbalancerurl\`].OutputValue" --output text)
 echo "Source endpoint: $source_endpoint"
-vpc_id=$(aws cloudformation describe-stacks --stack-name "opensearch-network-stack-ec2-source-$STAGE" --query "Stacks[0].Outputs[?contains(OutputValue, 'vpc')].OutputValue" --output text)
+vpc_id=$(aws cloudformation describe-stacks --stack-name "$SOURCE_NETWORK_STACK_NAME" --query "Stacks[0].Outputs[?contains(OutputValue, 'vpc')].OutputValue" --output text)
 echo "VPC ID: $vpc_id"
 
-# Replace source specific placeholders
-sed -i -e "s/<VPC_ID>/$vpc_id/g" "$TMP_DIR_PATH/generatedCDKContext.json"
-sed -i -e "s/<SOURCE_CLUSTER_ENDPOINT>/$source_endpoint/g" "$TMP_DIR_PATH/generatedCDKContext.json"
+# Replace source dependent placeholders in CDK context
+sed -i -e "s/<VPC_ID>/$vpc_id/g" "$GEN_CONTEXT_FILE"
+sed -i -e "s/<SOURCE_CLUSTER_ENDPOINT>/$source_endpoint/g" "$GEN_CONTEXT_FILE"
 
 if [ "$CLEAN_UP_ALL" = true ] ; then
-  #TODO adjust for file
   clean_up_all "$cdk_context" "$vpc_id"
   exit 0
 fi
@@ -283,7 +286,7 @@ if [ "$SKIP_MIGRATION_DEPLOY" = false ] ; then
     exit 1
   fi
   npm install
-  cdk deploy "*" --c contextFile="$TMP_DIR_PATH/generatedCDKContext.json" --c contextId="$MIGRATION_CONTEXT_ID" --require-approval never --concurrency 3
+  cdk deploy "*" --c contextFile="$GEN_CONTEXT_FILE" --c contextId="$MIGRATION_CONTEXT_ID" --require-approval never --concurrency 3
   if [ $? -ne 0 ]; then
     echo "Error: deploying migration stacks failed, exiting."
     exit 1

From d7977fffc7fa9dfdc5748e03564720c247c27b30 Mon Sep 17 00:00:00 2001
From: Tanner Lewis <lewijacn@amazon.com>
Date: Mon, 13 May 2024 11:26:21 -0400
Subject: [PATCH 3/5] Minor updates after testing

Signed-off-by: Tanner Lewis <lewijacn@amazon.com>
---
 .../lib/stack-composer.ts                     |  7 ++++++-
 .../test/stack-composer.test.ts               |  1 -
 test/awsE2ESolutionSetup.sh                   | 19 ++++++++++++-------
 3 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts
index 6033b5e8c..e7fe30869 100644
--- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts
+++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts
@@ -103,7 +103,12 @@ export class StackComposer {
         const contextFile = scope.node.tryGetContext("contextFile")
         if (contextFile) {
             const fileString = readFileSync(contextFile, 'utf-8');
-            const fileJSON = JSON.parse(fileString)
+            let fileJSON
+            try {
+                fileJSON = JSON.parse(fileString)
+            } catch (error) {
+                throw new Error(`Unable to parse context file ${contextFile} into JSON with following error: ${error}`);
+            }
             const contextBlock = fileJSON[contextId]
             if (!contextBlock) {
                 throw new Error(`No CDK context block found for contextId '${contextId}' in file ${contextFile}`)
diff --git a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts
index 2282508e8..99063884f 100644
--- a/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts
+++ b/deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts
@@ -251,7 +251,6 @@ test('Test that loading context via a file errors if file does not exist', () =>
     expect(createStackFunc).toThrow()
 })
 
-//TODO add better error here
 test('Test that loading context via a file errors if file is not proper json', () => {
 
     const contextOptions = {
diff --git a/test/awsE2ESolutionSetup.sh b/test/awsE2ESolutionSetup.sh
index 0ec93d5bf..5e1f75805 100755
--- a/test/awsE2ESolutionSetup.sh
+++ b/test/awsE2ESolutionSetup.sh
@@ -90,8 +90,7 @@ create_service_linked_roles () {
 }
 
 clean_up_all () {
-  cdk_context_var=$1
-  vpc_id=$2
+  vpc_id=$1
   default_sg=$(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$vpc_id" "Name=group-name,Values=default" --query "SecurityGroups[*].GroupId" --output text)
   instance_ids=($(aws ec2 describe-instances --filters "Name=tag:Name,Values=$SOURCE_INFRA_STACK_NAME/*" 'Name=instance-state-name,Values=running' --query 'Reservations[*].Instances[*].InstanceId' --output text))
   # Revert source cluster back to default SG to remove added SGs
@@ -115,8 +114,14 @@ bootstrap_region () {
 
 usage() {
   echo ""
-  echo "Script to setup E2E AWS infrastructure for an ES 7.10.2 source cluster running on EC2, as well as "
-  echo "an OpenSearch Service Domain as the target cluster, and the Migration infrastructure for simulating a migration from source to target."
+  echo "Script to setup E2E AWS infrastructure for simulating a test environment with a source cluster on EC2, the "
+  echo "opensearch-migrations tooling, and a target cluster. Once this script has deployed these resources, it will add "
+  echo "any needed security groups to the source cluster nodes as well as install and start the capture proxy on the "
+  echo "source cluster nodes. The source cluster, migration tooling, and target cluster can all be customized by use "
+  echo "of CDK context in the provided 'context-file' option"
+  echo ""
+  echo "The following placeholder values are replaced before a source deployment: <STAGE>, and the following"
+  echo "placeholder values are replaced after a source deployment: <SOURCE_CLUSTER_ENDPOINT> <VPC_ID>"
   echo ""
   echo "Usage: "
   echo "  ./awsE2ESolutionSetup.sh <>"
@@ -139,7 +144,7 @@ usage() {
   exit 1
 }
 
-STAGE='script-test'
+STAGE='aws-integ'
 RUN_POST_ACTIONS=false
 CREATE_SLR=false
 BOOTSTRAP_REGION=false
@@ -249,7 +254,7 @@ if [ ! -d "opensearch-cluster-cdk" ]; then
 else
   echo "Repo already exists, skipping clone."
 fi
-cd opensearch-cluster-cdk && git pull && git checkout tanner-migration-testing
+cd opensearch-cluster-cdk && git pull && git checkout migration-es
 npm install
 if [ "$BOOTSTRAP_REGION" = true ] ; then
   bootstrap_region
@@ -274,7 +279,7 @@ sed -i -e "s/<VPC_ID>/$vpc_id/g" "$GEN_CONTEXT_FILE"
 sed -i -e "s/<SOURCE_CLUSTER_ENDPOINT>/$source_endpoint/g" "$GEN_CONTEXT_FILE"
 
 if [ "$CLEAN_UP_ALL" = true ] ; then
-  clean_up_all "$cdk_context" "$vpc_id"
+  clean_up_all "$vpc_id"
   exit 0
 fi
 

From c40989b75a89eea349b7dbb8c09c997db8c3c263 Mon Sep 17 00:00:00 2001
From: Tanner Lewis <lewijacn@amazon.com>
Date: Mon, 13 May 2024 12:24:49 -0400
Subject: [PATCH 4/5] Minor validations added to e2e script

Signed-off-by: Tanner Lewis <lewijacn@amazon.com>
---
 test/awsE2ESolutionSetup.sh | 17 ++++++++++++++++-
 test/defaultCDKContext.json |  1 -
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/test/awsE2ESolutionSetup.sh b/test/awsE2ESolutionSetup.sh
index 5e1f75805..6a522e852 100755
--- a/test/awsE2ESolutionSetup.sh
+++ b/test/awsE2ESolutionSetup.sh
@@ -68,6 +68,20 @@ prepare_source_nodes_for_capture () {
   done
 }
 
+validate_required_options () {
+  suffix_required_value='ec2-source-<STAGE>'
+  source_infra_suffix=$(jq ".[\"$SOURCE_CONTEXT_ID\"].suffix" "$GEN_CONTEXT_FILE" -r)
+  source_network_suffix=$(jq ".[\"$SOURCE_CONTEXT_ID\"].networkStackSuffix" "$GEN_CONTEXT_FILE" -r)
+  if [ "$source_infra_suffix" != "$suffix_required_value" ]; then
+    echo "Error: source CDK context must include the 'suffix' option with a value of '$suffix_required_value', however found a value of '$source_infra_suffix', exiting."
+    exit 1
+  fi
+  if [ "$source_network_suffix" != "$suffix_required_value" ]; then
+    echo "Error: source CDK context must include the 'networkStackSuffix' option with a value of '$suffix_required_value', however found a value of '$source_network_suffix', exiting."
+    exit 1
+  fi
+}
+
 restore_and_record () {
   deploy_stage=$1
   source_lb_endpoint=$(aws cloudformation describe-stacks --stack-name "$SOURCE_INFRA_STACK_NAME" --query "Stacks[0].Outputs[?OutputKey==\`loadbalancerurl\`].OutputValue" --output text)
@@ -247,6 +261,7 @@ GEN_CONTEXT_FILE="$TMP_DIR_PATH/generatedCDKContext.json"
 # Replace preliminary placeholders in CDK context into a generated context file
 mkdir -p "$TMP_DIR_PATH"
 cp $CONTEXT_FILE "$GEN_CONTEXT_FILE"
+validate_required_options
 sed -i -e "s/<STAGE>/$STAGE/g" "$GEN_CONTEXT_FILE"
 
 if [ ! -d "opensearch-cluster-cdk" ]; then
@@ -254,7 +269,7 @@ if [ ! -d "opensearch-cluster-cdk" ]; then
 else
   echo "Repo already exists, skipping clone."
 fi
-cd opensearch-cluster-cdk && git pull && git checkout migration-es
+cd opensearch-cluster-cdk && git pull && git checkout migration-es && git pull
 npm install
 if [ "$BOOTSTRAP_REGION" = true ] ; then
   bootstrap_region
diff --git a/test/defaultCDKContext.json b/test/defaultCDKContext.json
index 620b57d5f..5453c2969 100644
--- a/test/defaultCDKContext.json
+++ b/test/defaultCDKContext.json
@@ -16,7 +16,6 @@
     "migrationConsoleEnableOSI": true
   },
   "source-single-node-ec2": {
-    "stage": "<STAGE>",
     "suffix": "ec2-source-<STAGE>",
     "networkStackSuffix": "ec2-source-<STAGE>",
     "distVersion": "7.10.2",

From 719cab1ddf3a8807ae962641d84550d3cfa8e17c Mon Sep 17 00:00:00 2001
From: Tanner Lewis <lewijacn@amazon.com>
Date: Tue, 14 May 2024 10:17:38 -0400
Subject: [PATCH 5/5] Add OSIS SLR in optional initial setup function

Signed-off-by: Tanner Lewis <lewijacn@amazon.com>
---
 test/awsE2ESolutionSetup.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/awsE2ESolutionSetup.sh b/test/awsE2ESolutionSetup.sh
index 6a522e852..872c81e8c 100755
--- a/test/awsE2ESolutionSetup.sh
+++ b/test/awsE2ESolutionSetup.sh
@@ -97,10 +97,12 @@ restore_and_record () {
   aws ecs update-service --cluster "migration-${deploy_stage}-ecs-cluster" --service "migration-${deploy_stage}-traffic-replayer-default" --desired-count 0 > /dev/null 2>&1
 }
 
-# One-time required service-linked-role creation for AWS accounts which do not have these roles
+# One-time required service-linked-role creation for AWS accounts which do not have these roles, will ignore/fail if
+# any of these roles already exist
 create_service_linked_roles () {
   aws iam create-service-linked-role --aws-service-name opensearchservice.amazonaws.com
   aws iam create-service-linked-role --aws-service-name ecs.amazonaws.com
+  aws iam create-service-linked-role --aws-service-name osis.amazonaws.com
 }
 
 clean_up_all () {