Extend E2E script to allow CDK customization

.gitignore

@@ -8,6 +8,7 @@ __pycache__
 .python-version
 logs
 .vscode/
+tmp/
 
 # Build files
 plugins/opensearch/loggable-transport-netty4/.gradle/

deployment/cdk/opensearch-service-migration/lib/lambda/msk-ordered-endpoints-handler.ts

@@ -29,6 +29,14 @@ export const handler = async (event: any, context: Context) => {
     console.log('Lambda is invoked with event:' + JSON.stringify(event));
     console.log('Lambda is invoked with context:' + JSON.stringify(context));
 
+    if (event.RequestType === 'Delete') {
+        console.log('Skipping processing for Delete event...')
+        return {
+            statusCode: 200,
+            UniqueId: "getMSKBrokers"
+        }
+    }
+
     if (!process.env.MSK_ARN) {
         throw Error(`Missing at least one required environment variable [MSK_ARN: ${process.env.MSK_ARN}]`)
     }

deployment/cdk/opensearch-service-migration/lib/stack-composer.ts

@@ -1,5 +1,6 @@
 import {Construct} from "constructs";
 import {Duration, Stack, StackProps} from "aws-cdk-lib";
+import {readFileSync} from 'fs';
 import {OpenSearchDomainStack} from "./opensearch-domain-stack";
 import {EngineVersion, TLSSecurityPolicy} from "aws-cdk-lib/aws-opensearchservice";
 import * as defaultValuesJson from "../default-values.json"
@@ -98,6 +99,36 @@ export class StackComposer {
         }
     }
 
+    private parseContextBlock(scope: Construct, contextId: string) {
+        const contextFile = scope.node.tryGetContext("contextFile")
+        if (contextFile) {
+            const fileString = readFileSync(contextFile, 'utf-8');
+            let fileJSON
+            try {
+                fileJSON = JSON.parse(fileString)
+            } catch (error) {
+                throw new Error(`Unable to parse context file ${contextFile} into JSON with following error: ${error}`);
+            }
+            const contextBlock = fileJSON[contextId]
+            if (!contextBlock) {
+                throw new Error(`No CDK context block found for contextId '${contextId}' in file ${contextFile}`)
+            }
+            return contextBlock
+        }
+
+        let contextJSON = scope.node.tryGetContext(contextId)
+        if (!contextJSON) {
+            throw new Error(`No CDK context block found for contextId '${contextId}'`)
+        }
+        // For a context block to be provided as a string (as in the case of providing via command line) it will need to be properly escaped
+        // to be captured. This requires JSON to parse twice, 1. Returns a normal JSON string with no escaping 2. Returns a JSON object for use
+        if (typeof contextJSON === 'string') {
+            contextJSON = JSON.parse(JSON.parse(contextJSON))
+        }
+        return contextJSON
+
+    }
+
     constructor(scope: Construct, props: StackComposerProps) {
 
         const defaultValues: { [x: string]: (any); } = defaultValuesJson
@@ -108,18 +139,11 @@ export class StackComposer {
         if (!contextId) {
             throw new Error("Required context field 'contextId' not provided")
         }
-        let contextJSON = scope.node.tryGetContext(contextId)
-        if (!contextJSON) {
-            throw new Error(`No CDK context block found for contextId '${contextId}'`)
-        }
+        const contextJSON = this.parseContextBlock(scope, contextId)
         console.log('Received following context block for deployment: ')
         console.log(contextJSON)
         console.log('End of context block.')
-        // For a context block to be provided as a string (as in the case of providing via command line) it will need to be properly escaped
-        // to be captured. This requires JSON to parse twice, 1. Returns a normal JSON string with no escaping 2. Returns a JSON object for use
-        if (typeof contextJSON === 'string') {
-            contextJSON = JSON.parse(JSON.parse(contextJSON))
-        }
+
         const stage = this.getContextForType('stage', 'string', defaultValues, contextJSON)
 
         let version: EngineVersion

deployment/cdk/opensearch-service-migration/test/common-utilities.test.ts

@@ -22,7 +22,7 @@ test('Test valid fargate cpu arch strings can be parsed', () => {
 test('Test invalid fargate cpu arch strings throws error', () => {
     const cpuArch = "arm32"
     const getArchFunction = () => validateFargateCpuArch(cpuArch)
-    expect(getArchFunction).toThrowError()
+    expect(getArchFunction).toThrow()
 })
 
 test('Test detected fargate cpu arch is valid', () => {

deployment/cdk/opensearch-service-migration/test/network-stack.test.ts

@@ -99,17 +99,17 @@ test('Test valid imported target cluster endpoint having port and ending in slas
 test('Test target cluster endpoint with no protocol throws error', () => {
     const inputTargetEndpoint = "vpc-domain-abcdef.us-east-1.es.amazonaws.com:443/"
     const validateAndFormatURL = () => NetworkStack.validateAndReturnFormattedHttpURL(inputTargetEndpoint)
-    expect(validateAndFormatURL).toThrowError()
+    expect(validateAndFormatURL).toThrow()
 })
 
 test('Test target cluster endpoint with path throws error', () => {
     const inputTargetEndpoint = "https://vpc-domain-abcdef.us-east-1.es.amazonaws.com:443/indexes"
     const validateAndFormatURL = () => NetworkStack.validateAndReturnFormattedHttpURL(inputTargetEndpoint)
-    expect(validateAndFormatURL).toThrowError()
+    expect(validateAndFormatURL).toThrow()
 })
 
 test('Test invalid target cluster endpoint throws error', () => {
     const inputTargetEndpoint = "vpc-domain-abcdef"
     const validateAndFormatURL = () => NetworkStack.validateAndReturnFormattedHttpURL(inputTargetEndpoint)
-    expect(validateAndFormatURL).toThrowError()
+    expect(validateAndFormatURL).toThrow()
 })

deployment/cdk/opensearch-service-migration/test/resources/invalid-context-file.json

@@ -0,0 +1,7 @@
+"unit-test-1": {
+"stage": "unittest",
+"vpcEnabled": true,
+"migrationAssistanceEnabled": true,
+"kafkaBrokerServiceEnabled": true,
+"otelCollectorEnabled": false
+}

deployment/cdk/opensearch-service-migration/test/resources/sample-context-file.json

@@ -0,0 +1,9 @@
+{
+  "unit-test-1": {
+    "stage": "unittest",
+    "vpcEnabled": true,
+    "migrationAssistanceEnabled": true,
+    "kafkaBrokerServiceEnabled": true,
+    "otelCollectorEnabled": false
+  }
+}

deployment/cdk/opensearch-service-migration/test/stack-composer.test.ts

@@ -1,8 +1,9 @@
 import {Template} from "aws-cdk-lib/assertions";
 import {OpenSearchDomainStack} from "../lib/opensearch-domain-stack";
-import {createStackComposer} from "./test-utils";
+import {createStackComposer, createStackComposerOnlyPassedContext} from "./test-utils";
 import {App} from "aws-cdk-lib";
 import {StackComposer} from "../lib/stack-composer";
+import {KafkaStack, OpenSearchContainerStack, OtelCollectorStack} from "../lib";
 
 test('Test empty string provided for a parameter which has a default value, uses the default value', () => {
 
@@ -27,7 +28,7 @@ test('Test invalid engine version format throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test ES 7.10 engine version format is parsed', () => {
@@ -117,7 +118,7 @@ test('Test access policy missing Statement throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test access policy with empty Statement array throws error', () => {
@@ -128,7 +129,7 @@ test('Test access policy with empty Statement array throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test access policy with empty Statement block throws error', () => {
@@ -139,7 +140,7 @@ test('Test access policy with empty Statement block throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test access policy with improper Statement throws error', () => {
@@ -151,7 +152,7 @@ test('Test access policy with improper Statement throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test invalid TLS security policy throws error', () => {
@@ -162,7 +163,7 @@ test('Test invalid TLS security policy throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test invalid EBS volume type throws error', () => {
@@ -173,7 +174,7 @@ test('Test invalid EBS volume type throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test invalid domain removal policy type throws error', () => {
@@ -184,7 +185,7 @@ test('Test invalid domain removal policy type throws error', () => {
 
     const createStackFunc = () => createStackComposer(contextOptions)
 
-    expect(createStackFunc).toThrowError()
+    expect(createStackFunc).toThrow()
 })
 
 test('Test that app registry association is created when migrationsAppRegistryARN is provided', () => {
@@ -223,3 +224,53 @@ test('Test that with analytics and assistance stacks enabled, creates one opense
     const domainStacks = openSearchStacks.stacks.filter((s) => s instanceof OpenSearchDomainStack)
     expect(domainStacks.length).toEqual(1)
 })
+
+
+test('Test that loading context via a file is successful', () => {
+
+    const contextOptions = {
+        contextFile: './test/resources/sample-context-file.json',
+        contextId: 'unit-test-1'
+    }
+    const stacks =  createStackComposerOnlyPassedContext(contextOptions)
+    const kafkaContainerStack = stacks.stacks.filter((s) => s instanceof KafkaStack)
+    expect(kafkaContainerStack.length).toEqual(1)
+    const otelContainerStack = stacks.stacks.filter((s) => s instanceof OtelCollectorStack)
+    expect(otelContainerStack.length).toEqual(0)
+})
+
+test('Test that loading context via a file errors if file does not exist', () => {
+
+    const contextOptions = {
+        contextFile: './test/resources/missing-file.json',
+        contextId: 'unit-test-1'
+    }
+
+    const createStackFunc = () => createStackComposerOnlyPassedContext(contextOptions)
+
+    expect(createStackFunc).toThrow()
+})
+
+test('Test that loading context via a file errors if file is not proper json', () => {
+
+    const contextOptions = {
+        contextFile: './test/resources/invalid-context-file.json',
+        contextId: 'unit-test-1'
+    }
+
+    const createStackFunc = () => createStackComposerOnlyPassedContext(contextOptions)
+
+    expect(createStackFunc).toThrow()
+})
+
+test('Test that loading context via a file errors if contextId does not exist', () => {
+
+    const contextOptions = {
+        contextFile: './test/resources/sample-context-file.json',
+        contextId: 'unit-test-fake'
+    }
+
+    const createStackFunc = () => createStackComposerOnlyPassedContext(contextOptions)
+
+    expect(createStackFunc).toThrow()
+})

deployment/cdk/opensearch-service-migration/test/test-utils.ts

@@ -14,4 +14,14 @@ export function createStackComposer(contextBlock: { [x: string]: (any); }) {
         env: {account: "test-account", region: "us-east-1"},
         migrationsSolutionVersion: "1.0.0"
     })
-}
+}
+
+export function createStackComposerOnlyPassedContext(contextBlock: { [x: string]: (any); }) {
+    const app = new App({
+        context: contextBlock
+    })
+    return new StackComposer(app, {
+        env: {account: "test-account", region: "us-east-1"},
+        migrationsSolutionVersion: "1.0.0"
+    })
+}