Add/Modify OSI options on Migration Console (#643)

This change makes the following minor modifications for our OSI migration setup for the migration console

*Removes incorrect separate OSI pipeline IAM roles for source and target clusters. This must be the same role and have adjusted accordingly
*Add option to include-index-regex. This allows a user to specify multiple selection regexes that we will use to select which indices to migrate when configuring the template for OSI. As a default we will still try to move everything except system indices if not specified.
*Properly hookup missing CW log settings following OSI required conventions
*Minor cleanup in spots of OSI script

Signed-off-by: Tanner Lewis <lewijacn@amazon.com>

.flake8

@@ -12,4 +12,6 @@ exclude = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules,.venv,*/cd
 
 # F401 - Unused imports -- this is the only way to have a file-wide rule exception
 per-file-ignores =
-    experimental/upgrades/upgrade_testing_framework/steps/__init__.py:F401
+    experimental/upgrades/upgrade_testing_framework/steps/__init__.py:F401
+    # W605 - invalid escape sequence -- this is a required regex pattern for OSI template
+    TrafficCapture/dockerSolution/src/main/docker/migrationConsole/osiMigration.py:W605

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/README.md

@@ -0,0 +1,50 @@
+# Migration Console
+The accessible control hub for all things migrations
+
+
+## Running Python tests
+
+
+### Installing Requirements
+
+To isolate the Python environment for the project from your local machine, create virtual environment like so:
+```
+python3 -m venv .venv
+source .venv/bin/activate
+```
+
+You can exit the Python virtual environment and remove its resources like so:
+```
+deactivate
+rm -rf .venv
+```
+
+Install developer requirements for osiMigration library like so:
+```
+pip install -r lib/osiMigrationLib/dev-requirements.txt
+```
+
+
+### Unit Tests
+
+Unit tests can be run from this current `migrationConsole/` directory using:
+
+```shell
+python -m unittest
+```
+
+### Coverage
+
+_Code coverage_ metrics can be generated after a unit-test run. A report can either be printed on the command line:
+
+```shell
+python -m coverage report --omit "*/test/*"
+```
+
+or generated as HTML:
+
+```shell
+python -m coverage html --omit "*/test/*"
+```
+
+Note that the `--omit` parameter must be specified to avoid tracking code coverage on unit test code itself.

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/osiMigrationLib/dev-requirements.txt

@@ -0,0 +1,2 @@
+-r requirements.txt
+coverage

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/osiPipelineTemplate.yaml

@@ -8,20 +8,21 @@ historical-data-migration:
   source:
     opensearch:
       hosts:
-      - <SOURCE_CLUSTER_ENDPOINT_PLACEHOLDER>
+        - <SOURCE_CLUSTER_ENDPOINT_PLACEHOLDER>
       indices:
-        # Indices to exclude - exclude system indices by default
-        exclude:
-        - index_name_regex: \.*
+        # As a default this will be populated to move all indices except system indices (those that start with '.'),
+        # but tool also allows passing regex strings of indices to include, in which case all indices matching the regex
+        # will be migrated
+        <INDEX_SELECTION_OPTIONS_PLACEHOLDER>
       <SOURCE_AUTH_OPTIONS_PLACEHOLDER>
 
   # Target cluster configuration
   sink:
-  - opensearch:
-      hosts:
-      - <TARGET_CLUSTER_ENDPOINT_PLACEHOLDER>
-      # Derive index name from record metadata
-      index: ${getMetadata("opensearch-index")}
-      # Use the same document ID as the source cluster document
-      document_id: ${getMetadata("opensearch-document_id")}
-      <TARGET_AUTH_OPTIONS_PLACEHOLDER>
+    - opensearch:
+        hosts:
+          - <TARGET_CLUSTER_ENDPOINT_PLACEHOLDER>
+        # Derive index name from record metadata
+        index: ${getMetadata("opensearch-index")}
+        # Use the same document ID as the source cluster document
+        document_id: ${getMetadata("opensearch-document_id")}
+        <TARGET_AUTH_OPTIONS_PLACEHOLDER>

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/test/resources/basicAuthSourceAndSigv4Target.yaml

@@ -0,0 +1,38 @@
+version: "2"
+# NOTE: Placeholder values will be automatically populated and do not need to be changed
+pipeline_configurations:
+  aws:
+    secrets:
+      source-secret-config:
+        secret_id: unit-test-secret
+        region: us-west-2
+        sts_role_arn: arn=arn:aws:iam::123456789012:role/OSMigrations-aws-integ-us--osisPipelineRole123
+
+historical-data-migration:
+
+  # Source cluster configuration
+  source:
+    opensearch:
+      hosts:
+        - https://vpc-test-123.com
+      indices:
+        # As a default this will be populated to move all indices except system indices (those that start with '.'),
+        # but tool also allows passing regex strings of indices to include, in which case all indices matching the regex
+        # will be migrated
+        exclude:
+          - index_name_regex: \.*
+      username: "${{aws_secrets:source-secret-config:username}}"
+      password: "${{aws_secrets:source-secret-config:password}}"
+
+  # Target cluster configuration
+  sink:
+    - opensearch:
+        hosts:
+          - https://vpc-test-456.com
+        # Derive index name from record metadata
+        index: ${getMetadata("opensearch-index")}
+        # Use the same document ID as the source cluster document
+        document_id: ${getMetadata("opensearch-document_id")}
+        aws:
+          region: us-west-2
+          sts_role_arn: arn=arn:aws:iam::123456789012:role/OSMigrations-aws-integ-us--osisPipelineRole123

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/test/resources/basicAuthSourceWithMultipleIndexInclusionRule.yaml

@@ -0,0 +1,40 @@
+version: "2"
+# NOTE: Placeholder values will be automatically populated and do not need to be changed
+pipeline_configurations:
+  aws:
+    secrets:
+      source-secret-config:
+        secret_id: unit-test-secret
+        region: us-west-2
+        sts_role_arn: arn=arn:aws:iam::123456789012:role/OSMigrations-aws-integ-us--osisPipelineRole123
+
+historical-data-migration:
+
+  # Source cluster configuration
+  source:
+    opensearch:
+      hosts:
+        - https://vpc-test-123.com
+      indices:
+        # As a default this will be populated to move all indices except system indices (those that start with '.'),
+        # but tool also allows passing regex strings of indices to include, in which case all indices matching the regex
+        # will be migrated
+        include:
+          - index_name_regex: index*
+          - index_name_regex: .*
+          - index_name_regex: sam[a-z]+
+      username: "${{aws_secrets:source-secret-config:username}}"
+      password: "${{aws_secrets:source-secret-config:password}}"
+
+  # Target cluster configuration
+  sink:
+    - opensearch:
+        hosts:
+          - https://vpc-test-456.com
+        # Derive index name from record metadata
+        index: ${getMetadata("opensearch-index")}
+        # Use the same document ID as the source cluster document
+        document_id: ${getMetadata("opensearch-document_id")}
+        aws:
+          region: us-west-2
+          sts_role_arn: arn=arn:aws:iam::123456789012:role/OSMigrations-aws-integ-us--osisPipelineRole123

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/test/resources/basicAuthSourceWithSingleIndexInclusionRule.yaml

@@ -0,0 +1,38 @@
+version: "2"
+# NOTE: Placeholder values will be automatically populated and do not need to be changed
+pipeline_configurations:
+  aws:
+    secrets:
+      source-secret-config:
+        secret_id: unit-test-secret
+        region: us-west-2
+        sts_role_arn: arn=arn:aws:iam::123456789012:role/OSMigrations-aws-integ-us--osisPipelineRole123
+
+historical-data-migration:
+
+  # Source cluster configuration
+  source:
+    opensearch:
+      hosts:
+        - https://vpc-test-123.com
+      indices:
+        # As a default this will be populated to move all indices except system indices (those that start with '.'),
+        # but tool also allows passing regex strings of indices to include, in which case all indices matching the regex
+        # will be migrated
+        include:
+          - index_name_regex: index*
+      username: "${{aws_secrets:source-secret-config:username}}"
+      password: "${{aws_secrets:source-secret-config:password}}"
+
+  # Target cluster configuration
+  sink:
+    - opensearch:
+        hosts:
+          - https://vpc-test-456.com
+        # Derive index name from record metadata
+        index: ${getMetadata("opensearch-index")}
+        # Use the same document ID as the source cluster document
+        document_id: ${getMetadata("opensearch-document_id")}
+        aws:
+          region: us-west-2
+          sts_role_arn: arn=arn:aws:iam::123456789012:role/OSMigrations-aws-integ-us--osisPipelineRole123

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/test/resources/sigv4SourceAndSigv4Target.yaml

@@ -0,0 +1,33 @@
+version: "2"
+# NOTE: Placeholder values will be automatically populated and do not need to be changed
+
+
+historical-data-migration:
+
+  # Source cluster configuration
+  source:
+    opensearch:
+      hosts:
+        - https://vpc-test-123.com
+      indices:
+        # As a default this will be populated to move all indices except system indices (those that start with '.'),
+        # but tool also allows passing regex strings of indices to include, in which case all indices matching the regex
+        # will be migrated
+        exclude:
+          - index_name_regex: \.*
+      aws:
+        region: us-west-2
+        sts_role_arn: arn=arn:aws:iam::123456789012:role/OSMigrations-aws-integ-us--osisPipelineRole123
+
+  # Target cluster configuration
+  sink:
+    - opensearch:
+        hosts:
+          - https://vpc-test-456.com
+        # Derive index name from record metadata
+        index: ${getMetadata("opensearch-index")}
+        # Use the same document ID as the source cluster document
+        document_id: ${getMetadata("opensearch-document_id")}
+        aws:
+          region: us-west-2
+          sts_role_arn: arn=arn:aws:iam::123456789012:role/OSMigrations-aws-integ-us--osisPipelineRole123

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/test/testOSIMigration.py

@@ -0,0 +1,97 @@
+import unittest
+
+import osiMigration
+
+# These values should map to static template values in the resources directory
+SOURCE_ENDPOINT = 'https://vpc-test-123.com'
+TARGET_ENDPOINT = 'https://vpc-test-456.com'
+PIPELINE_ROLE_ARN = 'arn=arn:aws:iam::123456789012:role/OSMigrations-aws-integ-us--osisPipelineRole123'
+AWS_REGION = 'us-west-2'
+SECRET_NAME = 'unit-test-secret'
+INDEX_INCLUSION_RULE_1 = 'index*'
+INDEX_INCLUSION_RULE_2 = '.*'
+INDEX_INCLUSION_RULE_3 = 'sam[a-z]+'
+
+
+class TestOSIMigration(unittest.TestCase):
+    def test_construct_config_sigv4_source_and_sigv4_target(self):
+        generated_config = osiMigration.construct_pipeline_config(
+            pipeline_config_file_path='./osiPipelineTemplate.yaml',
+            source_endpoint=SOURCE_ENDPOINT,
+            target_endpoint=TARGET_ENDPOINT,
+            source_auth_type='SIGV4',
+            target_auth_type='SIGV4',
+            pipeline_role_arn=PIPELINE_ROLE_ARN,
+            aws_region=AWS_REGION)
+        with open('./test/resources/sigv4SourceAndSigv4Target.yaml', "r") as expected_file:
+            expected_file_contents = expected_file.read()
+        self.assertEqual(generated_config, expected_file_contents)
+
+    def test_construct_config_basic_auth_source_and_sigv4_target(self):
+        generated_config = osiMigration.construct_pipeline_config(
+            pipeline_config_file_path='./osiPipelineTemplate.yaml',
+            source_endpoint=SOURCE_ENDPOINT,
+            target_endpoint=TARGET_ENDPOINT,
+            source_auth_type='BASIC_AUTH',
+            source_auth_secret=SECRET_NAME,
+            target_auth_type='SIGV4',
+            pipeline_role_arn=PIPELINE_ROLE_ARN,
+            aws_region=AWS_REGION)
+        with open('./test/resources/basicAuthSourceAndSigv4Target.yaml', "r") as expected_file:
+            expected_file_contents = expected_file.read()
+        self.assertEqual(generated_config, expected_file_contents)
+
+    def test_construct_config_basic_auth_source_with_single_index_inclusion_rules(self):
+        generated_config = osiMigration.construct_pipeline_config(
+            pipeline_config_file_path='./osiPipelineTemplate.yaml',
+            source_endpoint=SOURCE_ENDPOINT,
+            target_endpoint=TARGET_ENDPOINT,
+            source_auth_type='BASIC_AUTH',
+            source_auth_secret=SECRET_NAME,
+            include_index_regex_list=[INDEX_INCLUSION_RULE_1],
+            target_auth_type='SIGV4',
+            pipeline_role_arn=PIPELINE_ROLE_ARN,
+            aws_region=AWS_REGION)
+        with open('./test/resources/basicAuthSourceWithSingleIndexInclusionRule.yaml', "r") as expected_file:
+            expected_file_contents = expected_file.read()
+        self.assertEqual(generated_config, expected_file_contents)
+
+    def test_construct_config_basic_auth_source_with_multiple_index_inclusion_rules(self):
+        generated_config = osiMigration.construct_pipeline_config(
+            pipeline_config_file_path='./osiPipelineTemplate.yaml',
+            source_endpoint=SOURCE_ENDPOINT,
+            target_endpoint=TARGET_ENDPOINT,
+            source_auth_type='BASIC_AUTH',
+            source_auth_secret=SECRET_NAME,
+            include_index_regex_list=[INDEX_INCLUSION_RULE_1, INDEX_INCLUSION_RULE_2, INDEX_INCLUSION_RULE_3],
+            target_auth_type='SIGV4',
+            pipeline_role_arn=PIPELINE_ROLE_ARN,
+            aws_region=AWS_REGION)
+        with open('./test/resources/basicAuthSourceWithMultipleIndexInclusionRule.yaml', "r") as expected_file:
+            expected_file_contents = expected_file.read()
+        self.assertEqual(generated_config, expected_file_contents)
+
+    def test_construct_config_throws_error_if_secret_not_provided_for_basic_auth(self):
+        with self.assertRaises(osiMigration.InvalidAuthParameters):
+            osiMigration.construct_pipeline_config(
+                pipeline_config_file_path='./osiPipelineTemplate.yaml',
+                source_endpoint=SOURCE_ENDPOINT,
+                target_endpoint=TARGET_ENDPOINT,
+                source_auth_type='BASIC_AUTH',
+                target_auth_type='SIGV4',
+                pipeline_role_arn=PIPELINE_ROLE_ARN,
+                aws_region=AWS_REGION)
+
+    def test_construct_config_throws_error_if_pipeline_role_not_provided_for_sigv4(self):
+        with self.assertRaises(osiMigration.InvalidAuthParameters):
+            osiMigration.construct_pipeline_config(
+                pipeline_config_file_path='./osiPipelineTemplate.yaml',
+                source_endpoint=SOURCE_ENDPOINT,
+                target_endpoint=TARGET_ENDPOINT,
+                source_auth_type='SIGV4',
+                target_auth_type='SIGV4',
+                aws_region=AWS_REGION)
+
+
+if __name__ == '__main__':
+    unittest.main()

deployment/cdk/opensearch-service-migration/lib/service-stacks/migration-console-stack.ts

@@ -92,7 +92,17 @@ export class MigrationConsoleStack extends MigrationServiceCore {
                 "iam:PassRole"
             ]
         })
-        return [osiManagementPolicy, passPipelineRolePolicy]
+        const configureLogGroupPolicy = new PolicyStatement({
+            effect: Effect.ALLOW,
+            resources: ["*"],
+            actions: [
+                "logs:CreateLogDelivery",
+                "logs:PutResourcePolicy",
+                "logs:DescribeResourcePolicies",
+                "logs:DescribeLogGroups"
+            ]
+        })
+        return [osiManagementPolicy, passPipelineRolePolicy, configureLogGroupPolicy]
     }
 
     constructor(scope: Construct, id: string, props: MigrationConsoleProps) {
@@ -218,7 +228,8 @@ export class MigrationConsoleStack extends MigrationServiceCore {
             const osiLogGroup = new LogGroup(this, 'OSILogGroup',  {
                 retention: RetentionDays.ONE_MONTH,
                 removalPolicy: RemovalPolicy.DESTROY,
-                logGroupName: `/migration/${props.stage}/${props.defaultDeployId}/openSearchIngestion`
+                // Naming requirement from OSI
+                logGroupName: `/aws/vendedlogs/osi-${props.stage}-${props.defaultDeployId}`
             });
             new StringParameter(this, 'SSMParameterOSIPipelineLogGroupName', {
                 description: 'OpenSearch Migration Parameter for OpenSearch Ingestion Pipeline Log Group Name',