From 5c9e29485212dd7f013482a431d764f13407dcac Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Tue, 18 Jun 2024 13:26:49 -0600 Subject: [PATCH 1/4] Add RFS on ECS yaml generation to cdk Signed-off-by: Mikayla Thompson --- .../lib/console_link/README.md | 2 ++ .../console_link/models/backfill_rfs.py | 1 + .../lib/console_link/tests/test_backfill.py | 1 + .../lib/migration-services-yaml.ts | 29 ++++++++++++++++++- .../reindex-from-snapshot-stack.ts | 9 +++++- .../lib/stack-composer.ts | 2 ++ 6 files changed, 42 insertions(+), 2 deletions(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md index 76cd62bc4..7b1ecc480 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/README.md @@ -96,6 +96,7 @@ There is also a block that specifies the deployment type. Exactly one of the fol - `socket`: optional, path to mounted docker socket, defaults to `/var/run/docker.sock` - `ecs`: + - `cluster_name`: required, name of the ECS cluster containing the RFS service - `service_name`: required, name of the ECS service for RFS - `aws_region`: optional. if not provided, the usual rules are followed for determining aws region. (`AWS_DEFAULT_REGION`, `~/.aws/config`, etc.) @@ -114,6 +115,7 @@ backfill: snapshot_name: "def" scale: 3 ecs: + cluster_name: migration-aws-integ-ecs-cluster service_name: migration-aws-integ-reindex-from-snapshot aws-region: us-east-1 ``` diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/backfill_rfs.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/backfill_rfs.py index 67806c76a..fd5dd5bb5 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/backfill_rfs.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/backfill_rfs.py @@ -20,6 +20,7 @@ ECS_RFS_SCHEMA = { "type": "dict", "schema": { + "cluster_name": {"type": "string", "required": True}, "service_name": {"type": "string", "required": True}, "aws_region": {"type": "string", "required": False} } diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_backfill.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_backfill.py index 5626cdc51..6d54de2b7 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_backfill.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_backfill.py @@ -95,6 +95,7 @@ def test_get_backfill_valid_ecs_rfs(): ecs_rfs_config = { "reindex_from_snapshot": { "ecs": { + "cluster_name": "migration-aws-integ-ecs-cluster", "service_name": "migration-aws-integ-reindex-from-snapshot" } } diff --git a/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts b/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts index b410d61e0..0f7b6ddf2 100644 --- a/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts +++ b/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts @@ -10,16 +10,43 @@ export class MetricsSourceYaml { cloudwatch? : object | null = null; } +export class ECSService { + cluster_name: string | undefined; + service_name: string | undefined; + aws_region: string | undefined; + constructor() { + this.cluster_name = undefined; + this.service_name = undefined; + this.aws_region = undefined; + } +} + +export class RFSBackfillYaml { + ecs: ECSService; + constructor() { + this.ecs = new ECSService(); + } +} + +export class OSIBackfillYaml { + opensearch_ingestion = { + aws_region: null, + } + +} + export class ServicesYaml { source_cluster: ClusterYaml; target_cluster: ClusterYaml; metrics_source: MetricsSourceYaml = new MetricsSourceYaml(); + backfill: RFSBackfillYaml | OSIBackfillYaml; stringify(): string { return yaml.stringify({ source_cluster: this.source_cluster, target_cluster: this.target_cluster, - metrics_source: this.metrics_source + metrics_source: this.metrics_source, + backfill: this.backfill, }, { 'nullStr': '' diff --git a/deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts b/deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts index 97669f55f..9a8b49c4f 100644 --- a/deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/service-stacks/reindex-from-snapshot-stack.ts @@ -10,6 +10,7 @@ import { createOpenSearchIAMAccessPolicy, createOpenSearchServerlessIAMAccessPolicy } from "../common-utilities"; +import { RFSBackfillYaml } from "../migration-services-yaml"; export interface ReindexFromSnapshotProps extends StackPropsExt { @@ -21,6 +22,7 @@ export interface ReindexFromSnapshotProps extends StackPropsExt { } export class ReindexFromSnapshotStack extends MigrationServiceCore { + rfsBackfillYaml: RFSBackfillYaml; constructor(scope: Construct, id: string, props: ReindexFromSnapshotProps) { super(scope, id, props) @@ -60,6 +62,11 @@ export class ReindexFromSnapshotStack extends MigrationServiceCore { taskMemoryLimitMiB: 4096, ...props }); + + this.rfsBackfillYaml = new RFSBackfillYaml(); + this.rfsBackfillYaml.ecs.cluster_name = `migration-${props.stage}-ecs-cluster`; + this.rfsBackfillYaml.ecs.service_name = `migration-${props.stage}-reindex-from-snapshot`; + } -} \ No newline at end of file +} diff --git a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts index a4bb3d749..01fcd1857 100644 --- a/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts +++ b/deployment/cdk/opensearch-service-migration/lib/stack-composer.ts @@ -428,6 +428,8 @@ export class StackComposer { }) this.addDependentStacks(reindexFromSnapshotStack, [migrationStack, openSearchStack, osContainerStack]) this.stacks.push(reindexFromSnapshotStack) + servicesYaml.backfill = reindexFromSnapshotStack.rfsBackfillYaml; + } let captureProxyESStack From b636ad1860cdd89549b8a7661c22d4176ae2fcc7 Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Tue, 18 Jun 2024 14:27:30 -0600 Subject: [PATCH 2/4] Clean up adding to YAML and tests Signed-off-by: Mikayla Thompson --- .../lib/migration-services-yaml.ts | 9 +++--- .../test/migration-services-yaml.test.ts | 28 +++++++++++++++++-- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts b/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts index 0f7b6ddf2..9ac04c729 100644 --- a/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts +++ b/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts @@ -29,9 +29,6 @@ export class RFSBackfillYaml { } export class OSIBackfillYaml { - opensearch_ingestion = { - aws_region: null, - } } @@ -46,7 +43,11 @@ export class ServicesYaml { source_cluster: this.source_cluster, target_cluster: this.target_cluster, metrics_source: this.metrics_source, - backfill: this.backfill, + backfill: this.backfill + ? (this.backfill instanceof RFSBackfillYaml + ? { reindex_from_snapshot: this.backfill } + : { opensearch_ingestion: this.backfill }) + : undefined }, { 'nullStr': '' diff --git a/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts b/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts index a22a53ccf..46f04d739 100644 --- a/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts @@ -1,4 +1,4 @@ -import { ClusterYaml, ServicesYaml } from "../lib/migration-services-yaml" +import { ClusterYaml, RFSBackfillYaml, ServicesYaml } from "../lib/migration-services-yaml" test('Test default servicesYaml can be stringified', () => { const servicesYaml = new ServicesYaml(); @@ -8,7 +8,7 @@ test('Test default servicesYaml can be stringified', () => { expect(yaml).toBe("metrics_source:\n cloudwatch:\n"); }) -test('Test servicesYaml with cluster can be stringified', () => { +test('Test servicesYaml with target cluster can be stringified', () => { let servicesYaml = new ServicesYaml(); const cluster: ClusterYaml = { 'endpoint': 'https://abc.com', 'no_auth': '' }; servicesYaml.target_cluster = cluster; @@ -18,7 +18,7 @@ test('Test servicesYaml with cluster can be stringified', () => { expect(yaml).toBe(`target_cluster:\n endpoint: ${cluster.endpoint}\n no_auth: ""\nmetrics_source:\n cloudwatch:\n`); }) -test('Test servicesYaml with cluster can be stringified', () => { +test('Test servicesYaml with source and target cluster can be stringified', () => { let servicesYaml = new ServicesYaml(); const targetCluster: ClusterYaml = { 'endpoint': 'https://abc.com', 'no_auth': '' }; servicesYaml.target_cluster = targetCluster; @@ -33,3 +33,25 @@ test('Test servicesYaml with cluster can be stringified', () => { const sourceClusterYaml = `source_cluster:\n endpoint: ${sourceCluster.endpoint}\n basic_auth:\n user: ${sourceClusterUser}\n password: ${sourceClusterPassword}\n` expect(yaml).toBe(`${sourceClusterYaml}target_cluster:\n endpoint: ${targetCluster.endpoint}\n no_auth: ""\nmetrics_source:\n cloudwatch:\n`); }) + +test('Test servicesYaml with rfs backfill can be stringified', () => { + const clusterName = "migration-cluster-name"; + const serviceName = "rfs-service-name"; + const region = "us-east-1" + let servicesYaml = new ServicesYaml(); + servicesYaml.backfill = new RFSBackfillYaml(); + servicesYaml.backfill.ecs.cluster_name = clusterName; + servicesYaml.backfill.ecs.service_name = serviceName; + servicesYaml.backfill.ecs.aws_region = region; + + expect(servicesYaml.backfill).toBeDefined(); + expect(servicesYaml.backfill.ecs).toBeDefined(); + const yaml = servicesYaml.stringify(); + expect(yaml).toBe(`metrics_source:\n cloudwatch:\nbackfill:\n reindex_from_snapshot:\n ecs:\n cluster_name: ${clusterName}\n service_name: ${serviceName}\n aws_region: ${region}\n`); +}) + +test('Test servicesYaml without backfill does not include backend section', () => { + let servicesYaml = new ServicesYaml(); + const yaml = servicesYaml.stringify(); + expect(yaml).toBe(`metrics_source:\n cloudwatch:\n`); +}) \ No newline at end of file From a15d440a96b27c2a4a91c3af7e2d4c1e3ed33d27 Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Tue, 18 Jun 2024 14:33:28 -0600 Subject: [PATCH 3/4] Fix test implementation Signed-off-by: Mikayla Thompson --- .../lib/console_link/tests/test_backfill_logic.py | 1 + .../test/migration-services-yaml.test.ts | 13 ++++++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_backfill_logic.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_backfill_logic.py index 801bcb55a..17a31d15d 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_backfill_logic.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_backfill_logic.py @@ -27,6 +27,7 @@ def ecs_rfs_backfill() -> ECSRFSBackfill: ecs_rfs_config = { "reindex_from_snapshot": { "ecs": { + "cluster_name": "migration-aws-integ-ecs-cluster", "service_name": "migration-aws-integ-reindex-from-snapshot" } } diff --git a/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts b/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts index 46f04d739..2f7a38f91 100644 --- a/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts +++ b/deployment/cdk/opensearch-service-migration/test/migration-services-yaml.test.ts @@ -39,13 +39,16 @@ test('Test servicesYaml with rfs backfill can be stringified', () => { const serviceName = "rfs-service-name"; const region = "us-east-1" let servicesYaml = new ServicesYaml(); - servicesYaml.backfill = new RFSBackfillYaml(); - servicesYaml.backfill.ecs.cluster_name = clusterName; - servicesYaml.backfill.ecs.service_name = serviceName; - servicesYaml.backfill.ecs.aws_region = region; + let rfsBackfillYaml = new RFSBackfillYaml(); + rfsBackfillYaml.ecs.cluster_name = clusterName; + rfsBackfillYaml.ecs.service_name = serviceName; + rfsBackfillYaml.ecs.aws_region = region; + servicesYaml.backfill = rfsBackfillYaml; + + expect(servicesYaml.backfill).toBeDefined(); expect(servicesYaml.backfill).toBeDefined(); - expect(servicesYaml.backfill.ecs).toBeDefined(); + expect(servicesYaml.backfill instanceof RFSBackfillYaml).toBeTruthy(); const yaml = servicesYaml.stringify(); expect(yaml).toBe(`metrics_source:\n cloudwatch:\nbackfill:\n reindex_from_snapshot:\n ecs:\n cluster_name: ${clusterName}\n service_name: ${serviceName}\n aws_region: ${region}\n`); }) From 1bd15c0ba2ee18119cdc78e96a34f39f886400d2 Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Tue, 18 Jun 2024 16:29:59 -0600 Subject: [PATCH 4/4] Add toDict for the backfill options Signed-off-by: Mikayla Thompson --- .../lib/migration-services-yaml.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts b/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts index 9ac04c729..eb5e8d95c 100644 --- a/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts +++ b/deployment/cdk/opensearch-service-migration/lib/migration-services-yaml.ts @@ -26,10 +26,20 @@ export class RFSBackfillYaml { constructor() { this.ecs = new ECSService(); } + + toDict() { + return { + reindex_from_snapshot: {ecs: this.ecs} + }; + } } export class OSIBackfillYaml { - + toDict() { + return { + opensearch_ingestion: null + }; + } } export class ServicesYaml { @@ -43,11 +53,7 @@ export class ServicesYaml { source_cluster: this.source_cluster, target_cluster: this.target_cluster, metrics_source: this.metrics_source, - backfill: this.backfill - ? (this.backfill instanceof RFSBackfillYaml - ? { reindex_from_snapshot: this.backfill } - : { opensearch_ingestion: this.backfill }) - : undefined + backfill: this.backfill?.toDict() }, { 'nullStr': ''