Merge branch 'main' into console-lib-snapshot-for-filesystem

Signed-off-by: Mikayla Thompson <thomika@amazon.com>
opensearch-project · Jun 24, 2024 · 2be3d70 · 2be3d70
2 parents 862e15c + 5925602
commit 2be3d70
Show file tree

Hide file tree

Showing 8 changed files with 178 additions and 82 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -173,6 +173,16 @@ jobs:
         env:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
 
+  fetch-migration-docker-build:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./FetchMigration
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build Docker Image
+        run: docker build -t migrations/fetch-migration -f Dockerfile .
+
   all-ci-checks-pass:
     needs:
       - cdk-tests
@@ -181,7 +191,9 @@ jobs:
       - python-e2e-tests
       - python-lint
       - python-tests
+      - fetch-migration-docker-build
     runs-on: ubuntu-latest
     steps:
       - run: |
           echo '## :heavy_check_mark: All continous integration checks pass' >> $GITHUB_STEP_SUMMARY
+
diff --git a/.whitesource b/.whitesource
@@ -1,14 +1,15 @@
 {
   "scanSettings": {
-    "configMode": "AUTO",
+    "configMode": "LOCAL",
     "configExternalURL": "",
     "projectToken": "",
     "baseBranches": []
   },
   "checkRunSettings": {
     "vulnerableCheckRunConclusionLevel": "failure",
     "displayMode": "diff",
-    "useMendCheckNames": true
+    "useMendCheckNames": true,
+    "strictMode" : "failure"
   },
   "issueSettings": {
     "minSeverityLevel": "LOW",
@@ -19,4 +20,4 @@
       "enabled": true
     }
   }
-}
+}
diff --git a/CreateSnapshot/src/main/java/com/rfs/CreateSnapshot.java b/CreateSnapshot/src/main/java/com/rfs/CreateSnapshot.java
@@ -2,20 +2,19 @@
 
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
 
-import com.rfs.common.UsernamePassword;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 
-import com.rfs.common.ConnectionDetails;
+import com.rfs.common.FileSystemSnapshotCreator;
 import com.rfs.common.OpenSearchClient;
+import com.rfs.common.S3SnapshotCreator;
 import com.rfs.common.SnapshotCreator;
 import com.rfs.common.TryHandlePhaseFailure;
-import com.rfs.common.S3SnapshotCreator;
 import com.rfs.worker.SnapshotRunner;
 
-import java.util.Optional;
 import java.util.function.Function;
 
 @Slf4j
@@ -26,13 +25,18 @@ public static class Args {
                 description = "The name of the snapshot to migrate")
         public String snapshotName;
 
+        @Parameter(names = {"--file-system-repo-path"},
+                required = false,
+                description = "The full path to the snapshot repo on the file system.")
+        public String fileSystemRepoPath;
+
         @Parameter(names = {"--s3-repo-uri"},
-                required = true,
+                required = false,
                 description = "The S3 URI of the snapshot repo, like: s3://my-bucket/dir1/dir2")
         public String s3RepoUri;
 
         @Parameter(names = {"--s3-region"},
-                required = true,
+                required = false,
                 description = "The AWS Region the S3 bucket is in, like: us-east-2"
         )
         public String s3Region;
@@ -70,12 +74,25 @@ public static void main(String[] args) throws Exception {
                 .build()
                 .parse(args);
 
-        log.info("Running CreateSnapshot with " + String.join(" ", args));
-        run(c -> new S3SnapshotCreator(arguments.snapshotName, c, arguments.s3RepoUri, arguments.s3Region),
-                new OpenSearchClient(arguments.sourceHost, arguments.sourceUser, arguments.sourcePass, arguments.sourceInsecure));
+        if (arguments.fileSystemRepoPath == null && arguments.s3RepoUri == null) {
+            throw new ParameterException("Either file-system-repo-path or s3-repo-uri must be set");
+        }
+        if (arguments.fileSystemRepoPath != null && arguments.s3RepoUri != null) {
+            throw new ParameterException("Only one of file-system-repo-path and s3-repo-uri can be set");
+        }
+        if (arguments.s3RepoUri != null && arguments.s3Region == null) {
+            throw new ParameterException("If an s3 repo is being used, s3-region must be set");
+        }
+
+        log.info("Running CreateSnapshot with {}", String.join(" ", args));
+        run(c -> ((arguments.fileSystemRepoPath != null)
+                        ? new FileSystemSnapshotCreator(arguments.snapshotName, c, arguments.fileSystemRepoPath)
+                        : new S3SnapshotCreator(arguments.snapshotName, c, arguments.s3RepoUri, arguments.s3Region)),
+                new OpenSearchClient(arguments.sourceHost, arguments.sourceUser, arguments.sourcePass, arguments.sourceInsecure)
+        );
     }
 
-    public static void run(Function<OpenSearchClient,SnapshotCreator> snapshotCreatorFactory,
+    public static void run(Function<OpenSearchClient, SnapshotCreator> snapshotCreatorFactory,
                            OpenSearchClient openSearchClient)
             throws Exception {
         TryHandlePhaseFailure.executeWithTryCatch(() -> {

diff --git a/FetchMigration/Dockerfile b/FetchMigration/Dockerfile
@@ -1,5 +1,5 @@
 FROM opensearchproject/data-prepper:2.5.0
-COPY python/Pipfile .
+COPY python/Pipfile python/Pipfile.lock ./
 
 # Install dependencies to local user directory
 RUN apt -y update

diff --git a/MetadataMigration/src/main/java/com/rfs/MetadataMigration.java b/MetadataMigration/src/main/java/com/rfs/MetadataMigration.java
@@ -2,21 +2,23 @@
 
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
+import com.beust.jcommander.ParameterException;
 
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
 
 import com.rfs.common.ClusterVersion;
 import com.rfs.common.ConnectionDetails;
+import com.rfs.common.FileSystemRepo;
 import com.rfs.common.GlobalMetadata;
 import com.rfs.common.IndexMetadata;
 import com.rfs.common.OpenSearchClient;
-import com.rfs.common.S3Uri;
 import com.rfs.common.S3Repo;
+import com.rfs.common.S3Uri;
+import com.rfs.common.SnapshotRepo;
 import com.rfs.common.SourceRepo;
 import com.rfs.common.TryHandlePhaseFailure;
-import com.rfs.common.SnapshotRepo;
 import com.rfs.transformers.TransformFunctions;
 import com.rfs.transformers.Transformer;
 import com.rfs.version_es_7_10.GlobalMetadataFactory_ES_7_10;
@@ -35,13 +37,16 @@ public static class Args {
         @Parameter(names = {"--snapshot-name"}, description = "The name of the snapshot to migrate", required = true)
         public String snapshotName;
 
-        @Parameter(names = {"--s3-local-dir"}, description = "The absolute path to the directory on local disk to download S3 files to", required = true)
+        @Parameter(names = {"--file-system-repo-path"}, required = false, description = "The full path to the snapshot repo on the file system.")
+        public String fileSystemRepoPath;
+
+        @Parameter(names = {"--s3-local-dir"}, description = "The absolute path to the directory on local disk to download S3 files to", required = false)
         public String s3LocalDirPath;
 
-        @Parameter(names = {"--s3-repo-uri"}, description = "The S3 URI of the snapshot repo, like: s3://my-bucket/dir1/dir2", required = true)
+        @Parameter(names = {"--s3-repo-uri"}, description = "The S3 URI of the snapshot repo, like: s3://my-bucket/dir1/dir2", required = false)
         public String s3RepoUri;
 
-        @Parameter(names = {"--s3-region"}, description = "The AWS Region the S3 bucket is in, like: us-east-2", required = true)
+        @Parameter(names = {"--s3-region"}, description = "The AWS Region the S3 bucket is in, like: us-east-2", required = false)
         public String s3Region;
 
         @Parameter(names = {"--target-host"}, description = "The target host and port (e.g. http://localhost:9200)", required = true)
@@ -80,8 +85,19 @@ public static void main(String[] args) throws Exception {
                 .build()
                 .parse(args);
 
+        if (arguments.fileSystemRepoPath == null && arguments.s3RepoUri == null) {
+            throw new ParameterException("Either file-system-repo-path or s3-repo-uri must be set");
+        }
+        if (arguments.fileSystemRepoPath != null && arguments.s3RepoUri != null) {
+            throw new ParameterException("Only one of file-system-repo-path and s3-repo-uri can be set");
+        }
+        if ((arguments.s3RepoUri != null) && (arguments.s3Region == null || arguments.s3LocalDirPath == null)) {
+            throw new ParameterException("If an s3 repo is being used, s3-region and s3-local-dir-path must be set");
+        }
+
         final String snapshotName = arguments.snapshotName;
-        final Path s3LocalDirPath = Paths.get(arguments.s3LocalDirPath);
+        final Path fileSystemRepoPath = arguments.fileSystemRepoPath != null ? Paths.get(arguments.fileSystemRepoPath): null;
+        final Path s3LocalDirPath = arguments.s3LocalDirPath != null ? Paths.get(arguments.s3LocalDirPath) : null;
         final String s3RepoUri = arguments.s3RepoUri;
         final String s3Region = arguments.s3Region;
         final String targetHost = arguments.targetHost;
@@ -93,11 +109,15 @@ public static void main(String[] args) throws Exception {
 
         final ConnectionDetails targetConnection = new ConnectionDetails(targetHost, targetUser, targetPass);
 
+
+
         TryHandlePhaseFailure.executeWithTryCatch(() -> {
             log.info("Running RfsWorker");
             OpenSearchClient targetClient = new OpenSearchClient(targetConnection);
 
-            final SourceRepo sourceRepo = S3Repo.create(s3LocalDirPath, new S3Uri(s3RepoUri), s3Region);
+            final SourceRepo sourceRepo = fileSystemRepoPath != null
+                    ? new FileSystemRepo(fileSystemRepoPath)
+                    : S3Repo.create(s3LocalDirPath, new S3Uri(s3RepoUri), s3Region);
             final SnapshotRepo.Provider repoDataProvider = new SnapshotRepoProvider_ES_7_10(sourceRepo);
             final GlobalMetadata.Factory metadataFactory = new GlobalMetadataFactory_ES_7_10(repoDataProvider);
             final GlobalMetadataCreator_OS_2_11 metadataCreator = new GlobalMetadataCreator_OS_2_11(targetClient, List.of(), componentTemplateAllowlist, indexTemplateAllowlist);

diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/Pipfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/Pipfile
@@ -4,11 +4,12 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-requests = "*"
+requests = ">=2.32.3"
 boto3 = "*"
 pyyaml = "*"
 Click = "*"
 cerberus = "*"
+awscli = "*"
 
 [dev-packages]
 pytest = "*"