From 864eb6ea47c7b3213bd0047768fef96fc46fac81 Mon Sep 17 00:00:00 2001 From: Greg Schohn Date: Tue, 14 May 2024 09:07:09 -0400 Subject: [PATCH 1/7] Jenkins improvements for local testing. Setup a docker-compose environment to run jenkins and have a couple build options to build with local (current) changes. I'm also setting up/testing some changes for loading the Jenkinsfile from the repo. Signed-off-by: Greg Schohn --- Jenkinsfile | 78 ++++++++++++++++++++++++++++ jenkinsdocker/Dockerfile | 28 ++++++++++ jenkinsdocker/buildAll.xml | 42 +++++++++++++++ jenkinsdocker/buildFromLocal.xml | 69 ++++++++++++++++++++++++ jenkinsdocker/config.yml | 40 ++++++++++++++ jenkinsdocker/configuration.yaml | 21 ++++++++ jenkinsdocker/copyGitTrackedFiles.sh | 26 ++++++++++ jenkinsdocker/docker-compose.yml | 21 ++++++++ jenkinsdocker/plugins.txt | 70 +++++++++++++++++++++++++ 9 files changed, 395 insertions(+) create mode 100644 Jenkinsfile create mode 100644 jenkinsdocker/Dockerfile create mode 100644 jenkinsdocker/buildAll.xml create mode 100644 jenkinsdocker/buildFromLocal.xml create mode 100644 jenkinsdocker/config.yml create mode 100644 jenkinsdocker/configuration.yaml create mode 100755 jenkinsdocker/copyGitTrackedFiles.sh create mode 100644 jenkinsdocker/docker-compose.yml create mode 100644 jenkinsdocker/plugins.txt diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..7b1344bce --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,78 @@ +pipeline { + environment { + // GIT_URL = 'https://github.com/mikaylathompson/opensearch-migrations.git' + GIT_URL = 'https://github.com/opensearch-project/opensearch-migrations.git' + GIT_BRANCH = 'main' + STAGE = 'aws-integ' + } + + agent any + + parameters { + booleanParam(name: 'USE_LOCAL_WORKSPACE', defaultValue: false, description: 'Use local workspace for the build') + string(name: 'BRANCH_NAME', defaultValue: 'main', description: 'Branch to build from') + } + + stages { + stage('Checkout') { + agent any + steps { + script { + if (params.USE_LOCAL_WORKSPACE) { + sh "/copyGitTrackedFiles.sh /opensearch-migrations-src ." + } else { + git branch: "${params.BRANCH_NAME}", url: "${env.GIT_URL}" + } + } + } + } + + stage('Test Caller Identity') { + agent any + steps { + sh 'aws sts get-caller-identity' + } + } + + stage('Build') { + agent any + steps { + timeout(time: 1, unit: 'HOURS') { + dir('TrafficCapture') { + sh './gradlew build -x test' + } + } + } + } + + stage('Deploy') { + steps { + dir('test') { + sh 'sudo usermod -aG docker $USER' + sh 'sudo newgrp docker' + sh "sudo ./awsE2ESolutionSetup.sh --stage ${env.STAGE} --migrations-git-url ${env.GIT_URL} --migrations-git-branch ${env.GIT_BRANCH}" + } + } + } + + stage('Integ Tests') { + steps { + dir('test') { + script { + def time = new Date().getTime() + def uniqueId = "integ_min_${time}_${currentBuild.number}" + sh "sudo ./awsRunIntegTests.sh --stage ${env.STAGE} --migrations-git-url ${env.GIT_URL} --migrations-git-branch ${env.GIT_BRANCH} --unique-id ${uniqueId}" + } + } + + } + } + } +// post { +// always { +// dir('test') { +// sh "sudo ./awsE2ESolutionSetup.sh --stage ${env.STAGE} --run-post-actions" +// } +// } +// } +} \ No newline at end of file diff --git a/jenkinsdocker/Dockerfile b/jenkinsdocker/Dockerfile new file mode 100644 index 000000000..3bd34b265 --- /dev/null +++ b/jenkinsdocker/Dockerfile @@ -0,0 +1,28 @@ +FROM jenkins/jenkins:lts + +USER root +RUN apt -y update +RUN apt -y install software-properties-common +RUN add-apt-repository 'deb http://deb.debian.org/debian bullseye main contrib non-free' +RUN add-apt-repository 'deb http://deb.debian.org/debian bullseye-backports main contrib non-free' +RUN apt -y install openjdk-11-jdk +RUN apt -y install gradle +RUN apt -y install docker +RUN apt -y install docker.io +RUN apt -y install npm +RUN npm install -g aws-cdk +RUN apt -y install parallel +RUN apt -y install vim less +RUN apt -y install python3 python3-pip +RUN apt -y install awscli + +ENV PATH=/usr/lib/jvm/java-11-openjdk-amd64/bin:$PATH +ENV JAVA_OPTS="-Djenkins.install.runSetupWizard=false" + +COPY plugins.txt /usr/share/jenkins/ref/plugins.txt +RUN jenkins-plugin-cli -f /usr/share/jenkins/ref/plugins.txt + +COPY copyGitTrackedFiles.sh . +RUN chmod ugo+x copyGitTrackedFiles.sh +RUN mkdir -p /var/jenkins_home/jobs/buildAll/builds; mkdir -p /var/jenkins_home/jobs/localWorkspace/builds +#COPY configuration.yaml /var/jenkins_home/casc_configs/configuration.yaml diff --git a/jenkinsdocker/buildAll.xml b/jenkinsdocker/buildAll.xml new file mode 100644 index 000000000..b26ef0ad2 --- /dev/null +++ b/jenkinsdocker/buildAll.xml @@ -0,0 +1,42 @@ + + + false + + + + false + false + + + + + false + project + false + + + + + + + 2 + + + /opensearch-migrations-src + + + + + */jenkinstests + + + false + + + + Jenkinsfile + true + + + false + \ No newline at end of file diff --git a/jenkinsdocker/buildFromLocal.xml b/jenkinsdocker/buildFromLocal.xml new file mode 100644 index 000000000..632a61a95 --- /dev/null +++ b/jenkinsdocker/buildFromLocal.xml @@ -0,0 +1,69 @@ + + + + + + + + + + + + false + + + + false + false + + + + + false + project + false + + + + + + + true + + + false + \ No newline at end of file diff --git a/jenkinsdocker/config.yml b/jenkinsdocker/config.yml new file mode 100644 index 000000000..5122b6a6b --- /dev/null +++ b/jenkinsdocker/config.yml @@ -0,0 +1,40 @@ +jenkins: + systemMessage: "Jenkins configured via JCasC" + +jobs: + - script: > + pipelineJob('example-pipeline') { + definition { + cpsScm { + scm { + git { + remote { + url('/opensearch-migrations-src') + } + branches('*/main') + scriptPath('jenkins/release.jenkinsFile') + } + } + } + } + } + +#credentials: +# system: +# domainCredentials: +# - credentials: +# - usernamePassword: +# id: "github-credentials" +# username: "user" +# password: "password" +# description: "GitHub Credentials" + +tool: + git: + installations: + - name: "Default" + home: "/usr/bin/git" + +#unclassified: +# location: +# url: "http://your-jenkins-url.com" diff --git a/jenkinsdocker/configuration.yaml b/jenkinsdocker/configuration.yaml new file mode 100644 index 000000000..e98e0be08 --- /dev/null +++ b/jenkinsdocker/configuration.yaml @@ -0,0 +1,21 @@ +jobs: + - script: | + pipelineJob('DSL_Pipeline') { + + def repo = '/opensearch-migrations-src' + def branch = 'main' + description("Build All Pipeline") + + definition { + cpsScm { + scm { + git { + remote { url(repo) } + branches(branch) + } + scriptPath('jenkinsFile') + extensions { } // required as otherwise it may try to tag the repo, which you may not want + } + } + } + } diff --git a/jenkinsdocker/copyGitTrackedFiles.sh b/jenkinsdocker/copyGitTrackedFiles.sh new file mode 100755 index 000000000..7e55d87bd --- /dev/null +++ b/jenkinsdocker/copyGitTrackedFiles.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +destination_dir=`cd $2; echo $PWD` +cd $1 || exit + +copy_files() { +# set -x + local file="$1" + local destination_dir="$2" + local relative_path="${file#./}" + + if git check-ignore -q "$relative_path"; then + echo "Ignoring: $relative_path" + else + mkdir -p "$destination_dir/$(dirname "$relative_path")" + cp -r "$relative_path" "$destination_dir/$relative_path" + fi +} + +export -f copy_files +git ls-files -z | parallel -0 copy_files {} "$destination_dir" diff --git a/jenkinsdocker/docker-compose.yml b/jenkinsdocker/docker-compose.yml new file mode 100644 index 000000000..052d4ee5d --- /dev/null +++ b/jenkinsdocker/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3.7' +services: + + jenkins: + build: + context: . + dockerfile: Dockerfile + volumes: + - ~/.aws:/root/.aws + - ..:/opensearch-migrations-src + - ./buildAll.xml:/var/jenkins_home/jobs/buildAll/config.xml + - ./buildFromLocal.xml:/var/jenkins_home/jobs/localWorkspace/config.xml + - /var/run/docker.sock:/var/run/docker.sock + environment: + - ADMIN_PASSWORD=admin + #- repoPath=/opensearch-migrations-src +# - CASC_JENKINS_CONFIG=/var/jenkins_home/casc_configs/configuration.yaml + #- repoPath=https://github.com/opensearch-project/opensearch-migrations + ports: + - "8080:8080" + - "50000:50000" diff --git a/jenkinsdocker/plugins.txt b/jenkinsdocker/plugins.txt new file mode 100644 index 000000000..ebe267f81 --- /dev/null +++ b/jenkinsdocker/plugins.txt @@ -0,0 +1,70 @@ +git +configuration-as-code +ec2 +ansicolor +audit-trail +aws-secrets-manager-credentials-provider +blueocean +build-with-parameters +build-timeout +build-timestamp +docker-custom-build-environment +description-setter +disable-github-multibranch-status +docker-workflow +adoptopenjdk +email-ext +environment-script +extensible-choice-parameter +external-monitor-job +generic-webhook-trigger +git-parameter +ghprb +gradle +hidden-parameter +ivy +javax-mail-api +ace-editor +handlebars +ownership +jobConfigHistory +job-dsl +job-import-plugin +job-restrictions +ldap +lockable-resources +login-theme +managed-scripts +nodelabelparameter +nvm-wrapper +oic-auth +jdk-tool +pam-auth +parameterized-scheduler +workflow-aggregator +pipeline-utility-steps +pipeline-utility-steps +pipeline-aws +pipeline-github-lib +pipeline-stage-view +postbuild-task +postbuildscript +powershell +preSCMbuildstep +project-description-setter +python +readonly-parameters +rebuild +role-strategy +s3 +saml +ssh-slaves +sshd +strict-crumb-issuer +support-core +throttle-concurrents +timestamper +validating-string-parameter +ws-cleanup +simple-theme-plugin +uno-choice From d990bc84bdd165ab614478d1234784655de91424 Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Tue, 14 May 2024 21:30:28 -0600 Subject: [PATCH 2/7] init console_link library Signed-off-by: Mikayla Thompson --- .../console_link/console_link/__init__.py | 0 .../console_link/console_link/cli.py | 29 ++++++++++ .../console_link/logic/clusters.py | 9 ++++ .../console_link/logic/instantiation.py | 11 ++++ .../console_link/models/cluster.py | 41 ++++++++++++++ .../console_link/models/replayerService.py | 53 +++++++++++++++++++ .../console_link/console_link/services.yaml | 16 ++++++ .../migrationConsole/console_link/setup.py | 31 +++++++++++ 8 files changed, 190 insertions(+) create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/__init__.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/clusters.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayerService.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/services.yaml create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/__init__.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py new file mode 100644 index 000000000..412c3fb3b --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py @@ -0,0 +1,29 @@ +import click +from console_link.logic.clusters import cat_indices +from console_link.logic.instantiation import Environment +from pprint import pprint + + +class Context(object): + def __init__(self, config_file) -> None: + self.config_file = config_file + self.env = Environment(config_file) + + +@click.group() +@click.option('--config-file', default='services.yaml', help='Path to config file') +@click.pass_context +def cli(ctx, config_file): + ctx.obj = Context(config_file) + + +@cli.command(name="cat-indices") +@click.pass_obj +def cat_indices_cmd(ctx): + """Simple program that calls `_cat/indices` on both a source and target cluster.""" + pprint(cat_indices(ctx.env)) + pass + + +if __name__ == '__main__': + cli() diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/clusters.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/clusters.py new file mode 100644 index 000000000..3267758c0 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/clusters.py @@ -0,0 +1,9 @@ +from console_link.logic.instantiation import Environment + + +def cat_indices(env: Environment): + cat_indices_path = '/_cat/indices?format=json' + # source_indices = env.source_cluster.call_api(cat_indices_path) + source_indices = {} + target_indices = env.target_cluster.call_api(cat_indices_path) + return source_indices, target_indices diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py new file mode 100644 index 000000000..c4bb18807 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py @@ -0,0 +1,11 @@ +from console_link.models.cluster import Cluster +import yaml + + +class Environment: + def __init__(self, config_file: str): + self.config_file = config_file + with open(self.config_file) as f: + self.config = yaml.safe_load(f) + self.source_cluster = Cluster(self.config['source_cluster']) + self.target_cluster = Cluster(self.config['target_cluster']) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py new file mode 100644 index 000000000..3d707dd09 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py @@ -0,0 +1,41 @@ +from typing import Dict, Optional +from enum import Enum +import requests +from requests.auth import HTTPBasicAuth + +requests.packages.urllib3.disable_warnings() + +AuthMethod = Enum("AuthMethod", ["BASIC", "SIGV4"]) +HttpMethod = Enum("HttpMethod", ["GET", "POST", "PUT", "DELETE"]) + + +class Cluster(): + """ + An abstract class that represents a elasticsearch or opensearch cluster. + """ + endpoint: str = "" + auth_type: Optional[AuthMethod] = None + auth_details: Optional[Dict] = None + + def __init__(self, config: Dict) -> None: + self.endpoint = config["endpoint"] + if self.endpoint.startswith("https"): + self.allow_insecure = config.get("allow_insecure", False) + self.auth_type = AuthMethod[config["authorization"]["type"].upper()] + self.auth_details = config["authorization"]["details"] + pass + + def call_api(self, path, method: HttpMethod = HttpMethod.GET) -> Dict: + """ + Calls an API on the source cluster. + """ + if self.auth_type == AuthMethod.BASIC: + auth = HTTPBasicAuth(self.auth_details["username"], self.auth_details["password"]) + elif self.auth_type is None: + auth = None + else: + raise NotImplementedError(f"Auth type {self.auth_type} not implemented") + + r = requests.request(method.name, f"{self.endpoint}{path}", verify=(not self.allow_insecure), auth=auth) + r.raise_for_status() + return r.json() diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayerService.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayerService.py new file mode 100644 index 000000000..8ddc2c19a --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayerService.py @@ -0,0 +1,53 @@ +from abc import ABC, abstractmethod +from typing import Dict, Optional + + +class AbstractReplayerService(ABC): + """ + An abstract class that represents a replayer. + """ + endpoint: str = "" + authDetails: Optional[Dict] = {"type": "None"} + + @abstractmethod + def get_config(self) -> Dict: + """ + Returns the configuration details of the replayer. + """ + pass + + @abstractmethod + def start_replayer(): + """ + Starts the replayer. + """ + pass + + @abstractmethod + def stop_replayer(): + """ + Stops the replayer. + """ + pass + + +class LocalReplayer(AbstractReplayerService): + def get_config(self) -> Dict: + return self.config + + def start_replayer(): + pass + + def stop_replayer(): + pass + + +class ECSReplayer(AbstractReplayerService): + def get_config(self) -> Dict: + return self.config + + def start_replayer(): + pass + + def stop_replayer(): + pass diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/services.yaml b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/services.yaml new file mode 100644 index 000000000..00859a2da --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/services.yaml @@ -0,0 +1,16 @@ +source_cluster: + endpoint: "https://capture-proxy-es:9200" + allow_insecure: true + authorization: + type: "basic" + details: + username: "admin" + password: "admin" +target_cluster: + endpoint: "https://opensearchtarget:9200" + allow_insecure: true + authorization: + type: "basic" + details: + username: "admin" + password: "myStrongPassword123!" \ No newline at end of file diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py new file mode 100644 index 000000000..b3a18263d --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py @@ -0,0 +1,31 @@ +from setuptools import setup, find_packages + +setup( + name='console_link', + version='1.0.0', + description='A Python module to create a console application from a Python script', + packages=find_packages(exclude=('tests')), + install_requires=[ + 'requests', + 'boto3', + 'pyyaml', + 'Click' + ], + entry_points={ + 'console_scripts': [ + 'console_link = console_link.cli:cli', + ], + }, + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'Topic :: Software Development :: Build Tools', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'License :: OSI Approved :: MIT License', + ], +) From 5f0b9f10a73eea9d70745a1e8ad6f66277052f4f Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Wed, 15 May 2024 07:56:51 -0600 Subject: [PATCH 3/7] fix bugs, clean up function Signed-off-by: Mikayla Thompson --- .../src/main/docker/docker-compose.yml | 2 ++ .../src/main/docker/migrationConsole/Dockerfile | 3 +++ .../console_link/console_link/cli.py | 8 +++++--- .../console_link/console_link/logic/clusters.py | 13 ++++++------- .../console_link/logic/instantiation.py | 6 ++++++ .../console_link/console_link/models/cluster.py | 6 +++--- .../console_link/{console_link => }/services.yaml | 0 7 files changed, 25 insertions(+), 13 deletions(-) rename TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/{console_link => }/services.yaml (100%) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 69ed9072d..89882519a 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -95,6 +95,8 @@ services: - migrations volumes: - sharedReplayerOutput:/shared-replayer-output + - ./migrationConsole/console_link/services.yaml:/etc/migration_services.yaml + - ./migrationConsole/console_link:/root/console_link environment: - MIGRATION_KAFKA_BROKER_ENDPOINTS=kafka:9092 # command: ./runTestBenchmarks.sh diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile index d16835af5..1d773e12f 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile @@ -41,4 +41,7 @@ RUN chmod ug+x /root/showFetchMigrationCommand.sh RUN chmod ug+x /root/osiMigration.py RUN chmod ug+x /root/kafka-tools/kafkaExport.sh +COPY console_link /root/console_link +RUN pip install -e /root/console_link/ + CMD tail -f /dev/null diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py index 412c3fb3b..194da3193 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py @@ -1,7 +1,6 @@ import click from console_link.logic.clusters import cat_indices from console_link.logic.instantiation import Environment -from pprint import pprint class Context(object): @@ -11,7 +10,7 @@ def __init__(self, config_file) -> None: @click.group() -@click.option('--config-file', default='services.yaml', help='Path to config file') +@click.option('--config-file', default='/etc/migration_services.yaml', help='Path to config file') @click.pass_context def cli(ctx, config_file): ctx.obj = Context(config_file) @@ -21,7 +20,10 @@ def cli(ctx, config_file): @click.pass_obj def cat_indices_cmd(ctx): """Simple program that calls `_cat/indices` on both a source and target cluster.""" - pprint(cat_indices(ctx.env)) + click.echo("SOURCE CLUSTER") + click.echo(cat_indices(ctx.env.source_cluster)) + click.echo("TARGET CLUSTER") + click.echo(cat_indices(ctx.env.target_cluster)) pass diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/clusters.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/clusters.py index 3267758c0..c399cce7e 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/clusters.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/clusters.py @@ -1,9 +1,8 @@ -from console_link.logic.instantiation import Environment +from console_link.models.cluster import Cluster -def cat_indices(env: Environment): - cat_indices_path = '/_cat/indices?format=json' - # source_indices = env.source_cluster.call_api(cat_indices_path) - source_indices = {} - target_indices = env.target_cluster.call_api(cat_indices_path) - return source_indices, target_indices +def cat_indices(cluster: Cluster, as_json=False): + as_json_suffix = "?format=json" if as_json else "" + cat_indices_path = f"/_cat/indices{as_json_suffix}" + r = cluster.call_api(cat_indices_path) + return r.json() if as_json else r.content diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py index c4bb18807..227d000c8 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py @@ -4,8 +4,14 @@ class Environment: def __init__(self, config_file: str): + # TODO: add validation of overall yaml structure here, and details in each component. + self.config_file = config_file with open(self.config_file) as f: self.config = yaml.safe_load(f) + self.source_cluster = Cluster(self.config['source_cluster']) + + # At some point, target and replayers should be stored as pairs, but for the time being + # we can probably assume one target cluster. self.target_cluster = Cluster(self.config['target_cluster']) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py index 3d707dd09..ae38b1c53 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py @@ -11,7 +11,7 @@ class Cluster(): """ - An abstract class that represents a elasticsearch or opensearch cluster. + An elasticcsearch or opensearch cluster. """ endpoint: str = "" auth_type: Optional[AuthMethod] = None @@ -27,7 +27,7 @@ def __init__(self, config: Dict) -> None: def call_api(self, path, method: HttpMethod = HttpMethod.GET) -> Dict: """ - Calls an API on the source cluster. + Calls an API on the cluster. """ if self.auth_type == AuthMethod.BASIC: auth = HTTPBasicAuth(self.auth_details["username"], self.auth_details["password"]) @@ -38,4 +38,4 @@ def call_api(self, path, method: HttpMethod = HttpMethod.GET) -> Dict: r = requests.request(method.name, f"{self.endpoint}{path}", verify=(not self.allow_insecure), auth=auth) r.raise_for_status() - return r.json() + return r diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/services.yaml b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/services.yaml similarity index 100% rename from TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/services.yaml rename to TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/services.yaml From 6f0c2ae9042dad474fd36b8c1f792d1f274532a0 Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Wed, 15 May 2024 11:09:43 -0600 Subject: [PATCH 4/7] Add some schema validation, fix replayer Signed-off-by: Mikayla Thompson --- .../console_link/console_link/cli.py | 12 +++- .../console_link/logic/__init__.py | 2 + .../console_link/logic/instantiation.py | 23 +++++++ .../console_link/models/cluster.py | 34 +++++++++- .../console_link/models/replayer.py | 66 +++++++++++++++++++ .../console_link/models/replayerService.py | 53 --------------- .../console_link/services.yaml | 4 +- .../migrationConsole/console_link/setup.py | 3 +- 8 files changed, 137 insertions(+), 60 deletions(-) create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/__init__.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayer.py delete mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayerService.py diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py index 194da3193..bd7f04d3d 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py @@ -1,5 +1,5 @@ import click -from console_link.logic.clusters import cat_indices +import console_link.logic as logic from console_link.logic.instantiation import Environment @@ -21,11 +21,17 @@ def cli(ctx, config_file): def cat_indices_cmd(ctx): """Simple program that calls `_cat/indices` on both a source and target cluster.""" click.echo("SOURCE CLUSTER") - click.echo(cat_indices(ctx.env.source_cluster)) + click.echo(logic.clusters.cat_indices(ctx.env.source_cluster)) click.echo("TARGET CLUSTER") - click.echo(cat_indices(ctx.env.target_cluster)) + click.echo(logic.clusters.cat_indices(ctx.env.target_cluster)) pass +@cli.command(name="start-replayer") +@click.pass_obj +def start_replayer_cmd(ctx): + logic.services.start_replayer(ctx.env.replayer) + + if __name__ == '__main__': cli() diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/__init__.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/__init__.py new file mode 100644 index 000000000..a79414ae6 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/__init__.py @@ -0,0 +1,2 @@ +import console_link.logic.clusters +import console_link.logic.instantiation diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py index 227d000c8..483540899 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py @@ -1,5 +1,25 @@ from console_link.models.cluster import Cluster import yaml +from cerberus import Validator + +SCHEMA = { + 'source_cluster': { + 'type': 'dict', + 'required': True + }, + 'target_cluster': { + 'type': 'dict', + 'required': True + }, + 'replayer': { + 'type': 'dict', + 'required': False + }, + 'backfill': { + 'type': 'dict', + 'required': False + } +} class Environment: @@ -9,6 +29,9 @@ def __init__(self, config_file: str): self.config_file = config_file with open(self.config_file) as f: self.config = yaml.safe_load(f) + v = Validator(SCHEMA) + if not v.validate(self.config): + raise ValueError(f"Invalid config file: {v.errors}") self.source_cluster = Cluster(self.config['source_cluster']) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py index ae38b1c53..1aa15170e 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py @@ -2,12 +2,38 @@ from enum import Enum import requests from requests.auth import HTTPBasicAuth +from cerberus import Validator -requests.packages.urllib3.disable_warnings() +requests.packages.urllib3.disable_warnings() AuthMethod = Enum("AuthMethod", ["BASIC", "SIGV4"]) HttpMethod = Enum("HttpMethod", ["GET", "POST", "PUT", "DELETE"]) +SCHEMA = { + 'endpoint': { + 'type': 'string', + 'required': True + }, + 'allow_insecure': { + 'type': 'boolean', + 'required': False + }, + 'authorization': { + 'type': 'dict', + 'required': False, + 'schema': { + 'type': { + 'type': 'string', + 'required': True, + 'allowed': [e.name.lower() for e in AuthMethod] + }, + 'details': { + 'type': 'dict' + } + } + } +} + class Cluster(): """ @@ -18,6 +44,10 @@ class Cluster(): auth_details: Optional[Dict] = None def __init__(self, config: Dict) -> None: + v = Validator(SCHEMA) + if not v.validate(config): + raise ValueError(f"Invalid config file for cluster: {v.errors}") + self.endpoint = config["endpoint"] if self.endpoint.startswith("https"): self.allow_insecure = config.get("allow_insecure", False) @@ -35,7 +65,7 @@ def call_api(self, path, method: HttpMethod = HttpMethod.GET) -> Dict: auth = None else: raise NotImplementedError(f"Auth type {self.auth_type} not implemented") - + r = requests.request(method.name, f"{self.endpoint}{path}", verify=(not self.allow_insecure), auth=auth) r.raise_for_status() return r diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayer.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayer.py new file mode 100644 index 000000000..b48714a97 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayer.py @@ -0,0 +1,66 @@ +from enum import Enum +from typing import Dict +import boto3 + +DeploymentType = Enum('DeploymentType', ['DOCKER', 'ECS']) + +SCHEMA = { + 'deployment_type': { + 'type': 'string', + 'required': True + } +} + + +class BaseReplayer(): + """ + The Replayer base class + """ + def __init__(self, config: Dict) -> None: + self.config = config + v = Validator(SCHEMA) + if not v.validate(config): + raise ValueError(f"Invalid config file for cluster: {v.errors}") + + def start_replayer(self): + """ + Starts the replayer. + """ + raise NotImplementedError + + def stop_replayer(self): + """ + Stops the replayer. + """ + raise NotImplementedError + + +class LocalReplayer(BaseReplayer): + def start_replayer(self): + pass + + def stop_replayer(self): + pass + + +class ECSReplayer(BaseReplayer): + client = boto3.client('ecs') + + def __init__(self, config: Dict) -> None: + super().__init__(config) + self.cluster_name = config['cluster_name'] + self.service_name = config['service_name'] + + def start_replayer(self) -> None: + self.client.update_service( + cluster=self.cluster_name, + service=self.service_name, + desiredCount=1 + ) + + def stop_replayer(self) -> None: + self.client.update_service( + cluster=self.cluster_name, + service=self.service_name, + desiredCount=0 + ) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayerService.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayerService.py deleted file mode 100644 index 8ddc2c19a..000000000 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayerService.py +++ /dev/null @@ -1,53 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Dict, Optional - - -class AbstractReplayerService(ABC): - """ - An abstract class that represents a replayer. - """ - endpoint: str = "" - authDetails: Optional[Dict] = {"type": "None"} - - @abstractmethod - def get_config(self) -> Dict: - """ - Returns the configuration details of the replayer. - """ - pass - - @abstractmethod - def start_replayer(): - """ - Starts the replayer. - """ - pass - - @abstractmethod - def stop_replayer(): - """ - Stops the replayer. - """ - pass - - -class LocalReplayer(AbstractReplayerService): - def get_config(self) -> Dict: - return self.config - - def start_replayer(): - pass - - def stop_replayer(): - pass - - -class ECSReplayer(AbstractReplayerService): - def get_config(self) -> Dict: - return self.config - - def start_replayer(): - pass - - def stop_replayer(): - pass diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/services.yaml b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/services.yaml index 00859a2da..6f1324336 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/services.yaml +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/services.yaml @@ -13,4 +13,6 @@ target_cluster: type: "basic" details: username: "admin" - password: "myStrongPassword123!" \ No newline at end of file + password: "myStrongPassword123!" +replayer: + deployment_type: "docker" diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py index b3a18263d..6e97c7448 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py @@ -9,7 +9,8 @@ 'requests', 'boto3', 'pyyaml', - 'Click' + 'Click', + 'cerberus' ], entry_points={ 'console_scripts': [ From 4534f6cd457e25aca3c959601d9d304ecbfdbd26 Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Wed, 15 May 2024 11:37:37 -0600 Subject: [PATCH 5/7] Start adding tests Signed-off-by: Mikayla Thompson --- .../console_link/models/cluster.py | 2 +- .../console_link/tests/requirements.txt | 1 + .../console_link/tests/test_cluster.py | 50 +++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/tests/requirements.txt create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/tests/test_cluster.py diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py index 1aa15170e..eafe13939 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py @@ -46,7 +46,7 @@ class Cluster(): def __init__(self, config: Dict) -> None: v = Validator(SCHEMA) if not v.validate(config): - raise ValueError(f"Invalid config file for cluster: {v.errors}") + raise ValueError("Invalid config file for cluster", v.errors) self.endpoint = config["endpoint"] if self.endpoint.startswith("https"): diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/tests/requirements.txt b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/tests/requirements.txt new file mode 100644 index 000000000..55b033e90 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/tests/requirements.txt @@ -0,0 +1 @@ +pytest \ No newline at end of file diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/tests/test_cluster.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/tests/test_cluster.py new file mode 100644 index 000000000..2b067244a --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/tests/test_cluster.py @@ -0,0 +1,50 @@ +import pytest +from console_link.models.cluster import Cluster + +# Define a valid cluster configuration +valid_cluster_config = { + 'endpoint': "https://opensearchtarget:9200", + 'allow_insecure': True, + 'authorization': { + 'type': "basic", + 'details': { + 'username': "admin", + 'password': "myStrongPassword123!" + } + } +} + + +def test_valid_cluster_config(): + cluster = Cluster(valid_cluster_config) + assert isinstance(cluster, Cluster) + + +def test_invalid_auth_type_refused(): + invalid_auth_type = { + 'endpoint': "https://opensearchtarget:9200", + 'authorization': { + 'type': "invalid_type", + } + } + with pytest.raises(ValueError) as excinfo: + Cluster(invalid_auth_type) + assert "Invalid config file for cluster" in excinfo.value.args[0] + assert excinfo.value.args[1]['authorization'] == [{'type': ['unallowed value invalid_type']}] + + +def test_missing_endpoint_refused(): + missing_endpoint = { + 'allow_insecure': True, + 'authorization': { + 'type': "basic", + 'details': { + 'username': "XXXXX", + 'password': "XXXXXXXXXXXXXXXXXXX!" + } + } + } + with pytest.raises(ValueError) as excinfo: + Cluster(missing_endpoint) + assert "Invalid config file for cluster" in excinfo.value.args[0] + assert excinfo.value.args[1]['endpoint'] == ['required field'] From 12e8554989a707556136ae72cb08e9dbebc5b168 Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Wed, 15 May 2024 11:41:41 -0600 Subject: [PATCH 6/7] Change cli name to just `console` Signed-off-by: Mikayla Thompson --- .../src/main/docker/migrationConsole/console_link/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py index 6e97c7448..305997728 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py @@ -14,7 +14,7 @@ ], entry_points={ 'console_scripts': [ - 'console_link = console_link.cli:cli', + 'console = console_link.cli:cli', ], }, classifiers=[ From dfea93540c568ac8625b5662bd44f65cbe08733e Mon Sep 17 00:00:00 2001 From: Mikayla Thompson Date: Thu, 16 May 2024 11:44:36 -0600 Subject: [PATCH 7/7] Address review comments & flake8 Signed-off-by: Mikayla Thompson --- .../src/main/docker/docker-compose.yml | 5 +++-- .../main/docker/migrationConsole/Dockerfile | 3 +-- .../console_link/logic/__init__.py | 2 -- .../console_link/console_link/__init__.py | 0 .../{ => lib}/console_link/console_link/cli.py | 0 .../console_link/logic/__init__.py | 2 ++ .../console_link/logic/clusters.py | 0 .../console_link/logic/instantiation.py | 2 +- .../console_link/models/cluster.py | 2 +- .../console_link/models/metrics_source.py | 0 .../console_link/models/replayer.py | 18 +++++++++++++++++- .../{ => lib}/console_link/services.yaml | 0 .../{ => lib}/console_link/setup.py | 0 .../console_link/tests/requirements.txt | 0 .../console_link/tests/test_cluster.py | 0 15 files changed, 25 insertions(+), 9 deletions(-) delete mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/__init__.py rename TrafficCapture/dockerSolution/src/main/docker/migrationConsole/{ => lib}/console_link/console_link/__init__.py (100%) rename TrafficCapture/dockerSolution/src/main/docker/migrationConsole/{ => lib}/console_link/console_link/cli.py (100%) create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/__init__.py rename TrafficCapture/dockerSolution/src/main/docker/migrationConsole/{ => lib}/console_link/console_link/logic/clusters.py (100%) rename TrafficCapture/dockerSolution/src/main/docker/migrationConsole/{ => lib}/console_link/console_link/logic/instantiation.py (94%) rename TrafficCapture/dockerSolution/src/main/docker/migrationConsole/{ => lib}/console_link/console_link/models/cluster.py (98%) create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metrics_source.py rename TrafficCapture/dockerSolution/src/main/docker/migrationConsole/{ => lib}/console_link/console_link/models/replayer.py (76%) rename TrafficCapture/dockerSolution/src/main/docker/migrationConsole/{ => lib}/console_link/services.yaml (100%) rename TrafficCapture/dockerSolution/src/main/docker/migrationConsole/{ => lib}/console_link/setup.py (100%) rename TrafficCapture/dockerSolution/src/main/docker/migrationConsole/{ => lib}/console_link/tests/requirements.txt (100%) rename TrafficCapture/dockerSolution/src/main/docker/migrationConsole/{ => lib}/console_link/tests/test_cluster.py (100%) diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 89882519a..65f96a500 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -95,8 +95,9 @@ services: - migrations volumes: - sharedReplayerOutput:/shared-replayer-output - - ./migrationConsole/console_link/services.yaml:/etc/migration_services.yaml - - ./migrationConsole/console_link:/root/console_link + - ./migrationConsole/lib/console_link/services.yaml:/etc/migration_services.yaml + # this is a convenience thing for testing -- it should be removed before this makes it to prod. + - ./migrationConsole/lib/console_link:/root/lib/console_link environment: - MIGRATION_KAFKA_BROKER_ENDPOINTS=kafka:9092 # command: ./runTestBenchmarks.sh diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile index 1d773e12f..ff2c6958d 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile @@ -41,7 +41,6 @@ RUN chmod ug+x /root/showFetchMigrationCommand.sh RUN chmod ug+x /root/osiMigration.py RUN chmod ug+x /root/kafka-tools/kafkaExport.sh -COPY console_link /root/console_link -RUN pip install -e /root/console_link/ +RUN pip install -e /root/lib/console_link/ CMD tail -f /dev/null diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/__init__.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/__init__.py deleted file mode 100644 index a79414ae6..000000000 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -import console_link.logic.clusters -import console_link.logic.instantiation diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/__init__.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/__init__.py similarity index 100% rename from TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/__init__.py rename to TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/__init__.py diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py similarity index 100% rename from TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/cli.py rename to TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/cli.py diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/__init__.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/__init__.py new file mode 100644 index 000000000..25a7cfb91 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/__init__.py @@ -0,0 +1,2 @@ +import console_link.logic.clusters +import console_link.logic.instantiation # noqa: F401 diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/clusters.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/clusters.py similarity index 100% rename from TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/clusters.py rename to TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/clusters.py diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/instantiation.py similarity index 94% rename from TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py rename to TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/instantiation.py index 483540899..2111d131a 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/logic/instantiation.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/logic/instantiation.py @@ -31,7 +31,7 @@ def __init__(self, config_file: str): self.config = yaml.safe_load(f) v = Validator(SCHEMA) if not v.validate(self.config): - raise ValueError(f"Invalid config file: {v.errors}") + raise ValueError("Invalid config file", v.errors) self.source_cluster = Cluster(self.config['source_cluster']) diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py similarity index 98% rename from TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py rename to TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py index eafe13939..d0b796f45 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/cluster.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/cluster.py @@ -55,7 +55,7 @@ def __init__(self, config: Dict) -> None: self.auth_details = config["authorization"]["details"] pass - def call_api(self, path, method: HttpMethod = HttpMethod.GET) -> Dict: + def call_api(self, path, method: HttpMethod = HttpMethod.GET) -> requests.Response: """ Calls an API on the cluster. """ diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metrics_source.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/metrics_source.py new file mode 100644 index 000000000..e69de29bb diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayer.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/replayer.py similarity index 76% rename from TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayer.py rename to TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/replayer.py index b48714a97..ede125ead 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/console_link/models/replayer.py +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/console_link/models/replayer.py @@ -1,6 +1,7 @@ from enum import Enum from typing import Dict import boto3 +from cerberus import Validator DeploymentType = Enum('DeploymentType', ['DOCKER', 'ECS']) @@ -20,7 +21,7 @@ def __init__(self, config: Dict) -> None: self.config = config v = Validator(SCHEMA) if not v.validate(config): - raise ValueError(f"Invalid config file for cluster: {v.errors}") + raise ValueError("Invalid config file for replayer", v.errors) def start_replayer(self): """ @@ -43,11 +44,26 @@ def stop_replayer(self): pass +ECS_SCHEMA = { + 'cluster_name': { + 'type': 'string', + 'required': True + }, + 'service_name': { + 'type': 'string', + 'required': True + } +} + + class ECSReplayer(BaseReplayer): client = boto3.client('ecs') def __init__(self, config: Dict) -> None: super().__init__(config) + v = Validator(ECS_SCHEMA) + if not v.validate(config): + raise ValueError("Invalid config file for replayer", v.errors) self.cluster_name = config['cluster_name'] self.service_name = config['service_name'] diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/services.yaml b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/services.yaml similarity index 100% rename from TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/services.yaml rename to TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/services.yaml diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/setup.py similarity index 100% rename from TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/setup.py rename to TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/setup.py diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/tests/requirements.txt b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/requirements.txt similarity index 100% rename from TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/tests/requirements.txt rename to TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/requirements.txt diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/tests/test_cluster.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cluster.py similarity index 100% rename from TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_link/tests/test_cluster.py rename to TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link/tests/test_cluster.py