From 374bd137037c08e930c9adb050534ac7e7a299ea Mon Sep 17 00:00:00 2001 From: Brian Presley Date: Mon, 5 Feb 2024 11:28:24 -0600 Subject: [PATCH] Add dango stack to migration console Signed-off-by: Brian Presley --- .../src/main/docker/docker-compose.yml | 3 + .../main/docker/migrationConsole/Dockerfile | 38 +++++- .../console_api/console_api/__init__.py | 0 .../console/console_api/console_api/asgi.py | 16 +++ .../console_api/orchestrator/__init__.py | 0 .../console_api/orchestrator/admin.py | 3 + .../console_api/orchestrator/apps.py | 6 + .../orchestrator/migrations/__init__.py | 0 .../console_api/orchestrator/models.py | 3 + .../console_api/orchestrator/serializers.py | 6 + .../console_api/orchestrator/tests.py | 3 + .../console_api/orchestrator/urls.py | 6 + .../console_api/orchestrator/views.py | 51 +++++++ .../console_api/console_api/settings.py | 125 ++++++++++++++++++ .../console/console_api/console_api/urls.py | 22 +++ .../console/console_api/console_api/wsgi.py | 16 +++ .../console/console_api/db.sqlite3 | 0 .../console/console_api/manage.py | 22 +++ .../migrationConsole/console/requirements.txt | 11 ++ .../src/main/docker/migrationConsole/init.sh | 9 ++ test/requirements.txt | 4 +- 21 files changed, 340 insertions(+), 4 deletions(-) create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/__init__.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/asgi.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/__init__.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/admin.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/apps.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/migrations/__init__.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/models.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/serializers.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/tests.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/urls.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/views.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/settings.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/urls.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/wsgi.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/db.sqlite3 create mode 100755 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/manage.py create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/requirements.txt create mode 100644 TrafficCapture/dockerSolution/src/main/docker/migrationConsole/init.sh diff --git a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml index 798ca1071..a16e4a5d1 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml +++ b/TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml @@ -130,6 +130,9 @@ services: - migrations volumes: - sharedReplayerOutput:/shared-replayer-output + ports: + #- "${PORT:-}:8000" + - "8000:8000" environment: - MIGRATION_KAFKA_BROKER_ENDPOINTS=kafka:9092 diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile index 69186c587..3258493b7 100644 --- a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile @@ -1,10 +1,35 @@ FROM ubuntu:jammy +ARG EXPERIMENTAL ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && \ - apt-get install -y --no-install-recommends python3.9 python3-pip python3-dev openjdk-11-jre-headless wget gcc libc-dev git curl vim jq unzip less && \ - pip3 install urllib3==1.25.11 opensearch-benchmark==1.1.0 awscurl tqdm + apt-get install -y --no-install-recommends \ + python3.9 \ + python3-pip \ + python3-dev \ + openjdk-11-jre-headless \ + wget \ + gcc \ + libc-dev \ + git \ + curl \ + vim \ + jq \ + unzip \ + less + + #pip3 install urllib3==1.25.11 opensearch-benchmark==1.1.0 awscurl tqdm + +# Experimental libraries needed for API +#RUN if [ "$EXPERIMENTAL" = "true" ]; then \ + #pip3 install \ + #django==5.0.1 \ + #djangorestframework==3.14.0 \ + #django-extensions==3.2.3 \ + #werkzeug==3.0.1; \ + #fi + # TODO upon the next release of opensearch-benchmark the awscli package should be installed by pip3, with the expected boto3 version upgrade resolving the current conflicts between opensearch-benchmark and awscli RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install && rm -rf aws awscliv2.zip RUN mkdir /root/kafka-tools @@ -26,4 +51,11 @@ RUN wget -qO- https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz | RUN wget -O kafka/libs/msk-iam-auth.jar https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.9/aws-msk-iam-auth-1.1.9-all.jar WORKDIR /root -CMD tail -f /dev/null +# Experimental +COPY console /console +RUN pip3 install -r /console/requirements.txt + + +COPY init.sh /init.sh +RUN chmod +x /init.sh +ENTRYPOINT ["/init.sh"] diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/__init__.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/asgi.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/asgi.py new file mode 100644 index 000000000..f0128bea7 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for console_api project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'console_api.settings') + +application = get_asgi_application() diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/__init__.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/admin.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/admin.py new file mode 100644 index 000000000..8c38f3f3d --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/apps.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/apps.py new file mode 100644 index 000000000..f25738e31 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class OrchestratorConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'orchestrator' diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/migrations/__init__.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/models.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/models.py new file mode 100644 index 000000000..71a836239 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/serializers.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/serializers.py new file mode 100644 index 000000000..7720ba062 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/serializers.py @@ -0,0 +1,6 @@ +from rest_framework import serializers + +class MigrationStatusSerializer(serializers.Serializer): + status = serializers.CharField(max_length=100) + details = serializers.CharField(max_length=1000) + timestamp = serializers.DateTimeField() diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/tests.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/urls.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/urls.py new file mode 100644 index 000000000..70bd1dfe9 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/urls.py @@ -0,0 +1,6 @@ +from django.urls import path +from . import views + +urlpatterns = [ + path('status', views.migration_status, name='migration-status'), +] diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/views.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/views.py new file mode 100644 index 000000000..b97348abf --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/views.py @@ -0,0 +1,51 @@ +from rest_framework.response import Response +from rest_framework.decorators import api_view +from .serializers import MigrationStatusSerializer +import datetime +import subprocess +import os + +@api_view(['GET']) +def migration_status(request): + + data = { + 'status': 'Completed', + 'details': 'Migration completed successfully.', + 'timestamp': datetime.datetime.now(datetime.timezone.utc) + } + + serializer = MigrationStatusSerializer(data=data) + + serializer.is_valid(raise_exception=True) + + return Response(serializer.data) + + +# TODO: Switch to POST +@api_view(['GET']) +def start_migration(request): + + # Use subprocess.Popen to run the command + #process = subprocess.Popen(['/root/catIndices.sh'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + full_load = os.getenv('FETCH_MIGRATION_COMMAND') + if not full_load: + return Response({"errors": ["Cannot execute full load command"]}, status=400) + + # TODO: Create replayer command + # command = os.getenv('START_REPLAYER') + process = subprocess.Popen(['ls'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + output, errors = process.communicate() + + # Check for errors + if process.returncode != 0: + return Response({"errors": errors}, status=400) + + # Split the output into a list of files + index_listing = output.split('\n') + + # Return the list of files in the response + return Response({"Index Listing\n": index_listing}) + +@api_view(['POST']) +def stop_migration(request): + pass \ No newline at end of file diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/settings.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/settings.py new file mode 100644 index 000000000..b635697a3 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/settings.py @@ -0,0 +1,125 @@ +""" +Django settings for console_api project. + +Generated by 'django-admin startproject' using Django 5.0.1. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/5.0/ref/settings/ +""" + +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'django-insecure-26h*wo1qzffhpum=bn#8d(7e8mo-w9fr6*wdy#%izy#5^85-a9' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ +# 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'rest_framework', + 'django_extensions', +# 'django.contrib.staticfiles', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'console_api.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'console_api.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/5.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': BASE_DIR / 'db.sqlite3', + } +} + + +# Password validation +# https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/5.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/5.0/howto/static-files/ + +STATIC_URL = 'static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/urls.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/urls.py new file mode 100644 index 000000000..8373bb84e --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/urls.py @@ -0,0 +1,22 @@ +""" +URL configuration for console_api project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/5.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import include, path + +urlpatterns = [ + path("orchestrator/", include("console_api.orchestrator.urls")), +] diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/wsgi.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/wsgi.py new file mode 100644 index 000000000..709f97c31 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for console_api project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'console_api.settings') + +application = get_wsgi_application() diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/db.sqlite3 b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/db.sqlite3 new file mode 100644 index 000000000..e69de29bb diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/manage.py b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/manage.py new file mode 100755 index 000000000..e3b602047 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'console_api.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/requirements.txt b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/requirements.txt new file mode 100644 index 000000000..66074f5fb --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/requirements.txt @@ -0,0 +1,11 @@ +awscurl==0.32 +boto3==1.10.32 +botocore==1.13.50 +Django==5.0.1 +django-extensions==3.2.3 +djangorestframework==3.14.0 +opensearch-benchmark +opensearch-py==2.2.0 +tqdm==4.66.1 +urllib3==1.25.11 +Werkzeug \ No newline at end of file diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/init.sh b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/init.sh new file mode 100644 index 000000000..57d21f9e3 --- /dev/null +++ b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/init.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +if [ "$EXPERIMENTAL" = "true" ]; then + python3 /console/console_api/manage.py runserver_plus 0.0.0.0:8000 +else + # TODO: Replace command below with "tail -f /dev/null" once env variable used. + python3 /console/console_api/manage.py runserver_plus 0.0.0.0:8000 +fi + diff --git a/test/requirements.txt b/test/requirements.txt index eab51fc68..15dfff039 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -9,4 +9,6 @@ pytest-xdist==3.3.1 requests>==2.31.0 urllib3>==2.0.7 requests_aws4auth -boto3 \ No newline at end of file +boto3 +pytz +tzdata \ No newline at end of file