Add dango stack to migration console

Signed-off-by: Brian Presley <bjpres@amazon.com>

TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml

@@ -130,6 +130,9 @@ services:
       - migrations
     volumes:
       - sharedReplayerOutput:/shared-replayer-output
+    ports:
+        #- "${PORT:-}:8000"
+      - "8000:8000"
     environment:
       - MIGRATION_KAFKA_BROKER_ENDPOINTS=kafka:9092
 

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/Dockerfile

@@ -1,10 +1,35 @@
 FROM ubuntu:jammy
+ARG EXPERIMENTAL
 
 ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends python3.9 python3-pip python3-dev openjdk-11-jre-headless wget gcc libc-dev git curl vim jq unzip less && \
-    pip3 install urllib3==1.25.11 opensearch-benchmark==1.1.0 awscurl tqdm
+    apt-get install -y --no-install-recommends \
+    python3.9 \
+    python3-pip \
+    python3-dev \
+    openjdk-11-jre-headless \
+    wget \
+    gcc \
+    libc-dev \
+    git \
+    curl \
+    vim \
+    jq \
+    unzip \
+    less 
+
+    #pip3 install urllib3==1.25.11 opensearch-benchmark==1.1.0 awscurl tqdm
+
+# Experimental libraries needed for API
+#RUN if [ "$EXPERIMENTAL" = "true" ]; then \
+    #pip3 install \
+        #django==5.0.1 \
+        #djangorestframework==3.14.0 \
+        #django-extensions==3.2.3 \
+        #werkzeug==3.0.1; \
+    #fi
+
 # TODO upon the next release of opensearch-benchmark the awscli package should be installed by pip3, with the expected boto3 version upgrade resolving the current conflicts between opensearch-benchmark and awscli
 RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && unzip awscliv2.zip && ./aws/install && rm -rf aws awscliv2.zip
 RUN mkdir /root/kafka-tools
@@ -26,4 +51,11 @@ RUN wget -qO- https://archive.apache.org/dist/kafka/3.6.0/kafka_2.13-3.6.0.tgz |
 RUN wget -O kafka/libs/msk-iam-auth.jar https://github.com/aws/aws-msk-iam-auth/releases/download/v1.1.9/aws-msk-iam-auth-1.1.9-all.jar
 WORKDIR /root
 
-CMD tail -f /dev/null
+# Experimental
+COPY console /console
+RUN pip3 install -r /console/requirements.txt
+
+
+COPY init.sh /init.sh
+RUN chmod +x /init.sh
+ENTRYPOINT ["/init.sh"]

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/asgi.py

@@ -0,0 +1,16 @@
+"""
+ASGI config for console_api project.
+
+It exposes the ASGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.0/howto/deployment/asgi/
+"""
+
+import os
+
+from django.core.asgi import get_asgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'console_api.settings')
+
+application = get_asgi_application()

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class OrchestratorConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'orchestrator'

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/serializers.py

@@ -0,0 +1,6 @@
+from rest_framework import serializers
+
+class MigrationStatusSerializer(serializers.Serializer):
+    status = serializers.CharField(max_length=100)
+    details = serializers.CharField(max_length=1000)
+    timestamp = serializers.DateTimeField()

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/urls.py

@@ -0,0 +1,6 @@
+from django.urls import path
+from . import views
+
+urlpatterns = [
+    path('status', views.migration_status, name='migration-status'),
+]

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/orchestrator/views.py

@@ -0,0 +1,51 @@
+from rest_framework.response import Response
+from rest_framework.decorators import api_view
+from .serializers import MigrationStatusSerializer
+import datetime
+import subprocess
+import os
+
+@api_view(['GET'])
+def migration_status(request):
+
+    data = {
+        'status': 'Completed',
+        'details': 'Migration completed successfully.',
+        'timestamp': datetime.datetime.now(datetime.timezone.utc)
+    }
+
+    serializer = MigrationStatusSerializer(data=data)
+
+    serializer.is_valid(raise_exception=True)
+
+    return Response(serializer.data)
+
+
+# TODO: Switch to POST 
+@api_view(['GET'])
+def start_migration(request):
+
+    # Use subprocess.Popen to run the command
+    #process = subprocess.Popen(['/root/catIndices.sh'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    full_load = os.getenv('FETCH_MIGRATION_COMMAND')
+    if not full_load:
+        return Response({"errors": ["Cannot execute full load command"]}, status=400)
+
+    # TODO: Create replayer command
+    # command = os.getenv('START_REPLAYER')
+    process = subprocess.Popen(['ls'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+    output, errors = process.communicate()
+
+    # Check for errors
+    if process.returncode != 0:
+        return Response({"errors": errors}, status=400)
+
+    # Split the output into a list of files
+    index_listing = output.split('\n')
+
+    # Return the list of files in the response
+    return Response({"Index Listing\n": index_listing})
+
+@api_view(['POST'])
+def stop_migration(request):
+    pass

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/settings.py

@@ -0,0 +1,125 @@
+"""
+Django settings for console_api project.
+
+Generated by 'django-admin startproject' using Django 5.0.1.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.0/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/5.0/ref/settings/
+"""
+
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/5.0/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'django-insecure-26h*wo1qzffhpum=bn#8d(7e8mo-w9fr6*wdy#%izy#5^85-a9'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = [
+#    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'rest_framework',
+    'django_extensions',
+#    'django.contrib.staticfiles',
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'console_api.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'console_api.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/5.0/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': BASE_DIR / 'db.sqlite3',
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/5.0/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/5.0/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/5.0/howto/static-files/
+
+STATIC_URL = 'static/'
+
+# Default primary key field type
+# https://docs.djangoproject.com/en/5.0/ref/settings/#default-auto-field
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/urls.py

@@ -0,0 +1,22 @@
+"""
+URL configuration for console_api project.
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/5.0/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import include, path
+
+urlpatterns = [
+    path("orchestrator/", include("console_api.orchestrator.urls")),
+]

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/console_api/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for console_api project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/5.0/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'console_api.settings')
+
+application = get_wsgi_application()

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/console_api/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'console_api.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console/requirements.txt

@@ -0,0 +1,11 @@
+awscurl==0.32
+boto3==1.10.32
+botocore==1.13.50
+Django==5.0.1
+django-extensions==3.2.3
+djangorestframework==3.14.0
+opensearch-benchmark
+opensearch-py==2.2.0
+tqdm==4.66.1
+urllib3==1.25.11
+Werkzeug

TrafficCapture/dockerSolution/src/main/docker/migrationConsole/init.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+if [ "$EXPERIMENTAL" = "true" ]; then
+    python3 /console/console_api/manage.py runserver_plus 0.0.0.0:8000
+else
+    # TODO: Replace command below with "tail -f /dev/null" once env variable used.
+    python3 /console/console_api/manage.py runserver_plus 0.0.0.0:8000
+fi
+

test/requirements.txt

@@ -9,4 +9,6 @@ pytest-xdist==3.3.1
 requests>==2.31.0
 urllib3>==2.0.7
 requests_aws4auth
-boto3
+boto3
+pytz
+tzdata