FEAT: Run Task from Github Action

.github/actions/run_task/action.yaml

@@ -1,5 +1,5 @@
-name: Run LAMP ECS Task in Environment
-description: Run an existing task in the LAMP ECS Cluster
+name: Manually Run ECS Task
+description: Run an existing task in an existing AWS Service and Cluster
 
 inputs:
   role-to-assume:
@@ -9,26 +9,25 @@ inputs:
     description: AWS region to use
     required: true
     default: us-east-1
-  environment:
-    description: LAMP environment
+  cluster:
+    description: ECS Cluster for Service
     required: true
-  task_name:
-    description: name of the task to run in kebab case
+  service:
+    description: ECS Service for task to run
     required: true
 
 runs:
   using: composite
   steps:
     - name: Setup AWS Credentials
-      uses: aws-actions/configure-aws-credentials@v1
+      uses: aws-actions/configure-aws-credentials@v4
       with:
         role-to-assume: ${{ inputs.role-to-assume }}
         aws-region: ${{ inputs.aws-region }}
-    - name: Run ECS Task
-      id: run-ecs
-      run: ${{ github.action_path }}/run_task.sh
+        mask-aws-account-id: true
+    - run: ${{ github.action_path }}/run_task.sh
       shell: bash
       env:
         AWS_REGION: ${{ inputs.aws-region }}
-        TASK_NAME: ${{ inputs.task_name }}
-        ENVIRONMENT: ${{ inputs.environment }}
+        CLUSTER: ${{ inputs.cluster }}
+        SERVICE: ${{ inputs.service }}

.github/actions/run_task/run_task.sh

@@ -4,59 +4,36 @@ set -e -u
 # uncomment to debug
 # set -x
 
-# Run an ECS Task in the LAMP ECS Cluster
+# Run an ECS Task from the provided CLUSTER and SERVICE
 
 # required environment varialbes
-# - TASK_NAME
-# - ENVIRONMENT
-
-# Get the VPC ID based on the VPC name
-if [ "$ENVIRONMENT" == "staging" ] || [ "$ENVIRONMENT" == "dev" ]; then
-    VPC_NAME="vpc-dev-ctd-itd"
-elif [ "$ENVIRONMENT" == "prod" ]; then
-    VPC_NAME="vpc-prod-ctd-itd"
-else
-    echo "Invalid environment: $ENVIRONMENT"
-    exit 1
-fi
-
-echo "Retrieving VPC ID for ${VPC_NAME}"
-
-VPC_ID=$(aws ec2 describe-vpcs \
-  --filters "Name=tag:Name,Values=${VPC_NAME}" \
-  --query "Vpcs[0].VpcId" \
-  --output text)
-
-echo "VPC ID: ${VPC_ID}"
-
-# Get the subnet ID based on the VPC ID
-echo "Retrieving SUBNET ID"
-
-SUBNET_ID=$(aws ec2 describe-subnets \
-  --filters "Name=vpc-id,Values=${VPC_ID}" \
-  --query "Subnets[0].SubnetId" \
-  --output text)
-
-echo "SUBNET ID: ${SUBNET_ID}"
-
-# Get the Security Group Id that can run the task.
-SG_NAME="lamp-${TASK_NAME}-${ENVIRONMENT}-ecs-service"
-echo "Retrieving SECURITY GROUP ID for ${SG_NAME}"
-
-SECURITY_GROUP_ID=$(aws ec2 describe-security-groups \
-  --filters "Name=group-name,Values=${SG_NAME}" \
-  --query "SecurityGroups[0].GroupId" \
-  --output text)
-
-echo "SECURITY GROUP ID: ${SECURITY_GROUP_ID}"
-
-# Build the Task Definition from the Task Name and Env
-TASK_DEFINITION="lamp-${TASK_NAME}-${ENVIRONMENT}"
+# - CLUSTER
+# - SERVICE
+
+# Get the Security Groups that can run the task.
+echo "Retrieving SecurityGroups for SERVICE:${SERVICE} in CLUSTER:${CLUSTER}"
+SECURITY_GROUPS=$(aws ecs describe-services \
+  --services $SERVICE \
+  --cluster $CLUSTER \
+  --query services[0].networkConfiguration.awsvpcConfiguration.securityGroups \
+  --output text \
+  | sed 's/\t/,/g')
+echo "SECURITY GROUPS: ${SECURITY_GROUPS}"
+
+# Get the Subnets that the task runs on.
+echo "Retrieving subnets for SERVICE:${SERVICE} in CLUSTER:${CLUSTER}"
+SUBNETS=$(aws ecs describe-services \
+  --services $SERVICE \
+  --cluster $CLUSTER \
+  --query services[0].networkConfiguration.awsvpcConfiguration.subnets \
+  --output text \
+  | sed 's/\t/,/g')
+echo "SUBNETS: ${SUBNETS}"
 
 # Run the ECS task
 aws ecs run-task \
-  --cluster lamp \
-  --task-definition $TASK_DEFINITION \
+  --cluster $CLUSTER \
+  --task-definition $SERVICE \
   --launch-type FARGATE \
   --count 1 \
-  --network-configuration "awsvpcConfiguration={subnets=[$SUBNET_ID],securityGroups=[$SECURITY_GROUP_ID],assignPublicIp=ENABLED}"
+  --network-configuration "awsvpcConfiguration={subnets=[$SUBNETS],securityGroups=[$SECURITY_GROUPS],assignPublicIp=DISABLED}"

.github/workflows/ad_hoc_deploy_run.yml

@@ -10,28 +10,31 @@ on:
           - dev
           - staging
           - prod
+    secrets:
+      AWS_ROLE_ARN:
+        description: AWS_ROLE_ARN
+        required: true
 
 jobs:
-  deploy:
-    uses: ./.github/workflows/deploy-base.yaml
-    with:
-      # pass the inputs from the workflow dispatch through to the deploy base. the booleans are
-      # converted to strings, so flip them back using fromJson function
-      environment: ${{ github.event.inputs.environment }}
-      deploy-ad-hoc: true
-    secrets: inherit
+  # deploy:
+  #   uses: ./.github/workflows/deploy-base.yaml
+  #   with:
+  #     # pass the inputs from the workflow dispatch through to the deploy base. the booleans are
+  #     # converted to strings, so flip them back using fromJson function
+  #     environment: ${{ github.event.inputs.environment }}
+  #     deploy-ad-hoc: true
+  #   secrets: inherit
   run_ad_hoc_task:
-    needs: deploy
     runs-on: ubuntu-latest
     permissions:
       id-token: write
       contents: read
     steps:
       - name: Checkout Branch
         uses: actions/checkout@v3
-      - name: Run Task Action
+      - name: Run Ad-Hoc Task
         uses: ./.github/actions/run_task
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
-          environment: ${{ github.event.inputs.environment }}
-          task_name: 'ad-hoc'
+          cluster: 'lamp'
+          service: lamp-ad-hoc-${{ inputs.environment }}

.github/workflows/deploy-base.yaml

@@ -136,13 +136,14 @@ jobs:
       - name: Deploy Ad-Hoc Process
         id: deploy-ad-hoc
         if: ${{ inputs.deploy-ad-hoc }}
-        uses: mbta/actions/deploy-scheduled-ecs@v2
+        uses: mbta/actions/deploy-ecs@v2
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
           ecs-cluster: lamp
           ecs-service: lamp-ad-hoc-${{ inputs.environment }}
           ecs-task-definition: lamp-ad-hoc-${{ inputs.environment }}
           docker-tag: ${{ steps.build-push.outputs.docker-tag }}
+          allow-zero-desired: true
       - uses: mbta/actions/notify-slack-deploy@v2
         if: ${{ !cancelled() }}
         with:

.github/workflows/run-task.yaml

@@ -27,14 +27,14 @@ jobs:
         uses: actions/checkout@v3
       - name: Generate Task Name
         run: |
-          if [ "${{ github.event.inputs.task }}" == "Tableau Publisher" ]; then
+          if [ "${{ inputs.task }}" == "Tableau Publisher" ]; then
             echo "task_name=tableau-publisher" >> $GITHUB_ENV
-          elif [ "${{ github.event.inputs.task }}" == "Transit Master Ingestion" ]; then
+          elif [ "${{ inputs.task }}" == "Transit Master Ingestion" ]; then
             echo "task_name=tm-ingestion" >> $GITHUB_ENV
           fi
       - name: Run Task Action
         uses: ./.github/actions/run_task
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
-          environment: ${{ inputs.environment }}
-          task_name: ${{ env.task_name }}
+          cluster: 'lamp'
+          service: lamp-${{ env.task_name }}-${{ inputs.environment }}

Dockerfile

@@ -54,4 +54,4 @@ COPY alembic.ini alembic.ini
 ARG VERSION="v0.0.0-unknown"
 RUN echo "VERSION = '${VERSION}'" > src/lamp_py/__version__.py
 
-RUN poetry install --no-dev --no-interaction --no-ansi -v
+RUN poetry install --only main --no-interaction --no-ansi -v

pyproject.toml

@@ -15,6 +15,7 @@ bus_performance_manager = 'lamp_py.bus_performance_manager.pipeline:start'
 seed_metadata = 'lamp_py.postgres.seed_metadata:run'
 hyper_update = 'lamp_py.tableau.pipeline:start_hyper_updates'
 transit_master_ingestion = 'lamp_py.ingestion_tm.pipeline:start'
+ad_hoc = 'lamp_py.ad_hoc.pipeline:start'
 
 [tool.poetry.dependencies]
 python = "^3.10"

src/lamp_py/ad_hoc/__init__.py

@@ -0,0 +1 @@
+"""location for all ad-hoc process runner scripts"""

src/lamp_py/ad_hoc/pipeline.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+
+import logging
+import os
+
+from lamp_py.aws.ecs import check_for_parallel_tasks
+from lamp_py.runtime_utils.env_validation import validate_environment
+
+from lamp_py.ad_hoc.runner_001 import runner
+
+logging.getLogger().setLevel("INFO")
+DESCRIPTION = """Entry Point For Ad-Hoc Runner"""
+
+
+def start() -> None:
+    """configure and start the ad-hoc runner"""
+    # configure the environment
+    os.environ["SERVICE_NAME"] = "ad_hoc"
+
+    validate_environment(
+        required_variables=[
+            "ARCHIVE_BUCKET",
+            "ERROR_BUCKET",
+            "INCOMING_BUCKET",
+            "PUBLIC_ARCHIVE_BUCKET",
+            "SPRINGBOARD_BUCKET",
+        ],
+        db_prefixes=["MD", "RPM"],
+    )
+
+    check_for_parallel_tasks()
+
+    # run the main method
+    runner()
+
+
+if __name__ == "__main__":
+    start()

src/lamp_py/ad_hoc/runner_001.py

@@ -0,0 +1,9 @@
+from lamp_py.runtime_utils.process_logger import ProcessLogger
+
+
+def runner() -> None:
+    """Test ad-hoc runner"""
+    logger = ProcessLogger("ad_hoc_runner")
+    logger.log_start()
+    logger.add_metadata(check_ran=True)
+    logger.log_complete()