diff --git a/.github/actions/run_task/action.yaml b/.github/actions/run_task/action.yaml index bc3f3d58..d1819fc6 100644 --- a/.github/actions/run_task/action.yaml +++ b/.github/actions/run_task/action.yaml @@ -1,5 +1,5 @@ -name: Run LAMP ECS Task in Environment -description: Run an existing task in the LAMP ECS Cluster +name: Manually Run ECS Task +description: Run an existing task in an existing AWS Service and Cluster inputs: role-to-assume: @@ -9,26 +9,25 @@ inputs: description: AWS region to use required: true default: us-east-1 - environment: - description: LAMP environment + cluster: + description: ECS Cluster for Service required: true - task_name: - description: name of the task to run in kebab case + service: + description: ECS Service for task to run required: true runs: using: composite steps: - name: Setup AWS Credentials - uses: aws-actions/configure-aws-credentials@v1 + uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ inputs.role-to-assume }} aws-region: ${{ inputs.aws-region }} - - name: Run ECS Task - id: run-ecs - run: ${{ github.action_path }}/run_task.sh + mask-aws-account-id: true + - run: ${{ github.action_path }}/run_task.sh shell: bash env: AWS_REGION: ${{ inputs.aws-region }} - TASK_NAME: ${{ inputs.task_name }} - ENVIRONMENT: ${{ inputs.environment }} + CLUSTER: ${{ inputs.cluster }} + SERVICE: ${{ inputs.service }} diff --git a/.github/actions/run_task/run_task.sh b/.github/actions/run_task/run_task.sh index e43ebd0f..1461b988 100755 --- a/.github/actions/run_task/run_task.sh +++ b/.github/actions/run_task/run_task.sh @@ -4,59 +4,36 @@ set -e -u # uncomment to debug # set -x -# Run an ECS Task in the LAMP ECS Cluster +# Run an ECS Task from the provided CLUSTER and SERVICE # required environment varialbes -# - TASK_NAME -# - ENVIRONMENT - -# Get the VPC ID based on the VPC name -if [ "$ENVIRONMENT" == "staging" ] || [ "$ENVIRONMENT" == "dev" ]; then - VPC_NAME="vpc-dev-ctd-itd" -elif [ "$ENVIRONMENT" == "prod" ]; then - VPC_NAME="vpc-prod-ctd-itd" -else - echo "Invalid environment: $ENVIRONMENT" - exit 1 -fi - -echo "Retrieving VPC ID for ${VPC_NAME}" - -VPC_ID=$(aws ec2 describe-vpcs \ - --filters "Name=tag:Name,Values=${VPC_NAME}" \ - --query "Vpcs[0].VpcId" \ - --output text) - -echo "VPC ID: ${VPC_ID}" - -# Get the subnet ID based on the VPC ID -echo "Retrieving SUBNET ID" - -SUBNET_ID=$(aws ec2 describe-subnets \ - --filters "Name=vpc-id,Values=${VPC_ID}" \ - --query "Subnets[0].SubnetId" \ - --output text) - -echo "SUBNET ID: ${SUBNET_ID}" - -# Get the Security Group Id that can run the task. -SG_NAME="lamp-${TASK_NAME}-${ENVIRONMENT}-ecs-service" -echo "Retrieving SECURITY GROUP ID for ${SG_NAME}" - -SECURITY_GROUP_ID=$(aws ec2 describe-security-groups \ - --filters "Name=group-name,Values=${SG_NAME}" \ - --query "SecurityGroups[0].GroupId" \ - --output text) - -echo "SECURITY GROUP ID: ${SECURITY_GROUP_ID}" - -# Build the Task Definition from the Task Name and Env -TASK_DEFINITION="lamp-${TASK_NAME}-${ENVIRONMENT}" +# - CLUSTER +# - SERVICE + +# Get the Security Groups that can run the task. +echo "Retrieving SecurityGroups for SERVICE:${SERVICE} in CLUSTER:${CLUSTER}" +SECURITY_GROUPS=$(aws ecs describe-services \ + --services $SERVICE \ + --cluster $CLUSTER \ + --query services[0].networkConfiguration.awsvpcConfiguration.securityGroups \ + --output text \ + | sed 's/\t/,/g') +echo "SECURITY GROUPS: ${SECURITY_GROUPS}" + +# Get the Subnets that the task runs on. +echo "Retrieving subnets for SERVICE:${SERVICE} in CLUSTER:${CLUSTER}" +SUBNETS=$(aws ecs describe-services \ + --services $SERVICE \ + --cluster $CLUSTER \ + --query services[0].networkConfiguration.awsvpcConfiguration.subnets \ + --output text \ + | sed 's/\t/,/g') +echo "SUBNETS: ${SUBNETS}" # Run the ECS task aws ecs run-task \ - --cluster lamp \ - --task-definition $TASK_DEFINITION \ + --cluster $CLUSTER \ + --task-definition $SERVICE \ --launch-type FARGATE \ --count 1 \ - --network-configuration "awsvpcConfiguration={subnets=[$SUBNET_ID],securityGroups=[$SECURITY_GROUP_ID],assignPublicIp=ENABLED}" + --network-configuration "awsvpcConfiguration={subnets=[$SUBNETS],securityGroups=[$SECURITY_GROUPS],assignPublicIp=DISABLED}" diff --git a/.github/workflows/ad_hoc_deploy_run.yml b/.github/workflows/ad_hoc_deploy_run.yml index 5a65905a..1f75d2eb 100644 --- a/.github/workflows/ad_hoc_deploy_run.yml +++ b/.github/workflows/ad_hoc_deploy_run.yml @@ -10,6 +10,10 @@ on: - dev - staging - prod + secrets: + AWS_ROLE_ARN: + description: AWS_ROLE_ARN + required: true jobs: deploy: @@ -21,7 +25,6 @@ jobs: deploy-ad-hoc: true secrets: inherit run_ad_hoc_task: - needs: deploy runs-on: ubuntu-latest permissions: id-token: write @@ -29,9 +32,9 @@ jobs: steps: - name: Checkout Branch uses: actions/checkout@v3 - - name: Run Task Action + - name: Run Ad-Hoc Task uses: ./.github/actions/run_task with: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - environment: ${{ github.event.inputs.environment }} - task_name: 'ad-hoc' + cluster: 'lamp' + service: lamp-ad-hoc-${{ inputs.environment }} diff --git a/.github/workflows/deploy-base.yaml b/.github/workflows/deploy-base.yaml index c9b123cb..0f083035 100644 --- a/.github/workflows/deploy-base.yaml +++ b/.github/workflows/deploy-base.yaml @@ -136,13 +136,14 @@ jobs: - name: Deploy Ad-Hoc Process id: deploy-ad-hoc if: ${{ inputs.deploy-ad-hoc }} - uses: mbta/actions/deploy-scheduled-ecs@v2 + uses: mbta/actions/deploy-ecs@v2 with: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} ecs-cluster: lamp ecs-service: lamp-ad-hoc-${{ inputs.environment }} ecs-task-definition: lamp-ad-hoc-${{ inputs.environment }} docker-tag: ${{ steps.build-push.outputs.docker-tag }} + allow-zero-desired: true - uses: mbta/actions/notify-slack-deploy@v2 if: ${{ !cancelled() }} with: diff --git a/.github/workflows/run-task.yaml b/.github/workflows/run-task.yaml index 3094a8d3..006b0e33 100644 --- a/.github/workflows/run-task.yaml +++ b/.github/workflows/run-task.yaml @@ -27,14 +27,14 @@ jobs: uses: actions/checkout@v3 - name: Generate Task Name run: | - if [ "${{ github.event.inputs.task }}" == "Tableau Publisher" ]; then + if [ "${{ inputs.task }}" == "Tableau Publisher" ]; then echo "task_name=tableau-publisher" >> $GITHUB_ENV - elif [ "${{ github.event.inputs.task }}" == "Transit Master Ingestion" ]; then + elif [ "${{ inputs.task }}" == "Transit Master Ingestion" ]; then echo "task_name=tm-ingestion" >> $GITHUB_ENV fi - name: Run Task Action uses: ./.github/actions/run_task with: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} - environment: ${{ inputs.environment }} - task_name: ${{ env.task_name }} + cluster: 'lamp' + service: lamp-${{ env.task_name }}-${{ inputs.environment }} diff --git a/Dockerfile b/Dockerfile index d72226d4..f8ffc60a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,4 +54,4 @@ COPY alembic.ini alembic.ini ARG VERSION="v0.0.0-unknown" RUN echo "VERSION = '${VERSION}'" > src/lamp_py/__version__.py -RUN poetry install --no-dev --no-interaction --no-ansi -v +RUN poetry install --only main --no-interaction --no-ansi -v diff --git a/pyproject.toml b/pyproject.toml index c04edd4b..4c223d88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ bus_performance_manager = 'lamp_py.bus_performance_manager.pipeline:start' seed_metadata = 'lamp_py.postgres.seed_metadata:run' hyper_update = 'lamp_py.tableau.pipeline:start_hyper_updates' transit_master_ingestion = 'lamp_py.ingestion_tm.pipeline:start' +ad_hoc = 'lamp_py.ad_hoc.pipeline:start' [tool.poetry.dependencies] python = "^3.10" diff --git a/src/lamp_py/ad_hoc/__init__.py b/src/lamp_py/ad_hoc/__init__.py new file mode 100644 index 00000000..075391e7 --- /dev/null +++ b/src/lamp_py/ad_hoc/__init__.py @@ -0,0 +1 @@ +"""location for all ad-hoc process runner scripts""" diff --git a/src/lamp_py/ad_hoc/pipeline.py b/src/lamp_py/ad_hoc/pipeline.py new file mode 100644 index 00000000..82d17b67 --- /dev/null +++ b/src/lamp_py/ad_hoc/pipeline.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +import logging +import os + +from lamp_py.aws.ecs import check_for_parallel_tasks +from lamp_py.runtime_utils.env_validation import validate_environment + +from lamp_py.ad_hoc.runner_001 import runner + +logging.getLogger().setLevel("INFO") +DESCRIPTION = """Entry Point For Ad-Hoc Runner""" + + +def start() -> None: + """configure and start the ad-hoc runner""" + # configure the environment + os.environ["SERVICE_NAME"] = "ad_hoc" + + validate_environment( + required_variables=[ + "ARCHIVE_BUCKET", + "ERROR_BUCKET", + "INCOMING_BUCKET", + "PUBLIC_ARCHIVE_BUCKET", + "SPRINGBOARD_BUCKET", + ], + db_prefixes=["MD", "RPM"], + ) + + check_for_parallel_tasks() + + # run the main method + runner() + + +if __name__ == "__main__": + start() diff --git a/src/lamp_py/ad_hoc/runner_001.py b/src/lamp_py/ad_hoc/runner_001.py new file mode 100644 index 00000000..854b9ed1 --- /dev/null +++ b/src/lamp_py/ad_hoc/runner_001.py @@ -0,0 +1,9 @@ +from lamp_py.runtime_utils.process_logger import ProcessLogger + + +def runner() -> None: + """Test ad-hoc runner""" + logger = ProcessLogger("ad_hoc_runner") + logger.log_start() + logger.add_metadata(check_ran=True) + logger.log_complete()