Destroy environment workflow (#1146)

Co-authored-by: Oren Zohar <85433724+oren-zohar@users.noreply.github.com>
elastic · Jul 26, 2023 · 7ca2442 · 7ca2442
1 parent e171e1f
commit 7ca2442
Show file tree

Hide file tree

Showing 3 changed files with 161 additions and 34 deletions.
diff --git a/.github/workflows/destroy-environment.yml b/.github/workflows/destroy-environment.yml
@@ -0,0 +1,54 @@
+name: Destroy Environment
+run-name: Destroying ${{ github.event.inputs.prefix }}* by @${{ github.actor }}
+
+on:
+  # Ability to execute on demand
+  workflow_dispatch:
+    inputs:
+      ec-api-key:
+        type: string
+        description: "Elastic Cloud API key"
+        required: true
+      prefix:
+        type: string
+        description: "Delete all environments starting with `prefix`"
+        required: true
+      ignore-prefix:
+        type: string
+        description: "Ignore all environments starting with `ignore-prefix`"
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_REGION: "eu-west-1"
+  ENV_PREFIX: ${{ github.event.inputs.prefix }}
+  ENV_IGNORE_PREFIX: ${{ github.event.inputs.ingore-prefix }}
+
+jobs:
+  Destroy:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 120
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Init Hermit
+        run: ./bin/hermit env -r >> $GITHUB_ENV
+        working-directory: ./
+
+      - name: Mask API Key
+        run: |
+          ec_api_key=$(jq -r '.inputs["ec-api-key"]' $GITHUB_EVENT_PATH)
+          echo "::add-mask::$ec_api_key"
+          echo "TF_VAR_ec_api_key=$ec_api_key" >> $GITHUB_ENV
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1-node16
+        with:
+          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_REGION }}
+
+      - name: Destroy Environment
+        run: |
+          just delete-cloud-env ${{ env.ENV_PREFIX }} '${{ env.ENV_IGNORE_PREFIX }}' "false"
diff --git a/deploy/test-environments/delete_env.sh b/deploy/test-environments/delete_env.sh
@@ -11,6 +11,38 @@ EOF
 
 # Defaults
 INTERACTIVE=true
+AWS_REGION="eu-west-1"  # Add your desired default AWS region here
+
+# Arrays to keep track of successful and failed deletions
+DELETED_ENVS=()
+FAILED_ENVS=()
+
+# Function to delete Terraform environment
+function delete_environment() {
+    local ENV=$1
+    echo "Deleting Terraform environment: $ENV"
+    tfstate="./$ENV-terraform.tfstate"
+
+    # Copy state file, destroy environment, and remove environment data from S3
+    if aws s3 cp $BUCKET/"$ENV"/terraform.tfstate "$tfstate" && \
+    terraform destroy -var="region=$AWS_REGION" -state "$tfstate" --auto-approve && \
+    aws s3 rm $BUCKET/"$ENV" --recursive; then
+        echo "Successfully deleted $ENV"
+        DELETED_ENVS+=("$ENV")
+    else
+        echo "Failed to delete $ENV"
+        FAILED_ENVS+=("$ENV")
+    fi
+
+    rm "$tfstate"
+}
+
+# Function to delete CloudFormation stack
+function delete_stack() {
+    local STACK_NAME=$1
+    echo "Deleting CloudFormation stack: $STACK_NAME"
+    aws cloudformation delete-stack --stack-name "$STACK_NAME" --region "$AWS_REGION"
+}
 
 # Parsing command-line arguments
 while [[ "$#" -gt 0 ]]; do
@@ -36,59 +68,63 @@ done
 
 BUCKET=s3://tf-state-bucket-test-infra
 ALL_ENVS=$(aws s3 ls $BUCKET/"$ENV_PREFIX" | awk '{print $2}' | sed 's/\///g')
+ALL_STACKS=$(aws cloudformation list-stacks --stack-status-filter "CREATE_COMPLETE" "UPDATE_COMPLETE" --region "$AWS_REGION" | jq -r '.StackSummaries[] | select(.StackName | startswith("'$ENV_PREFIX'") and (if "'$IGNORE_PREFIX'" != "" then .StackName | startswith("'$IGNORE_PREFIX'") | not else true end)) | .StackName')
 
 # Divide environments into those to be deleted and those to be skipped
-TO_DELETE=()
-TO_SKIP=()
+TO_DELETE_ENVS=()
+TO_SKIP_ENVS=()
 
-for ENV in $ALL_ENVS
-do
+for ENV in $ALL_ENVS; do
     if [[ -n "$IGNORE_PREFIX" && "$ENV" == "$IGNORE_PREFIX"* ]]; then
-        TO_SKIP+=("$ENV")
+        TO_SKIP_ENVS+=("$ENV")
     else
-        TO_DELETE+=("$ENV")
+        TO_DELETE_ENVS+=("$ENV")
     fi
 done
 
 # Print the lists of environments to be deleted and skipped
-echo "Environments to delete (${#TO_DELETE[@]}):"
-printf "%s\n" "${TO_DELETE[@]}"
+echo "Environments to delete (${#TO_DELETE_ENVS[@]}):"
+printf "%s\n" "${TO_DELETE_ENVS[@]}"
 
-echo "Environments to skip (${#TO_SKIP[@]}):"
-printf "%s\n" "${TO_SKIP[@]}"
+echo "Environments to skip (${#TO_SKIP_ENVS[@]}):"
+printf "%s\n" "${TO_SKIP_ENVS[@]}"
 
 # Ask for user confirmation if interactive mode is enabled
 if [ "$INTERACTIVE" = true ]; then
     read -r -p "Are you sure you want to delete these environments? (y/n): " confirm && [[ $confirm == [yY] || $confirm == [yY][eE][sS] ]] || exit 1
 fi
 
-# Arrays to keep track of successful and failed deletions
-DELETED_ENVS=()
-FAILED_ENVS=()
+# Delete the Terraform environments
+for ENV in "${TO_DELETE_ENVS[@]}"; do
+    delete_environment "$ENV"
+done
 
-# Delete the environments
-for ENV in "${TO_DELETE[@]}"
-do
-    echo "Deleting $ENV"
-    tfstate="./$ENV-terraform.tfstate"
+# Print summary of environment deletions
+echo "Successfully deleted environments (${#DELETED_ENVS[@]}):"
+printf "%s\n" "${DELETED_ENVS[@]}"
 
-    # Copy state file, destroy environment, and remove environment data from S3
-    if aws s3 cp $BUCKET/"$ENV"/terraform.tfstate "$tfstate" && \
-    terraform destroy -var="region=eu-west-1" -state "$tfstate" --auto-approve && \
-    aws s3 rm $BUCKET/"$ENV" --recursive; then
-        echo "Successfully deleted $ENV"
-        DELETED_ENVS+=("$ENV")
+echo "Failed to delete environments (${#FAILED_ENVS[@]}):"
+printf "%s\n" "${FAILED_ENVS[@]}"
+
+DELETED_STACKS=()
+FAILED_STACKS=()
+
+# Wait for the CloudFormation stacks to be deleted
+for STACK in $ALL_STACKS; do
+    delete_stack "$STACK"
+    aws cloudformation wait stack-delete-complete --stack-name "$STACK" --region "$AWS_REGION"
+    if [ $? -eq 0 ]; then
+        echo "Successfully deleted CloudFormation stack: $STACK"
+        DELETED_STACKS+=("$STACK")
     else
-        echo "Failed to delete $ENV"
-        FAILED_ENVS+=("$ENV")
+        echo "Failed to delete CloudFormation stack: $STACK"
+        FAILED_STACKS+=("$STACK")
     fi
-
-    rm "$tfstate"
 done
 
-# Print summary of deletions
-echo "Successfully deleted (${#DELETED_ENVS[@]}):"
-printf "%s\n" "${DELETED_ENVS[@]}"
+# Print summary of stacks deletions
+echo "Successfully deleted CloudFormation stacks (${#DELETED_STACKS[@]}):"
+printf "%s\n" "${DELETED_STACKS[@]}"
 
-echo "Failed to delete (${#FAILED_ENVS[@]}):"
-printf "%s\n" "${FAILED_ENVS[@]}"
+echo "Failed to delete CloudFormation stacks (${#FAILED_STACKS[@]}):"
+printf "%s\n" "${FAILED_STACKS[@]}"
diff --git a/dev-docs/Cloud-Env-Testing.md b/dev-docs/Cloud-Env-Testing.md
@@ -104,6 +104,43 @@ Follow these steps to log in to the created environment:
 
 If you wish to automatically delete the environment after the tests finish, set the `cleanup-env` input to `true`.
 
+In addition to the automatic cleanup, you can manually delete environments using the [Destroy Environment](https://github.com/elastic/cloudbeat/actions/workflows/destroy-environment.yml) workflow or by directly executing the `delete-cloud-env` command.
+
+### Destroy Environment Workflow
+
+The **Destroy Environment** GitHub workflow automates the process of cleaning up environments. When activated, it automatically performs the cleanup of environments, ensuring that all associated resources are correctly removed.
+
+#### How to Run the Flow
+
+Follow these steps to run the workflow:
+
+1. Go to [`Actions > Destroy Environment`](https://github.com/elastic/cloudbeat/actions/workflows/destroy-environment.yml)
+
+   ![Destroy Environment](https://github.com/gurevichdmitry/cloudbeat/assets/99176494/505d6553-7780-4450-83e9-3617f64442ad)
+
+2. Click the `Run workflow` button.
+
+   ![Run Workflow](https://github.com/gurevichdmitry/cloudbeat/assets/99176494/8965311c-eeac-492f-a564-a57c46854a3a)
+
+3. Complete the required input fields:
+
+    - `ec-api-key` (required): Use the [Production Elastic Cloud](https://cloud.elastic.co/home) API key.
+    - `prefix` (required): The prefix used to identify the environments to be deleted.
+
+   <img width="411" alt="Enter Inputs" src="https://github.com/elastic/cloudbeat/assets/99176494/04973b00-5411-4ace-ab3a-534371877c91">
+
+1. Optionally, modify other input value if required:
+
+    - `ignore-prefix` (optional): The prefix used to identify environments that should be excluded from deletion.
+
+   <img width="411" alt="Optional Inputs" src="https://github.com/elastic/cloudbeat/assets/99176494/aa89ad4e-fd32-461d-ab2d-3fee28094a9d">
+
+2. Click the `Run workflow` button to start.
+
+   ![Run Workflow](https://github.com/gurevichdmitry/cloudbeat/assets/99176494/64b554d5-70f0-4cf3-b2b9-8f8429d1fc07)
+
+### Manual Environment Deletion
+
 In addition to the automatic cleanup, you can manually delete environments using:
 
 ```bash
@@ -120,4 +157,4 @@ Before running the script, ensure that:
 - The `TF_VAR_ec_api_key` environment variable is set.
 
 **Note**: The script will ask for confirmation before deleting each environment, unless you set the `interactive` flag
-to `false`.
+to `false`.