Merge branch 'dg-refactor-python-project' of github.com:elastic/cloud…

…beat into dg-refactor-python-project
elastic · Jan 17, 2024 · 0397114 · 0397114
2 parents 4750616 + 697601e
commit 0397114
Show file tree

Hide file tree

Showing 4 changed files with 174 additions and 33 deletions.
diff --git a/.github/workflows/test-gcp-dm.yml b/.github/workflows/test-gcp-dm.yml
@@ -0,0 +1,124 @@
+name: GCP Deployment Manager Test
+
+on:
+  pull_request:
+    branches:
+      - main
+      - "[0-9]+.[0-9]+"
+    types: [opened, synchronize, reopened]
+    paths:
+      - "deploy/deployment-manager/compute_engine.py"
+      - "deploy/deployment-manager/compute_engine.py.schema"
+      - "deploy/deployment-manager/deploy.sh"
+      - "deploy/deployment-manager/set_env.sh"
+
+env:
+  WORKING_DIR: deploy/test-environments
+  TF_VAR_ec_api_key: ${{ secrets.EC_API_KEY }}
+  TF_VAR_ess_region: gcp-us-west2 # default region for testing deployments
+
+jobs:
+  Test-GCP-DM:
+    name: GCP Deployment Manager Test
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    permissions:
+      contents: "read"
+      id-token: "write"
+    steps:
+      - name: Set up unique deployment names
+        run: |
+          suffix="$(date +%s | tail -c 3)"
+          echo "TF_VAR_deployment_name=gcp-dm-ci-test-$suffix" >> $GITHUB_ENV
+          echo "GCP_DEPLOYMENT_NAME=ea-cspm-ci-dm-test-$suffix" >> $GITHUB_ENV
+
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Init Hermit
+        run: ./bin/hermit env -r >> $GITHUB_ENV
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          poetry --version
+
+      - id: google-auth
+        name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v1
+        with:
+          workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }} # this also sets the project name
+          service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+
+      - name: set TF_VAR_stack_version
+        run: |
+          version=$(grep defaultBeatVersion version/version.go | cut -f2 -d "\"")
+          echo "TF_VAR_stack_version=$version" >> $GITHUB_ENV
+
+      - name: Provision Test Environment (EC)
+        id: apply
+        if: success()
+        working-directory: ${{ env.WORKING_DIR }}
+        run: |
+          terraform -v
+          terraform init
+          terraform validate
+          terraform apply --auto-approve -target="module.ec_deployment" -target="module.ec_project"
+          terraform output
+          echo "KIBANA_URL=$(terraform output -raw kibana_url)" >> $GITHUB_ENV
+          echo "ES_URL=$(terraform output -raw elasticsearch_url)" >> $GITHUB_ENV
+          echo "ES_USER=$(terraform output -raw elasticsearch_username)" >> $GITHUB_ENV
+
+          export ES_PASSWORD=$(terraform output -raw elasticsearch_password)
+          echo "::add-mask::$ES_PASSWORD"
+          echo "ES_PASSWORD=$ES_PASSWORD" >> $GITHUB_ENV
+
+      - name: Install CSPM GCP integration
+        id: cspm-gcp-integration
+        working-directory: deploy/test-environments/fleet_api
+        env:
+          STACK_VERSION: ${{ env.ELK_VERSION }}
+          DEPLOYMENT_NAME: ${{env.GCP_DEPLOYMENT_NAME}}
+        run: |
+          poetry install
+          poetry run python src/install_cspm_gcp_integration.py
+
+      - name: Deploy CSPM GCP agent
+        id: cspm-gcp-agent
+        working-directory: deploy/deployment-manager
+        env:
+          DEPLOYMENT_LABELS: ${{ env.GCP_DEFAULT_TAGS }}
+        run: |
+          . ./set_env.sh && ./deploy.sh
+
+      - name: Check for findings
+        working-directory: ./tests
+        env:
+          USE_K8S: false
+        run: |
+          poetry install
+          poetry run pytest -k "cspm_gcp" --alluredir=./allure/results/ --clean-alluredir --maxfail=4
+
+      - name: Destory EC deployment
+        if: always()
+        working-directory: ${{ env.WORKING_DIR }}
+        run: |
+          terraform destroy --auto-approve -target="module.ec_deployment" -target="module.ec_project"
+
+      - name: Set up GCP Cloud SDK
+        if: always()
+        uses: "google-github-actions/setup-gcloud@v2"
+
+      - name: Delete GCP Deployment Manager deployment
+        if: always()
+        working-directory: ${{ env.WORKING_DIR }}
+        run: |
+          DEPLOYMENT=${{env.GCP_DEPLOYMENT_NAME}}
+          PROJECT_NAME=$(gcloud config get-value core/project)
+          PROJECT_NUMBER=$(gcloud projects list --filter="${PROJECT_NAME}" --format="value(PROJECT_NUMBER)")
+          ./delete_gcp_env.sh $PROJECT_NAME $PROJECT_NUMBER $DEPLOYMENT
diff --git a/deploy/deployment-manager/deploy.sh b/deploy/deployment-manager/deploy.sh
@@ -28,7 +28,7 @@
 #3. A dedicated network for the compute instance.
 #4. A service account bindings that associates the builtin roles with the service account.
 
-#In case the deployment encounters any issues and fails, the script will attempt to delete the deployment along with all the associated resources that were created during the process.
+# In case the deployment encounters any issues and fails, the script will attempt to delete the deployment along with all the associated resources that were created during the process.
 
 DEPLOYMENT_NAME=${DEPLOYMENT_NAME:-elastic-agent-cspm}
 ALLOW_SSH=${ALLOW_SSH:-false}

diff --git a/deploy/test-environments/delete_env.sh b/deploy/test-environments/delete_env.sh
@@ -160,40 +160,10 @@ printf "%s\n" "${DELETED_STACKS[@]}"
 echo "Failed to delete CloudFormation stacks (${#FAILED_STACKS[@]}):"
 printf "%s\n" "${FAILED_STACKS[@]}"
 
-DELETED_DEPLOYMENTS=()
-FAILED_DEPLOYMENTS=()
-
+# Delete GCP deployments
 PROJECT_NAME=$(gcloud config get-value core/project)
 PROJECT_NUMBER=$(gcloud projects list --filter="${PROJECT_NAME}" --format="value(PROJECT_NUMBER)")
-export PROJECT_NAME
-export PROJECT_NUMBER
-
-# Delete GCP Deployments
-for DEPLOYMENT in $ALL_GCP_DEPLOYMENTS; do
-    # Add the needed roles to delete the templates to the project using the deployment manager
-    gcloud projects add-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/iam.roleAdmin --no-user-output-enabled
-    gcloud projects add-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/resourcemanager.projectIamAdmin --no-user-output-enabled
-
-    if gcloud deployment-manager deployments delete "$DEPLOYMENT" -q; then
-        echo "Successfully deleted GCP deployment: $DEPLOYMENT"
-        DELETED_DEPLOYMENTS+=("$DEPLOYMENT")
-    else
-        echo "Failed to delete GCP deployment: $DEPLOYMENT"
-        FAILED_DEPLOYMENTS+=("$DEPLOYMENT")
-    fi
-
-    # Remove the roles required to deploy the DM templates
-    gcloud projects remove-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/iam.roleAdmin --no-user-output-enabled
-    gcloud projects remove-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/resourcemanager.projectIamAdmin --no-user-output-enabled
-
-done
-
-# Print summary of gcp deployments deletions
-echo "Successfully deleted GCP deployments (${#DELETED_DEPLOYMENTS[@]}):"
-printf "%s\n" "${DELETED_DEPLOYMENTS[@]}"
-
-echo "Failed to delete GCP deployments (${#FAILED_DEPLOYMENTS[@]}):"
-printf "%s\n" "${FAILED_DEPLOYMENTS[@]}"
+./delete_gcp_env.sh "$PROJECT_NAME" "$PROJECT_NUMBER" "$ALL_GCP_DEPLOYMENTS"
 
 # Delete Azure groups
 FAILED_AZURE_GROUPS=()

diff --git a/deploy/test-environments/delete_gcp_env.sh b/deploy/test-environments/delete_gcp_env.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+set -euo pipefail
+
+if [ "$#" -lt 3 ]; then
+    echo "Missing params. Usage: $0 PROJECT_NAME PROJECT_NUMBER DEPLOYMENT1,DEPLOYMENT2,..."
+    exit 1
+fi
+
+DELETED_DEPLOYMENTS=()
+FAILED_DEPLOYMENTS=()
+PROJECT_NAME=$1
+PROJECT_NUMBER=$2
+shift 2
+GCP_DEPLOYMENTS=("$@")
+
+echo "Project Name: $PROJECT_NAME"
+echo "Project Number: $PROJECT_NUMBER"
+echo "GCP Deployments: ${GCP_DEPLOYMENTS[*]}"
+
+for DEPLOYMENT in "${GCP_DEPLOYMENTS[@]}"; do
+    # Add the needed roles to delete the templates to the project using the deployment manager
+    gcloud projects add-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/iam.roleAdmin --no-user-output-enabled
+    gcloud projects add-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/resourcemanager.projectIamAdmin --no-user-output-enabled
+
+    if gcloud deployment-manager deployments delete "$DEPLOYMENT" -q; then
+        echo "Successfully deleted GCP deployment: $DEPLOYMENT"
+        DELETED_DEPLOYMENTS+=("$DEPLOYMENT")
+    else
+        echo "Failed to delete GCP deployment: $DEPLOYMENT"
+        FAILED_DEPLOYMENTS+=("$DEPLOYMENT")
+    fi
+
+    # Remove the roles required to deploy the DM templates
+    gcloud projects remove-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/iam.roleAdmin --no-user-output-enabled
+    gcloud projects remove-iam-policy-binding "${PROJECT_NAME}" --member=serviceAccount:"${PROJECT_NUMBER}"@cloudservices.gserviceaccount.com --role=roles/resourcemanager.projectIamAdmin --no-user-output-enabled
+
+done
+
+# Print summary of gcp deployments deletions
+echo "Successfully deleted GCP deployments (${#DELETED_DEPLOYMENTS[@]}):"
+printf "%s\n" "${DELETED_DEPLOYMENTS[@]}"
+
+if [ ${#FAILED_DEPLOYMENTS[@]} -gt 0 ]; then
+    echo "Failed to delete GCP deployments (${#FAILED_DEPLOYMENTS[@]}):"
+    printf "%s\n" "${FAILED_DEPLOYMENTS[@]}"
+    exit 1
+fi