From 01482b380aeb1d9e995ca29b3c86cf3417e40b1e Mon Sep 17 00:00:00 2001
From: Gareth <garethgeorge@google.com>
Date: Mon, 16 Oct 2023 21:32:31 -0700
Subject: [PATCH] chore: ensure that e2e tests cleanup clusters (#1057)

Ensures that the delete clusters flow executes even when

* cluster turnup succeeds partially (e.g. cluster created but various
kubectl apply operations fail)
 * other testing stages fail

This reduces resource waste in our integration testing project.
---
 ci/cloudbuild/test.yaml | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/ci/cloudbuild/test.yaml b/ci/cloudbuild/test.yaml
index db5b7c1f2..fb9760794 100644
--- a/ci/cloudbuild/test.yaml
+++ b/ci/cloudbuild/test.yaml
@@ -231,6 +231,12 @@ steps:
   - |
     set -eux
 
+    cleanup() {
+      echo $? > /workspace/${BUILD_ID}/_CREATE_GKE_CLUSTER
+      exit 0
+    }
+    trap cleanup EXIT
+
     if [ -z "${_FULL_RELEASE_BUCKET}" ]; then
       echo "_FULL_RELEASE_BUCKET is not set. Skipping GKE cluster creation..."
       exit 0
@@ -267,6 +273,12 @@ steps:
   - |
     set -euxo pipefail
 
+    cleanup() {
+      echo $? > /workspace/${BUILD_ID}/_DEPLOY_CERTS
+      exit 0
+    }
+    trap cleanup EXIT
+
     export CLOUDSDK_COMPUTE_ZONE=$(cat /workspace/${BUILD_ID}/_CLOUDSDK_COMPUTE_ZONE)
     export CLOUDSDK_CONTAINER_CLUSTER=$(cat /workspace/${BUILD_ID}/_CLOUDSDK_CONTAINER_CLUSTER)
     export CLOUDSDK_CORE_PROJECT=${PROJECT_ID}
@@ -307,6 +319,12 @@ steps:
   - |
     set -eux
 
+    cleanup() {
+      echo $? > /workspace/${BUILD_ID}/_STRESS_TESTS
+      exit 0
+    }
+    trap cleanup EXIT
+
     if [ "${_STRESS_TEST_BASELOAD_FACTOR}" -le "0" ]; then
       echo "_STRESS_TEST_BASELOAD_FACTOR is set to ${_STRESS_TEST_BASELOAD_FACTOR}. Skipping..."
       exit 0
@@ -425,6 +443,12 @@ steps:
   - |
     set -eu
 
+    cleanup() {
+      echo $? > /workspace/${BUILD_ID}/_ACCEPTANCE_TESTS
+      exit 0
+    }
+    trap cleanup EXIT
+
     if [ "${_SKIP_ACCEPTANCE_TESTS}" == "true" ]; then
       echo "Skipping acceptance tests..."
       exit 0
@@ -589,6 +613,10 @@ steps:
         fi
     }
 
+    check /workspace/${BUILD_ID}/_CREATE_GKE_CLUSTER "Create GKE cluster"
+    check /workspace/${BUILD_ID}/_DEPLOY_CERTS "Deploy certs"
+    check /workspace/${BUILD_ID}/_STRESS_TESTS "Stress test"
+    check /workspace/${BUILD_ID}/_ACCEPTANCE_TESTS "Acceptance tests"
     check /workspace/${BUILD_ID}/_INTEGRATION_EXIT_CODE "Integration tests"
     check /workspace/${BUILD_ID}/_DELETE_CLUSTER_EXIT_CODE "Delete GKE cluster"