feat: Add pre-upgrade jobs to run helm-mapkubeapis (#1439)

* feat(kubecost): Add pre-upgrade to run helm-mapkubeapi

* feat(kubecost): Use kommander chart version for kubetools image tag

* feat(kubecost): Use force true on pre-upgrade job

* feat(logging-operator): Add pre-upgrade job to run helm-mapkubeapis

* feat(gatekeeper): Add pre-upgrade job to run helm-mapkubeapis

* fix(gatekeeper): update release name

* fix(gatekeeper): add dependsOn in HR kustomization

* feat: Update release script to update the kubetools image tag

* chore: clean up todos

* refactor: typo

* fix: check if HR exists in pre-upgrade scripts

---------

Co-authored-by: Grace Do <xgrace@gmail.com>

hack/release/pkg/chartversion/chartversion.go

@@ -7,19 +7,30 @@ import (
 	"strings"
 
 	"github.com/drone/envsubst"
+
 	"github.com/mesosphere/kommander-applications/hack/release/pkg/constants"
 )
 
-const kommanderChartVersionTemplate = "${kommanderChartVersion:=%s}"
+const (
+	kommanderChartVersionTemplate = "${kommanderChartVersion:=%s}"
+	preUpgradePath                = "*/pre-upgrade/pre-upgrade.yaml"
+)
 
 var (
 	kommanderHelmReleasePathPattern        = filepath.Join(constants.KommanderAppPath, "*/kommander.yaml")
 	kommanderAppMgmtHelmReleasePathPattern = filepath.Join(constants.KommanderAppMgmtPath, "*/kommander-appmanagement.yaml")
-	kommanderOperatorPath                  = "./common/kommander-operator/helmrelease.yaml"
-	filesContainingKommanderVersion        = []string{
+	kommanderOperatorDefaultsCMPath        = "./common/kommander-operator/defaults/cm.yaml"
+	kubecostPreUpgradePath                 = filepath.Join(constants.KubecostPath, preUpgradePath)
+	gatekeeperPreUpgradePath               = filepath.Join(constants.GatekeeperPath, preUpgradePath)
+	loggingOperatorPreUpgradePath          = filepath.Join(constants.LoggingOperatorPath, preUpgradePath)
+
+	filesContainingKommanderVersion = []string{
 		kommanderHelmReleasePathPattern,
 		kommanderAppMgmtHelmReleasePathPattern,
-		kommanderOperatorPath,
+		kommanderOperatorDefaultsCMPath,
+		kubecostPreUpgradePath,
+		gatekeeperPreUpgradePath,
+		loggingOperatorPreUpgradePath,
 	}
 )
 

hack/release/pkg/chartversion/chartversion_test.go

@@ -168,3 +168,36 @@ func TestUpdateChartVersionsTooManyFiles(t *testing.T) {
 	err = UpdateChartVersions(tmpDir, updateToVersion)
 	assert.ErrorContains(t, err, "found > 1 match for HelmRelease path")
 }
+
+func TestUpdatePreUpgradeImages(t *testing.T) {
+	tmpDir, err := os.MkdirTemp("", "prerelease")
+	assert.Nil(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	// Make a copy of the current repo state to modify
+	err = cp.Copy(rootDir, tmpDir)
+	assert.Nil(t, err)
+
+	updateToVersion := "v1.0.0"
+	err = UpdateChartVersions(tmpDir, updateToVersion)
+	assert.Nil(t, err)
+
+	preUpgradePaths := []string{kubecostPreUpgradePath, gatekeeperPreUpgradePath, loggingOperatorPreUpgradePath}
+
+	for _, path := range preUpgradePaths {
+		t.Run(path, func(t *testing.T) {
+			updatedFile, err := filepath.Glob(filepath.Join(tmpDir, path))
+			assert.Nil(t, err)
+			assert.Len(t, updatedFile, 1)
+
+			content, err := os.ReadFile(updatedFile[0])
+			require.NoError(t, err)
+
+			assert.Equal(t,
+				1,
+				strings.Count(string(content), updateToVersion),
+			)
+		})
+	}
+
+}

hack/release/pkg/constants/constants.go

@@ -1,9 +1,12 @@
 package constants
 
 const (
-	KommanderAppPath     = "./services/kommander/"
-	KommanderAppMgmtPath = "./services/kommander-appmanagement/"
-  CAPIMateDefaultVersion = "v0.0.0-dev.0"
+	KommanderAppPath       = "./services/kommander/"
+	KommanderAppMgmtPath   = "./services/kommander-appmanagement/"
+	KubecostPath           = "./services/kubecost"
+	LoggingOperatorPath    = "./services/logging-operator"
+	GatekeeperPath         = "./services/gatekeeper"
+	CAPIMateDefaultVersion = "v0.0.0-dev.0"
 	// SemverRegexp validates any semver (taken verbatim from semver specs).
 	SemverRegexp = `v?(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?` //nolint:lll // it's not readable anyway
 )

services/gatekeeper/3.11.0/kustomization.yaml

@@ -4,3 +4,4 @@ resources:
   - release.yaml
   - constraints.yaml
   - constrainttemplates.yaml
+  - pre-upgrade.yaml

services/gatekeeper/3.11.0/pre-upgrade.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: gatekeeper-pre-upgrade
+  namespace: ${releaseNamespace}
+spec:
+  force: true
+  prune: true
+  wait: true
+  interval: 6h
+  retryInterval: 1m
+  path: ./services/gatekeeper/3.11.0/pre-upgrade
+  sourceRef:
+    kind: GitRepository
+    name: management
+    namespace: kommander-flux
+  timeout: 1m
+  # passing releaseNamespace to 2nd level configuration files for ability to configure namespace correctly in attached clusters
+  # Using `substituteFrom` with `substitution-vars` creates 2nd level resources in `kommander` namespace instead of workspace ns
+  postBuild:
+    substitute:
+      releaseNamespace: ${releaseNamespace}

services/gatekeeper/3.11.0/pre-upgrade/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - pre-upgrade.yaml

services/gatekeeper/3.11.0/pre-upgrade/pre-upgrade.yaml

@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gatekeeper-pre-upgrade
+  namespace: ${releaseNamespace}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: gatekeeper-pre-upgrade
+  namespace: ${releaseNamespace}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "watch", "list", "update", "patch", "create"]
+  - apiGroups: ["helm.toolkit.fluxcd.io"]
+    resources: ["helmreleases"]
+    verbs: ["get", "watch", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gatekeeper-pre-upgrade
+  namespace: ${releaseNamespace}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: gatekeeper-pre-upgrade
+subjects:
+  - kind: ServiceAccount
+    name: gatekeeper-pre-upgrade
+    namespace: ${releaseNamespace}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: gatekeeper-pre-upgrade
+  namespace: ${releaseNamespace}
+spec:
+  ttlSecondsAfterFinished: 100
+  template:
+    metadata:
+      name: gatekeeper-pre-upgrade
+    spec:
+      serviceAccountName: gatekeeper-pre-upgrade
+      restartPolicy: OnFailure
+      priorityClassName: system-cluster-critical
+      containers:
+        - name: kubetools
+          image: "mesosphere/kommander2-kubetools:${kommanderChartVersion:=v2.6.0-dev}"
+          command:
+            - sh
+            - "-c"
+            - |-
+              /bin/bash <<'EOF'
+              set -o nounset
+              set -o pipefail
+
+              kubectl get helmreleases.helm.toolkit.fluxcd.io -n ${releaseNamespace} gatekeeper
+              if [[ $? -ne 0 ]]; then
+                echo "Since the gatekeeper HelmRelease does not exist, this might not be an upgrade scenario. Exiting..."
+                exit 0
+              fi
+
+              helm mapkubeapis kommander-gatekeeper --namespace ${releaseNamespace}
+              EOF

services/gatekeeper/3.11.0/release.yaml

@@ -4,7 +4,9 @@ metadata:
   name: gatekeeper-release
   namespace: ${releaseNamespace}
 spec:
-  dependsOn: []
+  dependsOn:
+    - name: gatekeeper-pre-upgrade
+      namespace: ${releaseNamespace}
   force: false
   prune: true
   wait: true

services/kubecost/0.33.1/kustomization.yaml

@@ -1,4 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - pre-upgrade.yaml
   - kubecost.yaml

services/kubecost/0.33.1/pre-upgrade.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: kubecost-pre-upgrade
+  namespace: ${releaseNamespace}
+spec:
+  force: true
+  prune: true
+  wait: true
+  interval: 6h
+  retryInterval: 1m
+  path: ./services/kubecost/0.33.1/pre-upgrade
+  sourceRef:
+    kind: GitRepository
+    name: management
+    namespace: kommander-flux
+  timeout: 1m
+  # passing releaseNamespace to 2nd level configuration files for ability to configure namespace correctly in attached clusters
+  # Using `substituteFrom` with `substitution-vars` creates 2nd level resources in `kommander` namespace instead of workspace ns
+  postBuild:
+    substitute:
+      releaseNamespace: ${releaseNamespace}

services/kubecost/0.33.1/pre-upgrade/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - pre-upgrade.yaml

services/kubecost/0.33.1/pre-upgrade/pre-upgrade.yaml

@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubecost-pre-upgrade
+  namespace: ${releaseNamespace}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kubecost-pre-upgrade
+  namespace: ${releaseNamespace}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "watch", "list", "update", "patch", "create"]
+  - apiGroups: ["helm.toolkit.fluxcd.io"]
+    resources: ["helmreleases"]
+    verbs: ["get", "watch", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kubecost-pre-upgrade
+  namespace: ${releaseNamespace}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kubecost-pre-upgrade
+subjects:
+  - kind: ServiceAccount
+    name: kubecost-pre-upgrade
+    namespace: ${releaseNamespace}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kubecost-pre-upgrade
+  namespace: ${releaseNamespace}
+spec:
+  ttlSecondsAfterFinished: 100
+  template:
+    metadata:
+      name: kubecost-pre-upgrade
+    spec:
+      serviceAccountName: kubecost-pre-upgrade
+      restartPolicy: OnFailure
+      priorityClassName: dkp-high-priority
+      containers:
+        - name: kubetools
+          image: "mesosphere/kommander2-kubetools:${kommanderChartVersion:=v2.5.2-dev}"
+          command:
+            - sh
+            - "-c"
+            - |-
+              /bin/bash <<'EOF'
+              set -o nounset
+              set -o pipefail
+
+              kubectl get helmreleases.helm.toolkit.fluxcd.io -n ${releaseNamespace} kubecost
+              if [[ $? -ne 0 ]]; then
+                echo "Since the kubecost HelmRelease does not exist, this might not be an upgrade scenario. Exiting..."
+                exit 0
+              fi
+
+              helm mapkubeapis kubecost --namespace ${releaseNamespace}
+              EOF

services/logging-operator/3.17.10/kustomization.yaml

@@ -4,3 +4,4 @@ resources:
   - grafana-dashboards
   - logging-operator.yaml
   - logging-operator-logging.yaml
+  - pre-upgrade.yaml

services/logging-operator/3.17.10/pre-upgrade.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: logging-operator-pre-upgrade
+  namespace: ${releaseNamespace}
+spec:
+  force: true
+  prune: true
+  wait: true
+  interval: 6h
+  retryInterval: 1m
+  path: ./services/logging-operator/3.17.10/pre-upgrade
+  sourceRef:
+    kind: GitRepository
+    name: management
+    namespace: kommander-flux
+  timeout: 1m
+  # passing releaseNamespace to 2nd level configuration files for ability to configure namespace correctly in attached clusters
+  # Using `substituteFrom` with `substitution-vars` creates 2nd level resources in `kommander` namespace instead of workspace ns
+  postBuild:
+    substitute:
+      releaseNamespace: ${releaseNamespace}

services/logging-operator/3.17.10/pre-upgrade/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - pre-upgrade.yaml

services/logging-operator/3.17.10/pre-upgrade/pre-upgrade.yaml

@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: logging-operator-pre-upgrade
+  namespace: ${releaseNamespace}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: logging-operator-pre-upgrade
+  namespace: ${releaseNamespace}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "watch", "list", "update", "patch", "create"]
+  - apiGroups: ["helm.toolkit.fluxcd.io"]
+    resources: ["helmreleases"]
+    verbs: ["get", "watch", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: logging-operator-pre-upgrade
+  namespace: ${releaseNamespace}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: logging-operator-pre-upgrade
+subjects:
+  - kind: ServiceAccount
+    name: logging-operator-pre-upgrade
+    namespace: ${releaseNamespace}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: logging-operator-pre-upgrade
+  namespace: ${releaseNamespace}
+spec:
+  ttlSecondsAfterFinished: 100
+  template:
+    metadata:
+      name: logging-operator-pre-upgrade
+    spec:
+      serviceAccountName: logging-operator-pre-upgrade
+      restartPolicy: OnFailure
+      priorityClassName: dkp-critical-priority
+      containers:
+        - name: kubetools
+          image: "mesosphere/kommander2-kubetools:${kommanderChartVersion:=v2.6.0-dev}"
+          command:
+            - sh
+            - "-c"
+            - |-
+              /bin/bash <<'EOF'
+              set -o nounset
+              set -o pipefail
+
+              kubectl get helmreleases.helm.toolkit.fluxcd.io -n ${releaseNamespace} logging-operator
+              if [[ $? -ne 0 ]]; then
+                echo "Since the logging-operator HelmRelease does not exist, this might not be an upgrade scenario. Exiting..."
+                exit 0
+              fi
+
+              helm mapkubeapis logging-operator --namespace ${releaseNamespace}
+              EOF