feat: Add pre-upgrade jobs to run helm-mapkubeapis (#1439) (2.5)

hack/release/pkg/chartversion/chartversion.go

@@ -7,19 +7,30 @@ import (
 	"strings"
 
 	"github.com/drone/envsubst"
+
 	"github.com/mesosphere/kommander-applications/hack/release/pkg/constants"
 )
 
-const kommanderChartVersionTemplate = "${kommanderChartVersion:=%s}"
+const (
+	kommanderChartVersionTemplate = "${kommanderChartVersion:=%s}"
+	preUpgradePath                = "*/pre-upgrade/pre-upgrade.yaml"
+)
 
 var (
 	kommanderHelmReleasePathPattern        = filepath.Join(constants.KommanderAppPath, "*/kommander.yaml")
 	kommanderAppMgmtHelmReleasePathPattern = filepath.Join(constants.KommanderAppMgmtPath, "*/kommander-appmanagement.yaml")
 	kommanderOperatorPath                  = "./common/kommander-operator/helmrelease.yaml"
-	filesContainingKommanderVersion        = []string{
+	kubecostPreUpgradePath                 = filepath.Join(constants.KubecostPath, preUpgradePath)
+	gatekeeperPreUpgradePath               = filepath.Join(constants.GatekeeperPath, preUpgradePath)
+	loggingOperatorPreUpgradePath          = filepath.Join(constants.LoggingOperatorPath, preUpgradePath)
+
+	filesContainingKommanderVersion = []string{
 		kommanderHelmReleasePathPattern,
 		kommanderAppMgmtHelmReleasePathPattern,
 		kommanderOperatorPath,
+		kubecostPreUpgradePath,
+		gatekeeperPreUpgradePath,
+		loggingOperatorPreUpgradePath,
 	}
 )
 

hack/release/pkg/chartversion/chartversion_test.go

@@ -168,3 +168,36 @@ func TestUpdateChartVersionsTooManyFiles(t *testing.T) {
 	err = UpdateChartVersions(tmpDir, updateToVersion)
 	assert.ErrorContains(t, err, "found > 1 match for HelmRelease path")
 }
+
+func TestUpdatePreUpgradeImages(t *testing.T) {
+	tmpDir, err := os.MkdirTemp("", "prerelease")
+	assert.Nil(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	// Make a copy of the current repo state to modify
+	err = cp.Copy(rootDir, tmpDir)
+	assert.Nil(t, err)
+
+	updateToVersion := "v1.0.0"
+	err = UpdateChartVersions(tmpDir, updateToVersion)
+	assert.Nil(t, err)
+
+	preUpgradePaths := []string{kubecostPreUpgradePath, gatekeeperPreUpgradePath, loggingOperatorPreUpgradePath}
+
+	for _, path := range preUpgradePaths {
+		t.Run(path, func(t *testing.T) {
+			updatedFile, err := filepath.Glob(filepath.Join(tmpDir, path))
+			assert.Nil(t, err)
+			assert.Len(t, updatedFile, 1)
+
+			content, err := os.ReadFile(updatedFile[0])
+			require.NoError(t, err)
+
+			assert.Equal(t,
+				1,
+				strings.Count(string(content), updateToVersion),
+			)
+		})
+	}
+
+}

hack/release/pkg/constants/constants.go

@@ -1,9 +1,12 @@
 package constants
 
 const (
-	KommanderAppPath     = "./services/kommander/"
-	KommanderAppMgmtPath = "./services/kommander-appmanagement/"
-  CAPIMateDefaultVersion = "v0.0.0-dev.0"
+	KommanderAppPath       = "./services/kommander/"
+	KommanderAppMgmtPath   = "./services/kommander-appmanagement/"
+	KubecostPath           = "./services/kubecost"
+	LoggingOperatorPath    = "./services/logging-operator"
+	GatekeeperPath         = "./services/gatekeeper"
+	CAPIMateDefaultVersion = "v0.0.0-dev.0"
 	// SemverRegexp validates any semver (taken verbatim from semver specs).
 	SemverRegexp = `v?(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?` //nolint:lll // it's not readable anyway
 )

services/centralized-kubecost/0.33.1/defaults/cm.yaml → services/centralized-kubecost/0.33.2/defaults/cm.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: centralized-kubecost-0.33.1-d2iq-defaults
+  name: centralized-kubecost-0.33.2-d2iq-defaults
   namespace: ${releaseNamespace}
 data:
   values.yaml: |

services/centralized-kubecost/0.33.1/post-install-jobs.yaml → services/centralized-kubecost/0.33.2/post-install-jobs.yaml

@@ -9,7 +9,7 @@ spec:
   wait: true
   interval: 6h
   retryInterval: 1m
-  path: ./services/centralized-kubecost/0.33.1/post-install-jobs
+  path: ./services/centralized-kubecost/0.33.2/post-install-jobs
   dependsOn:
     - name: centralized-kubecost-release
   sourceRef:

services/centralized-kubecost/0.33.1/release.yaml → services/centralized-kubecost/0.33.2/release.yaml

@@ -9,7 +9,7 @@ spec:
   wait: true
   interval: 6h
   retryInterval: 1m
-  path: ./services/centralized-kubecost/0.33.1/release
+  path: ./services/centralized-kubecost/0.33.2/release
   sourceRef:
     kind: GitRepository
     name: management

services/centralized-kubecost/0.33.1/release/release.yaml → services/centralized-kubecost/0.33.2/release/release.yaml

@@ -28,7 +28,7 @@ spec:
   releaseName: centralized-kubecost
   valuesFrom:
     - kind: ConfigMap
-      name: centralized-kubecost-0.33.1-d2iq-defaults
+      name: centralized-kubecost-0.33.2-d2iq-defaults
   targetNamespace: kubecost
 ---
 apiVersion: v1

services/gatekeeper/3.11.1/kustomization.yaml

@@ -4,3 +4,4 @@ resources:
   - release.yaml
   - constraints.yaml
   - constrainttemplates.yaml
+  - pre-upgrade.yaml

services/gatekeeper/3.11.1/pre-upgrade.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: gatekeeper-pre-upgrade
+  namespace: ${releaseNamespace}
+spec:
+  force: true
+  prune: true
+  wait: true
+  interval: 6h
+  retryInterval: 1m
+  path: ./services/gatekeeper/3.11.1/pre-upgrade
+  sourceRef:
+    kind: GitRepository
+    name: management
+    namespace: kommander-flux
+  timeout: 1m
+  # passing releaseNamespace to 2nd level configuration files for ability to configure namespace correctly in attached clusters
+  # Using `substituteFrom` with `substitution-vars` creates 2nd level resources in `kommander` namespace instead of workspace ns
+  postBuild:
+    substitute:
+      releaseNamespace: ${releaseNamespace}

services/gatekeeper/3.11.1/pre-upgrade/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - pre-upgrade.yaml

services/gatekeeper/3.11.1/pre-upgrade/pre-upgrade.yaml

@@ -0,0 +1,66 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gatekeeper-pre-upgrade
+  namespace: ${releaseNamespace}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: gatekeeper-pre-upgrade
+  namespace: ${releaseNamespace}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "watch", "list", "update", "patch", "create"]
+  - apiGroups: ["helm.toolkit.fluxcd.io"]
+    resources: ["helmreleases"]
+    verbs: ["get", "watch", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gatekeeper-pre-upgrade
+  namespace: ${releaseNamespace}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: gatekeeper-pre-upgrade
+subjects:
+  - kind: ServiceAccount
+    name: gatekeeper-pre-upgrade
+    namespace: ${releaseNamespace}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: gatekeeper-pre-upgrade
+  namespace: ${releaseNamespace}
+spec:
+  ttlSecondsAfterFinished: 100
+  template:
+    metadata:
+      name: gatekeeper-pre-upgrade
+    spec:
+      serviceAccountName: gatekeeper-pre-upgrade
+      restartPolicy: OnFailure
+      containers:
+        - name: kubetools
+          image: "mesosphere/kommander2-kubetools:${kommanderChartVersion:=v2.5.2-dev}"
+          command:
+            - sh
+            - "-c"
+            - |-
+              /bin/bash <<'EOF'
+              set -o nounset
+              set -o pipefail
+
+              kubectl get helmreleases.helm.toolkit.fluxcd.io -n ${releaseNamespace} gatekeeper
+              if [[ $? -ne 0 ]]; then
+                echo "Since the gatekeeper HelmRelease does not exist, this might not be an upgrade scenario. Exiting..."
+                exit 0
+              fi
+
+              # the gatekeeper releaseName is set to kommander-gatekeeper in the HelmRelease
+              helm mapkubeapis kommander-gatekeeper --namespace ${releaseNamespace}
+              EOF

services/gatekeeper/3.11.1/release.yaml

@@ -4,7 +4,9 @@ metadata:
   name: gatekeeper-release
   namespace: ${releaseNamespace}
 spec:
-  dependsOn: []
+  dependsOn:
+    - name: gatekeeper-pre-upgrade
+      namespace: ${releaseNamespace}
   force: false
   prune: true
   wait: true

services/kommander/0.5.2/defaults/cm.yaml

@@ -65,8 +65,8 @@ data:
           tag: ${kommanderLicensingControllerWebhookImageTag}
           repository: ${kommanderLicensingControllerWebhookImageRepository}
       defaultEnterpriseApps:
-        centralized-kubecost: "0.33.1"
-        kubecost: "0.33.1"
+        centralized-kubecost: "0.33.2"
+        kubecost: "0.33.2"
         kubecost-thanos-traefik: "0.0.1"
         centralized-grafana: "44.2.1"
         karma: "2.0.1"
@@ -106,7 +106,7 @@ data:
           reloader: "1.0.5"
           traefik: "20.8.0"
           kubernetes-dashboard: "6.0.0"
-          kubecost: "0.33.1"
+          kubecost: "0.33.2"
           kube-prometheus-stack: "44.2.1"
           prometheus-adapter: "4.0.2"
           prometheus-thanos-traefik: "0.0.1"

services/kubecost/0.33.1/defaults/cm.yaml → services/kubecost/0.33.2/defaults/cm.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: kubecost-0.33.1-d2iq-defaults
+  name: kubecost-0.33.2-d2iq-defaults
   namespace: ${releaseNamespace}
 data:
   values.yaml: |

services/kubecost/0.33.1/kubecost.yaml → services/kubecost/0.33.2/kubecost.yaml

@@ -28,7 +28,7 @@ spec:
   releaseName: kubecost
   valuesFrom:
     - kind: ConfigMap
-      name: kubecost-0.33.1-d2iq-defaults
+      name: kubecost-0.33.2-d2iq-defaults
   targetNamespace: ${releaseNamespace}
 ---
 apiVersion: v1

services/kubecost/0.33.1/kustomization.yaml → services/kubecost/0.33.2/kustomization.yaml

@@ -1,4 +1,5 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - pre-upgrade.yaml
   - kubecost.yaml

services/kubecost/0.33.2/pre-upgrade.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: kubecost-pre-upgrade
+  namespace: ${releaseNamespace}
+spec:
+  force: true
+  prune: true
+  wait: true
+  interval: 6h
+  retryInterval: 1m
+  path: ./services/kubecost/0.33.2/pre-upgrade
+  sourceRef:
+    kind: GitRepository
+    name: management
+    namespace: kommander-flux
+  timeout: 1m
+  # passing releaseNamespace to 2nd level configuration files for ability to configure namespace correctly in attached clusters
+  # Using `substituteFrom` with `substitution-vars` creates 2nd level resources in `kommander` namespace instead of workspace ns
+  postBuild:
+    substitute:
+      releaseNamespace: ${releaseNamespace}

services/kubecost/0.33.2/pre-upgrade/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - pre-upgrade.yaml

services/kubecost/0.33.2/pre-upgrade/pre-upgrade.yaml

@@ -0,0 +1,65 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kubecost-pre-upgrade
+  namespace: ${releaseNamespace}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: kubecost-pre-upgrade
+  namespace: ${releaseNamespace}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "watch", "list", "update", "patch", "create"]
+  - apiGroups: ["helm.toolkit.fluxcd.io"]
+    resources: ["helmreleases"]
+    verbs: ["get", "watch", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kubecost-pre-upgrade
+  namespace: ${releaseNamespace}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kubecost-pre-upgrade
+subjects:
+  - kind: ServiceAccount
+    name: kubecost-pre-upgrade
+    namespace: ${releaseNamespace}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kubecost-pre-upgrade
+  namespace: ${releaseNamespace}
+spec:
+  ttlSecondsAfterFinished: 100
+  template:
+    metadata:
+      name: kubecost-pre-upgrade
+    spec:
+      serviceAccountName: kubecost-pre-upgrade
+      restartPolicy: OnFailure
+      containers:
+        - name: kubetools
+          image: "mesosphere/kommander2-kubetools:${kommanderChartVersion:=v2.5.2-dev}"
+          command:
+            - sh
+            - "-c"
+            - |-
+              /bin/bash <<'EOF'
+              set -o nounset
+              set -o pipefail
+
+              kubectl get helmreleases.helm.toolkit.fluxcd.io -n ${releaseNamespace} kubecost
+              if [[ $? -ne 0 ]]; then
+                echo "Since the kubecost HelmRelease does not exist, this might not be an upgrade scenario. Exiting..."
+                exit 0
+              fi
+
+              helm mapkubeapis kubecost --namespace ${releaseNamespace}
+              EOF

services/logging-operator/3.17.11/kustomization.yaml

@@ -4,3 +4,4 @@ resources:
   - grafana-dashboards
   - logging-operator.yaml
   - logging-operator-logging.yaml
+  - pre-upgrade.yaml

services/logging-operator/3.17.11/pre-upgrade.yaml

@@ -0,0 +1,22 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  name: logging-operator-pre-upgrade
+  namespace: ${releaseNamespace}
+spec:
+  force: true
+  prune: true
+  wait: true
+  interval: 6h
+  retryInterval: 1m
+  path: ./services/logging-operator/3.17.11/pre-upgrade
+  sourceRef:
+    kind: GitRepository
+    name: management
+    namespace: kommander-flux
+  timeout: 1m
+  # passing releaseNamespace to 2nd level configuration files for ability to configure namespace correctly in attached clusters
+  # Using `substituteFrom` with `substitution-vars` creates 2nd level resources in `kommander` namespace instead of workspace ns
+  postBuild:
+    substitute:
+      releaseNamespace: ${releaseNamespace}