From a7c98bd9dd46c173c9455577077f785dd7ff5b4c Mon Sep 17 00:00:00 2001 From: Casey Buto Date: Tue, 1 Aug 2023 16:33:19 -0400 Subject: [PATCH] feat: Add pre-upgrade jobs to run helm-mapkubeapis (#1439) * feat(kubecost): Add pre-upgrade to run helm-mapkubeapi * feat(kubecost): Use kommander chart version for kubetools image tag * feat(kubecost): Use force true on pre-upgrade job * feat(logging-operator): Add pre-upgrade job to run helm-mapkubeapis * feat(gatekeeper): Add pre-upgrade job to run helm-mapkubeapis * fix(gatekeeper): update release name * fix(gatekeeper): add dependsOn in HR kustomization * feat: Update release script to update the kubetools image tag * chore: clean up todos * refactor: typo * fix: check if HR exists in pre-upgrade scripts --------- Co-authored-by: Grace Do --- hack/release/pkg/chartversion/chartversion.go | 15 ++++- .../pkg/chartversion/chartversion_test.go | 33 ++++++++++ hack/release/pkg/constants/constants.go | 9 ++- services/gatekeeper/3.11.0/kustomization.yaml | 1 + services/gatekeeper/3.11.0/pre-upgrade.yaml | 22 +++++++ .../3.11.0/pre-upgrade/kustomization.yaml | 4 ++ .../3.11.0/pre-upgrade/pre-upgrade.yaml | 65 +++++++++++++++++++ services/gatekeeper/3.11.0/release.yaml | 4 +- services/kubecost/0.33.1/kustomization.yaml | 1 + services/kubecost/0.33.1/pre-upgrade.yaml | 22 +++++++ .../0.33.1/pre-upgrade/kustomization.yaml | 4 ++ .../0.33.1/pre-upgrade/pre-upgrade.yaml | 65 +++++++++++++++++++ .../3.17.10/kustomization.yaml | 1 + .../logging-operator/3.17.10/pre-upgrade.yaml | 22 +++++++ .../3.17.10/pre-upgrade/kustomization.yaml | 4 ++ .../3.17.10/pre-upgrade/pre-upgrade.yaml | 65 +++++++++++++++++++ 16 files changed, 331 insertions(+), 6 deletions(-) create mode 100644 services/gatekeeper/3.11.0/pre-upgrade.yaml create mode 100644 services/gatekeeper/3.11.0/pre-upgrade/kustomization.yaml create mode 100644 services/gatekeeper/3.11.0/pre-upgrade/pre-upgrade.yaml create mode 100644 services/kubecost/0.33.1/pre-upgrade.yaml create mode 100644 services/kubecost/0.33.1/pre-upgrade/kustomization.yaml create mode 100644 services/kubecost/0.33.1/pre-upgrade/pre-upgrade.yaml create mode 100644 services/logging-operator/3.17.10/pre-upgrade.yaml create mode 100644 services/logging-operator/3.17.10/pre-upgrade/kustomization.yaml create mode 100644 services/logging-operator/3.17.10/pre-upgrade/pre-upgrade.yaml diff --git a/hack/release/pkg/chartversion/chartversion.go b/hack/release/pkg/chartversion/chartversion.go index b0458422b9..52f4eb736d 100644 --- a/hack/release/pkg/chartversion/chartversion.go +++ b/hack/release/pkg/chartversion/chartversion.go @@ -7,19 +7,30 @@ import ( "strings" "github.com/drone/envsubst" + "github.com/mesosphere/kommander-applications/hack/release/pkg/constants" ) -const kommanderChartVersionTemplate = "${kommanderChartVersion:=%s}" +const ( + kommanderChartVersionTemplate = "${kommanderChartVersion:=%s}" + preUpgradePath = "*/pre-upgrade/pre-upgrade.yaml" +) var ( kommanderHelmReleasePathPattern = filepath.Join(constants.KommanderAppPath, "*/kommander.yaml") kommanderAppMgmtHelmReleasePathPattern = filepath.Join(constants.KommanderAppMgmtPath, "*/kommander-appmanagement.yaml") kommanderOperatorPath = "./common/kommander-operator/helmrelease.yaml" - filesContainingKommanderVersion = []string{ + kubecostPreUpgradePath = filepath.Join(constants.KubecostPath, preUpgradePath) + gatekeeperPreUpgradePath = filepath.Join(constants.GatekeeperPath, preUpgradePath) + loggingOperatorPreUpgradePath = filepath.Join(constants.LoggingOperatorPath, preUpgradePath) + + filesContainingKommanderVersion = []string{ kommanderHelmReleasePathPattern, kommanderAppMgmtHelmReleasePathPattern, kommanderOperatorPath, + kubecostPreUpgradePath, + gatekeeperPreUpgradePath, + loggingOperatorPreUpgradePath, } ) diff --git a/hack/release/pkg/chartversion/chartversion_test.go b/hack/release/pkg/chartversion/chartversion_test.go index 767bb798fd..f00d9e4d37 100644 --- a/hack/release/pkg/chartversion/chartversion_test.go +++ b/hack/release/pkg/chartversion/chartversion_test.go @@ -168,3 +168,36 @@ func TestUpdateChartVersionsTooManyFiles(t *testing.T) { err = UpdateChartVersions(tmpDir, updateToVersion) assert.ErrorContains(t, err, "found > 1 match for HelmRelease path") } + +func TestUpdatePreUpgradeImages(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "prerelease") + assert.Nil(t, err) + defer os.RemoveAll(tmpDir) + + // Make a copy of the current repo state to modify + err = cp.Copy(rootDir, tmpDir) + assert.Nil(t, err) + + updateToVersion := "v1.0.0" + err = UpdateChartVersions(tmpDir, updateToVersion) + assert.Nil(t, err) + + preUpgradePaths := []string{kubecostPreUpgradePath, gatekeeperPreUpgradePath, loggingOperatorPreUpgradePath} + + for _, path := range preUpgradePaths { + t.Run(path, func(t *testing.T) { + updatedFile, err := filepath.Glob(filepath.Join(tmpDir, path)) + assert.Nil(t, err) + assert.Len(t, updatedFile, 1) + + content, err := os.ReadFile(updatedFile[0]) + require.NoError(t, err) + + assert.Equal(t, + 1, + strings.Count(string(content), updateToVersion), + ) + }) + } + +} diff --git a/hack/release/pkg/constants/constants.go b/hack/release/pkg/constants/constants.go index d6e721f8e6..dc7a32ee3b 100644 --- a/hack/release/pkg/constants/constants.go +++ b/hack/release/pkg/constants/constants.go @@ -1,9 +1,12 @@ package constants const ( - KommanderAppPath = "./services/kommander/" - KommanderAppMgmtPath = "./services/kommander-appmanagement/" - CAPIMateDefaultVersion = "v0.0.0-dev.0" + KommanderAppPath = "./services/kommander/" + KommanderAppMgmtPath = "./services/kommander-appmanagement/" + KubecostPath = "./services/kubecost" + LoggingOperatorPath = "./services/logging-operator" + GatekeeperPath = "./services/gatekeeper" + CAPIMateDefaultVersion = "v0.0.0-dev.0" // SemverRegexp validates any semver (taken verbatim from semver specs). SemverRegexp = `v?(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)\.(?P0|[1-9]\d*)(?:-(?P(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?` //nolint:lll // it's not readable anyway ) diff --git a/services/gatekeeper/3.11.0/kustomization.yaml b/services/gatekeeper/3.11.0/kustomization.yaml index f4818475c2..16f1478020 100644 --- a/services/gatekeeper/3.11.0/kustomization.yaml +++ b/services/gatekeeper/3.11.0/kustomization.yaml @@ -4,3 +4,4 @@ resources: - release.yaml - constraints.yaml - constrainttemplates.yaml + - pre-upgrade.yaml diff --git a/services/gatekeeper/3.11.0/pre-upgrade.yaml b/services/gatekeeper/3.11.0/pre-upgrade.yaml new file mode 100644 index 0000000000..f3c4c2678a --- /dev/null +++ b/services/gatekeeper/3.11.0/pre-upgrade.yaml @@ -0,0 +1,22 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 +kind: Kustomization +metadata: + name: gatekeeper-pre-upgrade + namespace: ${releaseNamespace} +spec: + force: true + prune: true + wait: true + interval: 6h + retryInterval: 1m + path: ./services/gatekeeper/3.11.0/pre-upgrade + sourceRef: + kind: GitRepository + name: management + namespace: kommander-flux + timeout: 1m + # passing releaseNamespace to 2nd level configuration files for ability to configure namespace correctly in attached clusters + # Using `substituteFrom` with `substitution-vars` creates 2nd level resources in `kommander` namespace instead of workspace ns + postBuild: + substitute: + releaseNamespace: ${releaseNamespace} diff --git a/services/gatekeeper/3.11.0/pre-upgrade/kustomization.yaml b/services/gatekeeper/3.11.0/pre-upgrade/kustomization.yaml new file mode 100644 index 0000000000..df8c392760 --- /dev/null +++ b/services/gatekeeper/3.11.0/pre-upgrade/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - pre-upgrade.yaml diff --git a/services/gatekeeper/3.11.0/pre-upgrade/pre-upgrade.yaml b/services/gatekeeper/3.11.0/pre-upgrade/pre-upgrade.yaml new file mode 100644 index 0000000000..60c7a90ea2 --- /dev/null +++ b/services/gatekeeper/3.11.0/pre-upgrade/pre-upgrade.yaml @@ -0,0 +1,65 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gatekeeper-pre-upgrade + namespace: ${releaseNamespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gatekeeper-pre-upgrade + namespace: ${releaseNamespace} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "watch", "list", "update", "patch", "create"] + - apiGroups: ["helm.toolkit.fluxcd.io"] + resources: ["helmreleases"] + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gatekeeper-pre-upgrade + namespace: ${releaseNamespace} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gatekeeper-pre-upgrade +subjects: + - kind: ServiceAccount + name: gatekeeper-pre-upgrade + namespace: ${releaseNamespace} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: gatekeeper-pre-upgrade + namespace: ${releaseNamespace} +spec: + ttlSecondsAfterFinished: 100 + template: + metadata: + name: gatekeeper-pre-upgrade + spec: + serviceAccountName: gatekeeper-pre-upgrade + restartPolicy: OnFailure + containers: + - name: kubetools + image: "mesosphere/kommander2-kubetools:${kommanderChartVersion:=v2.5.2-dev}" + command: + - sh + - "-c" + - |- + /bin/bash <<'EOF' + set -o nounset + set -o pipefail + + kubectl get helmreleases.helm.toolkit.fluxcd.io -n ${releaseNamespace} gatekeeper + if [[ $? -ne 0 ]]; then + echo "Since the gatekeeper HelmRelease does not exist, this might not be an upgrade scenario. Exiting..." + exit 0 + fi + + helm mapkubeapis kommander-gatekeeper --namespace ${releaseNamespace} + EOF diff --git a/services/gatekeeper/3.11.0/release.yaml b/services/gatekeeper/3.11.0/release.yaml index a2a770fb40..9fbbab7ca0 100644 --- a/services/gatekeeper/3.11.0/release.yaml +++ b/services/gatekeeper/3.11.0/release.yaml @@ -4,7 +4,9 @@ metadata: name: gatekeeper-release namespace: ${releaseNamespace} spec: - dependsOn: [] + dependsOn: + - name: gatekeeper-pre-upgrade + namespace: ${releaseNamespace} force: false prune: true wait: true diff --git a/services/kubecost/0.33.1/kustomization.yaml b/services/kubecost/0.33.1/kustomization.yaml index f867bff569..36f49ab66f 100644 --- a/services/kubecost/0.33.1/kustomization.yaml +++ b/services/kubecost/0.33.1/kustomization.yaml @@ -1,4 +1,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - pre-upgrade.yaml - kubecost.yaml diff --git a/services/kubecost/0.33.1/pre-upgrade.yaml b/services/kubecost/0.33.1/pre-upgrade.yaml new file mode 100644 index 0000000000..112a3c3f69 --- /dev/null +++ b/services/kubecost/0.33.1/pre-upgrade.yaml @@ -0,0 +1,22 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 +kind: Kustomization +metadata: + name: kubecost-pre-upgrade + namespace: ${releaseNamespace} +spec: + force: true + prune: true + wait: true + interval: 6h + retryInterval: 1m + path: ./services/kubecost/0.33.1/pre-upgrade + sourceRef: + kind: GitRepository + name: management + namespace: kommander-flux + timeout: 1m + # passing releaseNamespace to 2nd level configuration files for ability to configure namespace correctly in attached clusters + # Using `substituteFrom` with `substitution-vars` creates 2nd level resources in `kommander` namespace instead of workspace ns + postBuild: + substitute: + releaseNamespace: ${releaseNamespace} diff --git a/services/kubecost/0.33.1/pre-upgrade/kustomization.yaml b/services/kubecost/0.33.1/pre-upgrade/kustomization.yaml new file mode 100644 index 0000000000..df8c392760 --- /dev/null +++ b/services/kubecost/0.33.1/pre-upgrade/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - pre-upgrade.yaml diff --git a/services/kubecost/0.33.1/pre-upgrade/pre-upgrade.yaml b/services/kubecost/0.33.1/pre-upgrade/pre-upgrade.yaml new file mode 100644 index 0000000000..b666e5dd97 --- /dev/null +++ b/services/kubecost/0.33.1/pre-upgrade/pre-upgrade.yaml @@ -0,0 +1,65 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kubecost-pre-upgrade + namespace: ${releaseNamespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: kubecost-pre-upgrade + namespace: ${releaseNamespace} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "watch", "list", "update", "patch", "create"] + - apiGroups: ["helm.toolkit.fluxcd.io"] + resources: ["helmreleases"] + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: kubecost-pre-upgrade + namespace: ${releaseNamespace} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubecost-pre-upgrade +subjects: + - kind: ServiceAccount + name: kubecost-pre-upgrade + namespace: ${releaseNamespace} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: kubecost-pre-upgrade + namespace: ${releaseNamespace} +spec: + ttlSecondsAfterFinished: 100 + template: + metadata: + name: kubecost-pre-upgrade + spec: + serviceAccountName: kubecost-pre-upgrade + restartPolicy: OnFailure + containers: + - name: kubetools + image: "mesosphere/kommander2-kubetools:${kommanderChartVersion:=v2.5.2-dev}" + command: + - sh + - "-c" + - |- + /bin/bash <<'EOF' + set -o nounset + set -o pipefail + + kubectl get helmreleases.helm.toolkit.fluxcd.io -n ${releaseNamespace} kubecost + if [[ $? -ne 0 ]]; then + echo "Since the kubecost HelmRelease does not exist, this might not be an upgrade scenario. Exiting..." + exit 0 + fi + + helm mapkubeapis kubecost --namespace ${releaseNamespace} + EOF diff --git a/services/logging-operator/3.17.10/kustomization.yaml b/services/logging-operator/3.17.10/kustomization.yaml index 8102753a0f..6227cdcbe3 100644 --- a/services/logging-operator/3.17.10/kustomization.yaml +++ b/services/logging-operator/3.17.10/kustomization.yaml @@ -4,3 +4,4 @@ resources: - grafana-dashboards - logging-operator.yaml - logging-operator-logging.yaml + - pre-upgrade.yaml diff --git a/services/logging-operator/3.17.10/pre-upgrade.yaml b/services/logging-operator/3.17.10/pre-upgrade.yaml new file mode 100644 index 0000000000..bf97848b60 --- /dev/null +++ b/services/logging-operator/3.17.10/pre-upgrade.yaml @@ -0,0 +1,22 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1beta2 +kind: Kustomization +metadata: + name: logging-operator-pre-upgrade + namespace: ${releaseNamespace} +spec: + force: true + prune: true + wait: true + interval: 6h + retryInterval: 1m + path: ./services/logging-operator/3.17.10/pre-upgrade + sourceRef: + kind: GitRepository + name: management + namespace: kommander-flux + timeout: 1m + # passing releaseNamespace to 2nd level configuration files for ability to configure namespace correctly in attached clusters + # Using `substituteFrom` with `substitution-vars` creates 2nd level resources in `kommander` namespace instead of workspace ns + postBuild: + substitute: + releaseNamespace: ${releaseNamespace} diff --git a/services/logging-operator/3.17.10/pre-upgrade/kustomization.yaml b/services/logging-operator/3.17.10/pre-upgrade/kustomization.yaml new file mode 100644 index 0000000000..df8c392760 --- /dev/null +++ b/services/logging-operator/3.17.10/pre-upgrade/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - pre-upgrade.yaml diff --git a/services/logging-operator/3.17.10/pre-upgrade/pre-upgrade.yaml b/services/logging-operator/3.17.10/pre-upgrade/pre-upgrade.yaml new file mode 100644 index 0000000000..c985540b90 --- /dev/null +++ b/services/logging-operator/3.17.10/pre-upgrade/pre-upgrade.yaml @@ -0,0 +1,65 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: logging-operator-pre-upgrade + namespace: ${releaseNamespace} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: logging-operator-pre-upgrade + namespace: ${releaseNamespace} +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "watch", "list", "update", "patch", "create"] + - apiGroups: ["helm.toolkit.fluxcd.io"] + resources: ["helmreleases"] + verbs: ["get", "watch", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: logging-operator-pre-upgrade + namespace: ${releaseNamespace} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: logging-operator-pre-upgrade +subjects: + - kind: ServiceAccount + name: logging-operator-pre-upgrade + namespace: ${releaseNamespace} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: logging-operator-pre-upgrade + namespace: ${releaseNamespace} +spec: + ttlSecondsAfterFinished: 100 + template: + metadata: + name: logging-operator-pre-upgrade + spec: + serviceAccountName: logging-operator-pre-upgrade + restartPolicy: OnFailure + containers: + - name: kubetools + image: "mesosphere/kommander2-kubetools:${kommanderChartVersion:=v2.5.2-dev}" + command: + - sh + - "-c" + - |- + /bin/bash <<'EOF' + set -o nounset + set -o pipefail + + kubectl get helmreleases.helm.toolkit.fluxcd.io -n ${releaseNamespace} logging-operator + if [[ $? -ne 0 ]]; then + echo "Since the logging-operator HelmRelease does not exist, this might not be an upgrade scenario. Exiting..." + exit 0 + fi + + helm mapkubeapis logging-operator --namespace ${releaseNamespace} + EOF