K8SPS-348: add scheduled backups (#716)

* K8SPS-348: add scheduled backups https://perconadev.atlassian.net/browse/K8SPS-348 * add timeout to tests * fix unit-test * fix scheduled-backup tests --------- Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com> Co-authored-by: Inel Pandzic <inel.pandzic@percona.com>
percona · Aug 12, 2024 · 3387435 · 3387435
1 parent a111a9c
commit 3387435
Show file tree

Hide file tree

Showing 80 changed files with 1,977 additions and 24 deletions.
diff --git a/api/v1alpha1/perconaservermysql_types.go b/api/v1alpha1/perconaservermysql_types.go
@@ -26,6 +26,7 @@ import (
 
 	cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
 	"github.com/pkg/errors"
+	"github.com/robfig/cron/v3"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 	appsv1 "k8s.io/api/apps/v1"
@@ -206,6 +207,17 @@ type BackupSpec struct {
 	Resources                corev1.ResourceRequirements   `json:"resources,omitempty"`
 	Storages                 map[string]*BackupStorageSpec `json:"storages,omitempty"`
 	BackoffLimit             *int32                        `json:"backoffLimit,omitempty"`
+	Schedule                 []BackupSchedule              `json:"schedule,omitempty"`
+}
+
+type BackupSchedule struct {
+	// +kubebuilder:validation:Required
+	Name string `json:"name,omitempty"`
+	// +kubebuilder:validation:Required
+	Schedule string `json:"schedule,omitempty"`
+	Keep     int    `json:"keep,omitempty"`
+	// +kubebuilder:validation:Required
+	StorageName string `json:"storageName,omitempty"`
 }
 
 // Retrieves the initialization image for the backup.
@@ -543,6 +555,24 @@ func (cr *PerconaServerMySQL) CheckNSetDefaults(ctx context.Context, serverVersi
 		return errors.New("backup.image can't be empty")
 	}
 
+	scheduleNames := make(map[string]struct{}, len(cr.Spec.Backup.Schedule))
+	for _, sch := range cr.Spec.Backup.Schedule {
+		if _, ok := scheduleNames[sch.Name]; ok {
+			return errors.Errorf("scheduled backups should have different names: %s name is used by multiple schedules", sch.Name)
+		}
+		scheduleNames[sch.Name] = struct{}{}
+		_, ok := cr.Spec.Backup.Storages[sch.StorageName]
+		if !ok {
+			return errors.Errorf("storage %s doesn't exist", sch.StorageName)
+		}
+		if sch.Schedule != "" {
+			_, err := cron.ParseStandard(sch.Schedule)
+			if err != nil {
+				return errors.Wrap(err, "invalid schedule format")
+			}
+		}
+	}
+
 	if cr.Spec.MySQL.StartupProbe.InitialDelaySeconds == 0 {
 		cr.Spec.MySQL.StartupProbe.InitialDelaySeconds = 15
 	}

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/cmd/manager/main.go b/cmd/manager/main.go
@@ -22,17 +22,14 @@ import (
 	"strconv"
 	"strings"
 
-	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
-	// to ensure that exec-entrypoint and run can make use of them.
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
-
 	cmscheme "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned/scheme"
 	"github.com/go-logr/logr"
 	uzap "go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
@@ -49,7 +46,6 @@ import (
 	"github.com/percona/percona-server-mysql-operator/pkg/platform"
 	"github.com/percona/percona-server-mysql-operator/pkg/xtrabackup"
 	"github.com/percona/percona-server-mysql-operator/pkg/xtrabackup/storage"
-	//+kubebuilder:scaffold:imports
 )
 
 var (
@@ -158,6 +154,7 @@ func main() {
 		ServerVersion: serverVersion,
 		Recorder:      mgr.GetEventRecorderFor("ps-controller"),
 		ClientCmd:     cliCmd,
+		Crons:         ps.NewCronRegistry(),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ps-controller")
 		os.Exit(1)

diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml
@@ -178,6 +178,19 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
+                  schedule:
+                    items:
+                      properties:
+                        keep:
+                          type: integer
+                        name:
+                          type: string
+                        schedule:
+                          type: string
+                        storageName:
+                          type: string
+                      type: object
+                    type: array
                   serviceAccountName:
                     type: string
                   storages:

diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml
@@ -2074,6 +2074,19 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
+                  schedule:
+                    items:
+                      properties:
+                        keep:
+                          type: integer
+                        name:
+                          type: string
+                        schedule:
+                          type: string
+                        storageName:
+                          type: string
+                      type: object
+                    type: array
                   serviceAccountName:
                     type: string
                   storages:

diff --git a/deploy/cr.yaml b/deploy/cr.yaml
@@ -378,6 +378,15 @@ spec:
   backup:
     enabled: true
     image: perconalab/percona-server-mysql-operator:main-backup
+#    schedule:
+#      - name: "sat-night-backup"
+#        schedule: "0 0 * * 6"
+#        keep: 3
+#        storageName: s3-us-west
+#      - name: "daily-backup"
+#        schedule: "0 0 * * *"
+#        keep: 5
+#        storageName: s3
 #    backoffLimit: 6
     imagePullPolicy: Always
 #    initImage: perconalab/percona-server-mysql-operator:main

diff --git a/deploy/crd.yaml b/deploy/crd.yaml
@@ -2074,6 +2074,19 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
+                  schedule:
+                    items:
+                      properties:
+                        keep:
+                          type: integer
+                        name:
+                          type: string
+                        schedule:
+                          type: string
+                        storageName:
+                          type: string
+                      type: object
+                    type: array
                   serviceAccountName:
                     type: string
                   storages:

diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml
@@ -2074,6 +2074,19 @@ spec:
                           x-kubernetes-int-or-string: true
                         type: object
                     type: object
+                  schedule:
+                    items:
+                      properties:
+                        keep:
+                          type: integer
+                        name:
+                          type: string
+                        schedule:
+                          type: string
+                        storageName:
+                          type: string
+                      type: object
+                    type: array
                   serviceAccountName:
                     type: string
                   storages:

diff --git a/e2e-tests/run-distro.csv b/e2e-tests/run-distro.csv
@@ -10,6 +10,7 @@ gr-init-deploy
 gr-one-pod
 gr-recreate
 gr-scaling
+gr-scheduled-backup
 gr-security-context
 gr-self-healing
 gr-tls-cert-manager
@@ -21,6 +22,7 @@ one-pod
 operator-self-healing
 recreate
 scaling
+scheduled-backup
 service-per-pod
 sidecars
 smart-update

diff --git a/e2e-tests/run-minikube.csv b/e2e-tests/run-minikube.csv
@@ -9,6 +9,7 @@ gr-init-deploy
 gr-one-pod
 gr-recreate
 gr-scaling
+gr-scheduled-backup
 gr-self-healing
 gr-tls-cert-manager
 gr-users
@@ -17,6 +18,7 @@ init-deploy
 one-pod
 operator-self-healing
 recreate
+scheduled-backup
 sidecars
 smart-update
 tls-cert-manager

diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv
@@ -13,6 +13,7 @@ gr-init-deploy
 gr-one-pod
 gr-recreate
 gr-scaling
+gr-scheduled-backup
 gr-security-context
 gr-self-healing
 gr-tls-cert-manager
@@ -25,6 +26,7 @@ one-pod
 operator-self-healing
 recreate
 scaling
+scheduled-backup
 service-per-pod
 sidecars
 smart-update

diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv
@@ -13,6 +13,7 @@ gr-init-deploy
 gr-one-pod
 gr-recreate
 gr-scaling
+gr-scheduled-backup
 gr-security-context
 gr-self-healing
 gr-tls-cert-manager
@@ -25,6 +26,7 @@ one-pod
 operator-self-healing
 recreate
 scaling
+scheduled-backup
 service-per-pod
 sidecars
 smart-update

diff --git a/e2e-tests/tests/gr-scheduled-backup/00-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/00-assert.yaml
@@ -0,0 +1,9 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 150
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-secret
+type: Opaque
diff --git a/e2e-tests/tests/gr-scheduled-backup/00-minio-secret.yaml b/e2e-tests/tests/gr-scheduled-backup/00-minio-secret.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: minio-secret
+stringData:
+  AWS_ACCESS_KEY_ID: some-access$\n"-key
+  AWS_SECRET_ACCESS_KEY: some-$\n"secret-key
diff --git a/e2e-tests/tests/gr-scheduled-backup/01-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/01-assert.yaml
@@ -0,0 +1,26 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 150
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: perconaservermysqls.ps.percona.com
+spec:
+  group: ps.percona.com
+  names:
+    kind: PerconaServerMySQL
+    listKind: PerconaServerMySQLList
+    plural: perconaservermysqls
+    shortNames:
+    - ps
+    singular: perconaservermysql
+  scope: Namespaced
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: check-operator-deploy-status
+timeout: 120
+commands:
+  - script: kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1
diff --git a/e2e-tests/tests/gr-scheduled-backup/01-deploy-operator.yaml b/e2e-tests/tests/gr-scheduled-backup/01-deploy-operator.yaml
@@ -0,0 +1,17 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+      init_temp_dir # do this only in the first TestStep
+
+      apply_s3_storage_secrets
+      deploy_operator
+      deploy_non_tls_cluster_secrets
+      deploy_tls_cluster_secrets
+      deploy_client
+      deploy_minio
+    timeout: 300
diff --git a/e2e-tests/tests/gr-scheduled-backup/02-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/02-assert.yaml
@@ -0,0 +1,54 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 420
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: gr-scheduled-backup-mysql
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: gr-scheduled-backup-router
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  updatedReplicas: 3
+---
+apiVersion: ps.percona.com/v1alpha1
+kind: PerconaServerMySQL
+metadata:
+  name: gr-scheduled-backup
+spec:
+  backup:
+    backoffLimit: 3
+status:
+  conditions:
+  - reason: Initializing
+    status: "False"
+    type: Initializing
+  - reason: Ready
+    status: "True"
+    type: Ready
+  - message: InnoDB cluster successfully bootstrapped with 3 nodes
+    reason: InnoDBClusterBootstrapped
+    status: "True"
+    type: InnoDBClusterBootstrapped
+  mysql:
+    ready: 3
+    size: 3
+    state: ready
+  router:
+    ready: 3
+    size: 3
+    state: ready
+  state: ready