diff --git a/api/v1alpha1/perconaservermysql_types.go b/api/v1alpha1/perconaservermysql_types.go index 414480e5e..913d07a24 100644 --- a/api/v1alpha1/perconaservermysql_types.go +++ b/api/v1alpha1/perconaservermysql_types.go @@ -26,6 +26,7 @@ import ( cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" "github.com/pkg/errors" + "github.com/robfig/cron/v3" "golang.org/x/text/cases" "golang.org/x/text/language" appsv1 "k8s.io/api/apps/v1" @@ -206,6 +207,17 @@ type BackupSpec struct { Resources corev1.ResourceRequirements `json:"resources,omitempty"` Storages map[string]*BackupStorageSpec `json:"storages,omitempty"` BackoffLimit *int32 `json:"backoffLimit,omitempty"` + Schedule []BackupSchedule `json:"schedule,omitempty"` +} + +type BackupSchedule struct { + // +kubebuilder:validation:Required + Name string `json:"name,omitempty"` + // +kubebuilder:validation:Required + Schedule string `json:"schedule,omitempty"` + Keep int `json:"keep,omitempty"` + // +kubebuilder:validation:Required + StorageName string `json:"storageName,omitempty"` } // Retrieves the initialization image for the backup. @@ -543,6 +555,24 @@ func (cr *PerconaServerMySQL) CheckNSetDefaults(ctx context.Context, serverVersi return errors.New("backup.image can't be empty") } + scheduleNames := make(map[string]struct{}, len(cr.Spec.Backup.Schedule)) + for _, sch := range cr.Spec.Backup.Schedule { + if _, ok := scheduleNames[sch.Name]; ok { + return errors.Errorf("scheduled backups should have different names: %s name is used by multiple schedules", sch.Name) + } + scheduleNames[sch.Name] = struct{}{} + _, ok := cr.Spec.Backup.Storages[sch.StorageName] + if !ok { + return errors.Errorf("storage %s doesn't exist", sch.StorageName) + } + if sch.Schedule != "" { + _, err := cron.ParseStandard(sch.Schedule) + if err != nil { + return errors.Wrap(err, "invalid schedule format") + } + } + } + if cr.Spec.MySQL.StartupProbe.InitialDelaySeconds == 0 { cr.Spec.MySQL.StartupProbe.InitialDelaySeconds = 15 } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 2d48d7ed1..a7253a421 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -28,6 +28,21 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupSchedule) DeepCopyInto(out *BackupSchedule) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSchedule. +func (in *BackupSchedule) DeepCopy() *BackupSchedule { + if in == nil { + return nil + } + out := new(BackupSchedule) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { *out = *in @@ -63,6 +78,11 @@ func (in *BackupSpec) DeepCopyInto(out *BackupSpec) { *out = new(int32) **out = **in } + if in.Schedule != nil { + in, out := &in.Schedule, &out.Schedule + *out = make([]BackupSchedule, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupSpec. diff --git a/cmd/manager/main.go b/cmd/manager/main.go index a221da498..5605bee90 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -22,10 +22,6 @@ import ( "strconv" "strings" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" - cmscheme "github.com/cert-manager/cert-manager/pkg/client/clientset/versioned/scheme" "github.com/go-logr/logr" uzap "go.uber.org/zap" @@ -33,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" @@ -49,7 +46,6 @@ import ( "github.com/percona/percona-server-mysql-operator/pkg/platform" "github.com/percona/percona-server-mysql-operator/pkg/xtrabackup" "github.com/percona/percona-server-mysql-operator/pkg/xtrabackup/storage" - //+kubebuilder:scaffold:imports ) var ( @@ -158,6 +154,7 @@ func main() { ServerVersion: serverVersion, Recorder: mgr.GetEventRecorderFor("ps-controller"), ClientCmd: cliCmd, + Crons: ps.NewCronRegistry(), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ps-controller") os.Exit(1) diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index d50d86e25..5807d810b 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -178,6 +178,19 @@ spec: x-kubernetes-int-or-string: true type: object type: object + schedule: + items: + properties: + keep: + type: integer + name: + type: string + schedule: + type: string + storageName: + type: string + type: object + type: array serviceAccountName: type: string storages: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index b80686269..9c4861db3 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -2074,6 +2074,19 @@ spec: x-kubernetes-int-or-string: true type: object type: object + schedule: + items: + properties: + keep: + type: integer + name: + type: string + schedule: + type: string + storageName: + type: string + type: object + type: array serviceAccountName: type: string storages: diff --git a/deploy/cr.yaml b/deploy/cr.yaml index 20fa9d29e..d29ef947d 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -378,6 +378,15 @@ spec: backup: enabled: true image: perconalab/percona-server-mysql-operator:main-backup +# schedule: +# - name: "sat-night-backup" +# schedule: "0 0 * * 6" +# keep: 3 +# storageName: s3-us-west +# - name: "daily-backup" +# schedule: "0 0 * * *" +# keep: 5 +# storageName: s3 # backoffLimit: 6 imagePullPolicy: Always # initImage: perconalab/percona-server-mysql-operator:main diff --git a/deploy/crd.yaml b/deploy/crd.yaml index a229ed219..dd38e7549 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -2074,6 +2074,19 @@ spec: x-kubernetes-int-or-string: true type: object type: object + schedule: + items: + properties: + keep: + type: integer + name: + type: string + schedule: + type: string + storageName: + type: string + type: object + type: array serviceAccountName: type: string storages: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index d83d41d18..f940984b9 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -2074,6 +2074,19 @@ spec: x-kubernetes-int-or-string: true type: object type: object + schedule: + items: + properties: + keep: + type: integer + name: + type: string + schedule: + type: string + storageName: + type: string + type: object + type: array serviceAccountName: type: string storages: diff --git a/e2e-tests/run-distro.csv b/e2e-tests/run-distro.csv index 11f0d1ecb..c7f9bd121 100644 --- a/e2e-tests/run-distro.csv +++ b/e2e-tests/run-distro.csv @@ -10,6 +10,7 @@ gr-init-deploy gr-one-pod gr-recreate gr-scaling +gr-scheduled-backup gr-security-context gr-self-healing gr-tls-cert-manager @@ -21,6 +22,7 @@ one-pod operator-self-healing recreate scaling +scheduled-backup service-per-pod sidecars smart-update diff --git a/e2e-tests/run-minikube.csv b/e2e-tests/run-minikube.csv index c2dbee3f3..1251534e6 100644 --- a/e2e-tests/run-minikube.csv +++ b/e2e-tests/run-minikube.csv @@ -9,6 +9,7 @@ gr-init-deploy gr-one-pod gr-recreate gr-scaling +gr-scheduled-backup gr-self-healing gr-tls-cert-manager gr-users @@ -17,6 +18,7 @@ init-deploy one-pod operator-self-healing recreate +scheduled-backup sidecars smart-update tls-cert-manager diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index fa1b86161..c3fa9ff00 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -13,6 +13,7 @@ gr-init-deploy gr-one-pod gr-recreate gr-scaling +gr-scheduled-backup gr-security-context gr-self-healing gr-tls-cert-manager @@ -25,6 +26,7 @@ one-pod operator-self-healing recreate scaling +scheduled-backup service-per-pod sidecars smart-update diff --git a/e2e-tests/run-release.csv b/e2e-tests/run-release.csv index fa1b86161..c3fa9ff00 100644 --- a/e2e-tests/run-release.csv +++ b/e2e-tests/run-release.csv @@ -13,6 +13,7 @@ gr-init-deploy gr-one-pod gr-recreate gr-scaling +gr-scheduled-backup gr-security-context gr-self-healing gr-tls-cert-manager @@ -25,6 +26,7 @@ one-pod operator-self-healing recreate scaling +scheduled-backup service-per-pod sidecars smart-update diff --git a/e2e-tests/tests/gr-scheduled-backup/00-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/00-assert.yaml new file mode 100644 index 000000000..fecdb222e --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/00-assert.yaml @@ -0,0 +1,9 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 150 +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret +type: Opaque diff --git a/e2e-tests/tests/gr-scheduled-backup/00-minio-secret.yaml b/e2e-tests/tests/gr-scheduled-backup/00-minio-secret.yaml new file mode 100644 index 000000000..3c797f054 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/00-minio-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret +stringData: + AWS_ACCESS_KEY_ID: some-access$\n"-key + AWS_SECRET_ACCESS_KEY: some-$\n"secret-key diff --git a/e2e-tests/tests/gr-scheduled-backup/01-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/01-assert.yaml new file mode 100644 index 000000000..5f346cb51 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/01-assert.yaml @@ -0,0 +1,26 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 150 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: perconaservermysqls.ps.percona.com +spec: + group: ps.percona.com + names: + kind: PerconaServerMySQL + listKind: PerconaServerMySQLList + plural: perconaservermysqls + shortNames: + - ps + singular: perconaservermysql + scope: Namespaced +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-operator-deploy-status +timeout: 120 +commands: + - script: kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 diff --git a/e2e-tests/tests/gr-scheduled-backup/01-deploy-operator.yaml b/e2e-tests/tests/gr-scheduled-backup/01-deploy-operator.yaml new file mode 100644 index 000000000..273c0cc4c --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/01-deploy-operator.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + init_temp_dir # do this only in the first TestStep + + apply_s3_storage_secrets + deploy_operator + deploy_non_tls_cluster_secrets + deploy_tls_cluster_secrets + deploy_client + deploy_minio + timeout: 300 diff --git a/e2e-tests/tests/gr-scheduled-backup/02-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/02-assert.yaml new file mode 100644 index 000000000..96f7dc2bd --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/02-assert.yaml @@ -0,0 +1,54 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: gr-scheduled-backup-mysql +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: gr-scheduled-backup-router +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + updatedReplicas: 3 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-scheduled-backup +spec: + backup: + backoffLimit: 3 +status: + conditions: + - reason: Initializing + status: "False" + type: Initializing + - reason: Ready + status: "True" + type: Ready + - message: InnoDB cluster successfully bootstrapped with 3 nodes + reason: InnoDBClusterBootstrapped + status: "True" + type: InnoDBClusterBootstrapped + mysql: + ready: 3 + size: 3 + state: ready + router: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/gr-scheduled-backup/02-create-cluster.yaml b/e2e-tests/tests/gr-scheduled-backup/02-create-cluster.yaml new file mode 100644 index 000000000..7eb11a808 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/02-create-cluster.yaml @@ -0,0 +1,38 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr \ + | yq eval ".spec.backup.backoffLimit=3" - \ + | yq eval '.spec.backup.storages.minio.type="s3"' - \ + | yq eval '.spec.backup.storages.minio.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.storages.minio.s3.credentialsSecret="minio-secret"' - \ + | yq eval ".spec.backup.storages.minio.s3.endpointUrl=\"http://minio-service.${NAMESPACE}:9000\"" - \ + | yq eval '.spec.backup.storages.minio.s3.region="us-east-1"' - \ + | yq eval '.spec.backup.storages.aws-s3.type="s3"' - \ + | yq eval ".spec.backup.storages.aws-s3.verifyTLS=true" - \ + | yq eval '.spec.backup.storages.aws-s3.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.storages.aws-s3.s3.credentialsSecret="aws-s3-secret"' - \ + | yq eval '.spec.backup.storages.aws-s3.s3.region="us-east-1"' - \ + | yq eval '.spec.backup.storages.aws-s3.s3.prefix="ps"' - \ + | yq eval '.spec.backup.storages.gcp-cs.type="gcs"' - \ + | yq eval ".spec.backup.storages.gcp-cs.verifyTLS=true" - \ + | yq eval '.spec.backup.storages.gcp-cs.gcs.bucket="operator-testing"' - \ + | yq eval '.spec.backup.storages.gcp-cs.gcs.credentialsSecret="gcp-cs-secret"' - \ + | yq eval '.spec.backup.storages.gcp-cs.gcs.endpointUrl="https://storage.googleapis.com"' - \ + | yq eval '.spec.backup.storages.gcp-cs.gcs.prefix="ps"' - \ + | yq eval '.spec.backup.storages.azure-blob.type="azure"' - \ + | yq eval ".spec.backup.storages.azure-blob.verifyTLS=true" - \ + | yq eval '.spec.backup.storages.azure-blob.azure.containerName="operator-testing"' - \ + | yq eval '.spec.backup.storages.azure-blob.azure.credentialsSecret="azure-secret"' - \ + | yq eval '.spec.backup.storages.azure-blob.azure.prefix="ps"' - \ + | yq eval '.spec.mysql.clusterType="group-replication"' - \ + | yq eval ".spec.proxy.router.enabled=true" - \ + | yq eval ".spec.proxy.haproxy.enabled=false" - \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/gr-scheduled-backup/03-write-data.yaml b/e2e-tests/tests/gr-scheduled-backup/03-write-data.yaml new file mode 100644 index 000000000..bc82e7920 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/03-write-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100500)" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" diff --git a/e2e-tests/tests/gr-scheduled-backup/04-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/04-assert.yaml new file mode 100644 index 000000000..8fc3414ae --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/04-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +kind: PerconaServerMySQLBackup +apiVersion: ps.percona.com/v1alpha1 +status: + state: Running diff --git a/e2e-tests/tests/gr-scheduled-backup/04-engage-minio.yaml b/e2e-tests/tests/gr-scheduled-backup/04-engage-minio.yaml new file mode 100644 index 000000000..322997907 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/04-engage-minio.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: minio + schedule: '*/2 * * * *' + storageName: minio diff --git a/e2e-tests/tests/gr-scheduled-backup/05-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/05-assert.yaml new file mode 100644 index 000000000..2f9c703e3 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/05-assert.yaml @@ -0,0 +1,14 @@ + +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +kind: PerconaServerMySQLBackup +apiVersion: ps.percona.com/v1alpha1 +status: + state: Running +--- +kind: PerconaServerMySQLBackup +apiVersion: ps.percona.com/v1alpha1 +status: + state: Succeeded diff --git a/e2e-tests/tests/gr-scheduled-backup/06-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/06-assert.yaml new file mode 100644 index 000000000..fe5f51584 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/06-assert.yaml @@ -0,0 +1,104 @@ + +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: batch/v1 +kind: Job +metadata: + generation: 1 + labels: + app.kubernetes.io/component: xtrabackup + app.kubernetes.io/instance: gr-scheduled-backup + app.kubernetes.io/managed-by: percona-server-operator + app.kubernetes.io/name: percona-server + app.kubernetes.io/part-of: percona-server + ownerReferences: + - apiVersion: ps.percona.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: PerconaServerMySQLBackup +spec: + backoffLimit: 3 + completionMode: NonIndexed + completions: 1 + parallelism: 1 + suspend: false + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: xtrabackup + app.kubernetes.io/instance: gr-scheduled-backup + app.kubernetes.io/managed-by: percona-server-operator + app.kubernetes.io/name: percona-server + app.kubernetes.io/part-of: percona-server +status: + ready: 0 + succeeded: 1 + uncountedTerminatedPods: {} +--- +apiVersion: batch/v1 +kind: Job +metadata: + generation: 1 + labels: + app.kubernetes.io/component: xtrabackup + app.kubernetes.io/instance: gr-scheduled-backup + app.kubernetes.io/managed-by: percona-server-operator + app.kubernetes.io/name: percona-server + app.kubernetes.io/part-of: percona-server + ownerReferences: + - apiVersion: ps.percona.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: PerconaServerMySQLBackup +spec: + backoffLimit: 3 + completionMode: NonIndexed + completions: 1 + parallelism: 1 + suspend: false + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: xtrabackup + app.kubernetes.io/instance: gr-scheduled-backup + app.kubernetes.io/managed-by: percona-server-operator + app.kubernetes.io/name: percona-server + app.kubernetes.io/part-of: percona-server +status: + ready: 0 + succeeded: 1 + uncountedTerminatedPods: {} +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: minio +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/gr-scheduled-backup/06-stop-minio.yaml b/e2e-tests/tests/gr-scheduled-backup/06-stop-minio.yaml new file mode 100644 index 000000000..6bdb6028f --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/06-stop-minio.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: minio + schedule: '' + storageName: minio diff --git a/e2e-tests/tests/gr-scheduled-backup/07-change-keep.yaml b/e2e-tests/tests/gr-scheduled-backup/07-change-keep.yaml new file mode 100644 index 000000000..ed4ff5a22 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/07-change-keep.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-scheduled-backup +spec: + backup: + schedule: + - keep: 1 + name: minio + schedule: '' + storageName: minio diff --git a/e2e-tests/tests/gr-scheduled-backup/08-should-be-one-backup.yaml b/e2e-tests/tests/gr-scheduled-backup/08-should-be-one-backup.yaml new file mode 100644 index 000000000..21fa0bbfa --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/08-should-be-one-backup.yaml @@ -0,0 +1,21 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + sleep 30 + jobs_count=$(kubectl get job -n ${NAMESPACE} -o yaml | yq '.items | length') + bcp_count=$(kubectl get ps-backup -n ${NAMESPACE} -o yaml | yq '.items | length') + if [[ $jobs_count != 1 ]]; then + echo "There are $jobs_count jobs but should be 1" + exit 1 + fi + if [[ $bcp_count != 1 ]]; then + echo "There are $bcp_count ps-backups but should be 1" + exit 1 + fi + timeout: 180 diff --git a/e2e-tests/tests/gr-scheduled-backup/09-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/09-assert.yaml new file mode 100644 index 000000000..f3590a882 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/09-assert.yaml @@ -0,0 +1,33 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: aws-s3 +status: + state: Running +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/gr-scheduled-backup/09-engage-s3.yaml b/e2e-tests/tests/gr-scheduled-backup/09-engage-s3.yaml new file mode 100644 index 000000000..b2dc91e73 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/09-engage-s3.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: s3 + schedule: '*/2 * * * *' + storageName: aws-s3 diff --git a/e2e-tests/tests/gr-scheduled-backup/10-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/10-assert.yaml new file mode 100644 index 000000000..5cb37b78a --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/10-assert.yaml @@ -0,0 +1,33 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: aws-s3 +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/gr-scheduled-backup/10-stop-s3.yaml b/e2e-tests/tests/gr-scheduled-backup/10-stop-s3.yaml new file mode 100644 index 000000000..69dc5ea64 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/10-stop-s3.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: s3 + schedule: '' + storageName: aws-s3 diff --git a/e2e-tests/tests/gr-scheduled-backup/11-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/11-assert.yaml new file mode 100644 index 000000000..4b0f82884 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/11-assert.yaml @@ -0,0 +1,48 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: gcp-cs +status: + state: Running +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: aws-s3 +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/gr-scheduled-backup/11-engage-gcs.yaml b/e2e-tests/tests/gr-scheduled-backup/11-engage-gcs.yaml new file mode 100644 index 000000000..fda320eea --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/11-engage-gcs.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: s3 + schedule: '*/2 * * * *' + storageName: gcp-cs diff --git a/e2e-tests/tests/gr-scheduled-backup/12-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/12-assert.yaml new file mode 100644 index 000000000..0b2c462b4 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/12-assert.yaml @@ -0,0 +1,48 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: gcp-cs +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: aws-s3 +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/gr-scheduled-backup/12-stop-gcs.yaml b/e2e-tests/tests/gr-scheduled-backup/12-stop-gcs.yaml new file mode 100644 index 000000000..c8aba1284 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/12-stop-gcs.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: s3 + schedule: '' + storageName: gcp-cs diff --git a/e2e-tests/tests/gr-scheduled-backup/13-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/13-assert.yaml new file mode 100644 index 000000000..f6c49bf53 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/13-assert.yaml @@ -0,0 +1,63 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: azure-blob +status: + state: Running +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: gcp-cs +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: aws-s3 +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/gr-scheduled-backup/13-engage-azure.yaml b/e2e-tests/tests/gr-scheduled-backup/13-engage-azure.yaml new file mode 100644 index 000000000..189a0e7e3 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/13-engage-azure.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: azure + schedule: '*/2 * * * *' + storageName: azure-blob diff --git a/e2e-tests/tests/gr-scheduled-backup/14-assert.yaml b/e2e-tests/tests/gr-scheduled-backup/14-assert.yaml new file mode 100644 index 000000000..085a78906 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/14-assert.yaml @@ -0,0 +1,63 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: azure-blob +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: gcp-cs +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: aws-s3 +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: gr-scheduled-backup +spec: + clusterName: gr-scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/gr-scheduled-backup/14-stop-azure.yaml b/e2e-tests/tests/gr-scheduled-backup/14-stop-azure.yaml new file mode 100644 index 000000000..e75de40b9 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/14-stop-azure.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: azure + schedule: '' + storageName: azure-blob diff --git a/e2e-tests/tests/gr-scheduled-backup/15-should-be-4-backups.yaml b/e2e-tests/tests/gr-scheduled-backup/15-should-be-4-backups.yaml new file mode 100644 index 000000000..907016a31 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/15-should-be-4-backups.yaml @@ -0,0 +1,21 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + sleep 30 + jobs_count=$(kubectl get job -n ${NAMESPACE} -o yaml | yq '.items | length') + bcp_count=$(kubectl get ps-backup -n ${NAMESPACE} -o yaml | yq '.items | length') + if [[ $jobs_count != 4 ]]; then + echo "There are $jobs_count jobs but should be 1" + exit 1 + fi + if [[ $bcp_count != 4 ]]; then + echo "There are $bcp_count ps-backups but should be 1" + exit 1 + fi + timeout: 180 diff --git a/e2e-tests/tests/gr-scheduled-backup/16-delete-all-backups.yaml b/e2e-tests/tests/gr-scheduled-backup/16-delete-all-backups.yaml new file mode 100644 index 000000000..6183ad863 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/16-delete-all-backups.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + kubectl delete ps-backup --all -n "${NAMESPACE}" + sleep 60 + timeout: 180 diff --git a/e2e-tests/tests/gr-scheduled-backup/98-drop-finalizer.yaml b/e2e-tests/tests/gr-scheduled-backup/98-drop-finalizer.yaml new file mode 100644 index 000000000..b4004614a --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/98-drop-finalizer.yaml @@ -0,0 +1,5 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: gr-scheduled-backup + finalizers: [] diff --git a/e2e-tests/tests/gr-scheduled-backup/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/gr-scheduled-backup/99-remove-cluster-gracefully.yaml new file mode 100644 index 000000000..c911c7458 --- /dev/null +++ b/e2e-tests/tests/gr-scheduled-backup/99-remove-cluster-gracefully.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: ps.percona.com/v1alpha1 + kind: PerconaServerMySQL + metadata: + name: gr-scheduled-backup +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + destroy_operator + timeout: 120 diff --git a/e2e-tests/tests/scheduled-backup/00-assert.yaml b/e2e-tests/tests/scheduled-backup/00-assert.yaml new file mode 100644 index 000000000..fecdb222e --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/00-assert.yaml @@ -0,0 +1,9 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 150 +--- +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret +type: Opaque diff --git a/e2e-tests/tests/scheduled-backup/00-minio-secret.yaml b/e2e-tests/tests/scheduled-backup/00-minio-secret.yaml new file mode 100644 index 000000000..3c797f054 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/00-minio-secret.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret +stringData: + AWS_ACCESS_KEY_ID: some-access$\n"-key + AWS_SECRET_ACCESS_KEY: some-$\n"secret-key diff --git a/e2e-tests/tests/scheduled-backup/01-assert.yaml b/e2e-tests/tests/scheduled-backup/01-assert.yaml new file mode 100644 index 000000000..5f346cb51 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/01-assert.yaml @@ -0,0 +1,26 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 150 +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: perconaservermysqls.ps.percona.com +spec: + group: ps.percona.com + names: + kind: PerconaServerMySQL + listKind: PerconaServerMySQLList + plural: perconaservermysqls + shortNames: + - ps + singular: perconaservermysql + scope: Namespaced +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: check-operator-deploy-status +timeout: 120 +commands: + - script: kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1 diff --git a/e2e-tests/tests/scheduled-backup/01-deploy-operator.yaml b/e2e-tests/tests/scheduled-backup/01-deploy-operator.yaml new file mode 100644 index 000000000..273c0cc4c --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/01-deploy-operator.yaml @@ -0,0 +1,17 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + init_temp_dir # do this only in the first TestStep + + apply_s3_storage_secrets + deploy_operator + deploy_non_tls_cluster_secrets + deploy_tls_cluster_secrets + deploy_client + deploy_minio + timeout: 300 diff --git a/e2e-tests/tests/scheduled-backup/02-assert.yaml b/e2e-tests/tests/scheduled-backup/02-assert.yaml new file mode 100644 index 000000000..16a544c97 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/02-assert.yaml @@ -0,0 +1,60 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: scheduled-backup-mysql +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: scheduled-backup-orc +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: scheduled-backup-haproxy +status: + observedGeneration: 1 + replicas: 3 + readyReplicas: 3 + currentReplicas: 3 + updatedReplicas: 3 + collisionCount: 0 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: scheduled-backup + finalizers: + - percona.com/delete-mysql-pods-in-order +status: + haproxy: + ready: 3 + size: 3 + state: ready + mysql: + ready: 3 + size: 3 + state: ready + orchestrator: + ready: 3 + size: 3 + state: ready + state: ready diff --git a/e2e-tests/tests/scheduled-backup/02-create-cluster.yaml b/e2e-tests/tests/scheduled-backup/02-create-cluster.yaml new file mode 100644 index 000000000..2d3b2415c --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/02-create-cluster.yaml @@ -0,0 +1,40 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 10 +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + get_cr \ + | yq eval '.spec.mysql.clusterType="async"' - \ + | yq eval ".spec.mysql.size=3" - \ + | yq eval ".spec.proxy.haproxy.enabled=true" - \ + | yq eval ".spec.proxy.haproxy.size=3" - \ + | yq eval ".spec.orchestrator.enabled=true" - \ + | yq eval ".spec.orchestrator.size=3" - \ + | yq eval '.spec.backup.storages.minio.type="s3"' - \ + | yq eval '.spec.backup.storages.minio.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.storages.minio.s3.credentialsSecret="minio-secret"' - \ + | yq eval ".spec.backup.storages.minio.s3.endpointUrl=\"http://minio-service.${NAMESPACE}:9000\"" - \ + | yq eval '.spec.backup.storages.minio.s3.region="us-east-1"' - \ + | yq eval '.spec.backup.storages.aws-s3.type="s3"' - \ + | yq eval ".spec.backup.storages.aws-s3.verifyTLS=true" - \ + | yq eval '.spec.backup.storages.aws-s3.s3.bucket="operator-testing"' - \ + | yq eval '.spec.backup.storages.aws-s3.s3.credentialsSecret="aws-s3-secret"' - \ + | yq eval '.spec.backup.storages.aws-s3.s3.region="us-east-1"' - \ + | yq eval '.spec.backup.storages.aws-s3.s3.prefix="ps"' - \ + | yq eval '.spec.backup.storages.gcp-cs.type="gcs"' - \ + | yq eval ".spec.backup.storages.gcp-cs.verifyTLS=true" - \ + | yq eval '.spec.backup.storages.gcp-cs.gcs.bucket="operator-testing"' - \ + | yq eval '.spec.backup.storages.gcp-cs.gcs.credentialsSecret="gcp-cs-secret"' - \ + | yq eval '.spec.backup.storages.gcp-cs.gcs.endpointUrl="https://storage.googleapis.com"' - \ + | yq eval '.spec.backup.storages.gcp-cs.gcs.prefix="ps"' - \ + | yq eval '.spec.backup.storages.azure-blob.type="azure"' - \ + | yq eval ".spec.backup.storages.azure-blob.verifyTLS=true" - \ + | yq eval '.spec.backup.storages.azure-blob.azure.containerName="operator-testing"' - \ + | yq eval '.spec.backup.storages.azure-blob.azure.credentialsSecret="azure-secret"' - \ + | yq eval '.spec.backup.storages.azure-blob.azure.prefix="ps"' - \ + | kubectl -n "${NAMESPACE}" apply -f - diff --git a/e2e-tests/tests/scheduled-backup/03-write-data.yaml b/e2e-tests/tests/scheduled-backup/03-write-data.yaml new file mode 100644 index 000000000..bc82e7920 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/03-write-data.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + run_mysql \ + "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" + + run_mysql \ + "INSERT myDB.myTable (id) VALUES (100500)" \ + "-h $(get_haproxy_svc $(get_cluster_name)) -uroot -proot_password" diff --git a/e2e-tests/tests/scheduled-backup/04-assert.yaml b/e2e-tests/tests/scheduled-backup/04-assert.yaml new file mode 100644 index 000000000..8fc3414ae --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/04-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +kind: PerconaServerMySQLBackup +apiVersion: ps.percona.com/v1alpha1 +status: + state: Running diff --git a/e2e-tests/tests/scheduled-backup/04-engage-minio.yaml b/e2e-tests/tests/scheduled-backup/04-engage-minio.yaml new file mode 100644 index 000000000..0234e978d --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/04-engage-minio.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: minio + schedule: '*/2 * * * *' + storageName: minio diff --git a/e2e-tests/tests/scheduled-backup/05-assert.yaml b/e2e-tests/tests/scheduled-backup/05-assert.yaml new file mode 100644 index 000000000..2f9c703e3 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/05-assert.yaml @@ -0,0 +1,14 @@ + +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +kind: PerconaServerMySQLBackup +apiVersion: ps.percona.com/v1alpha1 +status: + state: Running +--- +kind: PerconaServerMySQLBackup +apiVersion: ps.percona.com/v1alpha1 +status: + state: Succeeded diff --git a/e2e-tests/tests/scheduled-backup/06-assert.yaml b/e2e-tests/tests/scheduled-backup/06-assert.yaml new file mode 100644 index 000000000..501f4f0cc --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/06-assert.yaml @@ -0,0 +1,104 @@ + +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: batch/v1 +kind: Job +metadata: + generation: 1 + labels: + app.kubernetes.io/component: xtrabackup + app.kubernetes.io/instance: scheduled-backup + app.kubernetes.io/managed-by: percona-server-operator + app.kubernetes.io/name: percona-server + app.kubernetes.io/part-of: percona-server + ownerReferences: + - apiVersion: ps.percona.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: PerconaServerMySQLBackup +spec: + backoffLimit: 6 + completionMode: NonIndexed + completions: 1 + parallelism: 1 + suspend: false + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: xtrabackup + app.kubernetes.io/instance: scheduled-backup + app.kubernetes.io/managed-by: percona-server-operator + app.kubernetes.io/name: percona-server + app.kubernetes.io/part-of: percona-server +status: + ready: 0 + succeeded: 1 + uncountedTerminatedPods: {} +--- +apiVersion: batch/v1 +kind: Job +metadata: + generation: 1 + labels: + app.kubernetes.io/component: xtrabackup + app.kubernetes.io/instance: scheduled-backup + app.kubernetes.io/managed-by: percona-server-operator + app.kubernetes.io/name: percona-server + app.kubernetes.io/part-of: percona-server + ownerReferences: + - apiVersion: ps.percona.com/v1alpha1 + blockOwnerDeletion: true + controller: true + kind: PerconaServerMySQLBackup +spec: + backoffLimit: 6 + completionMode: NonIndexed + completions: 1 + parallelism: 1 + suspend: false + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: xtrabackup + app.kubernetes.io/instance: scheduled-backup + app.kubernetes.io/managed-by: percona-server-operator + app.kubernetes.io/name: percona-server + app.kubernetes.io/part-of: percona-server +status: + ready: 0 + succeeded: 1 + uncountedTerminatedPods: {} +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: minio +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/scheduled-backup/06-stop-minio.yaml b/e2e-tests/tests/scheduled-backup/06-stop-minio.yaml new file mode 100644 index 000000000..e733bdc6a --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/06-stop-minio.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: minio + schedule: '' + storageName: minio diff --git a/e2e-tests/tests/scheduled-backup/07-change-keep.yaml b/e2e-tests/tests/scheduled-backup/07-change-keep.yaml new file mode 100644 index 000000000..4df2ba0df --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/07-change-keep.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: scheduled-backup +spec: + backup: + schedule: + - keep: 1 + name: minio + schedule: '' + storageName: minio diff --git a/e2e-tests/tests/scheduled-backup/08-should-be-one-backup.yaml b/e2e-tests/tests/scheduled-backup/08-should-be-one-backup.yaml new file mode 100644 index 000000000..21fa0bbfa --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/08-should-be-one-backup.yaml @@ -0,0 +1,21 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + sleep 30 + jobs_count=$(kubectl get job -n ${NAMESPACE} -o yaml | yq '.items | length') + bcp_count=$(kubectl get ps-backup -n ${NAMESPACE} -o yaml | yq '.items | length') + if [[ $jobs_count != 1 ]]; then + echo "There are $jobs_count jobs but should be 1" + exit 1 + fi + if [[ $bcp_count != 1 ]]; then + echo "There are $bcp_count ps-backups but should be 1" + exit 1 + fi + timeout: 180 diff --git a/e2e-tests/tests/scheduled-backup/09-assert.yaml b/e2e-tests/tests/scheduled-backup/09-assert.yaml new file mode 100644 index 000000000..8d5a2fb49 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/09-assert.yaml @@ -0,0 +1,33 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: aws-s3 +status: + state: Running +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/scheduled-backup/09-engage-s3.yaml b/e2e-tests/tests/scheduled-backup/09-engage-s3.yaml new file mode 100644 index 000000000..74628815c --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/09-engage-s3.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: s3 + schedule: '*/2 * * * *' + storageName: aws-s3 diff --git a/e2e-tests/tests/scheduled-backup/10-assert.yaml b/e2e-tests/tests/scheduled-backup/10-assert.yaml new file mode 100644 index 000000000..d09df0846 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/10-assert.yaml @@ -0,0 +1,33 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: aws-s3 +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/scheduled-backup/10-stop-s3.yaml b/e2e-tests/tests/scheduled-backup/10-stop-s3.yaml new file mode 100644 index 000000000..ae8d7261f --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/10-stop-s3.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: s3 + schedule: '' + storageName: aws-s3 diff --git a/e2e-tests/tests/scheduled-backup/11-assert.yaml b/e2e-tests/tests/scheduled-backup/11-assert.yaml new file mode 100644 index 000000000..614ec54b2 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/11-assert.yaml @@ -0,0 +1,48 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: gcp-cs +status: + state: Running +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: aws-s3 +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/scheduled-backup/11-engage-gcs.yaml b/e2e-tests/tests/scheduled-backup/11-engage-gcs.yaml new file mode 100644 index 000000000..a8a8d79bc --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/11-engage-gcs.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: s3 + schedule: '*/2 * * * *' + storageName: gcp-cs diff --git a/e2e-tests/tests/scheduled-backup/12-assert.yaml b/e2e-tests/tests/scheduled-backup/12-assert.yaml new file mode 100644 index 000000000..61bde3558 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/12-assert.yaml @@ -0,0 +1,48 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: gcp-cs +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: aws-s3 +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/scheduled-backup/12-stop-gcs.yaml b/e2e-tests/tests/scheduled-backup/12-stop-gcs.yaml new file mode 100644 index 000000000..c08bc0cb6 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/12-stop-gcs.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: s3 + schedule: '' + storageName: gcp-cs diff --git a/e2e-tests/tests/scheduled-backup/13-assert.yaml b/e2e-tests/tests/scheduled-backup/13-assert.yaml new file mode 100644 index 000000000..4e8bfced2 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/13-assert.yaml @@ -0,0 +1,63 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: azure-blob +status: + state: Running +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: gcp-cs +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: aws-s3 +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/scheduled-backup/13-engage-azure.yaml b/e2e-tests/tests/scheduled-backup/13-engage-azure.yaml new file mode 100644 index 000000000..6ec6312c3 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/13-engage-azure.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: azure + schedule: '*/2 * * * *' + storageName: azure-blob diff --git a/e2e-tests/tests/scheduled-backup/14-assert.yaml b/e2e-tests/tests/scheduled-backup/14-assert.yaml new file mode 100644 index 000000000..7a959ed8a --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/14-assert.yaml @@ -0,0 +1,63 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 420 +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: azure-blob +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: gcp-cs +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: aws-s3 +status: + state: Succeeded +--- +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQLBackup +metadata: + finalizers: + - percona.com/delete-backup + generation: 1 + labels: + percona.com/backup-type: cron + percona.com/cluster: scheduled-backup +spec: + clusterName: scheduled-backup + storageName: minio +status: + state: Succeeded diff --git a/e2e-tests/tests/scheduled-backup/14-stop-azure.yaml b/e2e-tests/tests/scheduled-backup/14-stop-azure.yaml new file mode 100644 index 000000000..1a7ea558d --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/14-stop-azure.yaml @@ -0,0 +1,11 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: scheduled-backup +spec: + backup: + schedule: + - keep: 3 + name: azure + schedule: '' + storageName: azure-blob diff --git a/e2e-tests/tests/scheduled-backup/15-should-be-4-backups.yaml b/e2e-tests/tests/scheduled-backup/15-should-be-4-backups.yaml new file mode 100644 index 000000000..907016a31 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/15-should-be-4-backups.yaml @@ -0,0 +1,21 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + sleep 30 + jobs_count=$(kubectl get job -n ${NAMESPACE} -o yaml | yq '.items | length') + bcp_count=$(kubectl get ps-backup -n ${NAMESPACE} -o yaml | yq '.items | length') + if [[ $jobs_count != 4 ]]; then + echo "There are $jobs_count jobs but should be 1" + exit 1 + fi + if [[ $bcp_count != 4 ]]; then + echo "There are $bcp_count ps-backups but should be 1" + exit 1 + fi + timeout: 180 diff --git a/e2e-tests/tests/scheduled-backup/16-delete-all-backups.yaml b/e2e-tests/tests/scheduled-backup/16-delete-all-backups.yaml new file mode 100644 index 000000000..6183ad863 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/16-delete-all-backups.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + kubectl delete ps-backup --all -n "${NAMESPACE}" + sleep 60 + timeout: 180 diff --git a/e2e-tests/tests/scheduled-backup/98-drop-finalizer.yaml b/e2e-tests/tests/scheduled-backup/98-drop-finalizer.yaml new file mode 100644 index 000000000..892b63617 --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/98-drop-finalizer.yaml @@ -0,0 +1,5 @@ +apiVersion: ps.percona.com/v1alpha1 +kind: PerconaServerMySQL +metadata: + name: scheduled-backup + finalizers: [] diff --git a/e2e-tests/tests/scheduled-backup/99-remove-cluster-gracefully.yaml b/e2e-tests/tests/scheduled-backup/99-remove-cluster-gracefully.yaml new file mode 100644 index 000000000..0820fd01b --- /dev/null +++ b/e2e-tests/tests/scheduled-backup/99-remove-cluster-gracefully.yaml @@ -0,0 +1,16 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: + - apiVersion: ps.percona.com/v1alpha1 + kind: PerconaServerMySQL + metadata: + name: scheduled-backup +commands: + - script: |- + set -o errexit + set -o xtrace + + source ../../functions + + destroy_operator + timeout: 120 diff --git a/go.mod b/go.mod index 1c421a6af..08a18fcd2 100644 --- a/go.mod +++ b/go.mod @@ -93,6 +93,7 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.15.0 // indirect + github.com/robfig/cron/v3 v3.0.1 github.com/rs/xid v1.5.0 // indirect github.com/sergi/go-diff v1.3.1 // indirect github.com/spf13/afero v1.11.0 // indirect diff --git a/go.sum b/go.sum index 003e9f685..44e0c2687 100644 --- a/go.sum +++ b/go.sum @@ -206,6 +206,8 @@ github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqSc github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/prometheus/procfs v0.15.0 h1:A82kmvXJq2jTu5YUhSGNlYoxh85zLnKgPz4bMZgI5Ek= github.com/prometheus/procfs v0.15.0/go.mod h1:Y0RJ/Y5g5wJpkTisOtqwDSo4HwhGmLB4VQSw2sQJLHk= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= diff --git a/pkg/controller/ps/backup.go b/pkg/controller/ps/backup.go new file mode 100644 index 000000000..7f3657178 --- /dev/null +++ b/pkg/controller/ps/backup.go @@ -0,0 +1,268 @@ +package ps + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "hash/crc32" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + "github.com/robfig/cron/v3" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + apiv1alpha1 "github.com/percona/percona-server-mysql-operator/api/v1alpha1" + "github.com/percona/percona-server-mysql-operator/pkg/naming" +) + +type cronRegistry struct { + crons *cron.Cron + backupJobs *sync.Map +} + +func NewCronRegistry() cronRegistry { + c := cronRegistry{ + crons: cron.New(), + backupJobs: new(sync.Map), + } + + c.crons.Start() + + return c +} + +type BackupScheduleJob struct { + apiv1alpha1.BackupSchedule + JobID cron.EntryID +} + +// AddFuncWithSeconds does the same as cron.AddFunc but changes the schedule so that the function will run the exact second that this method is called. +func (r *cronRegistry) addFuncWithSeconds(spec string, cmd func()) (cron.EntryID, error) { + schedule, err := cron.ParseStandard(spec) + if err != nil { + return 0, errors.Wrap(err, "failed to parse cron schedule") + } + schedule.(*cron.SpecSchedule).Second = uint64(1 << time.Now().Second()) + id := r.crons.Schedule(schedule, cron.FuncJob(cmd)) + return id, nil +} + +func (r *cronRegistry) deleteBackupJob(name string) { + job, ok := r.backupJobs.LoadAndDelete(name) + if !ok { + return + } + r.crons.Remove(job.(BackupScheduleJob).JobID) +} + +func (r *cronRegistry) stopBackupJob(name string) { + job, ok := r.backupJobs.Load(name) + if !ok { + return + } + r.crons.Remove(job.(BackupScheduleJob).JobID) +} + +func (r *cronRegistry) getBackupJob(bcp apiv1alpha1.BackupSchedule) BackupScheduleJob { + sch := BackupScheduleJob{} + schRaw, ok := r.backupJobs.Load(bcp.Name) + if ok { + sch = schRaw.(BackupScheduleJob) + } + return sch +} + +func (r *cronRegistry) addBackupJob(ctx context.Context, cl client.Client, cluster *apiv1alpha1.PerconaServerMySQL, bcp apiv1alpha1.BackupSchedule) error { + if bcp.Schedule == "" { + r.stopBackupJob(bcp.Name) + return nil + } + r.deleteBackupJob(bcp.Name) + jobID, err := r.addFuncWithSeconds(bcp.Schedule, r.createBackupJobFunc(ctx, cl, cluster, bcp)) + if err != nil { + return errors.Wrap(err, "add func") + } + + r.backupJobs.Store(bcp.Name, BackupScheduleJob{ + BackupSchedule: bcp, + JobID: jobID, + }) + return nil +} + +func (r *cronRegistry) createBackupJobFunc(ctx context.Context, cl client.Client, cluster *apiv1alpha1.PerconaServerMySQL, backupJob apiv1alpha1.BackupSchedule) func() { + log := logf.FromContext(ctx) + + return func() { + cr := &apiv1alpha1.PerconaServerMySQL{} + err := cl.Get(ctx, types.NamespacedName{Name: cluster.Name, Namespace: cluster.Namespace}, cr) + if err != nil { + if k8serrors.IsNotFound(err) { + log.Info("Cluster is not found. Deleting the job", "name", backupJob.Name, "cluster", cluster.Name, "namespace", cluster.Namespace) + r.deleteBackupJob(backupJob.Name) + return + } + log.Error(err, "failed to get cluster") + } + + bcp := &apiv1alpha1.PerconaServerMySQLBackup{ + ObjectMeta: metav1.ObjectMeta{ + Finalizers: []string{naming.FinalizerDeleteBackup}, + Namespace: cr.Namespace, + Name: generateBackupName(cr, backupJob), + Labels: map[string]string{ + naming.LabelBackupAncestor: backupJob.Name, + naming.LabelCluster: cr.Name, + naming.LabelBackupType: "cron", + }, + }, + Spec: apiv1alpha1.PerconaServerMySQLBackupSpec{ + ClusterName: cr.Name, + StorageName: backupJob.StorageName, + }, + } + err = cl.Create(ctx, bcp) + if err != nil { + log.Error(err, "failed to create backup") + } + } +} + +func (r *PerconaServerMySQLReconciler) reconcileScheduledBackup(ctx context.Context, cr *apiv1alpha1.PerconaServerMySQL) error { + log := logf.FromContext(ctx).WithName("reconcileScheduledBackup") + + backups := make(map[string]apiv1alpha1.BackupSchedule) + backupNamePrefix := backupJobClusterPrefix(cr.Namespace + "-" + cr.Name) + + for i, bcp := range cr.Spec.Backup.Schedule { + _, ok := cr.Spec.Backup.Storages[bcp.StorageName] + if !ok { + log.Info("Invalid storage name for backup", "backup name", cr.Spec.Backup.Schedule[i].Name, "storage name", bcp.StorageName) + continue + } + + bcp.Name = backupNamePrefix + "-" + bcp.Name + backups[bcp.Name] = bcp + + sch := r.Crons.getBackupJob(bcp) + if ok && sch.Schedule == bcp.Schedule && sch.StorageName == bcp.StorageName { + continue + } + + log.Info("Creating or updating backup job", "name", bcp.Name, "schedule", bcp.Schedule) + if err := r.Crons.addBackupJob(ctx, r.Client, cr, bcp); err != nil { + log.Error(err, "can't add backup job", "backup name", cr.Spec.Backup.Schedule[i].Name, "schedule", bcp.Schedule) + } + } + + r.Crons.backupJobs.Range(func(k, v interface{}) bool { + item := v.(BackupScheduleJob) + if !strings.HasPrefix(item.Name, backupNamePrefix) { + return true + } + + spec, ok := backups[item.Name] + if !ok { + log.Info("Deleting outdated backup job", "name", item.Name) + r.Crons.deleteBackupJob(item.Name) + return true + } + + if spec.Keep <= 0 { + return true + } + + oldBackups, err := r.oldScheduledBackups(ctx, cr, item.Name, spec.Keep) + if err != nil { + log.Error(err, "failed to list old backups", "name", item.Name) + return true + } + + for _, bcp := range oldBackups { + err = r.Delete(ctx, &bcp) + if err != nil { + log.Error(err, "failed to delete old backup", "name", bcp.Name) + } + } + + return true + }) + + return nil +} + +func backupJobClusterPrefix(clusterName string) string { + h := sha1.New() + h.Write([]byte(clusterName)) + return hex.EncodeToString(h.Sum(nil))[:5] +} + +func (r *PerconaServerMySQLReconciler) oldScheduledBackups(ctx context.Context, cr *apiv1alpha1.PerconaServerMySQL, ancestor string, keep int) ([]apiv1alpha1.PerconaServerMySQLBackup, error) { + bcpList := apiv1alpha1.PerconaServerMySQLBackupList{} + err := r.List(ctx, + &bcpList, + &client.ListOptions{ + Namespace: cr.Namespace, + LabelSelector: labels.SelectorFromSet(map[string]string{ + naming.LabelCluster: cr.Name, + naming.LabelBackupAncestor: ancestor, + }), + }, + ) + if err != nil { + return []apiv1alpha1.PerconaServerMySQLBackup{}, err + } + + if len(bcpList.Items) <= keep { + return []apiv1alpha1.PerconaServerMySQLBackup{}, nil + } + + backups := []apiv1alpha1.PerconaServerMySQLBackup{} + for _, bcp := range bcpList.Items { + if bcp.Status.State == apiv1alpha1.BackupSucceeded { + backups = append(backups, bcp) + } + } + + if len(backups) <= keep { + return []apiv1alpha1.PerconaServerMySQLBackup{}, nil + } + + sort.Slice(backups, func(i, j int) bool { + return backups[i].CreationTimestamp.Compare(backups[j].CreationTimestamp.Time) == -1 + }) + + backups = backups[:len(backups)-keep] + + return backups, nil +} + +func generateBackupName(cr *apiv1alpha1.PerconaServerMySQL, backupJob apiv1alpha1.BackupSchedule) string { + result := "cron-" + if len(cr.Name) > 16 { + result += cr.Name[:16] + } else { + result += cr.Name + } + storageName := backupJob.StorageName + if len(storageName) > 16 { + storageName = storageName[:16] + } + result += "-" + storageName + "-" + + tnow := time.Now() + result += tnow.Format("20060102150405") + + result += "-" + strconv.FormatUint(uint64(crc32.ChecksumIEEE([]byte(backupJob.Schedule))), 32)[:5] + + return result +} diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index 429b86daf..e98d0f120 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -64,6 +64,8 @@ type PerconaServerMySQLReconciler struct { ServerVersion *platform.ServerVersion Recorder record.EventRecorder ClientCmd clientcmd.Client + + Crons cronRegistry } //+kubebuilder:rbac:groups=ps.percona.com,resources=perconaservermysqls;perconaservermysqls/status;perconaservermysqls/finalizers,verbs=get;list;watch;create;update;patch;delete @@ -389,6 +391,9 @@ func (r *PerconaServerMySQLReconciler) doReconcile( if err := r.reconcileMySQLRouter(ctx, cr); err != nil { return errors.Wrap(err, "MySQL router") } + if err := r.reconcileScheduledBackup(ctx, cr); err != nil { + return errors.Wrap(err, "scheduled backup") + } if err := r.cleanupOutdated(ctx, cr); err != nil { return errors.Wrap(err, "cleanup outdated") } diff --git a/pkg/controller/ps/suite_test.go b/pkg/controller/ps/suite_test.go index 049283b87..cbb4f22c3 100644 --- a/pkg/controller/ps/suite_test.go +++ b/pkg/controller/ps/suite_test.go @@ -35,7 +35,6 @@ import ( psv1alpha1 "github.com/percona/percona-server-mysql-operator/api/v1alpha1" "github.com/percona/percona-server-mysql-operator/pkg/platform" - //+kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -104,6 +103,7 @@ func reconciler() *PerconaServerMySQLReconciler { ServerVersion: &platform.ServerVersion{ Platform: platform.PlatformKubernetes, }, + Crons: NewCronRegistry(), }) } diff --git a/pkg/controller/psbackup/controller.go b/pkg/controller/psbackup/controller.go index 5106da0f3..e6ea085c6 100644 --- a/pkg/controller/psbackup/controller.go +++ b/pkg/controller/psbackup/controller.go @@ -154,7 +154,7 @@ func (r *PerconaServerMySQLBackupReconciler) Reconcile(ctx context.Context, req } job := &batchv1.Job{} - nn = xtrabackup.NamespacedName(cr) + nn = xtrabackup.JobNamespacedName(cr) err := r.Client.Get(ctx, nn, job) if err != nil && !k8serrors.IsNotFound(err) { return rr, errors.Wrapf(err, "get job %v", nn.String()) diff --git a/pkg/naming/naming.go b/pkg/naming/naming.go index 0be103285..2b694ca8c 100644 --- a/pkg/naming/naming.go +++ b/pkg/naming/naming.go @@ -1,8 +1,8 @@ package naming const ( - annotationPrefix = "percona.com/" - annotationPrefixMysql = "mysql.percona.com/" + perconaPrefix = "percona.com/" + mysqlPerconaPrefix = "mysql.percona.com/" ) const ( @@ -14,15 +14,24 @@ const ( ) const ( - LabelMySQLPrimary = annotationPrefixMysql + "primary" - LabelExposed = annotationPrefix + "exposed" + LabelCluster = perconaPrefix + "cluster" ) const ( - FinalizerDeleteSSL = annotationPrefix + "delete-ssl" - FinalizerDeletePodsInOrder = annotationPrefix + "delete-mysql-pods-in-order" + LabelMySQLPrimary = mysqlPerconaPrefix + "primary" + LabelExposed = perconaPrefix + "exposed" +) + +const ( + LabelBackupType = perconaPrefix + "backup-type" + LabelBackupAncestor = perconaPrefix + "backup-ancestor" +) + +const ( + FinalizerDeleteSSL = perconaPrefix + "delete-ssl" + FinalizerDeletePodsInOrder = perconaPrefix + "delete-mysql-pods-in-order" - FinalizerDeleteBackup = annotationPrefix + "delete-backup" + FinalizerDeleteBackup = perconaPrefix + "delete-backup" ) type AnnotationKey string @@ -32,9 +41,9 @@ func (s AnnotationKey) String() string { } const ( - AnnotationSecretHash AnnotationKey = annotationPrefix + "last-applied-secret" - AnnotationConfigHash AnnotationKey = annotationPrefix + "configuration-hash" - AnnotationTLSHash AnnotationKey = annotationPrefix + "last-applied-tls" - AnnotationPasswordsUpdated AnnotationKey = annotationPrefix + "passwords-updated" - AnnotationLastConfigHash AnnotationKey = annotationPrefix + "last-config-hash" + AnnotationSecretHash AnnotationKey = perconaPrefix + "last-applied-secret" + AnnotationConfigHash AnnotationKey = perconaPrefix + "configuration-hash" + AnnotationTLSHash AnnotationKey = perconaPrefix + "last-applied-tls" + AnnotationPasswordsUpdated AnnotationKey = perconaPrefix + "passwords-updated" + AnnotationLastConfigHash AnnotationKey = perconaPrefix + "last-config-hash" ) diff --git a/pkg/xtrabackup/xtrabackup.go b/pkg/xtrabackup/xtrabackup.go index 2c48d537a..e7d74d690 100644 --- a/pkg/xtrabackup/xtrabackup.go +++ b/pkg/xtrabackup/xtrabackup.go @@ -9,6 +9,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" apiv1alpha1 "github.com/percona/percona-server-mysql-operator/api/v1alpha1" "github.com/percona/percona-server-mysql-operator/pkg/k8s" @@ -42,20 +43,55 @@ func DeleteName(cr *apiv1alpha1.PerconaServerMySQLBackup) string { return componentShortName + "-delete-" + cr.Name } -func NamespacedName(cr *apiv1alpha1.PerconaServerMySQLBackup) types.NamespacedName { - return types.NamespacedName{Name: Name(cr), Namespace: cr.Namespace} +func JobNamespacedName(cr *apiv1alpha1.PerconaServerMySQLBackup) types.NamespacedName { + return types.NamespacedName{Name: JobName(cr), Namespace: cr.Namespace} } func JobName(cr *apiv1alpha1.PerconaServerMySQLBackup) string { - return Name(cr) + return trimJobName(Name(cr)) } func RestoreJobName(cluster *apiv1alpha1.PerconaServerMySQL, cr *apiv1alpha1.PerconaServerMySQLRestore) string { - return RestoreName(cr) + return trimJobName(RestoreName(cr)) } func DeleteJobName(cr *apiv1alpha1.PerconaServerMySQLBackup) string { - return DeleteName(cr) + return trimJobName(DeleteName(cr)) +} + +// trimJobName trims the provided string to ensure it stays within the 63-character limit. +// The job name will be included in the "batch.kubernetes.io/job-name" label in the ".spec.template" section of the job. +// Labels have a maximum length of 63 characters, so this function ensures the job name fits within that limit. +// Also it ensures that +func trimJobName(name string) string { + trimLeft := func(name string) string { + for i := 0; i < len(name); i++ { + if (name[i] < 'a' || name[i] > 'z') && (name[i] < '0' || name[i] > '9') { + continue + } + return name[i:] + } + return "" + } + + trimRight := func(name string) string { + for i := len(name) - 1; i >= 0; i-- { + if (name[i] < 'a' || name[i] > 'z') && (name[i] < '0' || name[i] > '9') { + continue + } + return name[:i+1] + } + return "" + } + + name = trimLeft(name) + name = trimRight(name) + if len(name) > validation.DNS1035LabelMaxLength { + name = name[:validation.DNS1035LabelMaxLength] + name = trimRight(name) + } + + return name } func MatchLabels(cluster *apiv1alpha1.PerconaServerMySQL) map[string]string {