From 0a6a268c8c90a91ff7586e92fbc766261697a404 Mon Sep 17 00:00:00 2001 From: vmudadla Date: Thu, 21 Mar 2024 17:53:42 -0500 Subject: [PATCH] Add integration tests for external storage/db connections Fix pre-commit issues Update test --- .github/resources/mariadb/deployment.yaml | 74 +++++++++++++++++++ .github/resources/mariadb/kustomization.yaml | 8 ++ .github/resources/mariadb/pvc.yaml | 11 +++ .github/resources/mariadb/secret.yaml | 11 +++ .github/resources/mariadb/service.yaml | 16 ++++ .github/resources/minio/deployment.yaml | 54 ++++++++++++++ .github/resources/minio/kustomization.yaml | 8 ++ .github/resources/minio/pvc.yaml | 11 +++ .github/resources/minio/secret.yaml | 8 ++ .github/resources/minio/service.yaml | 16 ++++ .github/workflows/kind-integration.yml | 53 +++++++++++++- tests/dspa_v2_test.go | 27 ++++--- tests/resources/dspa-external-lite.yaml | 77 ++++++++++++++++++++ 13 files changed, 363 insertions(+), 11 deletions(-) create mode 100644 .github/resources/mariadb/deployment.yaml create mode 100644 .github/resources/mariadb/kustomization.yaml create mode 100644 .github/resources/mariadb/pvc.yaml create mode 100644 .github/resources/mariadb/secret.yaml create mode 100644 .github/resources/mariadb/service.yaml create mode 100644 .github/resources/minio/deployment.yaml create mode 100644 .github/resources/minio/kustomization.yaml create mode 100644 .github/resources/minio/pvc.yaml create mode 100644 .github/resources/minio/secret.yaml create mode 100644 .github/resources/minio/service.yaml create mode 100644 tests/resources/dspa-external-lite.yaml diff --git a/.github/resources/mariadb/deployment.yaml b/.github/resources/mariadb/deployment.yaml new file mode 100644 index 000000000..5c79d94e3 --- /dev/null +++ b/.github/resources/mariadb/deployment.yaml @@ -0,0 +1,74 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mariadb + namespace: test-mariadb + labels: + app: mariadb + component: data-science-pipelines +spec: + strategy: + type: Recreate # Need this since backing PVC is ReadWriteOnce, which creates resource lock condition in default Rolling strategy + selector: + matchLabels: + app: mariadb + component: data-science-pipelines + template: + metadata: + labels: + app: mariadb + component: data-science-pipelines + spec: + containers: + - name: mariadb + image: quay.io/centos7/mariadb-103-centos7:ea07c0dade9571d78a272b453fd2dea92077dc7f + ports: + - containerPort: 3306 + readinessProbe: + exec: + command: + - /bin/sh + - "-i" + - "-c" + - >- + MYSQL_PWD=$MYSQL_PASSWORD mysql -h 127.0.0.1 -u $MYSQL_USER -D + $MYSQL_DATABASE -e 'SELECT 1' + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: 3306 + timeoutSeconds: 1 + env: + - name: MYSQL_USER + value: "mlpipeline" + - name: MYSQL_PASSWORD + valueFrom: + secretKeyRef: + key: "password" + name: "ds-pipeline-db-test" + - name: MYSQL_DATABASE + value: "mlpipeline" + - name: MYSQL_ALLOW_EMPTY_PASSWORD + value: "true" + resources: + limits: + cpu: 20m + memory: 500Mi + requests: + cpu: 20m + memory: 100m + volumeMounts: + - name: mariadb-persistent-storage + mountPath: /var/lib/mysql + volumes: + - name: mariadb-persistent-storage + persistentVolumeClaim: + claimName: mariadb-test diff --git a/.github/resources/mariadb/kustomization.yaml b/.github/resources/mariadb/kustomization.yaml new file mode 100644 index 000000000..b62f2cc47 --- /dev/null +++ b/.github/resources/mariadb/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: test-mariadb +resources: + - deployment.yaml + - pvc.yaml + - secret.yaml + - service.yaml diff --git a/.github/resources/mariadb/pvc.yaml b/.github/resources/mariadb/pvc.yaml new file mode 100644 index 000000000..65aadb3bf --- /dev/null +++ b/.github/resources/mariadb/pvc.yaml @@ -0,0 +1,11 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mariadb-test +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 500Mi + volumeMode: Filesystem diff --git a/.github/resources/mariadb/secret.yaml b/.github/resources/mariadb/secret.yaml new file mode 100644 index 000000000..064995af0 --- /dev/null +++ b/.github/resources/mariadb/secret.yaml @@ -0,0 +1,11 @@ +kind: Secret +apiVersion: v1 +metadata: + name: ds-pipeline-db-test + namespace: test-mariadb + labels: + app: mariadb + component: data-science-pipelines +stringData: + password: password +type: Opaque diff --git a/.github/resources/mariadb/service.yaml b/.github/resources/mariadb/service.yaml new file mode 100644 index 000000000..38cce624f --- /dev/null +++ b/.github/resources/mariadb/service.yaml @@ -0,0 +1,16 @@ +kind: Service +apiVersion: v1 +metadata: + name: mariadb + namespace: test-mariadb + labels: + app: mariadb + component: data-science-pipelines +spec: + ports: + - port: 3306 + protocol: TCP + targetPort: 3306 + selector: + app: mariadb + component: data-science-pipelines diff --git a/.github/resources/minio/deployment.yaml b/.github/resources/minio/deployment.yaml new file mode 100644 index 000000000..ebcf31911 --- /dev/null +++ b/.github/resources/minio/deployment.yaml @@ -0,0 +1,54 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: minio +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + volumes: + - name: data + persistentVolumeClaim: + claimName: minio + containers: + - resources: + limits: + cpu: 20m + memory: 500Mi + requests: + cpu: 20m + memory: 100m + name: minio + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio + key: accesskey + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio + key: secretkey + ports: + - containerPort: 9000 + protocol: TCP + imagePullPolicy: IfNotPresent + volumeMounts: + - name: data + mountPath: /data + subPath: minio + image: 'quay.io/minio/minio:RELEASE.2023-10-16T04-13-43Z' + args: + - server + - /data + - --console-address + - ":9001" + strategy: + type: Recreate diff --git a/.github/resources/minio/kustomization.yaml b/.github/resources/minio/kustomization.yaml new file mode 100644 index 000000000..73f80550c --- /dev/null +++ b/.github/resources/minio/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: test-minio +resources: + - deployment.yaml + - service.yaml + - pvc.yaml + - secret.yaml diff --git a/.github/resources/minio/pvc.yaml b/.github/resources/minio/pvc.yaml new file mode 100644 index 000000000..a5a82039b --- /dev/null +++ b/.github/resources/minio/pvc.yaml @@ -0,0 +1,11 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: minio +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 500Mi + volumeMode: Filesystem diff --git a/.github/resources/minio/secret.yaml b/.github/resources/minio/secret.yaml new file mode 100644 index 000000000..109b44bf8 --- /dev/null +++ b/.github/resources/minio/secret.yaml @@ -0,0 +1,8 @@ +kind: Secret +apiVersion: v1 +metadata: + name: minio +stringData: + accesskey: accesskey + secretkey: secretkey +type: Opaque diff --git a/.github/resources/minio/service.yaml b/.github/resources/minio/service.yaml new file mode 100644 index 000000000..5149a515e --- /dev/null +++ b/.github/resources/minio/service.yaml @@ -0,0 +1,16 @@ +kind: Service +apiVersion: v1 +metadata: + name: minio +spec: + ports: + - name: https + protocol: TCP + port: 9000 + targetPort: 9000 + - name: console + protocol: TCP + port: 9001 + targetPort: 9001 + selector: + app: minio diff --git a/.github/workflows/kind-integration.yml b/.github/workflows/kind-integration.yml index 5dabafddc..d9eff2925 100644 --- a/.github/workflows/kind-integration.yml +++ b/.github/workflows/kind-integration.yml @@ -23,9 +23,15 @@ concurrency: env: IMAGE_REPO_DSPO: data-science-pipelines-operator DSPA_NAMESPACE: test-dspa + DSPA_EXTERNAL_NAMESPACE: test-dspa-external + MINIO_NAMESPACE: test-minio + MARIADB_NAMESPACE: test-mariadb DSPA_NAME: test-dspa + DSPA_EXTERNAL_NAME: test-dspa-external DSPA_DEPLOY_WAIT_TIMEOUT: 300 INTEGRATION_TESTS_DIR: ${{ github.workspace }}/tests + DSPA_PATH: ${{ github.workspace }}/tests/resources/dspa-lite.yaml + DSPA_EXTERNAL_PATH: ${{ github.workspace }}/tests/resources/dspa-external-lite.yaml jobs: dspo-tests: runs-on: ubuntu-20.04 @@ -76,21 +82,64 @@ jobs: make podman-push -e IMG="${DSPO_IMAGE}" make deploy-kind -e IMG="${DSPO_IMAGE}" - - name: Wait for DSPO + - name: Create Minio Namespace + run: | + kubectl create namespace ${{ env.MINIO_NAMESPACE }} + + - name: Deploy Minio + working-directory: ${{ github.workspace }}/.github/resources/minio + run: | + kustomize build . | oc -n ${{ env.MINIO_NAMESPACE }} apply -f - + + - name: Create MariaDB Namespace + run: | + kubectl create namespace ${{ env.MARIADB_NAMESPACE }} + + - name: Deploy MariaDB + working-directory: ${{ github.workspace }}/.github/resources/mariadb + run: | + kustomize build . | oc -n ${{ env.MARIADB_NAMESPACE }} apply -f - + + - name: Wait for Dependencies (DSPO, Minio, Mariadb) run: | kubectl wait -n opendatahub --timeout=60s --for=condition=Available=true deployment data-science-pipelines-operator-controller-manager + kubectl wait -n ${{ env.MARIADB_NAMESPACE }} --timeout=60s --for=condition=Available=true deployment mariadb + kubectl wait -n ${{ env.MINIO_NAMESPACE }} --timeout=60s --for=condition=Available=true deployment minio - name: Create DSPA Namespace run: | kubectl create namespace ${{ env.DSPA_NAMESPACE }} + - name: Create Namespace for DSPA with External connections + run: | + kubectl create namespace ${{ env.DSPA_EXTERNAL_NAMESPACE }} + + - name: Create Minio Secrets + run: | + kubectl create secret generic minio --from-literal=accesskey=accesskey --from-literal=secretkey=secretkey -n ${{ env.DSPA_EXTERNAL_NAMESPACE }} + + - name: Create MariaDB Secrets + run: | + password=$(kubectl get secret ds-pipeline-db-test -n test-mariadb -o jsonpath="{.data.password}" | base64 --decode) + kubectl create secret generic ds-pipeline-db-test --from-literal=password="$password" -n ${{ env.DSPA_EXTERNAL_NAMESPACE }} + - name: Run tests working-directory: ${{ github.workspace }} env: NAMESPACE: ${{ env.DSPA_NAMESPACE }} DSPA_NAME: ${{ env.DSPA_NAME }} + DSPA_PATH: ${{ env.DSPA_PATH }} + run: | + make integrationtest K8SAPISERVERHOST=$(oc whoami --show-server) DSPANAMESPACE=${NAMESPACE} DSPAPATH=${DSPA_PATH} + + - name: Run tests for DSPA with External Connections + working-directory: ${{ github.workspace }} + env: + NAMESPACE: ${{ env.DSPA_EXTERNAL_NAMESPACE }} + DSPA_NAME: ${{ env.DSPA_EXTERNAL_NAME }} + DSPA_EXTERNAL_PATH: ${{ env.DSPA_EXTERNAL_PATH }} run: | - make integrationtest K8SAPISERVERHOST=$(oc whoami --show-server) DSPANAMESPACE=${NAMESPACE} + make integrationtest K8SAPISERVERHOST=$(oc whoami --show-server) DSPANAMESPACE=${NAMESPACE} DSPAPATH=${DSPA_EXTERNAL_PATH} - name: Clean up run: | diff --git a/tests/dspa_v2_test.go b/tests/dspa_v2_test.go index bb2acf2ca..076fa02bf 100644 --- a/tests/dspa_v2_test.go +++ b/tests/dspa_v2_test.go @@ -31,6 +31,24 @@ import ( func (suite *IntegrationTestSuite) TestDSPADeployment() { podCount := 8 + if suite.DSPA.Spec.ObjectStorage.ExternalStorage != nil { + podCount = podCount - 1 + } + if suite.DSPA.Spec.Database.ExternalDB != nil { + podCount = podCount - 1 + } + deployments := []string{ + fmt.Sprintf("ds-pipeline-%s", suite.DSPA.Name), + fmt.Sprintf("ds-pipeline-persistenceagent-%s", suite.DSPA.Name), + fmt.Sprintf("ds-pipeline-scheduledworkflow-%s", suite.DSPA.Name), + } + + if suite.DSPA.Spec.ObjectStorage.ExternalStorage == nil && suite.DSPA.Spec.Database.ExternalDB == nil { + deployments = append(deployments, + fmt.Sprintf("mariadb-%s", suite.DSPA.Name), + fmt.Sprintf("minio-%s", suite.DSPA.Name), + ) + } suite.T().Run("with default MariaDB and Minio", func(t *testing.T) { t.Run(fmt.Sprintf("should have %d pods", podCount), func(t *testing.T) { podList := &corev1.PodList{} @@ -41,15 +59,6 @@ func (suite *IntegrationTestSuite) TestDSPADeployment() { require.NoError(t, err) assert.Equal(t, podCount, len(podList.Items)) }) - - deployments := []string{ - fmt.Sprintf("ds-pipeline-%s", suite.DSPA.Name), - fmt.Sprintf("ds-pipeline-persistenceagent-%s", suite.DSPA.Name), - fmt.Sprintf("ds-pipeline-scheduledworkflow-%s", suite.DSPA.Name), - fmt.Sprintf("mariadb-%s", suite.DSPA.Name), - fmt.Sprintf("minio-%s", suite.DSPA.Name), - } - for _, deployment := range deployments { t.Run(fmt.Sprintf("should have a ready %s deployment", deployment), func(t *testing.T) { testUtil.TestForSuccessfulDeployment(t, suite.Ctx, suite.DSPANamespace, deployment, suite.Clientmgr.k8sClient) diff --git a/tests/resources/dspa-external-lite.yaml b/tests/resources/dspa-external-lite.yaml new file mode 100644 index 000000000..975d16f83 --- /dev/null +++ b/tests/resources/dspa-external-lite.yaml @@ -0,0 +1,77 @@ +apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1 +kind: DataSciencePipelinesApplication +metadata: + name: test-dspa-external +spec: + dspVersion: v2 + apiServer: + deploy: true + enableOauth: false + enableSamplePipeline: true + cABundle: + configMapName: kube-root-ca.crt + configMapKey: ca.crt + resources: + limits: + cpu: 20m + memory: 500Mi + requests: + cpu: 20m + memory: 100m + scheduledWorkflow: + deploy: true + resources: + limits: + cpu: 20m + memory: 500Mi + requests: + cpu: 20m + memory: 100m + persistenceAgent: + deploy: true + resources: + limits: + cpu: 20m + memory: 500Mi + requests: + cpu: 20m + memory: 100Mi + mlmd: + deploy: true + envoy: + resources: + limits: + cpu: 20m + memory: 500Mi + requests: + cpu: 20m + memory: 100Mi + grpc: + resources: + limits: + cpu: 20m + memory: 500Mi + requests: + cpu: 20m + memory: 100Mi + database: + customExtraParams: '{"tls":"false"}' + externalDB: + host: mariadb.test-mariadb.svc.cluster.local + port: "3306" + username: mlpipeline + pipelineDBName: mlpipeline + passwordSecret: + name: ds-pipeline-db-test + key: password + objectStorage: + externalStorage: + bucket: mlpipeline + host: minio.test-minio.svc.cluster.local + port: "9000" + region: us-east-2 + s3CredentialsSecret: + accessKey: accesskey + secretKey: secretkey + secretName: minio + scheme: http