diff --git a/config/internal/apiserver/deployment.yaml.tmpl b/config/internal/apiserver/deployment.yaml.tmpl index ae0d064f4..47fe06b73 100644 --- a/config/internal/apiserver/deployment.yaml.tmpl +++ b/config/internal/apiserver/deployment.yaml.tmpl @@ -80,6 +80,12 @@ spec: value: "ds-pipeline-visualizationserver" - name: ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT value: "8888" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRET + value: "{{.ObjectStorageConnection.CredentialsSecret.SecretName}}" + - name: OBJECTSTORECONFIG_CREDENTIALSACCESSKEYKEY + value: "{{.ObjectStorageConnection.CredentialsSecret.AccessKey}}" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRETKEYKEY + value: "{{.ObjectStorageConnection.CredentialsSecret.SecretKey}}" - name: OBJECTSTORECONFIG_BUCKETNAME value: "{{.ObjectStorageConnection.Bucket}}" - name: OBJECTSTORECONFIG_ACCESSKEY diff --git a/config/internal/devtools/database.secret.yaml.tmpl b/config/internal/devtools/database.secret.yaml.tmpl new file mode 100644 index 000000000..62a9b4d2b --- /dev/null +++ b/config/internal/devtools/database.secret.yaml.tmpl @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "{{.DBConnection.CredentialsSecret.Name}}" + namespace: {{.Namespace}} + labels: + app: mariadb-{{.Name}} + component: data-science-pipelines +data: + password: {{.DBConnection.Password}} diff --git a/config/internal/devtools/storage.secret.yaml.tmpl b/config/internal/devtools/storage.secret.yaml.tmpl new file mode 100644 index 000000000..17192f27f --- /dev/null +++ b/config/internal/devtools/storage.secret.yaml.tmpl @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "{{.ObjectStorageConnection.CredentialsSecret.SecretName}}" + namespace: {{.Namespace}} + labels: + app: minio-{{.Name}} + component: data-science-pipelines +stringData: + host: "{{.ObjectStorageConnection.Host}}" + port: "{{.ObjectStorageConnection.Port}}" + secure: "{{.ObjectStorageConnection.Secure}}" +data: + accesskey: "{{.ObjectStorageConnection.AccessKeyID}}" + secretkey: "{{.ObjectStorageConnection.SecretAccessKey}}" diff --git a/config/internal/mariadb/secret.yaml.tmpl b/config/internal/mariadb/secret.yaml.tmpl index 62a9b4d2b..3293bc703 100644 --- a/config/internal/mariadb/secret.yaml.tmpl +++ b/config/internal/mariadb/secret.yaml.tmpl @@ -7,4 +7,4 @@ metadata: app: mariadb-{{.Name}} component: data-science-pipelines data: - password: {{.DBConnection.Password}} + {{.DBConnection.CredentialsSecret.Key}}: "{{.DBConnection.Password}}" diff --git a/config/internal/minio/secret.yaml.tmpl b/config/internal/minio/secret.yaml.tmpl index 17192f27f..e0cf555c1 100644 --- a/config/internal/minio/secret.yaml.tmpl +++ b/config/internal/minio/secret.yaml.tmpl @@ -11,5 +11,5 @@ stringData: port: "{{.ObjectStorageConnection.Port}}" secure: "{{.ObjectStorageConnection.Secure}}" data: - accesskey: "{{.ObjectStorageConnection.AccessKeyID}}" - secretkey: "{{.ObjectStorageConnection.SecretAccessKey}}" + {{.ObjectStorageConnection.CredentialsSecret.AccessKey}}: "{{.ObjectStorageConnection.AccessKeyID}}" + {{.ObjectStorageConnection.CredentialsSecret.SecretKey}}: "{{.ObjectStorageConnection.SecretAccessKey}}" diff --git a/controllers/config/defaults.go b/controllers/config/defaults.go index b13ef3c78..2b0c38f65 100644 --- a/controllers/config/defaults.go +++ b/controllers/config/defaults.go @@ -32,8 +32,9 @@ const ( ArtifactScriptConfigMapKey = "artifact_script" DSPServicePrefix = "ds-pipeline" - DBSecretNamePrefix = "ds-pipeline-db-" - DBSecretKey = "password" + DefaultDBSecretNamePrefix = "ds-pipeline-db-" + DefaultDBSecretKey = "password" + GeneratedDBPasswordLength = 12 MariaDBName = "mlpipeline" MariaDBHostPrefix = "mariadb" @@ -47,9 +48,11 @@ const ( MinioDefaultBucket = "mlpipeline" MinioPVCSize = "10Gi" - ObjectStorageSecretName = "mlpipeline-minio-artifact" // hardcoded in kfp-tekton - ObjectStorageAccessKey = "accesskey" - ObjectStorageSecretKey = "secretkey" + DefaultObjectStorageSecretNamePrefix = "ds-pipeline-s3-" + DefaultObjectStorageAccessKey = "accesskey" + DefaultObjectStorageSecretKey = "secretkey" + GeneratedObjectStorageAccessKeyLength = 16 + GeneratedObjectStorageSecretKeyLength = 24 MlmdGrpcPort = "8080" ) diff --git a/controllers/database.go b/controllers/database.go index 33a083a53..5ba313f49 100644 --- a/controllers/database.go +++ b/controllers/database.go @@ -20,6 +20,7 @@ import ( "database/sql" b64 "encoding/base64" "fmt" + _ "github.com/go-sql-driver/mysql" dspav1alpha1 "github.com/opendatahub-io/data-science-pipelines-operator/api/v1alpha1" "github.com/opendatahub-io/data-science-pipelines-operator/controllers/config" @@ -27,12 +28,11 @@ import ( const dbSecret = "mariadb/secret.yaml.tmpl" -var dbTemplates = []string{ +var mariadbTemplates = []string{ "mariadb/deployment.yaml.tmpl", "mariadb/pvc.yaml.tmpl", "mariadb/service.yaml.tmpl", "mariadb/mariadb-sa.yaml.tmpl", - dbSecret, } // extract to var for mocking in testing @@ -100,18 +100,22 @@ func (r *DSPAReconciler) ReconcileDatabase(ctx context.Context, dsp *dspav1alpha // Default DB is currently MariaDB as well, but storing these bools seperately in case that changes deployDefaultDB := !databaseSpecified || defaultDBRequired + externalDBCredentialsProvided := externalDBSpecified && (dsp.Spec.Database.ExternalDB.PasswordSecret != nil) + mariaDBCredentialsProvided := mariaDBSpecified && (dsp.Spec.Database.MariaDB.PasswordSecret != nil) + databaseCredentialsProvided := externalDBCredentialsProvided || mariaDBCredentialsProvided + // If external db is specified, it takes precedence if externalDBSpecified { - log.Info("Deploying external db secret.") - // If using external DB, we just need to create the secret - // for apiserver - err := r.Apply(dsp, params, dbSecret) - if err != nil { - return err - } + log.Info("Using externalDB, bypassing database deployment.") } else if deployMariaDB || deployDefaultDB { + if !databaseCredentialsProvided { + err := r.Apply(dsp, params, dbSecret) + if err != nil { + return err + } + } log.Info("Applying mariaDB resources.") - for _, template := range dbTemplates { + for _, template := range mariadbTemplates { err := r.Apply(dsp, params, template) if err != nil { return err diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go index 1d8d8d465..ab197d20e 100644 --- a/controllers/dspipeline_params.go +++ b/controllers/dspipeline_params.go @@ -128,28 +128,75 @@ func passwordGen(n int) string { return string(b) } +func (p *DSPAParams) RetrieveSecret(ctx context.Context, client client.Client, secretName, secretKey string, log logr.Logger) (string, error) { + secret := &v1.Secret{} + namespacedName := types.NamespacedName{ + Name: secretName, + Namespace: p.Namespace, + } + err := client.Get(ctx, namespacedName, secret) + if err != nil { + log.V(1).Info(fmt.Sprintf("Unable to retrieve secret [%s].", secretName)) + return "", err + } + return base64.StdEncoding.EncodeToString(secret.Data[secretKey]), nil +} + +func (p *DSPAParams) RetrieveOrCreateSecret(ctx context.Context, client client.Client, secretName, secretKey string, generatedPasswordLength int, log logr.Logger) (string, error) { + val, err := p.RetrieveSecret(ctx, client, secretName, secretKey, log) + if err != nil && apierrs.IsNotFound(err) { + generatedPass := passwordGen(generatedPasswordLength) + return base64.StdEncoding.EncodeToString([]byte(generatedPass)), nil + } else if err != nil { + log.Error(err, "Unable to create DB secret...") + return "", err + } + log.Info(fmt.Sprintf("Secret [%s] already exists, using stored value.", secretName)) + return val, nil +} + +func (p *DSPAParams) RetrieveOrCreateDBSecret(ctx context.Context, client client.Client, secret *dspa.SecretKeyValue, log logr.Logger) (string, error) { + dbPassword, err := p.RetrieveOrCreateSecret(ctx, client, secret.Name, secret.Key, config.GeneratedDBPasswordLength, log) + if err != nil { + return "", err + } + return dbPassword, nil + +} + +func (p *DSPAParams) RetrieveOrCreateObjectStoreSecret(ctx context.Context, client client.Client, secret *dspa.S3CredentialSecret, log logr.Logger) (string, string, error) { + accessKey, err := p.RetrieveOrCreateSecret(ctx, client, secret.SecretName, secret.AccessKey, config.GeneratedObjectStorageAccessKeyLength, log) + if err != nil { + return "", "", err + } + secretKey, err := p.RetrieveOrCreateSecret(ctx, client, secret.SecretName, secret.SecretKey, config.GeneratedObjectStorageSecretKeyLength, log) + if err != nil { + return "", "", err + } + return accessKey, secretKey, nil +} + // SetupDBParams Populates the DB connection Parameters. // If an external secret is specified, SetupDBParams will retrieve DB credentials from it. // If DSPO is managing a dynamically created secret, then SetupDBParams generates the creds. func (p *DSPAParams) SetupDBParams(ctx context.Context, dsp *dspa.DataSciencePipelinesApplication, client client.Client, log logr.Logger) error { usingExternalDB := p.UsingExternalDB(dsp) - - var customCreds *dspa.SecretKeyValue - - // Even if a secret is specified DSPO will deploy its own secret owned by DSPO - p.DBConnection.CredentialsSecret = &dspa.SecretKeyValue{ - Name: config.DBSecretNamePrefix + p.Name, - Key: config.DBSecretKey, - } - if usingExternalDB { // Assume validation for CR ensures these values exist p.DBConnection.Host = dsp.Spec.Database.ExternalDB.Host p.DBConnection.Port = dsp.Spec.Database.ExternalDB.Port p.DBConnection.Username = dsp.Spec.Database.ExternalDB.Username p.DBConnection.DBName = dsp.Spec.Database.ExternalDB.DBName - customCreds = dsp.Spec.Database.ExternalDB.PasswordSecret + p.DBConnection.CredentialsSecret = dsp.Spec.Database.ExternalDB.PasswordSecret + + // Retreive DB Password from specified secret. Ignore error if the secret simply doesn't exist (will be created later) + password, err := p.RetrieveSecret(ctx, client, p.DBConnection.CredentialsSecret.Name, p.DBConnection.CredentialsSecret.Key, log) + if err != nil && !apierrs.IsNotFound(err) { + log.Error(err, "Unexpected error encountered while fetching Database Secret") + return err + } + p.DBConnection.Password = password } else { // If no externalDB or mariaDB is specified, DSPO assumes // MariaDB deployment with defaults. @@ -163,6 +210,7 @@ func (p *DSPAParams) SetupDBParams(ctx context.Context, dsp *dspa.DataSciencePip PVCSize: resource.MustParse(config.MariaDBNamePVCSize), } } + // If MariaDB was specified, ensure missing fields are // populated with defaults. if p.MariaDB.Image == "" { @@ -180,59 +228,25 @@ func (p *DSPAParams) SetupDBParams(ctx context.Context, dsp *dspa.DataSciencePip p.DBConnection.Port = config.MariaDBHostPort p.DBConnection.Username = p.MariaDB.Username p.DBConnection.DBName = p.MariaDB.DBName - if p.MariaDB.PasswordSecret != nil { - customCreds = p.MariaDB.PasswordSecret - } - } - - // Secret where DB credentials reside on cluster - var credsSecretName string - var credsPasswordKey string - - customCredentialsSpecified := customCreds != nil - if customCredentialsSpecified { - credsSecretName = customCreds.Name - credsPasswordKey = customCreds.Key - } else { - credsSecretName = p.DBConnection.CredentialsSecret.Name - credsPasswordKey = p.DBConnection.CredentialsSecret.Key - } - dbSecret := &v1.Secret{} - namespacedName := types.NamespacedName{ - Name: credsSecretName, - Namespace: p.Namespace, - } - - createNewSecret := false - - // Attempt to fetch the specified DB secret - err := client.Get(ctx, namespacedName, dbSecret) - if err != nil && apierrs.IsNotFound(err) { - if !customCredentialsSpecified { - generatedPass := passwordGen(12) - p.DBConnection.Password = base64.StdEncoding.EncodeToString([]byte(generatedPass)) - createNewSecret = true + // If custom DB Secret provided, use its values. Otherwise generate a default + if p.MariaDB.PasswordSecret != nil { + p.DBConnection.CredentialsSecret = p.MariaDB.PasswordSecret } else { - log.Error(err, fmt.Sprintf("DB secret [%s] was specified in CR but does not exist.", - credsSecretName)) + p.DBConnection.CredentialsSecret = &dspa.SecretKeyValue{ + Name: config.DefaultDBSecretNamePrefix + p.Name, + Key: config.DefaultDBSecretKey, + } + } + dbPassword, err := p.RetrieveOrCreateDBSecret(ctx, client, p.DBConnection.CredentialsSecret, log) + if err != nil { return err } - } else if err != nil { - log.Error(err, "Unable to fetch DB secret...") - return err + p.DBConnection.Password = dbPassword } - - // Password was dynamically generated, no need to retrieve it from fetched secret - if createNewSecret { - return nil - } - - p.DBConnection.Password = base64.StdEncoding.EncodeToString(dbSecret.Data[credsPasswordKey]) - if p.DBConnection.Password == "" { return fmt.Errorf(fmt.Sprintf("DB Password from secret [%s] for key [%s] was not successfully retrieved, "+ - "ensure that the secret with this key exist.", credsSecretName, credsPasswordKey)) + "ensure that the secret with this key exist.", p.DBConnection.CredentialsSecret.Name, p.DBConnection.CredentialsSecret.Key)) } return nil } @@ -243,16 +257,6 @@ func (p *DSPAParams) SetupDBParams(ctx context.Context, dsp *dspa.DataSciencePip func (p *DSPAParams) SetupObjectParams(ctx context.Context, dsp *dspa.DataSciencePipelinesApplication, client client.Client, log logr.Logger) error { usingExternalObjectStorage := p.UsingExternalStorage(dsp) - - var customCreds *dspa.S3CredentialSecret - - // Even if a secret is specified DSPO will deploy its own secret owned by DSPO - p.ObjectStorageConnection.CredentialsSecret = &dspa.S3CredentialSecret{ - SecretName: config.ObjectStorageSecretName, - AccessKey: config.ObjectStorageAccessKey, - SecretKey: config.ObjectStorageSecretKey, - } - if usingExternalObjectStorage { // Assume validation for CR ensures these values exist p.ObjectStorageConnection.Bucket = dsp.Spec.ObjectStorage.ExternalStorage.Bucket @@ -271,7 +275,21 @@ func (p *DSPAParams) SetupObjectParams(ctx context.Context, dsp *dspa.DataScienc // Port can be empty, which is fine. p.ObjectStorageConnection.Port = dsp.Spec.ObjectStorage.ExternalStorage.Port - customCreds = dsp.Spec.ObjectStorage.ExternalStorage.S3CredentialSecret + p.ObjectStorageConnection.CredentialsSecret = dsp.Spec.ObjectStorage.ExternalStorage.S3CredentialSecret + + // Retrieve ObjStore Creds from specified secret. Ignore error if the secret simply doesn't exist (will be created later) + accesskey, err := p.RetrieveSecret(ctx, client, p.ObjectStorageConnection.CredentialsSecret.SecretName, p.ObjectStorageConnection.CredentialsSecret.AccessKey, log) + if err != nil && !apierrs.IsNotFound(err) { + log.Error(err, "Unexpected error encountered while fetching Object Storage Secret") + return err + } + secretkey, err := p.RetrieveSecret(ctx, client, p.ObjectStorageConnection.CredentialsSecret.SecretName, p.ObjectStorageConnection.CredentialsSecret.SecretKey, log) + if err != nil && !apierrs.IsNotFound(err) { + log.Error(err, "Unexpected error encountered while fetching Object Storage Secret") + return err + } + p.ObjectStorageConnection.AccessKeyID = accesskey + p.ObjectStorageConnection.SecretAccessKey = secretkey } else { if p.Minio == nil { return fmt.Errorf("either [spec.objectStorage.minio] or [spec.objectStorage.externalStorage] " + @@ -301,8 +319,20 @@ func (p *DSPAParams) SetupObjectParams(ctx context.Context, dsp *dspa.DataScienc p.ObjectStorageConnection.Secure = util.BoolPointer(false) if p.Minio.S3CredentialSecret != nil { - customCreds = p.Minio.S3CredentialSecret + p.ObjectStorageConnection.CredentialsSecret = p.Minio.S3CredentialSecret + } else { + p.ObjectStorageConnection.CredentialsSecret = &dspa.S3CredentialSecret{ + SecretName: config.DefaultObjectStorageSecretNamePrefix + p.Name, + AccessKey: config.DefaultObjectStorageAccessKey, + SecretKey: config.DefaultObjectStorageSecretKey, + } + } + accessKey, secretKey, err := p.RetrieveOrCreateObjectStoreSecret(ctx, client, p.ObjectStorageConnection.CredentialsSecret, log) + if err != nil { + return err } + p.ObjectStorageConnection.AccessKeyID = accessKey + p.ObjectStorageConnection.SecretAccessKey = secretKey } endpoint := fmt.Sprintf( @@ -321,62 +351,12 @@ func (p *DSPAParams) SetupObjectParams(ctx context.Context, dsp *dspa.DataScienc p.ObjectStorageConnection.Endpoint = endpoint - // Secret where credentials reside on cluster - var credsSecretName string - var credsAccessKey string - var credsSecretKey string - - customCredentialsSpecified := customCreds != nil - if customCredentialsSpecified { - credsSecretName = customCreds.SecretName - credsAccessKey = customCreds.AccessKey - credsSecretKey = customCreds.SecretKey - } else { - credsSecretName = p.ObjectStorageConnection.CredentialsSecret.SecretName - credsAccessKey = p.ObjectStorageConnection.CredentialsSecret.AccessKey - credsSecretKey = p.ObjectStorageConnection.CredentialsSecret.SecretKey - } - - storageSecret := &v1.Secret{} - namespacedName := types.NamespacedName{ - Name: credsSecretName, - Namespace: p.Namespace, - } - - createNewSecret := false - - // Attempt to fetch the specified storage secret - err := client.Get(ctx, namespacedName, storageSecret) - if err != nil && apierrs.IsNotFound(err) { - if !customCredentialsSpecified { - generatedPass := passwordGen(16) - p.ObjectStorageConnection.AccessKeyID = base64.StdEncoding.EncodeToString([]byte(generatedPass)) - generatedPass = passwordGen(24) - p.ObjectStorageConnection.SecretAccessKey = base64.StdEncoding.EncodeToString([]byte(generatedPass)) - createNewSecret = true - } else { - log.Error(err, fmt.Sprintf("Storage secret [%s] was specified in CR but does not exist.", - credsSecretName)) - return err - } - } else if err != nil { - log.Error(err, "Unable to fetch Storage secret...") - return err - } - - // Password was dynamically generated, no need to retrieve it from fetched secret - if createNewSecret { - return nil - } - - p.ObjectStorageConnection.AccessKeyID = base64.StdEncoding.EncodeToString(storageSecret.Data[credsAccessKey]) - p.ObjectStorageConnection.SecretAccessKey = base64.StdEncoding.EncodeToString(storageSecret.Data[credsSecretKey]) - if p.ObjectStorageConnection.AccessKeyID == "" || p.ObjectStorageConnection.SecretAccessKey == "" { return fmt.Errorf(fmt.Sprintf("Object Storage Password from secret [%s] for keys [%s, %s] was not "+ - "successfully retrieved, ensure that the secret with this key exist.", credsSecretName, credsAccessKey, credsSecretKey)) + "successfully retrieved, ensure that the secret with this key exist.", + p.ObjectStorageConnection.CredentialsSecret.SecretName, + p.ObjectStorageConnection.CredentialsSecret.AccessKey, p.ObjectStorageConnection.CredentialsSecret.SecretKey)) } - return nil } diff --git a/controllers/storage.go b/controllers/storage.go index fedc3a2b9..a03b3e063 100644 --- a/controllers/storage.go +++ b/controllers/storage.go @@ -22,23 +22,23 @@ import ( "encoding/base64" "errors" "fmt" + "net/http" + "github.com/go-logr/logr" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" dspav1alpha1 "github.com/opendatahub-io/data-science-pipelines-operator/api/v1alpha1" "github.com/opendatahub-io/data-science-pipelines-operator/controllers/config" "github.com/opendatahub-io/data-science-pipelines-operator/controllers/util" - "net/http" ) const storageSecret = "minio/secret.yaml.tmpl" -var storageTemplates = []string{ +var minioTemplates = []string{ "minio/deployment.yaml.tmpl", "minio/pvc.yaml.tmpl", "minio/service.yaml.tmpl", "minio/minio-sa.yaml.tmpl", - storageSecret, } func joinHostPort(host, port string) (string, error) { @@ -195,18 +195,23 @@ func (r *DSPAReconciler) ReconcileStorage(ctx context.Context, dsp *dspav1alpha1 minioSpecified := !storageSpecified || dsp.Spec.ObjectStorage.Minio != nil deployMinio := !storageSpecified || (minioSpecified && dsp.Spec.ObjectStorage.Minio.Deploy) + externalStorageCredentialsProvided := externalStorageSpecified && (dsp.Spec.ObjectStorage.ExternalStorage.S3CredentialSecret != nil) + minioCredentialsProvided := minioSpecified && (dsp.Spec.ObjectStorage.Minio.S3CredentialSecret != nil) + storageCredentialsProvided := externalStorageCredentialsProvided || minioCredentialsProvided + // If external storage is specified, it takes precedence if externalStorageSpecified { - log.Info("Deploying external storage secret.") - // If using external storage, we just need to create the secret - // for apiserver - err := r.Apply(dsp, params, storageSecret) - if err != nil { - return err - } + log.Info("Using externalStorage, bypassing object storage deployment.") } else if deployMinio { + log.Info("No S3 storage credential reference provided, so using managed secret") + if !storageCredentialsProvided { + err := r.Apply(dsp, params, storageSecret) + if err != nil { + return err + } + } log.Info("Applying object storage resources.") - for _, template := range storageTemplates { + for _, template := range minioTemplates { err := r.Apply(dsp, params, template) if err != nil { return err @@ -224,7 +229,7 @@ func (r *DSPAReconciler) ReconcileStorage(ctx context.Context, dsp *dspav1alpha1 } } } else { - log.Info("No externalstorage detected, and minio disabled. " + + log.Info("No externalStorage detected, and minio disabled. " + "skipping application of storage Resources") return nil } diff --git a/controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml index fa277a796..23d162fc7 100644 --- a/controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml @@ -72,18 +72,24 @@ spec: value: "ds-pipeline-visualizationserver" - name: ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT value: "8888" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRET + value: "ds-pipeline-s3-testdsp0" + - name: OBJECTSTORECONFIG_CREDENTIALSACCESSKEYKEY + value: "accesskey" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRETKEYKEY + value: "secretkey" - name: OBJECTSTORECONFIG_BUCKETNAME value: "mlpipeline" - name: OBJECTSTORECONFIG_ACCESSKEY valueFrom: secretKeyRef: key: "accesskey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp0" - name: OBJECTSTORECONFIG_SECRETACCESSKEY valueFrom: secretKeyRef: key: "secretkey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp0" - name: OBJECTSTORECONFIG_SECURE value: "false" - name: MINIO_SERVICE_SERVICE_HOST diff --git a/controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml index 5c1263828..3376a4c22 100644 --- a/controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml @@ -72,18 +72,24 @@ spec: value: "ds-pipeline-visualizationserver" - name: ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT value: "8888" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRET + value: "ds-pipeline-s3-testdsp2" + - name: OBJECTSTORECONFIG_CREDENTIALSACCESSKEYKEY + value: "accesskey" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRETKEYKEY + value: "secretkey" - name: OBJECTSTORECONFIG_BUCKETNAME value: "mlpipeline" - name: OBJECTSTORECONFIG_ACCESSKEY valueFrom: secretKeyRef: key: "accesskey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp2" - name: OBJECTSTORECONFIG_SECRETACCESSKEY valueFrom: secretKeyRef: key: "secretkey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp2" - name: OBJECTSTORECONFIG_SECURE value: "false" - name: MINIO_SERVICE_SERVICE_HOST diff --git a/controllers/testdata/declarative/case_2/expected/created/minio_deployment.yaml b/controllers/testdata/declarative/case_2/expected/created/minio_deployment.yaml index 8bf87f744..c31501585 100644 --- a/controllers/testdata/declarative/case_2/expected/created/minio_deployment.yaml +++ b/controllers/testdata/declarative/case_2/expected/created/minio_deployment.yaml @@ -31,12 +31,12 @@ spec: valueFrom: secretKeyRef: key: "accesskey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp2" - name: MINIO_SECRET_KEY valueFrom: secretKeyRef: key: "secretkey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp2" image: minio:test2 name: minio ports: diff --git a/controllers/testdata/declarative/case_2/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_2/expected/created/mlpipelines-ui_deployment.yaml index 4430c6509..2b3e5d1cb 100644 --- a/controllers/testdata/declarative/case_2/expected/created/mlpipelines-ui_deployment.yaml +++ b/controllers/testdata/declarative/case_2/expected/created/mlpipelines-ui_deployment.yaml @@ -35,12 +35,12 @@ spec: valueFrom: secretKeyRef: key: "accesskey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp2" - name: MINIO_SECRET_KEY valueFrom: secretKeyRef: key: "secretkey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp2" - name: ALLOW_CUSTOM_VISUALIZATIONS value: "true" - name: ARGO_ARCHIVE_LOGS diff --git a/controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml index 0b617788d..e87aa42ee 100644 --- a/controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml @@ -29,8 +29,8 @@ spec: - name: DBCONFIG_PASSWORD valueFrom: secretKeyRef: - key: "password" - name: "ds-pipeline-db-testdsp3" + key: "testpswkey3" + name: "testdbpswsecretname3" - name: DBCONFIG_DBNAME value: "testdbname3" - name: DBCONFIG_HOST @@ -72,18 +72,24 @@ spec: value: "ds-pipeline-visualizationserver" - name: ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT value: "8888" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRET + value: "teststoragesecretname3" + - name: OBJECTSTORECONFIG_CREDENTIALSACCESSKEYKEY + value: "testaccesskey3" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRETKEYKEY + value: "testsecretkey3" - name: OBJECTSTORECONFIG_BUCKETNAME value: "testbucket3" - name: OBJECTSTORECONFIG_ACCESSKEY valueFrom: secretKeyRef: - key: "accesskey" - name: "mlpipeline-minio-artifact" + key: "testaccesskey3" + name: "teststoragesecretname3" - name: OBJECTSTORECONFIG_SECRETACCESSKEY valueFrom: secretKeyRef: - key: "secretkey" - name: "mlpipeline-minio-artifact" + key: "testsecretkey3" + name: "teststoragesecretname3" - name: OBJECTSTORECONFIG_SECURE value: "true" - name: MINIO_SERVICE_SERVICE_HOST diff --git a/controllers/testdata/declarative/case_3/expected/created/database_secret.yaml b/controllers/testdata/declarative/case_3/expected/created/database_secret.yaml deleted file mode 100644 index 9f2630148..000000000 --- a/controllers/testdata/declarative/case_3/expected/created/database_secret.yaml +++ /dev/null @@ -1,11 +0,0 @@ -kind: Secret -apiVersion: v1 -metadata: - name: ds-pipeline-db-testdsp3 - namespace: default - labels: - app: mariadb-extrenal-storage - component: data-science-pipelines -data: - password: dGVzdGRic2VjcmV0cHN3dmFsdWUz -type: Opaque diff --git a/controllers/testdata/declarative/case_3/expected/not_created/database_secret.yaml b/controllers/testdata/declarative/case_3/expected/not_created/database_secret.yaml new file mode 100644 index 000000000..ec3365d45 --- /dev/null +++ b/controllers/testdata/declarative/case_3/expected/not_created/database_secret.yaml @@ -0,0 +1,12 @@ +kind: Secret +apiVersion: v1 +metadata: + # todo: remove todo- from name this should actually be checked for but causes failures because previous tests don't clean up properly + name: todo-ds-pipeline-db-testdsp3 + namespace: namespace3 + labels: + app: mariadb-extrenal-storage + component: data-science-pipelines +data: + password: dGVzdGRic2VjcmV0cHN3dmFsdWUz +type: Opaque diff --git a/controllers/testdata/declarative/case_3/expected/created/storage_secret.yaml b/controllers/testdata/declarative/case_3/expected/not_created/storage_secret.yaml similarity index 60% rename from controllers/testdata/declarative/case_3/expected/created/storage_secret.yaml rename to controllers/testdata/declarative/case_3/expected/not_created/storage_secret.yaml index 36d77f689..6cc5af7a5 100644 --- a/controllers/testdata/declarative/case_3/expected/created/storage_secret.yaml +++ b/controllers/testdata/declarative/case_3/expected/not_created/storage_secret.yaml @@ -1,8 +1,9 @@ kind: Secret apiVersion: v1 metadata: - name: mlpipeline-minio-artifact - namespace: default + # todo: remove todo- this should actually be checked for but causes failures because previous tests don't clean up properly + name: todo-ds-pipeline-s3-testdsp3 + namespace: namespace3 labels: app: minio-extrenal-storage component: data-science-pipelines diff --git a/controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml index 94524294c..04e0af2ea 100644 --- a/controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml @@ -72,18 +72,24 @@ spec: value: "ds-pipeline-visualizationserver" - name: ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT value: "8888" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRET + value: "ds-pipeline-s3-testdsp4" + - name: OBJECTSTORECONFIG_CREDENTIALSACCESSKEYKEY + value: "accesskey" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRETKEYKEY + value: "secretkey" - name: OBJECTSTORECONFIG_BUCKETNAME value: "mlpipeline" - name: OBJECTSTORECONFIG_ACCESSKEY valueFrom: secretKeyRef: key: "accesskey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp4" - name: OBJECTSTORECONFIG_SECRETACCESSKEY valueFrom: secretKeyRef: key: "secretkey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp4" - name: OBJECTSTORECONFIG_SECURE value: "false" - name: MINIO_SERVICE_SERVICE_HOST diff --git a/controllers/testdata/declarative/case_4/expected/created/minio_deployment.yaml b/controllers/testdata/declarative/case_4/expected/created/minio_deployment.yaml index df5699578..0ea9304b2 100644 --- a/controllers/testdata/declarative/case_4/expected/created/minio_deployment.yaml +++ b/controllers/testdata/declarative/case_4/expected/created/minio_deployment.yaml @@ -31,12 +31,12 @@ spec: valueFrom: secretKeyRef: key: "accesskey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp4" - name: MINIO_SECRET_KEY valueFrom: secretKeyRef: key: "secretkey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp4" image: this-minio-image-from-cr-should-be-used:test4 name: minio ports: diff --git a/controllers/testdata/declarative/case_4/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_4/expected/created/mlpipelines-ui_deployment.yaml index 131b3cca3..15a5bdb5f 100644 --- a/controllers/testdata/declarative/case_4/expected/created/mlpipelines-ui_deployment.yaml +++ b/controllers/testdata/declarative/case_4/expected/created/mlpipelines-ui_deployment.yaml @@ -35,12 +35,12 @@ spec: valueFrom: secretKeyRef: key: "accesskey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp4" - name: MINIO_SECRET_KEY valueFrom: secretKeyRef: key: "secretkey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp4" - name: ALLOW_CUSTOM_VISUALIZATIONS value: "true" - name: ARGO_ARCHIVE_LOGS diff --git a/controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml index 92f6ac5b9..90c68bdbe 100644 --- a/controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml @@ -72,18 +72,24 @@ spec: value: "ds-pipeline-visualizationserver" - name: ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT value: "8888" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRET + value: "ds-pipeline-s3-testdsp5" + - name: OBJECTSTORECONFIG_CREDENTIALSACCESSKEYKEY + value: "accesskey" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRETKEYKEY + value: "secretkey" - name: OBJECTSTORECONFIG_BUCKETNAME value: "mlpipeline" - name: OBJECTSTORECONFIG_ACCESSKEY valueFrom: secretKeyRef: key: "accesskey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp5" - name: OBJECTSTORECONFIG_SECRETACCESSKEY valueFrom: secretKeyRef: key: "secretkey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp5" - name: OBJECTSTORECONFIG_SECURE value: "false" - name: MINIO_SERVICE_SERVICE_HOST diff --git a/controllers/testdata/declarative/case_5/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_5/expected/created/mlpipelines-ui_deployment.yaml index da6900631..5d8c14e0a 100644 --- a/controllers/testdata/declarative/case_5/expected/created/mlpipelines-ui_deployment.yaml +++ b/controllers/testdata/declarative/case_5/expected/created/mlpipelines-ui_deployment.yaml @@ -35,12 +35,12 @@ spec: valueFrom: secretKeyRef: key: "accesskey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp5" - name: MINIO_SECRET_KEY valueFrom: secretKeyRef: key: "secretkey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp5" - name: ALLOW_CUSTOM_VISUALIZATIONS value: "true" - name: ARGO_ARCHIVE_LOGS diff --git a/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml index 96c5967e8..897a3a3b8 100644 --- a/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml @@ -78,18 +78,24 @@ spec: value: "ds-pipeline-visualizationserver" - name: ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT value: "8888" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRET + value: "ds-pipeline-s3-testdsp6" + - name: OBJECTSTORECONFIG_CREDENTIALSACCESSKEYKEY + value: "accesskey" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRETKEYKEY + value: "secretkey" - name: OBJECTSTORECONFIG_BUCKETNAME value: "mlpipeline" - name: OBJECTSTORECONFIG_ACCESSKEY valueFrom: secretKeyRef: key: "accesskey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp6" - name: OBJECTSTORECONFIG_SECRETACCESSKEY valueFrom: secretKeyRef: key: "secretkey" - name: "mlpipeline-minio-artifact" + name: "ds-pipeline-s3-testdsp6" - name: OBJECTSTORECONFIG_SECURE value: "false" - name: MINIO_SERVICE_SERVICE_HOST diff --git a/controllers/testdata/declarative/case_6/expected/created/configmap_artifact_script.yaml b/controllers/testdata/declarative/case_6/expected/created/configmap_artifact_script.yaml index 2cbb8402e..60baaf581 100644 --- a/controllers/testdata/declarative/case_6/expected/created/configmap_artifact_script.yaml +++ b/controllers/testdata/declarative/case_6/expected/created/configmap_artifact_script.yaml @@ -38,5 +38,5 @@ metadata: name: ds-pipeline-artifact-script-testdsp6 namespace: default labels: - app: ds-pipeline-testdsp5 + app: ds-pipeline-testdsp6 component: data-science-pipelines