From ecc93f81b974f30640aadbf2c54198f8b3138600 Mon Sep 17 00:00:00 2001 From: Giulio Frasca Date: Tue, 23 May 2023 03:19:40 -0400 Subject: [PATCH] Create DevTools in DSPA CR, Add Database/Storage Secret Creation --- api/v1alpha1/dspipeline_types.go | 12 +++++ api/v1alpha1/zz_generated.deepcopy.go | 20 +++++++ ...b.io_datasciencepipelinesapplications.yaml | 11 ++++ .../database.secret.yaml.tmpl} | 0 .../storage.secret.yaml.tmpl} | 0 controllers/database.go | 1 - controllers/developer_tools.go | 53 +++++++++++++++++++ controllers/dspipeline_controller.go | 5 ++ controllers/dspipeline_params.go | 2 + controllers/storage.go | 1 - .../declarative/case_3/deploy/02_cr.yaml | 3 ++ 11 files changed, 106 insertions(+), 2 deletions(-) rename config/internal/{mariadb/secret.yaml.tmpl => devtools/database.secret.yaml.tmpl} (100%) rename config/internal/{minio/secret.yaml.tmpl => devtools/storage.secret.yaml.tmpl} (100%) create mode 100644 controllers/developer_tools.go diff --git a/api/v1alpha1/dspipeline_types.go b/api/v1alpha1/dspipeline_types.go index 89b0b59a4..1bd469976 100644 --- a/api/v1alpha1/dspipeline_types.go +++ b/api/v1alpha1/dspipeline_types.go @@ -35,6 +35,9 @@ type DSPASpec struct { *MlPipelineUI `json:"mlpipelineUI"` // +kubebuilder:validation:Required *ObjectStorage `json:"objectStorage"` + // DevTools enables non-production tools and utilities for Data Science Pipelines Operator developers + // +kubebuilder:validation:Optional + *DevTools `json:"devtools"` } type APIServer struct { @@ -164,6 +167,15 @@ type Minio struct { Image string `json:"image"` } +type DevTools struct { + // +kubebuilder:default:=false + // +kubebuilder:validation:Optional + EnableDatabaseSecret bool `json:"enableDatabaseSecret"` + // +kubebuilder:default:=false + // +kubebuilder:validation:Optional + EnableStorageSecret bool `json:"enableStorageSecret"` +} + // ResourceRequirements structures compute resource requirements. // Replaces ResourceRequirements from corev1 which also includes optional storage field. // We handle storage field separately, and should not include it as a subfield for Resources. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 8d32f5029..af0077df0 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -99,6 +99,11 @@ func (in *DSPASpec) DeepCopyInto(out *DSPASpec) { *out = new(ObjectStorage) (*in).DeepCopyInto(*out) } + if in.DevTools != nil { + in, out := &in.DevTools, &out.DevTools + *out = new(DevTools) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DSPASpec. @@ -217,6 +222,21 @@ func (in *Database) DeepCopy() *Database { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DevTools) DeepCopyInto(out *DevTools) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DevTools. +func (in *DevTools) DeepCopy() *DevTools { + if in == nil { + return nil + } + out := new(DevTools) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExternalDB) DeepCopyInto(out *ExternalDB) { *out = *in diff --git a/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml b/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml index e11642bf2..56ade90e0 100644 --- a/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml +++ b/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml @@ -235,6 +235,17 @@ spec: type: string type: object type: object + devtools: + description: DevTools enables non-production tools and utilities for + Data Science Pipelines Operator developers + properties: + enableDatabaseSecret: + default: false + type: boolean + enableStorageSecret: + default: false + type: boolean + type: object mlpipelineUI: properties: configMap: diff --git a/config/internal/mariadb/secret.yaml.tmpl b/config/internal/devtools/database.secret.yaml.tmpl similarity index 100% rename from config/internal/mariadb/secret.yaml.tmpl rename to config/internal/devtools/database.secret.yaml.tmpl diff --git a/config/internal/minio/secret.yaml.tmpl b/config/internal/devtools/storage.secret.yaml.tmpl similarity index 100% rename from config/internal/minio/secret.yaml.tmpl rename to config/internal/devtools/storage.secret.yaml.tmpl diff --git a/controllers/database.go b/controllers/database.go index ae1e44537..984a8106a 100644 --- a/controllers/database.go +++ b/controllers/database.go @@ -26,7 +26,6 @@ var dbTemplates = []string{ "mariadb/pvc.yaml.tmpl", "mariadb/sa.yaml.tmpl", "mariadb/service.yaml.tmpl", - "mariadb/secret.yaml.tmpl", } func (r *DSPAReconciler) ReconcileDatabase(ctx context.Context, dsp *dspav1alpha1.DataSciencePipelinesApplication, diff --git a/controllers/developer_tools.go b/controllers/developer_tools.go new file mode 100644 index 000000000..4031b83be --- /dev/null +++ b/controllers/developer_tools.go @@ -0,0 +1,53 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + dspav1alpha1 "github.com/opendatahub-io/data-science-pipelines-operator/api/v1alpha1" +) + +const dbSecret = "devtools/database.secret.yaml.tmpl" +const storageSecret = "devtools/storage.secret.yaml.tmpl" + +func (r *DSPAReconciler) ReconcileDevtools(dsp *dspav1alpha1.DataSciencePipelinesApplication, params *DSPAParams) error { + + log := r.Log.WithValues("namespace", dsp.Namespace).WithValues("dspa_name", dsp.Name) + + if dsp.Spec.DevTools != nil { + log.Info("Applying DevTool Resources") + + dbSecretEnabled := dsp.Spec.DevTools.EnableDatabaseSecret + storageSecretEnabled := dsp.Spec.DevTools.EnableStorageSecret + + if dbSecretEnabled { + log.Info("Database secret creation requested") + err := r.Apply(dsp, params, dbSecret) + if err != nil { + return err + } + } + + if storageSecretEnabled { + log.Info("Object Storage secret creation requested") + err := r.Apply(dsp, params, storageSecret) + if err != nil { + return err + } + } + log.Info("Finished applying DevTool Resources") + } + return nil +} diff --git a/controllers/dspipeline_controller.go b/controllers/dspipeline_controller.go index 730bb7b84..2a6010a8b 100644 --- a/controllers/dspipeline_controller.go +++ b/controllers/dspipeline_controller.go @@ -278,6 +278,11 @@ func (r *DSPAReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. return ctrl.Result{}, err } + err = r.ReconcileDevtools(dspa, params) + if err != nil { + return ctrl.Result{}, err + } + log.Info("Updating CR status") // Refresh DSPA before updating err = r.Get(ctx, req.NamespacedName, dspa) diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go index 09dbf61a1..8a5f127b2 100644 --- a/controllers/dspipeline_params.go +++ b/controllers/dspipeline_params.go @@ -46,6 +46,7 @@ type DSPAParams struct { MlPipelineUI *dspa.MlPipelineUI MariaDB *dspa.MariaDB Minio *dspa.Minio + DevTools *dspa.DevTools DBConnection ObjectStorageConnection } @@ -360,6 +361,7 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip p.MlPipelineUI = dsp.Spec.MlPipelineUI.DeepCopy() p.MariaDB = dsp.Spec.MariaDB.DeepCopy() p.Minio = dsp.Spec.Minio.DeepCopy() + p.DevTools = dsp.Spec.DevTools.DeepCopy() p.OAuthProxy = config.GetStringConfigWithDefault(config.OAuthProxyImagePath, config.DefaultImageValue) // TODO: If p. is nil we should create defaults diff --git a/controllers/storage.go b/controllers/storage.go index 48719ce7f..1f782f4a6 100644 --- a/controllers/storage.go +++ b/controllers/storage.go @@ -26,7 +26,6 @@ var storageTemplates = []string{ "minio/deployment.yaml.tmpl", "minio/pvc.yaml.tmpl", "minio/service.yaml.tmpl", - "minio/secret.yaml.tmpl", } // ReconcileStorage will set up Storage Connection. diff --git a/controllers/testdata/declarative/case_3/deploy/02_cr.yaml b/controllers/testdata/declarative/case_3/deploy/02_cr.yaml index 0b63880e7..4122613fe 100644 --- a/controllers/testdata/declarative/case_3/deploy/02_cr.yaml +++ b/controllers/testdata/declarative/case_3/deploy/02_cr.yaml @@ -34,3 +34,6 @@ spec: mlpipelineUI: deploy: false image: frontend:test3 + devtools: + enableDatabaseSecret: true + enableStorageSecret: true