From 1b07bff7edf8425df061ebfe99386012c6b50045 Mon Sep 17 00:00:00 2001 From: Humair Khan Date: Sun, 21 Jan 2024 19:51:51 -0500 Subject: [PATCH 1/3] remove unused v2 images. Signed-off-by: Humair Khan --- config/base/kustomization.yaml | 21 ------------------- config/base/params.env | 3 --- config/configmaps/files/config.yaml | 3 --- .../apiserver/default/deployment.yaml.tmpl | 14 ++++++------- config/manager/manager.yaml | 6 ------ controllers/dspipeline_params.go | 6 +++--- .../created/apiserver_deployment.yaml | 12 +++++------ .../created/apiserver_deployment.yaml | 12 +++++------ .../created/apiserver_deployment.yaml | 12 +++++------ .../created/apiserver_deployment.yaml | 12 +++++------ .../created/apiserver_deployment.yaml | 12 +++++------ .../created/apiserver_deployment.yaml | 12 +++++------ .../created/apiserver_deployment.yaml | 4 ---- 13 files changed, 46 insertions(+), 83 deletions(-) diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml index 9e464abac..dc17f39b1 100644 --- a/config/base/kustomization.yaml +++ b/config/base/kustomization.yaml @@ -147,13 +147,6 @@ vars: apiVersion: v1 fieldref: fieldpath: data.IMAGESV2_ARGO_APISERVER - - name: IMAGESV2_ARGO_ARTIFACT - objref: - kind: ConfigMap - name: dspo-parameters - apiVersion: v1 - fieldref: - fieldpath: data.IMAGESV2_ARGO_ARTIFACT - name: IMAGESV2_ARGO_PERSISTENCEAGENT objref: kind: ConfigMap @@ -168,20 +161,6 @@ vars: apiVersion: v1 fieldref: fieldpath: data.IMAGESV2_ARGO_SCHEDULEDWORKFLOW - - name: IMAGESV2_ARGO_CACHE - objref: - kind: ConfigMap - name: dspo-parameters - apiVersion: v1 - fieldref: - fieldpath: data.IMAGESV2_ARGO_CACHE - - name: IMAGESV2_ARGO_MOVERESULTSIMAGE - objref: - kind: ConfigMap - name: dspo-parameters - apiVersion: v1 - fieldref: - fieldpath: data.IMAGESV2_ARGO_MOVERESULTSIMAGE - name: IMAGESV2_ARGO_MLMDENVOY objref: kind: ConfigMap diff --git a/config/base/params.env b/config/base/params.env index 603d23a95..a435ba54d 100644 --- a/config/base/params.env +++ b/config/base/params.env @@ -8,14 +8,11 @@ IMAGES_MLMDGRPC=quay.io/opendatahub/ds-pipelines-metadata-grpc:v1.6.1 IMAGES_MLMDWRITER=quay.io/opendatahub/ds-pipelines-metadata-writer:v1.6.1 IMAGES_CRDVIEWER=gcr.io/ml-pipeline/viewer-crd-controller:2.0.0-rc.2 IMAGESV2_ARGO_APISERVER=quay.io/opendatahub/ds-pipelines-api-server:latest -IMAGESV2_ARGO_ARTIFACT=quay.io/opendatahub/ds-pipelines-artifact-manager:latest IMAGESV2_ARGO_PERSISTENCEAGENT=quay.io/opendatahub/ds-pipelines-persistenceagent:latest IMAGESV2_ARGO_SCHEDULEDWORKFLOW=quay.io/opendatahub/ds-pipelines-scheduledworkflow:latest IMAGESV2_ARGO_MLMDENVOY=quay.io/opendatahub/ds-pipelines-metadata-envoy:latest IMAGESV2_ARGO_MLMDGRPC=quay.io/opendatahub/ds-pipelines-metadata-grpc:latest IMAGESV2_ARGO_MLMDWRITER=quay.io/opendatahub/ds-pipelines-metadata-writer:latest -IMAGESV2_ARGO_CACHE=registry.access.redhat.com/ubi8/ubi-minimal:8.8 -IMAGESV2_ARGO_MOVERESULTSIMAGE=registry.access.redhat.com/ubi8/ubi-micro:8.8 IMAGESV2_ARGO_WORKFLOWCONTROLLER=quay.io/opendatahub/ds-pipelines-argo-workflowcontroller:3.3.10-upstream IMAGESV2_ARGO_ARGOEXEC=quay.io/opendatahub/ds-pipelines-argo-argoexec:3.3.10-upstream V2_LAUNCHER_IMAGE=quay.io/opendatahub/ds-pipelines-launcher:latest diff --git a/config/configmaps/files/config.yaml b/config/configmaps/files/config.yaml index 81c0d7993..83cd51f43 100644 --- a/config/configmaps/files/config.yaml +++ b/config/configmaps/files/config.yaml @@ -14,9 +14,6 @@ Images: ImagesV2: Argo: ApiServer: $(IMAGESV2_ARGO_APISERVER) - Artifact: $(IMAGESV2_ARGO_ARTIFACT) - Cache: $(IMAGESV2_ARGO_CACHE) - MoveResultsImage: $(IMAGESV2_ARGO_MOVERESULTSIMAGE) PersistentAgent: $(IMAGESV2_ARGO_PERSISTENCEAGENT) ScheduledWorkflow: $(IMAGESV2_ARGO_SCHEDULEDWORKFLOW) MlmdEnvoy: $(IMAGESV2_ARGO_MLMDENVOY) diff --git a/config/internal/apiserver/default/deployment.yaml.tmpl b/config/internal/apiserver/default/deployment.yaml.tmpl index e309bb70e..b94c09b74 100644 --- a/config/internal/apiserver/default/deployment.yaml.tmpl +++ b/config/internal/apiserver/default/deployment.yaml.tmpl @@ -83,10 +83,6 @@ spec: value: "{{.ObjectStorageConnection.Host}}" - name: MINIO_SERVICE_SERVICE_PORT value: "{{.ObjectStorageConnection.Port}}" - - name: CACHE_IMAGE - value: "{{.APIServer.CacheImage}}" - - name: MOVERESULTS_IMAGE - value: "{{.APIServer.MoveResultsImage}}" - name: V2_LAUNCHER_IMAGE value: "{{.APIServer.ArgoLauncherImage}}" - name: V2_DRIVER_IMAGE @@ -96,7 +92,7 @@ spec: {{ if .MLMD.Deploy }} - name: METADATA_GRPC_SERVICE_SERVICE_HOST value: "ds-pipeline-metadata-grpc-{{.Name}}.{{.Namespace}}.svc.cluster.local" - {{ if.MLMD.GRPC.Port }} + {{ if .MLMD.GRPC.Port }} - name: METADATA_GRPC_SERVICE_SERVICE_PORT value: "{{.MLMD.GRPC.Port}}" {{ end }} @@ -129,6 +125,12 @@ spec: ## Tekton-Specific Env Vars ## - name: EXECUTIONTYPE value: PipelineRun + - name: CACHE_IMAGE + value: "{{.APIServer.CacheImage}}" + - name: MOVERESULTS_IMAGE + value: "{{.APIServer.MoveResultsImage}}" + - name: ARTIFACT_IMAGE + value: "{{.APIServer.ArtifactImage}}" - name: ARTIFACT_BUCKET value: "{{.ObjectStorageConnection.Bucket}}" - name: ARTIFACT_ENDPOINT @@ -138,8 +140,6 @@ spec: configMapKeyRef: key: "{{ .APIServer.ArtifactScriptConfigMap.Key }}" name: "{{ .APIServer.ArtifactScriptConfigMap.Name }}" - - name: ARTIFACT_IMAGE - value: "{{.APIServer.ArtifactImage}}" - name: ARCHIVE_LOGS value: "{{.APIServer.ArchiveLogs}}" - name: TRACK_ARTIFACTS diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index db9280fb4..9811e1ae1 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -63,16 +63,10 @@ spec: value: $(IMAGES_CRDVIEWER) - name: IMAGESV2_ARGO_APISERVER value: $(IMAGESV2_ARGO_APISERVER) - - name: IMAGESV2_ARGO_ARTIFACT - value: $(IMAGESV2_ARGO_ARTIFACT) - name: IMAGESV2_ARGO_PERSISTENCEAGENT value: $(IMAGESV2_ARGO_PERSISTENCEAGENT) - name: IMAGESV2_ARGO_SCHEDULEDWORKFLOW value: $(IMAGESV2_ARGO_SCHEDULEDWORKFLOW) - - name: IMAGESV2_ARGO_CACHE - value: $(IMAGESV2_ARGO_CACHE) - - name: IMAGESV2_ARGO_MOVERESULTSIMAGE - value: $(IMAGESV2_ARGO_MOVERESULTSIMAGE) - name: IMAGESV2_ARGO_MLMDENVOY value: $(IMAGESV2_ARGO_MLMDENVOY) - name: IMAGESV2_ARGO_MLMDGRPC diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go index dd4dfc736..c4a16b690 100644 --- a/controllers/dspipeline_params.go +++ b/controllers/dspipeline_params.go @@ -488,9 +488,9 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip if p.APIServer != nil { APIServerImagePath := p.GetImageForComponent(dsp, config.APIServerImagePath, config.APIServerImagePathV2Argo, config.APIServerImagePathV2Tekton) - APIServerArtifactImagePath := p.GetImageForComponent(dsp, config.APIServerArtifactImagePath, config.APIServerArtifactImagePathV2Argo, config.APIServerArtifactImagePathV2Tekton) - APIServerCacheImagePath := p.GetImageForComponent(dsp, config.APIServerCacheImagePath, config.APIServerCacheImagePathV2Argo, config.APIServerCacheImagePathV2Tekton) - APIServerMoveResultsImagePath := p.GetImageForComponent(dsp, config.APIServerMoveResultsImagePath, config.APIServerMoveResultsImagePathV2Argo, config.APIServerMoveResultsImagePathV2Tekton) + APIServerArtifactImagePath := config.APIServerArtifactImagePath + APIServerCacheImagePath := config.APIServerCacheImagePath + APIServerMoveResultsImagePath := config.APIServerMoveResultsImagePath APIServerArgoLauncherImagePath := config.APIServerArgoLauncherImagePathV2Argo APIServerArgoDriverImagePath := config.APIServerArgoDriverImagePathV2Argo diff --git a/controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml index 410f7291c..553cba541 100644 --- a/controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml @@ -71,10 +71,6 @@ spec: value: "minio-testdsp0.default.svc.cluster.local" - name: MINIO_SERVICE_SERVICE_PORT value: "9000" - - name: CACHE_IMAGE - value: "ubi-minimal:test0" - - name: MOVERESULTS_IMAGE - value: "busybox:test0" - name: V2_LAUNCHER_IMAGE value: "argolauncherimage:test0" - name: V2_DRIVER_IMAGE @@ -85,6 +81,12 @@ spec: value: "8887" - name: EXECUTIONTYPE value: PipelineRun + - name: CACHE_IMAGE + value: "ubi-minimal:test0" + - name: MOVERESULTS_IMAGE + value: "busybox:test0" + - name: ARTIFACT_IMAGE + value: "artifact-manager:test0" - name: ARTIFACT_BUCKET value: "mlpipeline" - name: ARTIFACT_ENDPOINT @@ -94,8 +96,6 @@ spec: configMapKeyRef: key: "artifact_script" name: "ds-pipeline-artifact-script-testdsp0" - - name: ARTIFACT_IMAGE - value: "artifact-manager:test0" - name: ARCHIVE_LOGS value: "false" - name: TRACK_ARTIFACTS diff --git a/controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml index 0b1b13624..67037dbc0 100644 --- a/controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml @@ -71,10 +71,6 @@ spec: value: "minio-testdsp2.default.svc.cluster.local" - name: MINIO_SERVICE_SERVICE_PORT value: "9000" - - name: CACHE_IMAGE - value: "ubi-minimal:test2" - - name: MOVERESULTS_IMAGE - value: "busybox:test2" - name: V2_LAUNCHER_IMAGE value: "argolauncherimage:test2" - name: V2_DRIVER_IMAGE @@ -85,6 +81,12 @@ spec: value: "8887" - name: EXECUTIONTYPE value: PipelineRun + - name: CACHE_IMAGE + value: "ubi-minimal:test2" + - name: MOVERESULTS_IMAGE + value: "busybox:test2" + - name: ARTIFACT_IMAGE + value: "artifact-manager:test2" - name: ARTIFACT_BUCKET value: "mlpipeline" - name: ARTIFACT_ENDPOINT @@ -94,8 +96,6 @@ spec: configMapKeyRef: key: "artifact_script" name: "ds-pipeline-artifact-script-testdsp2" - - name: ARTIFACT_IMAGE - value: "artifact-manager:test2" - name: ARCHIVE_LOGS value: "false" - name: TRACK_ARTIFACTS diff --git a/controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml index 8f09908ae..48ffd9e53 100644 --- a/controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml @@ -71,10 +71,6 @@ spec: value: "teststoragehost3" - name: MINIO_SERVICE_SERVICE_PORT value: "80" - - name: CACHE_IMAGE - value: ubi-minimal:test3 - - name: MOVERESULTS_IMAGE - value: busybox:test3 - name: V2_LAUNCHER_IMAGE value: "argolauncherimage:test3" - name: V2_DRIVER_IMAGE @@ -85,6 +81,12 @@ spec: value: "8887" - name: EXECUTIONTYPE value: PipelineRun + - name: CACHE_IMAGE + value: ubi-minimal:test3 + - name: MOVERESULTS_IMAGE + value: busybox:test3 + - name: ARTIFACT_IMAGE + value: artifact-manager:test3 - name: ARTIFACT_BUCKET value: "testbucket3" - name: ARTIFACT_ENDPOINT @@ -94,8 +96,6 @@ spec: configMapKeyRef: key: "somekey" name: "doesnotexist" - - name: ARTIFACT_IMAGE - value: artifact-manager:test3 - name: ARCHIVE_LOGS value: "false" - name: TRACK_ARTIFACTS diff --git a/controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml index c8600c208..208d49808 100644 --- a/controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml @@ -71,10 +71,6 @@ spec: value: "minio-testdsp4.default.svc.cluster.local" - name: MINIO_SERVICE_SERVICE_PORT value: "9000" - - name: CACHE_IMAGE - value: "this-ubi-minimal-image-from-cr-should-be-used:test4" - - name: MOVERESULTS_IMAGE - value: "this-busybox-image-from-cr-should-be-used:test4" - name: V2_LAUNCHER_IMAGE value: "this-argolauncher-image-from-cr-should-be-used:test4" - name: V2_DRIVER_IMAGE @@ -85,6 +81,12 @@ spec: value: "8887" - name: EXECUTIONTYPE value: PipelineRun + - name: CACHE_IMAGE + value: "this-ubi-minimal-image-from-cr-should-be-used:test4" + - name: MOVERESULTS_IMAGE + value: "this-busybox-image-from-cr-should-be-used:test4" + - name: ARTIFACT_IMAGE + value: "this-artifact-manager-image-from-cr-should-be-used:test4" - name: ARTIFACT_BUCKET value: "mlpipeline" - name: ARTIFACT_ENDPOINT @@ -94,8 +96,6 @@ spec: configMapKeyRef: key: "artifact_script" name: "ds-pipeline-artifact-script-testdsp4" - - name: ARTIFACT_IMAGE - value: "this-artifact-manager-image-from-cr-should-be-used:test4" - name: ARCHIVE_LOGS value: "false" - name: TRACK_ARTIFACTS diff --git a/controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml index fa2cd0cd1..2836a968e 100644 --- a/controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml @@ -71,10 +71,6 @@ spec: value: "minio-testdsp5.default.svc.cluster.local" - name: MINIO_SERVICE_SERVICE_PORT value: "9000" - - name: CACHE_IMAGE - value: "ubi-minimal:test5" - - name: MOVERESULTS_IMAGE - value: "busybox:test5" - name: V2_LAUNCHER_IMAGE value: "argolauncherimage:test5" - name: V2_DRIVER_IMAGE @@ -89,6 +85,12 @@ spec: value: "8887" - name: EXECUTIONTYPE value: PipelineRun + - name: CACHE_IMAGE + value: "ubi-minimal:test5" + - name: MOVERESULTS_IMAGE + value: "busybox:test5" + - name: ARTIFACT_IMAGE + value: "artifact-manager:test5" - name: ARTIFACT_BUCKET value: "mlpipeline" - name: ARTIFACT_ENDPOINT @@ -98,8 +100,6 @@ spec: configMapKeyRef: key: "artifact_script" name: "ds-pipeline-artifact-script-testdsp5" - - name: ARTIFACT_IMAGE - value: "artifact-manager:test5" - name: ARCHIVE_LOGS value: "false" - name: TRACK_ARTIFACTS diff --git a/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml index fb65830ab..425388bf7 100644 --- a/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml @@ -77,10 +77,6 @@ spec: value: "minio-testdsp6.default.svc.cluster.local" - name: MINIO_SERVICE_SERVICE_PORT value: "9000" - - name: CACHE_IMAGE - value: "ubi-minimal:test6" - - name: MOVERESULTS_IMAGE - value: "busybox:test6" - name: V2_LAUNCHER_IMAGE value: "argolauncherimage:test6" - name: V2_DRIVER_IMAGE @@ -91,6 +87,12 @@ spec: value: "8887" - name: EXECUTIONTYPE value: PipelineRun + - name: CACHE_IMAGE + value: "ubi-minimal:test6" + - name: MOVERESULTS_IMAGE + value: "busybox:test6" + - name: ARTIFACT_IMAGE + value: "artifact-manager:test6" - name: ARTIFACT_BUCKET value: "mlpipeline" - name: ARTIFACT_ENDPOINT @@ -100,8 +102,6 @@ spec: configMapKeyRef: key: "artifact_script" name: "ds-pipeline-artifact-script-testdsp6" - - name: ARTIFACT_IMAGE - value: "artifact-manager:test6" - name: ARCHIVE_LOGS value: "false" - name: TRACK_ARTIFACTS diff --git a/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml index bbd8fb1d0..9bc3f5d2a 100644 --- a/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml @@ -71,10 +71,6 @@ spec: value: "minio-testdsp7.default.svc.cluster.local" - name: MINIO_SERVICE_SERVICE_PORT value: "9000" - - name: CACHE_IMAGE - value: "ubi-minimal:test7" - - name: MOVERESULTS_IMAGE - value: "busybox:test7" - name: V2_LAUNCHER_IMAGE value: "argolauncherimage:test7" - name: V2_DRIVER_IMAGE From 8fd761461a8de7412a7614579be6bee79af76d97 Mon Sep 17 00:00:00 2001 From: Humair Khan Date: Sun, 21 Jan 2024 20:20:15 -0500 Subject: [PATCH 2/3] change default argo ns to opendatahub. Signed-off-by: Humair Khan --- Makefile | 2 +- config/overlays/make-argodeploy/kustomization.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ab283eac2..a404df2af 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,7 @@ OPERATOR_NS ?= opendatahub # Namespace to deploy v2 infrastructure V2INFRA_NS ?= openshift-pipelines # Namespace to deploy argo infrastructure -ARGO_NS ?= argo +ARGO_NS ?= opendatahub # Integration Test ENVvars KUBECONFIGPATH ?= $(HOME)/.kube/config diff --git a/config/overlays/make-argodeploy/kustomization.yaml b/config/overlays/make-argodeploy/kustomization.yaml index 28933e3c5..43a08329f 100644 --- a/config/overlays/make-argodeploy/kustomization.yaml +++ b/config/overlays/make-argodeploy/kustomization.yaml @@ -1,6 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -namespace: argo +namespace: opendatahub resources: - ../../argo From 2ec61f910235a727b9b1315466ea44caf28031c5 Mon Sep 17 00:00:00 2001 From: Humair Khan Date: Sun, 21 Jan 2024 20:21:44 -0500 Subject: [PATCH 3/3] expose requeue time var. Signed-off-by: Humair Khan --- config/base/kustomization.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml index dc17f39b1..386a89393 100644 --- a/config/base/kustomization.yaml +++ b/config/base/kustomization.yaml @@ -112,6 +112,13 @@ vars: apiVersion: v1 fieldref: fieldpath: data.ZAP_LOG_LEVEL + - name: DSPO_REQUEUE_TIME + objref: + kind: ConfigMap + name: dspo-parameters + apiVersion: v1 + fieldref: + fieldpath: data.DSPO_REQUEUE_TIME - name: MAX_CONCURRENT_RECONCILES objref: kind: ConfigMap