diff --git a/api/v1alpha1/dspipeline_types.go b/api/v1alpha1/dspipeline_types.go index da3b4d6ba..5a6bee3c7 100644 --- a/api/v1alpha1/dspipeline_types.go +++ b/api/v1alpha1/dspipeline_types.go @@ -107,7 +107,9 @@ type APIServer struct { // Default: true // +kubebuilder:default:=true // +kubebuilder:validation:Optional - AutoUpdatePipelineDefaultVersion bool `json:"autoUpdatePipelineDefaultVersion"` + AutoUpdatePipelineDefaultVersion bool `json:"autoUpdatePipelineDefaultVersion"` + ArgoLauncherImage string `json:"argoLauncherImage,omitempty"` + ArgoDriverImage string `json:"argoDriverImage,omitempty"` // Specify custom Pod resource requirements for this component. Resources *ResourceRequirements `json:"resources,omitempty"` diff --git a/config/base/kustomization.yaml b/config/base/kustomization.yaml index 756c4bd89..b5321e096 100644 --- a/config/base/kustomization.yaml +++ b/config/base/kustomization.yaml @@ -203,5 +203,19 @@ vars: apiVersion: v1 fieldref: fieldpath: data.IMAGESV2_ARGO_WORKFLOWCONTROLLER + - name: V2_LAUNCHER_IMAGE + objref: + kind: ConfigMap + name: dspo-parameters + apiVersion: v1 + fieldref: + fieldpath: data.V2_LAUNCHER_IMAGE + - name: V2_DRIVER_IMAGE + objref: + kind: ConfigMap + name: dspo-parameters + apiVersion: v1 + fieldref: + fieldpath: data.V2_DRIVER_IMAGE configurations: - params.yaml diff --git a/config/base/params.env b/config/base/params.env index befd3e162..83524c475 100644 --- a/config/base/params.env +++ b/config/base/params.env @@ -21,6 +21,8 @@ IMAGESV2_ARGO_MLMDWRITER=gcr.io/ml-pipeline/metadata-writer:2.0.2 IMAGESV2_ARGO_CACHE=registry.access.redhat.com/ubi8/ubi-minimal:8.7 IMAGESV2_ARGO_MOVERESULTSIMAGE=registry.access.redhat.com/ubi8/ubi-micro:8.7 IMAGESV2_ARGO_WORKFLOWCONTROLLER=gcr.io/ml-pipeline/workflow-controller:v3.3.10-license-compliance +V2_LAUNCHER_IMAGE=gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 +V2_DRIVER_IMAGE=gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707 ZAP_LOG_LEVEL=info MAX_CONCURRENT_RECONCILES=10 DSPO_HEALTHCHECK_DATABASE_CONNECTIONTIMEOUT=15s diff --git a/config/configmaps/files/config.yaml b/config/configmaps/files/config.yaml index a1d9fe056..b68c61810 100644 --- a/config/configmaps/files/config.yaml +++ b/config/configmaps/files/config.yaml @@ -22,6 +22,8 @@ ImagesV2: MlmdEnvoy: $(IMAGESV2_ARGO_MLMDENVOY) MlmdGRPC: $(IMAGESV2_ARGO_MLMDGRPC) WorkflowController: $(IMAGESV2_ARGO_WORKFLOWCONTROLLER) + ArgoLauncherImage: $(V2_LAUNCHER_IMAGE) + ArgoDriverImage: $(V2_DRIVER_IMAGE) DSPO: HealthCheck: Database: diff --git a/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml b/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml index 0519d7926..fbc0b4ec0 100644 --- a/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml +++ b/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml @@ -48,6 +48,10 @@ spec: default: false description: 'Default: false' type: boolean + argoDriverImage: + type: string + argoLauncherImage: + type: string artifactImage: type: string artifactScriptConfigMap: diff --git a/config/internal/apiserver/default/deployment.yaml.tmpl b/config/internal/apiserver/default/deployment.yaml.tmpl index 30f9bd191..7b1c48ebb 100644 --- a/config/internal/apiserver/default/deployment.yaml.tmpl +++ b/config/internal/apiserver/default/deployment.yaml.tmpl @@ -83,6 +83,10 @@ spec: value: "{{.APIServer.CacheImage}}" - name: MOVERESULTS_IMAGE value: "{{.APIServer.MoveResultsImage}}" + - name: V2_LAUNCHER_IMAGE + value: "{{.APIServer.ArgoLauncherImage}}" + - name: V2_DRIVER_IMAGE + value: "{{.APIServer.ArgoDriverImage}}" ## Env Vars to only include if MLMD Deployed ## {{ if .MLMD }} {{ if .MLMD.Deploy }} diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index a5fcad2b0..5685d8a85 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -79,6 +79,10 @@ spec: value: $(IMAGESV2_ARGO_MLMDGRPC) - name: IMAGESV2_ARGO_WORKFLOWCONTROLLER value: $(IMAGESV2_ARGO_WORKFLOWCONTROLLER) + - name: V2_LAUNCHER_IMAGE + value: $(V2_LAUNCHER_IMAGE) + - name: V2_DRIVER_IMAGE + value: $(V2_DRIVER_IMAGE) - name: ZAP_LOG_LEVEL value: $(ZAP_LOG_LEVEL) - name: MAX_CONCURRENT_RECONCILES diff --git a/config/samples/dspa_simple_v2.yaml b/config/samples/dspa_simple_v2.yaml index 489ec65e1..ba3266939 100644 --- a/config/samples/dspa_simple_v2.yaml +++ b/config/samples/dspa_simple_v2.yaml @@ -6,7 +6,9 @@ spec: dspVersion: v2 apiServer: deploy: true - image: gcr.io/ml-pipeline/api-server:2.0.2 + image: gcr.io/ml-pipeline/api-server:2.0.5 + argoLauncherImage: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + argoDriverImage: gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707 persistenceAgent: deploy: true image: gcr.io/ml-pipeline/persistenceagent:2.0.2 diff --git a/controllers/config/defaults.go b/controllers/config/defaults.go index 78c132e70..e7b6bd590 100644 --- a/controllers/config/defaults.go +++ b/controllers/config/defaults.go @@ -78,14 +78,16 @@ const ( // DSPV2-Argo Image Paths const ( - APIServerImagePathV2Argo = "ImagesV2.Argo.ApiServer" - APIServerArtifactImagePathV2Argo = "ImagesV2.Argo.Artifact" - APIServerCacheImagePathV2Argo = "ImagesV2.Argo.Cache" - APIServerMoveResultsImagePathV2Argo = "ImagesV2.Argo.MoveResultsImage" - PersistenceAgentImagePathV2Argo = "ImagesV2.Argo.PersistentAgent" - ScheduledWorkflowImagePathV2Argo = "ImagesV2.Argo.ScheduledWorkflow" - MlmdEnvoyImagePathV2Argo = "ImagesV2.Argo.MlmdEnvoy" - MlmdGRPCImagePathV2Argo = "ImagesV2.Argo.MlmdGRPC" + APIServerImagePathV2Argo = "ImagesV2.Argo.ApiServer" + APIServerArtifactImagePathV2Argo = "ImagesV2.Argo.Artifact" + APIServerCacheImagePathV2Argo = "ImagesV2.Argo.Cache" + APIServerMoveResultsImagePathV2Argo = "ImagesV2.Argo.MoveResultsImage" + APIServerArgoLauncherImagePathV2Argo = "ImagesV2.Argo.ArgoLauncherImage" + APIServerArgoDriverImagePathV2Argo = "ImagesV2.Argo.ArgoDriverImage" + PersistenceAgentImagePathV2Argo = "ImagesV2.Argo.PersistentAgent" + ScheduledWorkflowImagePathV2Argo = "ImagesV2.Argo.ScheduledWorkflow" + MlmdEnvoyImagePathV2Argo = "ImagesV2.Argo.MlmdEnvoy" + MlmdGRPCImagePathV2Argo = "ImagesV2.Argo.MlmdGRPC" ) // DSPV2-Tekton Image Paths diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go index 3d6e3d1c3..ae4ad339a 100644 --- a/controllers/dspipeline_params.go +++ b/controllers/dspipeline_params.go @@ -483,16 +483,22 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip APIServerArtifactImagePath := p.GetImageForComponent(dsp, config.APIServerArtifactImagePath, config.APIServerArtifactImagePathV2Argo, config.APIServerArtifactImagePathV2Tekton) APIServerCacheImagePath := p.GetImageForComponent(dsp, config.APIServerCacheImagePath, config.APIServerCacheImagePathV2Argo, config.APIServerCacheImagePathV2Tekton) APIServerMoveResultsImagePath := p.GetImageForComponent(dsp, config.APIServerMoveResultsImagePath, config.APIServerMoveResultsImagePathV2Argo, config.APIServerMoveResultsImagePathV2Tekton) + APIServerArgoLauncherImagePath := config.APIServerArgoLauncherImagePathV2Argo + APIServerArgoDriverImagePath := config.APIServerArgoDriverImagePathV2Argo serverImageFromConfig := config.GetStringConfigWithDefault(APIServerImagePath, config.DefaultImageValue) artifactImageFromConfig := config.GetStringConfigWithDefault(APIServerArtifactImagePath, config.DefaultImageValue) cacheImageFromConfig := config.GetStringConfigWithDefault(APIServerCacheImagePath, config.DefaultImageValue) moveResultsImageFromConfig := config.GetStringConfigWithDefault(APIServerMoveResultsImagePath, config.DefaultImageValue) + argoLauncherImageFromConfig := config.GetStringConfigWithDefault(APIServerArgoLauncherImagePath, config.DefaultImageValue) + argoDriverImageFromConfig := config.GetStringConfigWithDefault(APIServerArgoDriverImagePath, config.DefaultImageValue) setStringDefault(serverImageFromConfig, &p.APIServer.Image) setStringDefault(artifactImageFromConfig, &p.APIServer.ArtifactImage) setStringDefault(cacheImageFromConfig, &p.APIServer.CacheImage) setStringDefault(moveResultsImageFromConfig, &p.APIServer.MoveResultsImage) + setStringDefault(argoLauncherImageFromConfig, &p.APIServer.ArgoLauncherImage) + setStringDefault(argoDriverImageFromConfig, &p.APIServer.ArgoDriverImage) setResourcesDefault(config.APIServerResourceRequirements, &p.APIServer.Resources) diff --git a/controllers/testdata/declarative/case_0/deploy/cr.yaml b/controllers/testdata/declarative/case_0/deploy/cr.yaml index 4b027be8e..55cde1561 100644 --- a/controllers/testdata/declarative/case_0/deploy/cr.yaml +++ b/controllers/testdata/declarative/case_0/deploy/cr.yaml @@ -3,6 +3,9 @@ kind: DataSciencePipelinesApplication metadata: name: testdsp0 spec: + apiServer: + argoLauncherImage: argolauncherimage:test0 + argoDriverImage: argodriverimage:test0 objectStorage: minio: image: minio:test0 diff --git a/controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml index 61185ee35..410f7291c 100644 --- a/controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml @@ -75,6 +75,10 @@ spec: value: "ubi-minimal:test0" - name: MOVERESULTS_IMAGE value: "busybox:test0" + - name: V2_LAUNCHER_IMAGE + value: "argolauncherimage:test0" + - name: V2_DRIVER_IMAGE + value: "argodriverimage:test0" - name: ML_PIPELINE_SERVICE_HOST value: ds-pipeline-testdsp0.default.svc.cluster.local - name: ML_PIPELINE_SERVICE_PORT_GRPC diff --git a/controllers/testdata/declarative/case_2/deploy/cr.yaml b/controllers/testdata/declarative/case_2/deploy/cr.yaml index 597f328d6..501af511f 100644 --- a/controllers/testdata/declarative/case_2/deploy/cr.yaml +++ b/controllers/testdata/declarative/case_2/deploy/cr.yaml @@ -11,6 +11,8 @@ spec: artifactImage: artifact-manager:test2 cacheImage: ubi-minimal:test2 moveResultsImage: busybox:test2 + argoLauncherImage: argolauncherimage:test2 + argoDriverImage: argodriverimage:test2 injectDefaultScript: true stripEOF: true enableOauth: true diff --git a/controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml index 8fe35e13e..0b1b13624 100644 --- a/controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml @@ -75,6 +75,10 @@ spec: value: "ubi-minimal:test2" - name: MOVERESULTS_IMAGE value: "busybox:test2" + - name: V2_LAUNCHER_IMAGE + value: "argolauncherimage:test2" + - name: V2_DRIVER_IMAGE + value: "argodriverimage:test2" - name: ML_PIPELINE_SERVICE_HOST value: ds-pipeline-testdsp2.default.svc.cluster.local - name: ML_PIPELINE_SERVICE_PORT_GRPC diff --git a/controllers/testdata/declarative/case_3/deploy/02_cr.yaml b/controllers/testdata/declarative/case_3/deploy/02_cr.yaml index 0b63880e7..0d7beca3f 100644 --- a/controllers/testdata/declarative/case_3/deploy/02_cr.yaml +++ b/controllers/testdata/declarative/case_3/deploy/02_cr.yaml @@ -10,6 +10,8 @@ spec: name: doesnotexist key: "somekey" deploy: true + argoLauncherImage: argolauncherimage:test3 + argoDriverImage: argodriverimage:test3 persistenceAgent: {} scheduledWorkflow: {} database: diff --git a/controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml index 3174377ba..8f09908ae 100644 --- a/controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml @@ -75,6 +75,10 @@ spec: value: ubi-minimal:test3 - name: MOVERESULTS_IMAGE value: busybox:test3 + - name: V2_LAUNCHER_IMAGE + value: "argolauncherimage:test3" + - name: V2_DRIVER_IMAGE + value: "argodriverimage:test3" - name: ML_PIPELINE_SERVICE_HOST value: ds-pipeline-testdsp3.default.svc.cluster.local - name: ML_PIPELINE_SERVICE_PORT_GRPC diff --git a/controllers/testdata/declarative/case_4/deploy/00_cr.yaml b/controllers/testdata/declarative/case_4/deploy/00_cr.yaml index fa9eb1402..a2d7e9f4c 100644 --- a/controllers/testdata/declarative/case_4/deploy/00_cr.yaml +++ b/controllers/testdata/declarative/case_4/deploy/00_cr.yaml @@ -12,6 +12,8 @@ spec: artifactImage: this-artifact-manager-image-from-cr-should-be-used:test4 cacheImage: this-ubi-minimal-image-from-cr-should-be-used:test4 moveResultsImage: this-busybox-image-from-cr-should-be-used:test4 + argoLauncherImage: this-argolauncher-image-from-cr-should-be-used:test4 + argoDriverImage: this-argodriver-image-from-cr-should-be-used:test4 injectDefaultScript: true stripEOF: true enableOauth: true diff --git a/controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml index 3c56b5494..c8600c208 100644 --- a/controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml @@ -75,6 +75,10 @@ spec: value: "this-ubi-minimal-image-from-cr-should-be-used:test4" - name: MOVERESULTS_IMAGE value: "this-busybox-image-from-cr-should-be-used:test4" + - name: V2_LAUNCHER_IMAGE + value: "this-argolauncher-image-from-cr-should-be-used:test4" + - name: V2_DRIVER_IMAGE + value: "this-argodriver-image-from-cr-should-be-used:test4" - name: ML_PIPELINE_SERVICE_HOST value: ds-pipeline-testdsp4.default.svc.cluster.local - name: ML_PIPELINE_SERVICE_PORT_GRPC diff --git a/controllers/testdata/declarative/case_5/deploy/cr.yaml b/controllers/testdata/declarative/case_5/deploy/cr.yaml index 98dd89e0a..2aba42877 100644 --- a/controllers/testdata/declarative/case_5/deploy/cr.yaml +++ b/controllers/testdata/declarative/case_5/deploy/cr.yaml @@ -3,6 +3,9 @@ kind: DataSciencePipelinesApplication metadata: name: testdsp5 spec: + apiServer: + argoLauncherImage: argolauncherimage:test5 + argoDriverImage: argodriverimage:test5 objectStorage: minio: image: minio:test5 diff --git a/controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml index 39cb194fc..fa2cd0cd1 100644 --- a/controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml @@ -75,6 +75,10 @@ spec: value: "ubi-minimal:test5" - name: MOVERESULTS_IMAGE value: "busybox:test5" + - name: V2_LAUNCHER_IMAGE + value: "argolauncherimage:test5" + - name: V2_DRIVER_IMAGE + value: "argodriverimage:test5" - name: METADATA_GRPC_SERVICE_SERVICE_HOST value: "ds-pipeline-metadata-grpc-testdsp5.default.svc.cluster.local" - name: METADATA_GRPC_SERVICE_SERVICE_PORT diff --git a/controllers/testdata/declarative/case_6/deploy/01_cr.yaml b/controllers/testdata/declarative/case_6/deploy/01_cr.yaml index a3e909ecd..291a3b0f8 100644 --- a/controllers/testdata/declarative/case_6/deploy/01_cr.yaml +++ b/controllers/testdata/declarative/case_6/deploy/01_cr.yaml @@ -9,6 +9,8 @@ spec: cABundle: configMapName: testcabundleconfigmap6 configMapKey: testcabundleconfigmapkey6.crt + argoLauncherImage: argolauncherimage:test6 + argoDriverImage: argodriverimage:test6 persistenceAgent: deploy: false scheduledWorkflow: diff --git a/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml index f7893dccc..fb65830ab 100644 --- a/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml @@ -81,6 +81,10 @@ spec: value: "ubi-minimal:test6" - name: MOVERESULTS_IMAGE value: "busybox:test6" + - name: V2_LAUNCHER_IMAGE + value: "argolauncherimage:test6" + - name: V2_DRIVER_IMAGE + value: "argodriverimage:test6" - name: ML_PIPELINE_SERVICE_HOST value: ds-pipeline-testdsp6.default.svc.cluster.local - name: ML_PIPELINE_SERVICE_PORT_GRPC diff --git a/controllers/testdata/declarative/case_7/deploy/cr.yaml b/controllers/testdata/declarative/case_7/deploy/cr.yaml index 787109894..a21e56490 100644 --- a/controllers/testdata/declarative/case_7/deploy/cr.yaml +++ b/controllers/testdata/declarative/case_7/deploy/cr.yaml @@ -12,6 +12,8 @@ spec: artifactImage: artifact-manager:test7 cacheImage: ubi-minimal:test7 moveResultsImage: busybox:test7 + argoLauncherImage: argolauncherimage:test7 + argoDriverImage: argodriverimage:test7 injectDefaultScript: true stripEOF: true enableOauth: true diff --git a/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml index a0545c34a..bbd8fb1d0 100644 --- a/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml @@ -75,6 +75,10 @@ spec: value: "ubi-minimal:test7" - name: MOVERESULTS_IMAGE value: "busybox:test7" + - name: V2_LAUNCHER_IMAGE + value: "argolauncherimage:test7" + - name: V2_DRIVER_IMAGE + value: "argodriverimage:test7" - name: METADATA_GRPC_SERVICE_SERVICE_HOST value: "ds-pipeline-metadata-grpc-testdsp7.default.svc.cluster.local" - name: METADATA_GRPC_SERVICE_SERVICE_PORT diff --git a/kfdef/kfdef.yaml b/kfdef/kfdef.yaml index 0620f6449..e3ebb61fa 100644 --- a/kfdef/kfdef.yaml +++ b/kfdef/kfdef.yaml @@ -52,6 +52,10 @@ spec: value: gcr.io/tfx-oss-public/ml_metadata_store_server:1.14.0 - name: IMAGESV2_ARGO_WORKFLOWCONTROLLER value: gcr.io/ml-pipeline/workflow-controller:v3.3.10-license-compliance + - name: V2_LAUNCHER_IMAGE + value: gcr.io/ml-pipeline/kfp-launcher@sha256:80cf120abd125db84fa547640fd6386c4b2a26936e0c2b04a7d3634991a850a4 + - name: V2_DRIVER_IMAGE + value: gcr.io/ml-pipeline/kfp-driver@sha256:8e60086b04d92b657898a310ca9757631d58547e76bbbb8bfc376d654bef1707 repoRef: name: manifests path: config