From 61fec0b1ed4cf848387aec204b125ffd3280a53d Mon Sep 17 00:00:00 2001 From: Maroun Touma Date: Wed, 23 Oct 2024 15:25:49 -0500 Subject: [PATCH] Use extra for spark Signed-off-by: Maroun Touma --- .make.defaults | 2 +- kfp/kfp_ray_components/Dockerfile | 10 +++++----- .../shared_workflow_support/pyproject.toml | 2 +- transforms/universal/doc_id/spark/pyproject.toml | 2 +- transforms/universal/filter/spark/pyproject.toml | 2 +- transforms/universal/noop/spark/pyproject.toml | 2 +- transforms/universal/profiler/spark/pyproject.toml | 2 +- transforms/universal/resize/spark/pyproject.toml | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.make.defaults b/.make.defaults index 0447f2463..a8dd6fdda 100644 --- a/.make.defaults +++ b/.make.defaults @@ -441,7 +441,7 @@ ifeq ($(USE_REPO_LIB_SRC), 1) $(MAKE) PIP_TARGET=data-prep-toolkit[spark] .defaults.pip-uninstall; \ $(MAKE) PIP_TARGET=data-prep-toolkit .defaults.pip-uninstall; \ $(MAKE) PYTHON_PROJECT_BUILD_DIR=$(DPK_PYTHON_BUILD_DIR) .defaults.install-src-venv; \ - $(MAKE) PYTHON_PROJECT_BUILD_DIR=$(DPK_SPARK_BUILD_DIR) .defaults.install-src-venv; \ + $(MAKE) PYTHON_PROJECT_BUILD_DIR=$(DPK_SPARK_BUILD_DIR) PROJECT_BUILD_EXTRA=spark .defaults.install-src-venv; \ echo Installed source from Python and Spark processing libraries for `which $(PYTHON)` else @# Help: DO NOT install Python or Spark data processing library source into existing venv diff --git a/kfp/kfp_ray_components/Dockerfile b/kfp/kfp_ray_components/Dockerfile index 225ba6b09..4e3b71f71 100644 --- a/kfp/kfp_ray_components/Dockerfile +++ b/kfp/kfp_ray_components/Dockerfile @@ -6,12 +6,12 @@ FROM ${BASE_IMAGE} COPY requirements.txt requirements.txt RUN pip install --no-cache-dir -r requirements.txt -# Copy and install data processing libraries +ARG WHEEL_FILE_NAME + +# Copy and install data processing libraries # These are expected to be placed in the docker context before this is run (see the make image). -COPY --chown=ray:users data-processing-lib-python/ data-processing-lib-python/ -RUN cd data-processing-lib-python && pip install --no-cache-dir -e . -COPY --chown=ray:users data-processing-lib-ray/ data-processing-lib-ray/ -RUN cd data-processing-lib-ray && pip install --no-cache-dir -e . +COPY --chown=ray:users data-processing-dist data-processing-dist +RUN pip install data-processing-dist/${WHEEL_FILE_NAME}[ray] COPY --chown=ray:users shared_workflow_support_lib shared_workflow_support_lib/ RUN cd shared_workflow_support_lib && pip install --no-cache-dir -e . diff --git a/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml b/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml index b2dc963d7..df92a5e5a 100644 --- a/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml +++ b/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml @@ -14,7 +14,7 @@ authors = [ dependencies = [ "requests", "kubernetes", - "data-prep-toolkit-ray==0.2.2.dev1", + "data-prep-toolkit[ray]==0.2.2.dev1", ] [build-system] diff --git a/transforms/universal/doc_id/spark/pyproject.toml b/transforms/universal/doc_id/spark/pyproject.toml index 636bbf26e..312af5c96 100644 --- a/transforms/universal/doc_id/spark/pyproject.toml +++ b/transforms/universal/doc_id/spark/pyproject.toml @@ -10,7 +10,7 @@ authors = [ { name = "Boris Lublinsky", email = "blublinsk@ibm.com" }, ] dependencies = [ - "data-prep-toolkit-spark==0.2.2.dev1", + "data-prep-toolkit[spark]==0.2.2.dev1", ] [build-system] diff --git a/transforms/universal/filter/spark/pyproject.toml b/transforms/universal/filter/spark/pyproject.toml index 54a49893e..2f108e063 100644 --- a/transforms/universal/filter/spark/pyproject.toml +++ b/transforms/universal/filter/spark/pyproject.toml @@ -9,7 +9,7 @@ authors = [ { name = "Boris Lublinsky", email = "blublinsk@ibm.com" }, ] dependencies = [ - "data-prep-toolkit-spark==0.2.2.dev1", + "data-prep-toolkit[spark]==0.2.2.dev1", ] [project.optional-dependencies] diff --git a/transforms/universal/noop/spark/pyproject.toml b/transforms/universal/noop/spark/pyproject.toml index 5068ffa2f..cebc41cec 100644 --- a/transforms/universal/noop/spark/pyproject.toml +++ b/transforms/universal/noop/spark/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] dependencies = [ "dpk-noop-transform-python==0.2.2.dev1", - "data-prep-toolkit-spark==0.2.2.dev1", + "data-prep-toolkit[spark]==0.2.2.dev1", ] [build-system] diff --git a/transforms/universal/profiler/spark/pyproject.toml b/transforms/universal/profiler/spark/pyproject.toml index 34003b539..f0b294061 100644 --- a/transforms/universal/profiler/spark/pyproject.toml +++ b/transforms/universal/profiler/spark/pyproject.toml @@ -10,7 +10,7 @@ authors = [ ] dependencies = [ "dpk-profiler-transform-python==0.2.2.dev1", - "data-prep-toolkit-spark==0.2.2.dev1", + "data-prep-toolkit[spark]==0.2.2.dev1", ] [build-system] diff --git a/transforms/universal/resize/spark/pyproject.toml b/transforms/universal/resize/spark/pyproject.toml index 6b6d0f50b..6a0493882 100644 --- a/transforms/universal/resize/spark/pyproject.toml +++ b/transforms/universal/resize/spark/pyproject.toml @@ -11,7 +11,7 @@ authors = [ ] dependencies = [ "dpk-resize-transform-python==0.2.2.dev1", - "data-prep-toolkit-spark==0.2.2.dev1", + "data-prep-toolkit[spark]==0.2.2.dev1", ] [build-system]