Use extra for spark

Signed-off-by: Maroun Touma <touma@us.ibm.com>
IBM · Oct 23, 2024 · 61fec0b · 61fec0b
1 parent fdee6a7
commit 61fec0b
Show file tree

Hide file tree

Showing 8 changed files with 12 additions and 12 deletions.
diff --git a/.make.defaults b/.make.defaults
@@ -441,7 +441,7 @@ ifeq ($(USE_REPO_LIB_SRC), 1)
 	$(MAKE) PIP_TARGET=data-prep-toolkit[spark] .defaults.pip-uninstall;	\
 	$(MAKE) PIP_TARGET=data-prep-toolkit .defaults.pip-uninstall;	\
 	$(MAKE) PYTHON_PROJECT_BUILD_DIR=$(DPK_PYTHON_BUILD_DIR) .defaults.install-src-venv;	\
-	$(MAKE) PYTHON_PROJECT_BUILD_DIR=$(DPK_SPARK_BUILD_DIR) .defaults.install-src-venv;	\
+	$(MAKE) PYTHON_PROJECT_BUILD_DIR=$(DPK_SPARK_BUILD_DIR) PROJECT_BUILD_EXTRA=spark .defaults.install-src-venv;	\
 	echo Installed source from Python and Spark processing libraries for `which $(PYTHON)`
 else
 	@# Help: DO NOT install Python or Spark data processing library source into existing venv

diff --git a/kfp/kfp_ray_components/Dockerfile b/kfp/kfp_ray_components/Dockerfile
@@ -6,12 +6,12 @@ FROM ${BASE_IMAGE}
 COPY requirements.txt requirements.txt
 RUN pip install --no-cache-dir -r  requirements.txt
 
-# Copy and install data processing libraries
+ARG WHEEL_FILE_NAME
+
+# Copy and install data processing libraries 
 # These are expected to be placed in the docker context before this is run (see the make image).
-COPY --chown=ray:users data-processing-lib-python/ data-processing-lib-python/
-RUN cd data-processing-lib-python && pip install --no-cache-dir -e .
-COPY --chown=ray:users data-processing-lib-ray/ data-processing-lib-ray/
-RUN cd data-processing-lib-ray    && pip install --no-cache-dir -e .
+COPY --chown=ray:users data-processing-dist data-processing-dist
+RUN  pip install data-processing-dist/${WHEEL_FILE_NAME}[ray]
 
 COPY --chown=ray:users shared_workflow_support_lib shared_workflow_support_lib/
 RUN cd shared_workflow_support_lib && pip install --no-cache-dir -e .

diff --git a/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml b/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml
@@ -14,7 +14,7 @@ authors = [
 dependencies = [
     "requests",
     "kubernetes",
-    "data-prep-toolkit-ray==0.2.2.dev1",
+    "data-prep-toolkit[ray]==0.2.2.dev1",
 ]
 
 [build-system]

diff --git a/transforms/universal/doc_id/spark/pyproject.toml b/transforms/universal/doc_id/spark/pyproject.toml
@@ -10,7 +10,7 @@ authors = [
     { name = "Boris Lublinsky", email = "blublinsk@ibm.com" },
 ]
 dependencies = [
-    "data-prep-toolkit-spark==0.2.2.dev1",
+    "data-prep-toolkit[spark]==0.2.2.dev1",
 ]
 
 [build-system]

diff --git a/transforms/universal/filter/spark/pyproject.toml b/transforms/universal/filter/spark/pyproject.toml
@@ -9,7 +9,7 @@ authors = [
     { name = "Boris Lublinsky", email = "blublinsk@ibm.com" },
 ]
 dependencies = [
-    "data-prep-toolkit-spark==0.2.2.dev1",
+    "data-prep-toolkit[spark]==0.2.2.dev1",
 ]
 
 [project.optional-dependencies]

diff --git a/transforms/universal/noop/spark/pyproject.toml b/transforms/universal/noop/spark/pyproject.toml
@@ -11,7 +11,7 @@ authors = [
 ]
 dependencies = [
     "dpk-noop-transform-python==0.2.2.dev1",
-    "data-prep-toolkit-spark==0.2.2.dev1",
+    "data-prep-toolkit[spark]==0.2.2.dev1",
 ]
 
 [build-system]

diff --git a/transforms/universal/profiler/spark/pyproject.toml b/transforms/universal/profiler/spark/pyproject.toml
@@ -10,7 +10,7 @@ authors = [
 ]
 dependencies = [
     "dpk-profiler-transform-python==0.2.2.dev1",
-    "data-prep-toolkit-spark==0.2.2.dev1",
+    "data-prep-toolkit[spark]==0.2.2.dev1",
 ]
 
 [build-system]

diff --git a/transforms/universal/resize/spark/pyproject.toml b/transforms/universal/resize/spark/pyproject.toml
@@ -11,7 +11,7 @@ authors = [
 ]
 dependencies = [
     "dpk-resize-transform-python==0.2.2.dev1",
-    "data-prep-toolkit-spark==0.2.2.dev1",
+    "data-prep-toolkit[spark]==0.2.2.dev1",
 ]
 
 [build-system]