Packaging modules as single package with extra (data-prep-kit[ray,spark] and data-prep-kit-tranforms[all, ray, etc.])

.make.defaults

@@ -234,6 +234,7 @@ __check_defined = \
 	mkdir ${LIB_NAME}
 	cp -p -R ${LIB_PATH}/src ${LIB_NAME}
 	cp -p -R ${LIB_PATH}/pyproject.toml ${LIB_NAME}
+	-cp -p -R ${LIB_PATH}/requirements.txt ${LIB_NAME}
 	cp -p -R ${LIB_PATH}/README.md ${LIB_NAME}
 
 # Build and image using the local Dockerfile and make the data-processing-lib/python
@@ -301,7 +302,10 @@ endif
 	if [ ! -z "$(EXTRA_INDEX_URL)" ]; then				\
 		extra_url='--extra-index-url $(EXTRA_INDEX_URL)';	\
 	fi;								\
-	pip install $(PIP_INSTALL_EXTRA_ARGS) $${extra_url}  -e $(PYTHON_PROJECT_DIR);
+	if [ -e $(PYTHON_PROJECT_DIR)/requirements.txt ]; then \
+		pip install -r  $(PYTHON_PROJECT_DIR)/requirements.txt; \
+	fi; \
+	pip install $(PIP_INSTALL_EXTRA_ARGS) $${extra_url}  -e $(PYTHON_PROJECT_DIR)
 	@echo Done installing source from $(PYTHON_PROJECT_DIR) into venv
 
 # Install local requirements last as it generally includes our lib source
@@ -344,6 +348,11 @@ endif
 .defaults.ray-lib-src-venv:: .defaults.create-venv .defaults.install-ray-lib-src-venv .defaults.install-local-requirements-venv
 	@# Help: Create the venv and install Ray library source, local dependencies and adjacent python source if present.
 
+# Install local requirements last as it generally includes our lib source
+.PHONY: .defaults.kfp-venv
+.defaults.kfp-venv:: .defaults.create-venv .defaults.install-ray-lib-src-venv
+	@# Help: Create the venv and install Ray library source, local dependencies and adjacent python source if present.
+
 # Install all source from the repo for a ray runtime transform into an existing venv 
 # And if there is an adjacent python dir (as for transforms), then also install that source
 .PHONY: .defaults.install-ray-lib-src-venv
@@ -627,7 +636,7 @@ MINIO_ADMIN_PWD= localminiosecretkey
 	rm -rf dist || true
 	rm -rf src/*egg-info || true
 	${PIP} install --upgrade build
-	${PYTHON} -m build
+	${PYTHON} -m build $(BUILD_WHEEL_ARG)
 
 # Publish the distribution in the dist directory, usually created with .defaults.build-dist  target
 .PHONY: .defaults.publish-dist 

data-processing-lib/Makefile

@@ -55,3 +55,8 @@ set-versions:
 	@# Help: Recursively $@ in all subdirs 
 	@$(MAKE) RULE=$@ .recurse
 
+
+build-pkg-dist:: 
+	$(MAKE) .defaults.build-dist BUILD_WHEEL_ARG=-w
+
+publish-dist :: .check-env .defaults.publish-dist

data-processing-lib/pyproject.toml

@@ -0,0 +1,44 @@
+[project]
+name = "data_prep_toolkit"
+version = "0.2.2.dev0"
+keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
+requires-python = ">=3.10"
+description = "Data Preparation Toolkit Library for Ray and Python"
+license = {text = "Apache-2.0"}
+readme = {file = "README.md", content-type = "text/markdown"}
+authors = [
+    { name = "David Wood", email = "dawood@us.ibm.com" },
+    { name = "Boris Lublinsky", email = "blublinsky@ibm.com" },
+]
+
+dynamic = ["dependencies", "optional-dependencies"]
+
+[project_urls]
+Repository = "https://github.com/IBM/data-prep-kit"
+Issues = "https://github.com/IBM/data-prep-kit/issues"
+Documentation = "https://ibm.github.io/data-prep-kit/"
+"Transform project" = "https://github.com/IBM/data-prep-kit/tree/dev/transforms/universal/noop"
+
+[build-system]
+requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools.dynamic.dependencies]
+file = ["requirements.txt"]
+
+[tool.setuptools.dynamic.optional-dependencies]
+dev = { file = ["requirements-dev.txt"]}
+ray = { file = ["requirements-ray.txt"]}
+spark = { file = ["requirements-spark.txt"]}
+
+[tool.setuptools.packages.find]
+where = ["python/src", "ray/src", "spark/src"]
+
+
+[tool.pytest.ini_options]
+# Currently we use low coverage since we have to run tests separately (see makefile)
+#addopts = "--cov --cov-report term-missing --cov-fail-under 25"
+markers = ["unit: unit tests", "integration: integration tests"]
+
+[tool.coverage.run]
+include = ["src/*"]

data-processing-lib/requirements-dev.txt

@@ -0,0 +1,9 @@
+twine
+pytest>=7.3.2
+pytest-dotenv>=0.5.2
+pytest-env>=1.0.0
+pre-commit>=3.3.2
+pytest-cov>=4.1.0
+pytest-mock>=3.10.0
+moto==5.0.5
+markupsafe==2.0.1

data-processing-lib/requirements-ray.txt

@@ -0,0 +1,3 @@
+ray[default]==2.24.0
+fastapi>=0.110.2
+pillow>=10.3.0

data-processing-lib/requirements-spark.txt

@@ -0,0 +1,2 @@
+pyspark>=3.5.2
+psutil>=6.0.0

data-processing-lib/requirements.txt

@@ -0,0 +1,6 @@
+  numpy < 1.29.0
+  pyarrow==16.1.0
+  boto3==1.34.69
+  argparse
+  mmh3
+  psutil

transforms/.make.workflows

@@ -52,7 +52,7 @@ endif
 
 ${WORKFLOW_VENV_ACTIVATE}: ${REPOROOT}/.make.versions ${REPOROOT}/kfp/kfp_ray_components/requirements.txt ${DPK_RAY_LIB_DIR} ${KFP_LIB_SRC_FILES} ${KFP_LIB_CONFIG_FILE} ${KFP_SHARED_LIB_SRC_FILES}
 	rm -rf ${REPOROOT}/transforms/venv
-	$(MAKE) -C ${REPOROOT}/transforms .defaults.ray-lib-src-venv
+	$(MAKE) -C ${REPOROOT}/transforms .defaults.kfp-venv
 	. ${WORKFLOW_VENV_ACTIVATE};     \
 	pip install -e $(REPOROOT)/kfp/kfp_support_lib/shared_workflow_support; \
 	pip install -e $(REPOROOT)/kfp/kfp_support_lib/$(WORKFLOW_SUPPORT_LIB); \

transforms/Makefile

@@ -79,3 +79,32 @@ workflow-upload::
 set-versions::
 	@# Help: Recursively make $@ in all subdirs
 	@$(MAKE) RULE=$@ .recurse
+
+
+build-pkg-dist::
+	## Most transforms today don't have a package name.... Need to fix that
+	## In the meantime, we will copy everything to a single folder
+	-rm -fr src
+	mkdir src
+	# Copy all the src folders recursively (not clear if they have subfolders)
+	for x in $(shell find . | grep '[ray| python]/src$$') ; do \
+		echo $$x ; \
+		if [ -d "$$x" ]; then \
+		   cp -r $$x/* src ; \
+		fi \
+	done
+	# Only needs to build the whl
+	$(MAKE) BUILD_WHEEL_ARG=-w .defaults.build-dist
+
+test-pkg-dist::
+	-rm -fr venv
+	python -m venv venv
+	source venv/bin/activate && $(PYTHON) -m pip  install '$(REPOROOT)/data-processing-lib/dist/data_prep_toolkit-$(DPK_VERSION)-py3-none-any.whl[dev,ray]'
+	source venv/bin/activate && $(PYTHON) -m pip  install 'dist/data_prep_toolkit_transforms-$(DPK_TRANSFORMS_VERSION)-py3-none-any.whl[all]'
+	for T in $(shell find . | grep '[ray| python]/test$$') ; do \
+        echo "running unit test on: $$T" ; \
+		source venv/bin/activate && $(PYTEST) $$T; \
+    done;
+	@# Help: Setup environment and run unit tests for all transforms
+
+

transforms/packaging/python/README.md → transforms/README-list.md

@@ -5,9 +5,14 @@
 The [transforms](https://github.com/IBM/data-prep-kit/blob/dev/transforms/README.md) are delivered as a standard pyton library available on pypi and can be installed using pip install:
 
 `python -m pip install data-prep-toolkit-transforms`
+or
+`python -m pip install data-prep-toolkit-transforms[ray]`
+
 
 installing the python transforms will also install  `data-prep-toolkit`
 
+installing the ray transforms will also install  `data-prep-toolkit[ray]`
+
 ## List of Transforms in current package
 
 Note: This list includes the transforms that were part of the release starting with data-prep-toolkit-transforms:0.2.1. This list may not always reflect up to date information. Users are encourage to raise an issue in git when they discover missing components or packages that are listed below but not in the current release they get from pypi.

transforms/code/code2parquet/python/Dockerfile

@@ -19,6 +19,7 @@ RUN cd data-processing-lib-python && pip install --no-cache-dir -e .
 
 COPY --chown=dpk:root src/ src/
 COPY --chown=dpk:root pyproject.toml pyproject.toml 
+COPY --chown=dpk:root requirements.txt requirements.txt
 RUN pip install --no-cache-dir -e .
 
 # copy the main() entry point to the image 

transforms/code/code2parquet/python/pyproject.toml

@@ -9,16 +9,15 @@ authors = [
     { name = "David Wood", email = "dawood@us.ibm.com" },
     { name = "Boris Lublinsky", email = "blublinsky@ibm.com" },
 ]
-dependencies = [
-    "data-prep-toolkit==0.2.2.dev0",
-    "parameterized",
-    "pandas",
-]
+dynamic = ["dependencies"]
 
 [build-system]
 requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+
 [project.optional-dependencies]
 dev = [
     "twine",

transforms/code/code2parquet/python/requirements.txt

@@ -0,0 +1,3 @@
+data-prep-toolkit==0.2.2.dev0
+parameterized
+pandas

transforms/code/code_quality/python/Dockerfile

@@ -19,6 +19,7 @@ RUN cd data-processing-lib-python && pip install --no-cache-dir -e .
 
 COPY --chown=dpk:root src/ src/
 COPY --chown=dpk:root pyproject.toml pyproject.toml 
+COPY --chown=dpk:root requirements.txt requirements.txt
 RUN pip install --no-cache-dir -e .
 
 #COPY requirements.txt requirements.txt

transforms/code/code_quality/python/pyproject.toml

@@ -8,16 +8,16 @@ readme = {file = "README.md", content-type = "text/markdown"}
 authors = [
     { name = "Shivdeep Singh", email = "shivdeep.singh@ibm.com" },
 ]
-dependencies = [
-    "data-prep-toolkit==0.2.2.dev0",
-    "bs4==0.0.2",
-    "transformers==4.38.2",
-]
+dynamic = ["dependencies"]
 
 [build-system]
 requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+
+
 [project.optional-dependencies]
 dev = [
     "twine",

transforms/code/code_quality/python/requirements.txt

@@ -0,0 +1,3 @@
+data-prep-toolkit==0.2.2.dev0
+bs4==0.0.2
+transformers==4.38.2

transforms/code/header_cleanser/python/Dockerfile

@@ -27,6 +27,7 @@ RUN cd data-processing-lib-python && pip install --no-cache-dir -e .
 
 COPY --chown=dpk:root src/ src/
 COPY --chown=dpk:root pyproject.toml pyproject.toml 
+COPY --chown=dpk:root requirements.txt requirements.txt
 RUN pip install --no-cache-dir -e .
 
 # copy source data

transforms/code/header_cleanser/python/pyproject.toml

@@ -8,15 +8,16 @@ readme = {file = "README.md", content-type = "text/markdown"}
 authors = [
     { name = "Yash kalathiya", email = "yashkalathiya164@gmail.com" },
 ]
-dependencies = [
-    "data-prep-toolkit==0.2.2.dev0",
-    "scancode-toolkit==32.1.0",
-]
+dynamic = ["dependencies"]
 
 [build-system]
 requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+
+
 [project.optional-dependencies]
 dev = [
     "twine",

transforms/code/header_cleanser/python/requirements.txt

@@ -0,0 +1,3 @@
+data-prep-toolkit==0.2.2.dev0
+scancode-toolkit==32.1.0 ; platform_system != 'Darwin'
+

transforms/code/license_select/python/Dockerfile

@@ -18,6 +18,7 @@ RUN cd data-processing-lib-python && pip install --no-cache-dir -e .
 
 COPY --chown=dpk:root src/ src/
 COPY --chown=dpk:root pyproject.toml pyproject.toml 
+COPY --chown=dpk:root requirements.txt requirements.txt
 RUN pip install --no-cache-dir -e .
 
 # copy source data

transforms/code/license_select/python/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dpk_license_select_transform_python"
-version = "0.2.1.dev0"
+version = "0.2.2.dev0"
 requires-python = ">=3.10"
 description = "License Select Python Transform"
 license = {text = "Apache-2.0"}
@@ -9,14 +9,16 @@ authors = [
     { name = "Shivdeep Singh", email = "shivdeep.singh@ibm.com" },
     { name = "Mark Lewis", email = "mark_lewis@uk.ibm.com" },
 ]
-dependencies = [
-    "data-prep-toolkit==0.2.1.dev0",
-]
+dynamic = ["dependencies"]
 
 [build-system]
 requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+
+
 [project.optional-dependencies]
 dev = [
     "twine",

transforms/code/license_select/python/requirements.txt

@@ -0,0 +1 @@
+data-prep-toolkit==0.2.2.dev0

transforms/code/license_select/ray/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dpk_license_select_transform_ray"
-version = "0.2.1.dev0"
+version = "0.2.2.dev0"
 requires-python = ">=3.10"
 description = "License Select Transform"
 license = {text = "Apache-2.0"}
@@ -10,8 +10,8 @@ authors = [
     { name = "Mark Lewis", email = "mark_lewis@uk.ibm.com" },
 ]
 dependencies = [
-    "dpk-license-select-transform-python==0.2.1.dev0",
-    "data-prep-toolkit-ray==0.2.1.dev0",
+    "dpk-license-select-transform-python==0.2.2.dev0",
+    "data-prep-toolkit-ray==0.2.2.dev0",
 ]
 
 [build-system]

transforms/code/proglang_select/python/Dockerfile

@@ -19,6 +19,7 @@ RUN cd data-processing-lib-python && pip install --no-cache-dir -e .
 
 COPY --chown=dpk:root src/ src/
 COPY --chown=dpk:root pyproject.toml pyproject.toml 
+COPY --chown=dpk:root requirements.txt requirements.txt
 RUN pip install --no-cache-dir -e .
 
 # copy the main() entry point to the image 

transforms/code/proglang_select/python/pyproject.toml

@@ -8,14 +8,16 @@ readme = {file = "README.md", content-type = "text/markdown"}
 authors = [
     { name = "Shivdeep Singh", email = "shivdeep.singh@ibm.com" },
 ]
-dependencies = [
-    "data-prep-toolkit==0.2.2.dev0",
-]
+dynamic = ["dependencies"]
 
 [build-system]
 requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+
+
 [project.optional-dependencies]
 dev = [
     "twine",

transforms/code/proglang_select/python/requirements.txt

@@ -0,0 +1 @@
+data-prep-toolkit==0.2.2.dev0

transforms/language/doc_chunk/python/Dockerfile

@@ -21,6 +21,7 @@ RUN cd data-processing-lib-python && pip install --no-cache-dir -e .
 
 COPY --chown=dpk:root src/ src/
 COPY --chown=dpk:root pyproject.toml pyproject.toml 
+COPY --chown=dpk:root requirements.txt requirements.txt
 RUN pip install ${PIP_INSTALL_EXTRA_ARGS} --no-cache-dir -e .
 
 # copy transform main() entry point to the image 

transforms/language/doc_chunk/python/pyproject.toml

@@ -10,16 +10,15 @@ authors = [
     { name = "Panos Vagenas", email = "pva@zurich.ibm.com" },
     { name = "Christoph Auer", email = "cau@zurich.ibm.com" },
 ]
-dependencies = [
-    "data-prep-toolkit==0.2.2.dev0",
-    "docling-core==1.3.0",
-    "llama-index-core>=0.11.0,<0.12.0",
-]
+dynamic = ["dependencies"]
 
 [build-system]
 requires = ["setuptools>=68.0.0", "wheel", "setuptools_scm[toml]>=7.1.0"]
 build-backend = "setuptools.build_meta"
 
+[tool.setuptools.dynamic]
+dependencies = {file = ["requirements.txt"]}
+
 [project.optional-dependencies]
 dev = [
     "twine",

transforms/language/doc_chunk/python/requirements.txt

@@ -0,0 +1,3 @@
+data-prep-toolkit==0.2.2.dev0
+docling-core==1.3.0
+llama-index-core>=0.11.0,<0.12.0