Move transform version numbers out of .make.versions to transform-spe…

…cific file (#657) * push noop version numbers down into noop/transform.config Signed-off-by: David Wood <dawood@us.ibm.com> * move USE_REPO_LIB_SRC into Makefiles Signed-off-by: David Wood <dawood@us.ibm.com> * remove USE_REPO_LIB_SRC from transform.config Signed-off-by: David Wood <dawood@us.ibm.com> * move transform version numbers out of .make.versions for all transforms Signed-off-by: David Wood <dawood@us.ibm.com> * fix Makefile typo on Include and add config for repo_level_ordering and fdedup Signed-off-by: David Wood <dawood@us.ibm.com> * fix version numbers for repo_level_ordering and fdedup Signed-off-by: David Wood <dawood@us.ibm.com> * remove commented out versions from .make.versions Signed-off-by: David Wood <dawood@us.ibm.com> * Enable sudo in header_cleanser Dockerfile Signed-off-by: David Wood <dawood@us.ibm.com> * fix html2parquet/ray/Makefile set-versions Signed-off-by: David Wood <dawood@us.ibm.com> * remove redundant requirements from html2parquet/ray requirements Signed-off-by: David Wood <dawood@us.ibm.com> * disable header_cleanser kfp test Signed-off-by: David Wood <dawood@us.ibm.com> * correct header_cleanser Makefile renames Signed-off-by: David Wood <dawood@us.ibm.com> * fix .defaults.copy-lib to also copy requirements.txt' Signed-off-by: David Wood <dawood@us.ibm.com> --------- Signed-off-by: David Wood <dawood@us.ibm.com>
IBM · Oct 9, 2024 · efc1162 · efc1162
1 parent e581989
commit efc1162
Show file tree

Hide file tree

Showing 97 changed files with 1,071 additions and 217 deletions.
diff --git a/.make.defaults b/.make.defaults
@@ -235,6 +235,10 @@ __check_defined = \
 	cp -p -R ${LIB_PATH}/src ${LIB_NAME}
 	cp -p -R ${LIB_PATH}/pyproject.toml ${LIB_NAME}
 	cp -p -R ${LIB_PATH}/README.md ${LIB_NAME}
+	if [ -e ${LIB_PATH}/requirements.txt ]; then		\
+	    cp -p ${LIB_PATH}/requirements.txt ${LIB_NAME};	\
+	fi
+
 
 # Build and image using the local Dockerfile and make the data-processing-lib/python
 # available in the current directory for use by the Dockerfile (i.e. to install the library).
@@ -591,8 +595,9 @@ MINIO_ADMIN_PWD= localminiosecretkey
 # Updates the versions references to our repo source as defined in .make.versions
 .PHONY: .defaults.__update-toml-lib-dep-versions
 .defaults.__update-toml-lib-dep-versions:
+ifeq ($(USE_REPO_LIB_SRC), 1)
 	@# Help: Update pyproject.toml to depend on lib versions defined in .make.versions 
-	@if [ -e pyproject.toml ]; then					\
+	if [ -e pyproject.toml ]; then					\
 	    cat pyproject.toml | sed 									\
 		-e 's/"data-prep-toolkit-ray\([=><~][=]\).*"/"data-prep-toolkit-ray\1$(DPK_LIB_VERSION)"/' 	\
 		-e 's/"data-prep-toolkit-spark\([=><~][=]\).*"/"data-prep-toolkit-spark\1$(DPK_LIB_VERSION)"/' 	\
@@ -603,7 +608,7 @@ MINIO_ADMIN_PWD= localminiosecretkey
 		> tt.toml;				\
 	    mv tt.toml pyproject.toml;			\
 	 fi
-	@if [ -e requirements.txt ]; then					\
+	if [ -e requirements.txt ]; then					\
 	    cat requirements.txt | sed 									\
 		-e 's/data-prep-toolkit-ray\([=><~][=]\).*/data-prep-toolkit-ray\1$(DPK_LIB_VERSION)/' 	\
 		-e 's/data-prep-toolkit-transforms\([=><~][=]\).*/data-prep-toolkit-transforms\1$(DPK_TRANSFORMS_VERSION)/' 	\
@@ -615,6 +620,7 @@ MINIO_ADMIN_PWD= localminiosecretkey
 		> tt.txt;				\
 	    mv tt.txt requirements.txt;			\
 	 fi
+endif
 
 # Build the distribution, usually in preparation for publishing using ith the .defaults.publish-dist  target
 .PHONY: .defaults.build-dist 

diff --git a/.make.versions b/.make.versions
@@ -25,9 +25,9 @@ DPK_VERSION=$(DPK_MAJOR_VERSION).$(DPK_MINOR_VERSION).$(DPK_MICRO_VERSION)$(DPK_
 
 # publish docker images with latest tag
 ifeq ($(DPK_VERSION_SUFFIX), )
-	DOCKER_IMAGE_VERSION=$(DPK_VERSION)
+	DOCKER_IMAGE_VERSION?=$(DPK_VERSION)
 else
-	DOCKER_IMAGE_VERSION=latest
+	DOCKER_IMAGE_VERSION?=latest
 endif
 
 # Data prep lab wheel version
@@ -39,78 +39,6 @@ DPK_LIB_KFP_SHARED=$(DPK_VERSION)
 KFP_DOCKER_VERSION=$(DOCKER_IMAGE_VERSION)
 KFP_DOCKER_VERSION_v2=$(DOCKER_IMAGE_VERSION)
 
-# Begin transform versions/tags
-BLOCKLIST_VERSION=$(DPK_VERSION)
-
-DOC_ID_PYTHON_VERSION=$(DPK_VERSION)
-DOC_ID_RAY_VERSION=$(DPK_VERSION)
-DOC_ID_SPARK_VERSION=$(DPK_VERSION)
-
-EDEDUP_PYTHON_VERSION=$(DPK_VERSION)
-EDEDUP_RAY_VERSION=$(DPK_VERSION)
-
-FDEDUP_RAY_VERSION=$(DPK_VERSION)
-
-FILTER_PYTHON_VERSION=$(DPK_VERSION)
-FILTER_RAY_VERSION=$(DPK_VERSION)
-FILTER_SPARK_VERSION=$(DPK_VERSION)
-
-NOOP_PYTHON_VERSION=$(DPK_VERSION)
-NOOP_RAY_VERSION=$(DPK_VERSION)
-NOOP_SPARK_VERSION=$(DPK_VERSION)
-
-PROFILER_PYTHON_VERSION=$(DPK_VERSION)
-PROFILER_RAY_VERSION=$(DPK_VERSION)
-PROFILER_SPARK_VERSION=$(DPK_VERSION)
-
-RESIZE_PYTHON_VERSION=$(DPK_VERSION)
-RESIZE_RAY_VERSION=$(DPK_VERSION)
-RESIZE_SPARK_VERSION=$(DPK_VERSION)
-
-LANG_ID_PYTHON_VERSION=$(DPK_VERSION)
-LANG_ID_RAY_VERSION=$(DPK_VERSION)
-
-TOKENIZATION_RAY_VERSION=$(DPK_VERSION)
-TOKENIZATION_PYTHON_VERSION=$(DPK_VERSION)
-
-MALWARE_RAY_VERSION=$(DPK_VERSION)
-MALWARE_PYTHON_VERSION=$(DPK_VERSION)
-
-PROGLANG_SELECT_PYTHON_VERSION=$(DPK_VERSION)
-PROGLANG_SELECT_RAY_VERSION=$(DPK_VERSION)
-
-DOC_QUALITY_PYTHON_VERSION=$(DPK_VERSION)
-DOC_QUALITY_RAY_VERSION=$(DPK_VERSION)
-
-CODE_QUALITY_RAY_VERSION=$(DPK_VERSION)
-CODE_QUALITY_PYTHON_VERSION=$(DPK_VERSION)
-
-CODE2PARQUET_PYTHON_VERSION=$(DPK_VERSION)
-CODE2PARQUET_RAY_VERSION=$(DPK_VERSION)
-INGEST_TO_PARQUET_VERSION=$(DPK_VERSION)
-REPO_LVL_ORDER_RAY_VERSION=$(DPK_VERSION)
-
-PDF2PARQUET_PYTHON_VERSION=$(DPK_VERSION)
-PDF2PARQUET_RAY_VERSION=$(DPK_VERSION)
-
-DOC_CHUNK_PYTHON_VERSION=$(DPK_VERSION)
-DOC_CHUNK_RAY_VERSION=$(DPK_VERSION)
-
-TEXT_ENCODER_PYTHON_VERSION=$(DPK_VERSION)
-TEXT_ENCODER_RAY_VERSION=$(DPK_VERSION)
-
-HEADER_CLEANSER_PYTHON_VERSION=$(DPK_VERSION) 
-HEADER_CLEANSER_RAY_VERSION=$(DPK_VERSION)
-
-LICENSE_SELECT_PYTHON_VERSION=$(DPK_VERSION)
-LICENSE_SELECT_RAY_VERSION=$(DPK_VERSION)
-
-PII_REDACTOR_PYTHON_VERSION=$(DPK_VERSION)
-
-HTML2PARQUET_PYTHON_VERSION=$(DPK_VERSION)
-
-DPK_TRANSFORMS_VERSION=$(DPK_VERSION)
-
 ################## ################## ################## ################## ################## ##################
 # Begin versions that the repo depends on. 
 

diff --git a/transforms/code/code2parquet/kfp_ray/Makefile b/transforms/code/code2parquet/kfp_ray/Makefile
@@ -2,6 +2,9 @@ REPOROOT=${CURDIR}/../../../../
 WORKFLOW_VENV_ACTIVATE=${REPOROOT}/transforms/venv/bin/activate
 include $(REPOROOT)/transforms/.make.workflows
 
+# Include the common configuration for this transform
+include ../transform.config
+
 SRC_DIR=${CURDIR}/../ray/
 
 PYTHON_WF := $(shell find ./ -name '*_wf.py')

diff --git a/transforms/code/code2parquet/python/Makefile b/transforms/code/code2parquet/python/Makefile
@@ -1,16 +1,22 @@
 # Define the root of the local git clone for the common rules to be able 
 # know where they are running from.
 REPOROOT=../../../..
+
+# Set this, before including .make.defaults, to 
+#   1 if requirements reference the latest code in the data processing library 
+#     in this repo (that is not yet published to pypi).	 This is the default setting.
+#   0 if the transforms DPK dependencies are on wheels published to 
+#     pypi (e.g. data-prep-toolkit=0.2.1)
+#USE_REPO_LIB_SRC=1
+
 # Include a library of common .transform.* targets which most
 # transforms should be able to reuse.  However, feel free
 # to override/redefine the rules below. 
-
-# $(REPOROOT)/.make.versions file contains the versions
-
-TRANSFORM_NAME=code2parquet
-
 include $(REPOROOT)/transforms/.make.transforms
 
+# Include the common configuration for this transform
+include ../transform.config
+
 venv::	.transforms.python-venv
 
 test::	.transforms.python-test

diff --git a/transforms/code/code2parquet/ray/Makefile b/transforms/code/code2parquet/ray/Makefile
@@ -1,12 +1,21 @@
 # Define the root of the local git clone for the common rules to be able 
 # know where they are running from.
 REPOROOT=../../../..
+
+# Set this, before including .make.defaults, to 
+#   1 if requirements reference the latest code in the data processing library 
+#     in this repo (that is not yet published to pypi).	 This is the default setting.
+#   0 if the transforms DPK dependencies are on wheels published to 
+#     pypi (e.g. data-prep-toolkit=0.2.1)
+#USE_REPO_LIB_SRC=1
+
 # Include a library of common .transform.* targets which most
 # transforms should be able to reuse.  However, feel free
 # to override/redefine the rules below. 
 include $(REPOROOT)/transforms/.make.transforms
 
-TRANSFORM_NAME=code2parquet
+# Include the common configuration for this transform
+include ../transform.config
 
 BASE_IMAGE=${RAY_BASE_IMAGE}
 venv::	.transforms.ray-venv

diff --git a/transforms/code/code2parquet/transform.config b/transforms/code/code2parquet/transform.config
@@ -0,0 +1,20 @@
+#
+# This is intended to be included across the Makefiles provided within
+# a given transform's directory tree,  so must use compatible syntax.
+#
+################################################################################
+# This defines the name of the transform and is used to match against
+# expected files and is used to define the transform's image name. 
+TRANSFORM_NAME=code2parquet
+
+################################################################################
+# This defines the transforms' version number as would be used
+# when publishing the wheel.  In general, only the micro version
+# number should be advanced relative to the DPK_VERSION. 
+#
+# If you change the versions numbers, be sure to run "make set-versions" to 
+# update version numbers across the transform (e.g., pyproject.toml).
+CODE2PARQUET_PYTHON_VERSION=$(DPK_VERSION)
+CODE2PARQUET_RAY_VERSION=$(CODE2PARQUET_PYTHON_VERSION)
+CODE2PARQUET_SPARK_VERSION=$(CODE2PARQUET_PYTHON_VERSION)
+
diff --git a/transforms/code/code_quality/kfp_ray/Makefile b/transforms/code/code_quality/kfp_ray/Makefile
@@ -2,6 +2,9 @@ REPOROOT=${CURDIR}/../../../../
 WORKFLOW_VENV_ACTIVATE=${REPOROOT}/transforms/venv/bin/activate
 include $(REPOROOT)/transforms/.make.workflows
 
+# Include the common configuration for this transform
+include ../transform.config
+
 SRC_DIR=${CURDIR}/../ray/
 
 PYTHON_WF := $(shell find ./ -name '*_wf.py')
@@ -48,4 +51,4 @@ workflow-test: workflow-build
 workflow-upload: workflow-build
 	@for file in $(YAML_WF); do \
 		$(MAKE) .workflows.upload-pipeline PIPELINE_FILE=$$file; \
-	done
+	done
diff --git a/transforms/code/code_quality/python/Makefile b/transforms/code/code_quality/python/Makefile
@@ -1,10 +1,21 @@
-
-# Define the root of the local git clone for the common rules to be able
+# Define the root of the local git clone for the common rules to be able 
 # know where they are running from.
 REPOROOT=../../../..
+
+# Set this, before including .make.defaults, to 
+#   1 if requirements reference the latest code in the data processing library 
+#     in this repo (that is not yet published to pypi).	 This is the default setting.
+#   0 if the transforms DPK dependencies are on wheels published to 
+#     pypi (e.g. data-prep-toolkit=0.2.1)
+#USE_REPO_LIB_SRC=1
+
+# Include a library of common .transform.* targets which most
+# transforms should be able to reuse.  However, feel free
+# to override/redefine the rules below. 
 include $(REPOROOT)/transforms/.make.transforms
 
-TRANSFORM_NAME=code_quality
+# Include the common configuration for this transform
+include ../transform.config
 
 # Use default rule inherited from makefile.common
 clean:: .transforms.clean

diff --git a/transforms/code/code_quality/ray/Makefile b/transforms/code/code_quality/ray/Makefile
@@ -1,10 +1,21 @@
-
-# Define the root of the local git clone for the common rules to be able
+# Define the root of the local git clone for the common rules to be able 
 # know where they are running from.
 REPOROOT=../../../..
+
+# Set this, before including .make.defaults, to 
+#   1 if requirements reference the latest code in the data processing library 
+#     in this repo (that is not yet published to pypi).	 This is the default setting.
+#   0 if the transforms DPK dependencies are on wheels published to 
+#     pypi (e.g. data-prep-toolkit=0.2.1)
+#USE_REPO_LIB_SRC=1
+
+# Include a library of common .transform.* targets which most
+# transforms should be able to reuse.  However, feel free
+# to override/redefine the rules below. 
 include $(REPOROOT)/transforms/.make.transforms
 
-TRANSFORM_NAME=code_quality
+# Include the common configuration for this transform
+include ../transform.config
 
 BASE_IMAGE=${RAY_BASE_IMAGE}
 

diff --git a/transforms/code/code_quality/transform.config b/transforms/code/code_quality/transform.config
@@ -0,0 +1,20 @@
+#
+# This is intended to be included across the Makefiles provided within
+# a given transform's directory tree,  so must use compatible syntax.
+#
+################################################################################
+# This defines the name of the transform and is used to match against
+# expected files and is used to define the transform's image name. 
+TRANSFORM_NAME=code_quality
+
+################################################################################
+# This defines the transforms' version number as would be used
+# when publishing the wheel.  In general, only the micro version
+# number should be advanced relative to the DPK_VERSION. 
+#
+# If you change the versions numbers, be sure to run "make set-versions" to 
+# update version numbers across the transform (e.g., pyproject.toml).
+CODE_QUALITY_PYTHON_VERSION=$(DPK_VERSION)
+CODE_QUALITY_RAY_VERSION=$(CODE_QUALITY_PYTHON_VERSION)
+CODE_QUALITY_SPARK_VERSION=$(CODE_QUALITY_PYTHON_VERSION)
+
diff --git a/...rms/code/header_cleanser/kfp_ray/Makefile → ...er_cleanser/kfp_ray/Makefile.disable-cicd b/...rms/code/header_cleanser/kfp_ray/Makefile → ...er_cleanser/kfp_ray/Makefile.disable-cicd
@@ -2,6 +2,9 @@ REPOROOT=${CURDIR}/../../../../
 WORKFLOW_VENV_ACTIVATE=${REPOROOT}/transforms/venv/bin/activate
 include $(REPOROOT)/transforms/.make.workflows
 
+# Include the common configuration for this transform
+include ../transform.config
+
 SRC_DIR=${CURDIR}/../ray/
 
 PYTHON_WF := $(shell find ./ -name '*_wf.py')
@@ -48,4 +51,4 @@ workflow-test: workflow-build
 workflow-upload: workflow-build
 	@for file in $(YAML_WF); do \
 		$(MAKE) .workflows.upload-pipeline PIPELINE_FILE=$$file; \
-	done
+	done
diff --git a/transforms/code/header_cleanser/python/Makefile b/transforms/code/header_cleanser/python/Makefile
@@ -1,10 +1,21 @@
-
-# Define the root of the local git clone for the common rules to be able
+# Define the root of the local git clone for the common rules to be able 
 # know where they are running from.
 REPOROOT=../../../..
+
+# Set this, before including .make.defaults, to 
+#   1 if requirements reference the latest code in the data processing library 
+#     in this repo (that is not yet published to pypi).	 This is the default setting.
+#   0 if the transforms DPK dependencies are on wheels published to 
+#     pypi (e.g. data-prep-toolkit=0.2.1)
+#USE_REPO_LIB_SRC=1
+
+# Include a library of common .transform.* targets which most
+# transforms should be able to reuse.  However, feel free
+# to override/redefine the rules below. 
 include $(REPOROOT)/transforms/.make.transforms
 
-TRANSFORM_NAME=header_cleanser
+# Include the common configuration for this transform
+include ../transform.config
 
 # Use default rule inherited from makefile.common
 clean:: .transforms.clean

diff --git a/transforms/code/header_cleanser/ray/Dockerfile b/transforms/code/header_cleanser/ray/Dockerfile
@@ -17,9 +17,11 @@ COPY --chown=ray:users pyproject.toml pyproject.toml
 RUN pip install --no-cache-dir -e .
 
 # Install system dependencies, including libgomp1
+USER root
 RUN sudo apt-get update && sudo apt-get install -y \
     libgomp1 \
     && sudo rm -rf /var/lib/apt/lists/*
+User ray
 
 # copy source data
 COPY ./src/header_cleanser_transform_ray.py .
@@ -36,4 +38,4 @@ ENV PYTHONPATH /home/ray
 ARG BUILD_DATE
 ARG GIT_COMMIT
 LABEL build-date=$BUILD_DATE
-LABEL git-commit=$GIT_COMMIT
+LABEL git-commit=$GIT_COMMIT
diff --git a/transforms/code/header_cleanser/ray/Makefile b/transforms/code/header_cleanser/ray/Makefile
@@ -1,10 +1,21 @@
-
-# Define the root of the local git clone for the common rules to be able
+# Define the root of the local git clone for the common rules to be able 
 # know where they are running from.
 REPOROOT=../../../..
+
+# Set this, before including .make.defaults, to 
+#   1 if requirements reference the latest code in the data processing library 
+#     in this repo (that is not yet published to pypi).	 This is the default setting.
+#   0 if the transforms DPK dependencies are on wheels published to 
+#     pypi (e.g. data-prep-toolkit=0.2.1)
+#USE_REPO_LIB_SRC=1
+
+# Include a library of common .transform.* targets which most
+# transforms should be able to reuse.  However, feel free
+# to override/redefine the rules below. 
 include $(REPOROOT)/transforms/.make.transforms
 
-TRANSFORM_NAME=header_cleanser
+# Include the common configuration for this transform
+include ../transform.config
 
 BASE_IMAGE=${RAY_BASE_IMAGE}