Add .make.* to triggers for transform tests.

.github/workflows/Makefile

@@ -8,7 +8,6 @@ LANG_TRANSFORMS=doc_chunk doc_quality lang_id pdf2parquet pii_redactor text_enco
 
 
 # A list that holds transforms that should not be tested with KFP
-KFP_BLACK_LIST="doc_chunk,pdf2parquet,pii_redactor"
 
 transform-tests:
 	$(MAKE) TRANSFORM_SUBDIR=universal .transform-tests 
@@ -29,13 +28,19 @@ transform-tests:
 	done
 
 .transform-kfp-tests:
-	@for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do	\
+	@KFP_BLACK_LIST=$$(cd ../..; bash scripts/check-workflows.sh -show-kfp-black-list);		\
+	for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do	\
 		dir=$$(basename $$i);				\
-		z=$$(echo ${KFP_BLACK_LIST} | grep -v $$dir);     \
-		if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ] || [ -z "$$z" ]; then		\
+		yml=test-$(TRANSFORM_SUBDIR)-$$dir-kfp.yml;                                     \
+		if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ]; then		\
+			echo No kfp_ray directory for $$dir. Skipping generation of $$yml;                                                      \
+			continue;			\
+		fi;					\
+		z=$$(echo $${KFP_BLACK_LIST} | grep  $$dir);     \
+		if  [ ! -z "$$z" ]; then		\
+			echo $$dir is black listed. Skipping generation of $$yml;                                                      \
 			continue;			\
 		fi;					\
-		yml=test-$(TRANSFORM_SUBDIR)-$$dir-kfp.yml;                                     \
 		echo Generating $$yml;                                                      \
 		cat test-kfp-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml;    \
 	done

.github/workflows/deploy-docs.yml

@@ -8,7 +8,7 @@ on:
             - "releases/**"
 jobs:
     deploy:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         env:
             REPO_URL: "https://github.com/${{ github.repository }}"
             REPO_BRANCH: "dev"

.github/workflows/deploy-library.yml

@@ -14,7 +14,7 @@ permissions:
 jobs:
     build-package:
         name: Build Ray data processing libraries
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -30,7 +30,7 @@ jobs:
         name: Publish packages to test.pypi.org
         # disabled
         if: false
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         needs: build-package
 
         steps:
@@ -47,7 +47,7 @@ jobs:
 
     publish-pypi:
         name: Publish release to pypi.org
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         needs: build-package
         # disabled as of now
         if: false

.github/workflows/deploy-transforms.yml

@@ -9,7 +9,7 @@ on:
 jobs:
     build-images:
         name: Build and check images
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -23,7 +23,7 @@ jobs:
         name: Publish packages to quay.io
         # disabled
         if: false
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         needs: build-images
 
         steps:

.github/workflows/test-code-code2parquet-kfp.yml

@@ -12,6 +12,8 @@ on:
         tags:
             - "*"
         paths:
+            - ".make.*"
+            - "transforms/.make.workflow"
             - "transforms/code/code2parquet/**"
             - "!kfp/**" # This is tested in separate workflow
             - "!data-processing-lib/**" # This is tested in separate workflow
@@ -24,6 +26,8 @@ on:
             - "dev"
             - "releases/**"
         paths:
+            - ".make.*"
+            - "transforms/.make.workflow"
             - "transforms/code/code2parquet/**"
             - "!data-processing-lib/**" # This is tested in separate workflow
             - "!kfp/**" # This is tested in separate workflow
@@ -32,10 +36,9 @@ on:
             - "!**/images/**"
             - "!**.gitignore"
 
-
 jobs:
     test-kfp-v1:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -74,7 +77,7 @@ jobs:
                   echo "Run transforms/code/code2parquet completed"
 
     test-kfp-v2:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -108,7 +111,7 @@ jobs:
                   export KFPv2=1
                   make -C $K8S_SETUP_SCRIPTS setup
                   make -C kfp/kfp_support_lib test
-                  make -C transforms workflow-build
+                  make -C transforms/code/code2parquet workflow-build
                   source $K8S_SETUP_SCRIPTS/common.sh
                   make -C transforms/code/code2parquet workflow-test
                   header_text "Run transforms/code/code2parquet completed"

.github/workflows/test-code-code2parquet.yml

@@ -12,6 +12,8 @@ on:
         tags:
             - "*"
         paths:
+            - ".make.*"
+            - "transforms/.make.transforms"
             - "transforms/code/code2parquet/**"
             - "data-processing-lib/**"
             - "!transforms/code/code2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow
@@ -26,6 +28,8 @@ on:
             - "dev"
             - "releases/**"
         paths:
+            - ".make.*"
+            - "transforms/.make.transforms"
             - "transforms/code/code2parquet/**"
             - "data-processing-lib/**"
             - "!transforms/code/code2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow
@@ -42,7 +46,7 @@ jobs:
         # The images are pushed if it is a merge to dev branch or a new tag is created.
         # The latter being part of the release process.
         # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file.
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         outputs:
             publish_images: ${{ steps.version.outputs.publish_images }}
         steps:
@@ -59,7 +63,7 @@ jobs:
                   fi
                   echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT"
     test-src:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -81,7 +85,7 @@ jobs:
                   fi
     test-image:
         needs: [check_if_push_image]
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         timeout-minutes: 120
         env:
             DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}

.github/workflows/test-code-code_quality-kfp.yml

@@ -12,6 +12,8 @@ on:
         tags:
             - "*"
         paths:
+            - ".make.*"
+            - "transforms/.make.workflow"
             - "transforms/code/code_quality/**"
             - "!kfp/**" # This is tested in separate workflow
             - "!data-processing-lib/**" # This is tested in separate workflow
@@ -24,6 +26,8 @@ on:
             - "dev"
             - "releases/**"
         paths:
+            - ".make.*"
+            - "transforms/.make.workflow"
             - "transforms/code/code_quality/**"
             - "!data-processing-lib/**" # This is tested in separate workflow
             - "!kfp/**" # This is tested in separate workflow
@@ -32,10 +36,9 @@ on:
             - "!**/images/**"
             - "!**.gitignore"
 
-
 jobs:
     test-kfp-v1:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -74,7 +77,7 @@ jobs:
                   echo "Run transforms/code/code_quality completed"
 
     test-kfp-v2:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -108,7 +111,7 @@ jobs:
                   export KFPv2=1
                   make -C $K8S_SETUP_SCRIPTS setup
                   make -C kfp/kfp_support_lib test
-                  make -C transforms workflow-build
+                  make -C transforms/code/code_quality workflow-build
                   source $K8S_SETUP_SCRIPTS/common.sh
                   make -C transforms/code/code_quality workflow-test
                   header_text "Run transforms/code/code_quality completed"

.github/workflows/test-code-code_quality.yml

@@ -12,6 +12,8 @@ on:
         tags:
             - "*"
         paths:
+            - ".make.*"
+            - "transforms/.make.transforms"
             - "transforms/code/code_quality/**"
             - "data-processing-lib/**"
             - "!transforms/code/code_quality/**/kfp_ray/**" # This is/will be tested in separate workflow
@@ -26,6 +28,8 @@ on:
             - "dev"
             - "releases/**"
         paths:
+            - ".make.*"
+            - "transforms/.make.transforms"
             - "transforms/code/code_quality/**"
             - "data-processing-lib/**"
             - "!transforms/code/code_quality/**/kfp_ray/**" # This is/will be tested in separate workflow
@@ -42,7 +46,7 @@ jobs:
         # The images are pushed if it is a merge to dev branch or a new tag is created.
         # The latter being part of the release process.
         # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file.
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         outputs:
             publish_images: ${{ steps.version.outputs.publish_images }}
         steps:
@@ -59,7 +63,7 @@ jobs:
                   fi
                   echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT"
     test-src:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -81,7 +85,7 @@ jobs:
                   fi
     test-image:
         needs: [check_if_push_image]
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         timeout-minutes: 120
         env:
             DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}

.github/workflows/test-code-header_cleanser-kfp.yml

@@ -12,6 +12,8 @@ on:
         tags:
             - "*"
         paths:
+            - ".make.*"
+            - "transforms/.make.workflow"
             - "transforms/code/header_cleanser/**"
             - "!kfp/**" # This is tested in separate workflow
             - "!data-processing-lib/**" # This is tested in separate workflow
@@ -24,6 +26,8 @@ on:
             - "dev"
             - "releases/**"
         paths:
+            - ".make.*"
+            - "transforms/.make.workflow"
             - "transforms/code/header_cleanser/**"
             - "!data-processing-lib/**" # This is tested in separate workflow
             - "!kfp/**" # This is tested in separate workflow
@@ -32,10 +36,9 @@ on:
             - "!**/images/**"
             - "!**.gitignore"
 
-
 jobs:
     test-kfp-v1:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -74,7 +77,7 @@ jobs:
                   echo "Run transforms/code/header_cleanser completed"
 
     test-kfp-v2:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -108,7 +111,7 @@ jobs:
                   export KFPv2=1
                   make -C $K8S_SETUP_SCRIPTS setup
                   make -C kfp/kfp_support_lib test
-                  make -C transforms workflow-build
+                  make -C transforms/code/header_cleanser workflow-build
                   source $K8S_SETUP_SCRIPTS/common.sh
                   make -C transforms/code/header_cleanser workflow-test
                   header_text "Run transforms/code/header_cleanser completed"

.github/workflows/test-code-header_cleanser.yml

@@ -12,6 +12,8 @@ on:
         tags:
             - "*"
         paths:
+            - ".make.*"
+            - "transforms/.make.transforms"
             - "transforms/code/header_cleanser/**"
             - "data-processing-lib/**"
             - "!transforms/code/header_cleanser/**/kfp_ray/**" # This is/will be tested in separate workflow
@@ -26,6 +28,8 @@ on:
             - "dev"
             - "releases/**"
         paths:
+            - ".make.*"
+            - "transforms/.make.transforms"
             - "transforms/code/header_cleanser/**"
             - "data-processing-lib/**"
             - "!transforms/code/header_cleanser/**/kfp_ray/**" # This is/will be tested in separate workflow
@@ -42,7 +46,7 @@ jobs:
         # The images are pushed if it is a merge to dev branch or a new tag is created.
         # The latter being part of the release process.
         # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file.
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         outputs:
             publish_images: ${{ steps.version.outputs.publish_images }}
         steps:
@@ -59,7 +63,7 @@ jobs:
                   fi
                   echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT"
     test-src:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -81,7 +85,7 @@ jobs:
                   fi
     test-image:
         needs: [check_if_push_image]
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         timeout-minutes: 120
         env:
             DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}