Merge branch 'dev' into version-mgt

IBM · Oct 3, 2024 · 9ef4266 · 9ef4266
2 parents 85804ec + a5f86da
commit 9ef4266
Show file tree

Hide file tree

Showing 108 changed files with 5,171 additions and 2,667 deletions.
diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile
@@ -8,7 +8,6 @@ LANG_TRANSFORMS=doc_chunk doc_quality lang_id pdf2parquet pii_redactor text_enco
 
 
 # A list that holds transforms that should not be tested with KFP
-KFP_BLACK_LIST="doc_chunk,pdf2parquet,pii_redactor"
 
 transform-tests:
 	$(MAKE) TRANSFORM_SUBDIR=universal .transform-tests 
@@ -29,13 +28,19 @@ transform-tests:
 	done
 
 .transform-kfp-tests:
-	@for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do	\
+	@KFP_BLACK_LIST=$$(cd ../..; bash scripts/check-workflows.sh -show-kfp-black-list);		\
+	for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do	\
 		dir=$$(basename $$i);				\
-		z=$$(echo ${KFP_BLACK_LIST} | grep -v $$dir);     \
-		if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ] || [ -z "$$z" ]; then		\
+		yml=test-$(TRANSFORM_SUBDIR)-$$dir-kfp.yml;                                     \
+		if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ]; then		\
+			echo No kfp_ray directory for $$dir. Skipping generation of $$yml;                                                      \
+			continue;			\
+		fi;					\
+		z=$$(echo $${KFP_BLACK_LIST} | grep  $$dir);     \
+		if  [ ! -z "$$z" ]; then		\
+			echo $$dir is black listed. Skipping generation of $$yml;                                                      \
 			continue;			\
 		fi;					\
-		yml=test-$(TRANSFORM_SUBDIR)-$$dir-kfp.yml;                                     \
 		echo Generating $$yml;                                                      \
 		cat test-kfp-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml;    \
 	done

diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml
@@ -8,7 +8,7 @@ on:
             - "releases/**"
 jobs:
     deploy:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         env:
             REPO_URL: "https://github.com/${{ github.repository }}"
             REPO_BRANCH: "dev"

diff --git a/.github/workflows/deploy-library.yml b/.github/workflows/deploy-library.yml
@@ -14,7 +14,7 @@ permissions:
 jobs:
     build-package:
         name: Build Ray data processing libraries
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -30,7 +30,7 @@ jobs:
         name: Publish packages to test.pypi.org
         # disabled
         if: false
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         needs: build-package
 
         steps:
@@ -47,7 +47,7 @@ jobs:
 
     publish-pypi:
         name: Publish release to pypi.org
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         needs: build-package
         # disabled as of now
         if: false

diff --git a/.github/workflows/deploy-transforms.yml b/.github/workflows/deploy-transforms.yml
@@ -9,7 +9,7 @@ on:
 jobs:
     build-images:
         name: Build and check images
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -23,7 +23,7 @@ jobs:
         name: Publish packages to quay.io
         # disabled
         if: false
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         needs: build-images
 
         steps:

diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml
@@ -12,6 +12,8 @@ on:
         tags:
             - "*"
         paths:
+            - ".make.*"
+            - "transforms/.make.workflow"
             - "transforms/code/code2parquet/**"
             - "!kfp/**" # This is tested in separate workflow
             - "!data-processing-lib/**" # This is tested in separate workflow
@@ -24,6 +26,8 @@ on:
             - "dev"
             - "releases/**"
         paths:
+            - ".make.*"
+            - "transforms/.make.workflow"
             - "transforms/code/code2parquet/**"
             - "!data-processing-lib/**" # This is tested in separate workflow
             - "!kfp/**" # This is tested in separate workflow
@@ -32,10 +36,14 @@ on:
             - "!**/images/**"
             - "!**.gitignore"
 
+# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+    cancel-in-progress: true
 
 jobs:
     test-kfp-v1:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -51,30 +59,34 @@ jobs:
             - name: Test KFP libs (shared and v1) and run a workflow
               timeout-minutes: 120
               run: |
-                  export REPOROOT=$PWD
-                  export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
-                  source $K8S_SETUP_SCRIPTS/requirements.env
-                  export PATH=$PATH:/tmp/
-                  curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
-                  chmod 777 /tmp/kind
-                  curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-                  chmod 700 /tmp/get_helm.sh
-                  HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
-                  chmod 777 /tmp/helm
-                  curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
-                  chmod 777 /tmp/kubectl
-                  curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
-                  chmod +x /tmp/mc
-                  export DEPLOY_KUBEFLOW=1
-                  make -C $K8S_SETUP_SCRIPTS setup
-                  make -C kfp/kfp_support_lib test
-                  make -C transforms workflow-build
-                  source $K8S_SETUP_SCRIPTS/common.sh
-                  make -C transforms/code/code2parquet workflow-test
-                  echo "Run transforms/code/code2parquet completed"
+                  if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
+                      export REPOROOT=$PWD
+                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
+                      source $K8S_SETUP_SCRIPTS/requirements.env
+                      export PATH=$PATH:/tmp/
+                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
+                      chmod 777 /tmp/kind
+                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
+                      chmod 700 /tmp/get_helm.sh
+                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
+                      chmod 777 /tmp/helm
+                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
+                      chmod 777 /tmp/kubectl
+                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
+                      chmod +x /tmp/mc
+                      export DEPLOY_KUBEFLOW=1
+                      make -C $K8S_SETUP_SCRIPTS setup
+                      make -C kfp/kfp_support_lib test
+                      make -C transforms/code/code2parquet workflow-build
+                      source $K8S_SETUP_SCRIPTS/common.sh
+                      make -C transforms/code/code2parquet workflow-test
+                      echo "Run transforms/code/code2parquet completed"
+                   else
+                      echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray"
+                   fi
 
     test-kfp-v2:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -90,25 +102,29 @@ jobs:
             - name: Test KFP libs (shared and v2) and run a workflow
               timeout-minutes: 120
               run: |
-                  export REPOROOT=$PWD
-                  export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
-                  source $K8S_SETUP_SCRIPTS/requirements.env
-                  export PATH=$PATH:/tmp/
-                  curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
-                  chmod 777 /tmp/kind
-                  curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
-                  chmod 700 /tmp/get_helm.sh
-                  HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
-                  chmod 777 /tmp/helm
-                  curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
-                  chmod 777 /tmp/kubectl
-                  curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
-                  chmod +x /tmp/mc
-                  export DEPLOY_KUBEFLOW=1
-                  export KFPv2=1
-                  make -C $K8S_SETUP_SCRIPTS setup
-                  make -C kfp/kfp_support_lib test
-                  make -C transforms workflow-build
-                  source $K8S_SETUP_SCRIPTS/common.sh
-                  make -C transforms/code/code2parquet workflow-test
-                  header_text "Run transforms/code/code2parquet completed"
+                  if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
+                      export REPOROOT=$PWD
+                      export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
+                      source $K8S_SETUP_SCRIPTS/requirements.env
+                      export PATH=$PATH:/tmp/
+                      curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64
+                      chmod 777 /tmp/kind
+                      curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
+                      chmod 700 /tmp/get_helm.sh
+                      HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo
+                      chmod 777 /tmp/helm
+                      curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl
+                      chmod 777 /tmp/kubectl
+                      curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc
+                      chmod +x /tmp/mc
+                      export DEPLOY_KUBEFLOW=1
+                      export KFPv2=1
+                      make -C $K8S_SETUP_SCRIPTS setup
+                      make -C kfp/kfp_support_lib test
+                      make -C transforms/code/code2parquet workflow-build
+                      source $K8S_SETUP_SCRIPTS/common.sh
+                      make -C transforms/code/code2parquet workflow-test
+                      echo "Run transforms/code/code2parquet completed"
+                  else
+                      echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray"
+                  fi
diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml
@@ -12,6 +12,8 @@ on:
         tags:
             - "*"
         paths:
+            - ".make.*"
+            - "transforms/.make.transforms"
             - "transforms/code/code2parquet/**"
             - "data-processing-lib/**"
             - "!transforms/code/code2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow
@@ -26,6 +28,8 @@ on:
             - "dev"
             - "releases/**"
         paths:
+            - ".make.*"
+            - "transforms/.make.transforms"
             - "transforms/code/code2parquet/**"
             - "data-processing-lib/**"
             - "!transforms/code/code2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow
@@ -36,13 +40,18 @@ on:
             - "!**/images/**"
             - "!**.gitignore"
 
+# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+    cancel-in-progress: true
+
 jobs:
     check_if_push_image:
         # check whether the Docker images should be pushed to the remote repository
         # The images are pushed if it is a merge to dev branch or a new tag is created.
         # The latter being part of the release process.
         # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file.
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         outputs:
             publish_images: ${{ steps.version.outputs.publish_images }}
         steps:
@@ -59,7 +68,7 @@ jobs:
                   fi
                   echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT"
     test-src:
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v4
@@ -81,7 +90,7 @@ jobs:
                   fi
     test-image:
         needs: [check_if_push_image]
-        runs-on: ubuntu-22.04
+        runs-on: ubuntu-latest
         timeout-minutes: 120
         env:
             DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }}