From 3fe8d2c193c564e65b298d4ee7363a5cd79a4024 Mon Sep 17 00:00:00 2001 From: David Wood Date: Fri, 13 Sep 2024 15:45:51 -0400 Subject: [PATCH 01/82] disable test workflow when none code files change Signed-off-by: David Wood --- .github/workflows/test.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0546f21e8..ae48ffcc6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,12 +8,26 @@ on: - "releases/**" tags: - "*" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "data-processing-lib/doc/**" + - "**/.gitignore" + - "**/.dockerignore" pull_request: branches: - "dev" - "releases/**" + paths-ignore: + - "**.md" + - "examples/**" + - "doc/**" + - "data-processing-lib/doc/**" + - "**/.gitignore" + - "**/.dockerignore" env: - KFP_BLACK_LIST: "doc_chunk-ray,pdf2parquet-ray,pii_redactor" + KFP_BLACK_LIST: "doc_chunk-ray,pdf2parquet-ray,pii_redactor" jobs: check_if_push_images: From 86951f883b8f50012d65979d292b93100c7edb25 Mon Sep 17 00:00:00 2001 From: David Wood Date: Mon, 16 Sep 2024 07:57:25 -0400 Subject: [PATCH 02/82] one more path-ignore in test.yml Signed-off-by: David Wood --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ae48ffcc6..d879949fa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ on: paths-ignore: - "**.md" - "examples/**" - - "doc/**" + - "**/doc/**" - "data-processing-lib/doc/**" - "**/.gitignore" - "**/.dockerignore" From d368dc7e5780037eee293ff4601110be7b122647 Mon Sep 17 00:00:00 2001 From: David Wood Date: Mon, 16 Sep 2024 09:06:50 -0400 Subject: [PATCH 03/82] one more fix for path-ignore in test.yml Signed-off-by: David Wood --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d879949fa..004862cb0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,6 @@ on: - "**.md" - "examples/**" - "**/doc/**" - - "data-processing-lib/doc/**" - "**/.gitignore" - "**/.dockerignore" pull_request: @@ -23,7 +22,6 @@ on: - "**.md" - "examples/**" - "**/doc/**" - - "data-processing-lib/doc/**" - "**/.gitignore" - "**/.dockerignore" env: From 6d7c186512738223db5412569559be916cff6875 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 09:24:52 -0400 Subject: [PATCH 04/82] test universal transform separately Signed-off-by: David Wood --- .github/workflows/test-universal.yml | 103 +++++++++++++++++++++++++++ .github/workflows/test.yml | 100 +++++++++++++------------- 2 files changed, 153 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/test-universal.yml diff --git a/.github/workflows/test-universal.yml b/.github/workflows/test-universal.yml new file mode 100644 index 000000000..2339aeb1b --- /dev/null +++ b/.github/workflows/test-universal.yml @@ -0,0 +1,103 @@ +name: Build, Test and (Optionally) Push images + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "**/.gitignore" + - "**/.dockerignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "**/.gitignore" + - "**/.dockerignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-universal: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Universal Transforms + run: | + make -C transforms/universal DOCKER=docker test-src + test-universal-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Universal Transform Images + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal DOCKER=docker test-image + - name: + Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/universal publish diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 004862cb0..e28f3d40c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -115,23 +115,23 @@ jobs: - name: Test Language Transforms run: | make -C transforms/language DOCKER=docker test-src - test-universal: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test Universal Transforms - run: | - make -C transforms/universal DOCKER=docker test-src + # test-universal: + # runs-on: ubuntu-22.04 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Free up space in github runner + # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + # run: | + # df -h + # sudo rm -rf "/usr/local/share/boost" + # sudo rm -rf "$AGENT_TOOLSDIRECTORY" + # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + # df -h + # - name: Test Universal Transforms + # run: | + # make -C transforms/universal DOCKER=docker test-src test-tools: runs-on: ubuntu-22.04 steps: @@ -334,39 +334,39 @@ jobs: if: needs.check_if_push_images.outputs.publish_images == 'true' run: make -C transforms/language publish - test-universal-images: - needs: [check_if_push_images] - runs-on: ubuntu-22.04 - timeout-minutes: 120 - env: - DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test Universal Transform Images - run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal DOCKER=docker test-image - - name: - Print space - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - docker images - - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' - run: make -C transforms/universal publish + # test-universal-images: + # needs: [check_if_push_images] + # runs-on: ubuntu-22.04 + # timeout-minutes: 120 + # env: + # DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + # DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Free up space in github runner + # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + # run: | + # df -h + # sudo rm -rf /opt/ghc + # sudo rm -rf "/usr/local/share/boost" + # sudo rm -rf "$AGENT_TOOLSDIRECTORY" + # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + # df -h + # - name: Test Universal Transform Images + # run: | + # make -C data-processing-lib/spark DOCKER=docker image + # make -C transforms/universal DOCKER=docker test-image + # - name: + # Print space + # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + # run: | + # df -h + # docker images + # - name: Publish images + # if: needs.check_if_push_images.outputs.publish_images == 'true' + # run: make -C transforms/universal publish build-kfp-components: needs: [check_if_push_images] runs-on: ubuntu-22.04 From 35f9f316576d27fc58344758973143d70c3a1e8a Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 09:29:17 -0400 Subject: [PATCH 05/82] rename test universal workflow Signed-off-by: David Wood --- .github/workflows/test-universal.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-universal.yml b/.github/workflows/test-universal.yml index 2339aeb1b..ec7b31d43 100644 --- a/.github/workflows/test-universal.yml +++ b/.github/workflows/test-universal.yml @@ -1,4 +1,4 @@ -name: Build, Test and (Optionally) Push images +name: Test universal transforms - src and images on: workflow_dispatch: From 37cd7ef9d2e496255d18e1dd32193e4ab2d4a5a2 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 09:31:09 -0400 Subject: [PATCH 06/82] add comments to noop src to trigger new universal test workflow Signed-off-by: David Wood --- transforms/universal/noop/python/src/noop_transform.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transforms/universal/noop/python/src/noop_transform.py b/transforms/universal/noop/python/src/noop_transform.py index a4bf5bb5c..7daf8c690 100644 --- a/transforms/universal/noop/python/src/noop_transform.py +++ b/transforms/universal/noop/python/src/noop_transform.py @@ -64,6 +64,7 @@ def transform(self, table: pa.Table, file_name: str = None) -> tuple[list[pa.Tab class NOOPTransformConfiguration(TransformConfiguration): + # Adding a test comment for cicd-opt branch testing - daw3rd 9/17/2024. """ Provides support for configuring and using the associated Transform class include configuration with CLI args. From a7fa50bc8840737bdb1ae1e4929e987f49fdaaca Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 09:36:03 -0400 Subject: [PATCH 07/82] fix paths in test universal workflow Signed-off-by: David Wood --- .github/workflows/test-universal.yml | 33 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test-universal.yml b/.github/workflows/test-universal.yml index ec7b31d43..1fd050d7e 100644 --- a/.github/workflows/test-universal.yml +++ b/.github/workflows/test-universal.yml @@ -9,25 +9,25 @@ on: tags: - "*" paths: - - "transforms/universal" - paths-ignore: - - "**.md" - - "examples/**" - - "**/doc/**" - - "**/.gitignore" - - "**/.dockerignore" + - "transforms/universal/**" + # paths-ignore: + # - "**.md" + # - "examples/**" + # - "**/doc/**" + # - "**/.gitignore" + # - "**/.dockerignore" pull_request: branches: - "dev" - "releases/**" paths: - - "transforms/universal" - paths-ignore: - - "**.md" - - "examples/**" - - "**/doc/**" - - "**/.gitignore" - - "**/.dockerignore" + - "transforms/universal/**" + # paths-ignore: + # - "**.md" + # - "examples/**" + # - "**/doc/**" + # - "**/.gitignore" + # - "**/.dockerignore" jobs: check_if_push_images: @@ -92,9 +92,8 @@ jobs: run: | make -C data-processing-lib/spark DOCKER=docker image make -C transforms/universal DOCKER=docker test-image - - name: - Print space - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | df -h docker images From 300d06a15c180c2dbc148023be6b218ffd0371b6 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 09:39:00 -0400 Subject: [PATCH 08/82] addj back ignore paths in test universal workflow Signed-off-by: David Wood --- .github/workflows/test-universal.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-universal.yml b/.github/workflows/test-universal.yml index 1fd050d7e..b8dbf3a41 100644 --- a/.github/workflows/test-universal.yml +++ b/.github/workflows/test-universal.yml @@ -10,24 +10,24 @@ on: - "*" paths: - "transforms/universal/**" - # paths-ignore: - # - "**.md" - # - "examples/**" - # - "**/doc/**" - # - "**/.gitignore" - # - "**/.dockerignore" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "**/.gitignore" + - "**/.dockerignore" pull_request: branches: - "dev" - "releases/**" paths: - "transforms/universal/**" - # paths-ignore: - # - "**.md" - # - "examples/**" - # - "**/doc/**" - # - "**/.gitignore" - # - "**/.dockerignore" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "**/.gitignore" + - "**/.dockerignore" jobs: check_if_push_images: From 0154179b47732694956a430527dafa0deb0358fd Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 09:40:29 -0400 Subject: [PATCH 09/82] another noop comment Signed-off-by: David Wood --- transforms/universal/noop/python/src/noop_transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/universal/noop/python/src/noop_transform.py b/transforms/universal/noop/python/src/noop_transform.py index 7daf8c690..62f647955 100644 --- a/transforms/universal/noop/python/src/noop_transform.py +++ b/transforms/universal/noop/python/src/noop_transform.py @@ -64,7 +64,7 @@ def transform(self, table: pa.Table, file_name: str = None) -> tuple[list[pa.Tab class NOOPTransformConfiguration(TransformConfiguration): - # Adding a test comment for cicd-opt branch testing - daw3rd 9/17/2024. + # Adding another test comment for cicd-opt branch testing - daw3rd 9/17/2024. """ Provides support for configuring and using the associated Transform class include configuration with CLI args. From 5dd78e6d717942256481dd753530d7ac6ec718d0 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 09:44:45 -0400 Subject: [PATCH 10/82] move ignored paths to paths in univesal test workflow Signed-off-by: David Wood --- .github/workflows/test-universal.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-universal.yml b/.github/workflows/test-universal.yml index b8dbf3a41..0cd0374eb 100644 --- a/.github/workflows/test-universal.yml +++ b/.github/workflows/test-universal.yml @@ -10,24 +10,22 @@ on: - "*" paths: - "transforms/universal/**" - paths-ignore: - - "**.md" - - "examples/**" - - "**/doc/**" - - "**/.gitignore" - - "**/.dockerignore" + - "!**.md" + - "!examples/**" + - "!**/doc/**" + - "!**/.gitignore" + - "!**/.dockerignore" pull_request: branches: - "dev" - "releases/**" paths: - "transforms/universal/**" - paths-ignore: - - "**.md" - - "examples/**" - - "**/doc/**" - - "**/.gitignore" - - "**/.dockerignore" + - "!**.md" + - "!examples/**" + - "!**/doc/**" + - "!**/.gitignore" + - "!**/.dockerignore" jobs: check_if_push_images: From d91da0d1ce41c6d7f090ae3f6f5aeb1a367c329a Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 09:51:28 -0400 Subject: [PATCH 11/82] test-universal workflow name changes Signed-off-by: David Wood --- .github/workflows/test-universal.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-universal.yml b/.github/workflows/test-universal.yml index 0cd0374eb..8ca1aa694 100644 --- a/.github/workflows/test-universal.yml +++ b/.github/workflows/test-universal.yml @@ -11,7 +11,6 @@ on: paths: - "transforms/universal/**" - "!**.md" - - "!examples/**" - "!**/doc/**" - "!**/.gitignore" - "!**/.dockerignore" @@ -22,7 +21,6 @@ on: paths: - "transforms/universal/**" - "!**.md" - - "!examples/**" - "!**/doc/**" - "!**/.gitignore" - "!**/.dockerignore" @@ -49,7 +47,7 @@ jobs: publish_images='true' fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" - test-universal: + test-src: runs-on: ubuntu-22.04 steps: - name: Checkout @@ -66,7 +64,7 @@ jobs: - name: Test Universal Transforms run: | make -C transforms/universal DOCKER=docker test-src - test-universal-images: + test-images: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 From 1be3292b7330df22327b844ea9be5516ed955a35 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 09:53:42 -0400 Subject: [PATCH 12/82] noop comments Signed-off-by: David Wood --- transforms/universal/noop/python/src/noop_transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/universal/noop/python/src/noop_transform.py b/transforms/universal/noop/python/src/noop_transform.py index 62f647955..89e589cd7 100644 --- a/transforms/universal/noop/python/src/noop_transform.py +++ b/transforms/universal/noop/python/src/noop_transform.py @@ -64,7 +64,7 @@ def transform(self, table: pa.Table, file_name: str = None) -> tuple[list[pa.Tab class NOOPTransformConfiguration(TransformConfiguration): - # Adding another test comment for cicd-opt branch testing - daw3rd 9/17/2024. + # Adding yet another test comment for cicd-opt branch testing - daw3rd 9/17/2024. """ Provides support for configuring and using the associated Transform class include configuration with CLI args. From 54d48b6e746fb616b6939c5d608f81d58e0f410e Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 11:52:07 -0400 Subject: [PATCH 13/82] noop readme change' Signed-off-by: David Wood --- transforms/universal/noop/python/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/transforms/universal/noop/python/README.md b/transforms/universal/noop/python/README.md index d438c8de2..db76c07b5 100644 --- a/transforms/universal/noop/python/README.md +++ b/transforms/universal/noop/python/README.md @@ -61,3 +61,4 @@ To see results of the transform. To use the transform image to transform your data, please refer to the [running images quickstart](../../../../doc/quick-start/run-transform-image.md), substituting the name of this transform image and runtime as appropriate. + From 288ffdf00aad90816b13fa5f9584a0a40882f3ef Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 11:55:17 -0400 Subject: [PATCH 14/82] change test universal not paths Signed-off-by: David Wood --- .github/workflows/test-universal.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-universal.yml b/.github/workflows/test-universal.yml index 8ca1aa694..c64270711 100644 --- a/.github/workflows/test-universal.yml +++ b/.github/workflows/test-universal.yml @@ -10,20 +10,18 @@ on: - "*" paths: - "transforms/universal/**" - - "!**.md" + - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" - - "!**/.dockerignore" pull_request: branches: - "dev" - "releases/**" paths: - "transforms/universal/**" - - "!**.md" + - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" - - "!**/.dockerignore" jobs: check_if_push_images: From 668b70d07184ecb6a7dbe34d03fe449e311cfea1 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 12:25:32 -0400 Subject: [PATCH 15/82] disable all but new noop and doc_id test workflows Signed-off-by: David Wood --- .github/workflows/Makefile | 8 ++ .github/workflows/test-transform.template | 96 +++++++++++++++++++++ .github/workflows/test-universal-doc_id.yml | 96 +++++++++++++++++++++ .github/workflows/test-universal-noop.yml | 96 +++++++++++++++++++++ .github/workflows/test-universal.yml | 4 +- .github/workflows/test.yml | 2 + 6 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/Makefile create mode 100644 .github/workflows/test-transform.template create mode 100644 .github/workflows/test-universal-doc_id.yml create mode 100644 .github/workflows/test-universal-noop.yml diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile new file mode 100644 index 000000000..8d1835541 --- /dev/null +++ b/.github/workflows/Makefile @@ -0,0 +1,8 @@ + +UNIVERSAL_TRANSFORMS=noop doc_id + +transform-tests: + for i in $(UNIVERSAL_TRANSFORMS); do \ + cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?universal/$$i?g" > test-universal-$$i.yml; \ + done + diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template new file mode 100644 index 000000000..c6f42ffd8 --- /dev/null +++ b/.github/workflows/test-transform.template @@ -0,0 +1,96 @@ +name: Test - @TARGET_TRANSFORM_DIR@ + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "@TARGET_TRANSFORM_DIR@/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "@TARGET_TRANSFORM_DIR@/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Universal Transforms + run: | + make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Universal Transform Images + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C @TARGET_TRANSFORM_DIR@ publish diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml new file mode 100644 index 000000000..e8c451768 --- /dev/null +++ b/.github/workflows/test-universal-doc_id.yml @@ -0,0 +1,96 @@ +name: Test - universal/doc_id + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "universal/doc_id/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "universal/doc_id/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Universal Transforms + run: | + make -C universal/doc_id DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Universal Transform Images + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C universal/doc_id DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C universal/doc_id publish diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml new file mode 100644 index 000000000..341def5a5 --- /dev/null +++ b/.github/workflows/test-universal-noop.yml @@ -0,0 +1,96 @@ +name: Test - universal/noop + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "universal/noop/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "universal/noop/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Universal Transforms + run: | + make -C universal/noop DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Universal Transform Images + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C universal/noop DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C universal/noop publish diff --git a/.github/workflows/test-universal.yml b/.github/workflows/test-universal.yml index c64270711..829835e90 100644 --- a/.github/workflows/test-universal.yml +++ b/.github/workflows/test-universal.yml @@ -1,4 +1,4 @@ -name: Test universal transforms - src and images +name: Test - Universal transforms on: workflow_dispatch: @@ -9,6 +9,7 @@ on: tags: - "*" paths: + - "!**" - "transforms/universal/**" - "!**/**.md" - "!**/doc/**" @@ -18,6 +19,7 @@ on: - "dev" - "releases/**" paths: + - "!**" - "transforms/universal/**" - "!**/**.md" - "!**/doc/**" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e28f3d40c..9bc9f9992 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,6 +9,7 @@ on: tags: - "*" paths-ignore: + - "**" # temporary disable - "**.md" - "examples/**" - "**/doc/**" @@ -19,6 +20,7 @@ on: - "dev" - "releases/**" paths-ignore: + - "**" # temporary disable - "**.md" - "examples/**" - "**/doc/**" From aa24ee71d8e9ab4134791971f10c49e0dc8e3799 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 12:27:20 -0400 Subject: [PATCH 16/82] code change in noop Signed-off-by: David Wood --- transforms/universal/noop/python/src/noop_transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/universal/noop/python/src/noop_transform.py b/transforms/universal/noop/python/src/noop_transform.py index 89e589cd7..dc16e9b53 100644 --- a/transforms/universal/noop/python/src/noop_transform.py +++ b/transforms/universal/noop/python/src/noop_transform.py @@ -115,5 +115,5 @@ def apply_input_params(self, args: Namespace) -> bool: return False self.params = self.params | captured - self.logger.info(f"noop parameters are : {self.params}") + self.logger.info(f"noop parameters are:: {self.params}") return True From 1d72cf5c274cf6509f27bb708ce01b6098fbe9f6 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 12:30:25 -0400 Subject: [PATCH 17/82] remake test transforms Signed-off-by: David Wood --- .github/workflows/Makefile | 2 +- .github/workflows/test-universal-doc_id.yml | 12 ++++++------ .github/workflows/test-universal-noop.yml | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile index 8d1835541..d03c97505 100644 --- a/.github/workflows/Makefile +++ b/.github/workflows/Makefile @@ -3,6 +3,6 @@ UNIVERSAL_TRANSFORMS=noop doc_id transform-tests: for i in $(UNIVERSAL_TRANSFORMS); do \ - cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?universal/$$i?g" > test-universal-$$i.yml; \ + cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/universal/$$i?g" > test-universal-$$i.yml; \ done diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index e8c451768..20ee6eccd 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -1,4 +1,4 @@ -name: Test - universal/doc_id +name: Test - transforms/universal/doc_id on: workflow_dispatch: @@ -9,7 +9,7 @@ on: tags: - "*" paths: - - "universal/doc_id/**" + - "transforms/universal/doc_id/**" - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" @@ -18,7 +18,7 @@ on: - "dev" - "releases/**" paths: - - "universal/doc_id/**" + - "transforms/universal/doc_id/**" - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" @@ -61,7 +61,7 @@ jobs: df -h - name: Test Universal Transforms run: | - make -C universal/doc_id DOCKER=docker test-src + make -C transforms/universal/doc_id DOCKER=docker test-src test-images: needs: [check_if_push_images] runs-on: ubuntu-22.04 @@ -85,7 +85,7 @@ jobs: - name: Test Universal Transform Images run: | make -C data-processing-lib/spark DOCKER=docker image - make -C universal/doc_id DOCKER=docker test-image + make -C transforms/universal/doc_id DOCKER=docker test-image - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -93,4 +93,4 @@ jobs: docker images - name: Publish images if: needs.check_if_push_images.outputs.publish_images == 'true' - run: make -C universal/doc_id publish + run: make -C transforms/universal/doc_id publish diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 341def5a5..29de62d8e 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -1,4 +1,4 @@ -name: Test - universal/noop +name: Test - transforms/universal/noop on: workflow_dispatch: @@ -9,7 +9,7 @@ on: tags: - "*" paths: - - "universal/noop/**" + - "transforms/universal/noop/**" - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" @@ -18,7 +18,7 @@ on: - "dev" - "releases/**" paths: - - "universal/noop/**" + - "transforms/universal/noop/**" - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" @@ -61,7 +61,7 @@ jobs: df -h - name: Test Universal Transforms run: | - make -C universal/noop DOCKER=docker test-src + make -C transforms/universal/noop DOCKER=docker test-src test-images: needs: [check_if_push_images] runs-on: ubuntu-22.04 @@ -85,7 +85,7 @@ jobs: - name: Test Universal Transform Images run: | make -C data-processing-lib/spark DOCKER=docker image - make -C universal/noop DOCKER=docker test-image + make -C transforms/universal/noop DOCKER=docker test-image - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -93,4 +93,4 @@ jobs: docker images - name: Publish images if: needs.check_if_push_images.outputs.publish_images == 'true' - run: make -C universal/noop publish + run: make -C transforms/universal/noop publish From 6bee51b73a19dc0950c71561b74e2db497dbbdbc Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 12:50:16 -0400 Subject: [PATCH 18/82] add individual test transform workflows Signed-off-by: David Wood --- .github/workflows/Makefile | 14 +- .github/workflows/test-code-code2parquet.yml | 98 +++++++++ .github/workflows/test-code-code_quality.yml | 98 +++++++++ .../workflows/test-code-header_cleanser.yml | 98 +++++++++ ...st-universal.yml => test-code-malware.yml} | 20 +- .../workflows/test-code-proglang_select.yml | 98 +++++++++ .../test-code-repo_level_ordering.yml | 98 +++++++++ .github/workflows/test-language-doc_chunk.yml | 98 +++++++++ .../workflows/test-language-doc_quality.yml | 98 +++++++++ .github/workflows/test-language-lang_id.yml | 98 +++++++++ .../workflows/test-language-pdf2parquet.yml | 98 +++++++++ .../workflows/test-language-pii_redactor.yml | 98 +++++++++ .../workflows/test-language-text_encoder.yml | 98 +++++++++ .github/workflows/test-transform.template | 6 +- .github/workflows/test-universal-doc_id.yml | 6 +- .github/workflows/test-universal-ededup.yml | 98 +++++++++ .github/workflows/test-universal-fdedup.yml | 98 +++++++++ .github/workflows/test-universal-filter.yml | 98 +++++++++ .../workflows/test-universal-html2parquet.yml | 98 +++++++++ .github/workflows/test-universal-noop.yml | 6 +- .github/workflows/test-universal-profiler.yml | 98 +++++++++ .github/workflows/test-universal-resize.yml | 98 +++++++++ .../workflows/test-universal-tokenization.yml | 98 +++++++++ .github/workflows/test.yml | 194 +++++++++--------- 24 files changed, 1896 insertions(+), 114 deletions(-) create mode 100644 .github/workflows/test-code-code2parquet.yml create mode 100644 .github/workflows/test-code-code_quality.yml create mode 100644 .github/workflows/test-code-header_cleanser.yml rename .github/workflows/{test-universal.yml => test-code-malware.yml} (87%) create mode 100644 .github/workflows/test-code-proglang_select.yml create mode 100644 .github/workflows/test-code-repo_level_ordering.yml create mode 100644 .github/workflows/test-language-doc_chunk.yml create mode 100644 .github/workflows/test-language-doc_quality.yml create mode 100644 .github/workflows/test-language-lang_id.yml create mode 100644 .github/workflows/test-language-pdf2parquet.yml create mode 100644 .github/workflows/test-language-pii_redactor.yml create mode 100644 .github/workflows/test-language-text_encoder.yml create mode 100644 .github/workflows/test-universal-ededup.yml create mode 100644 .github/workflows/test-universal-fdedup.yml create mode 100644 .github/workflows/test-universal-filter.yml create mode 100644 .github/workflows/test-universal-html2parquet.yml create mode 100644 .github/workflows/test-universal-profiler.yml create mode 100644 .github/workflows/test-universal-resize.yml create mode 100644 .github/workflows/test-universal-tokenization.yml diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile index d03c97505..cf6a9b96d 100644 --- a/.github/workflows/Makefile +++ b/.github/workflows/Makefile @@ -1,7 +1,19 @@ -UNIVERSAL_TRANSFORMS=noop doc_id +# Directories in the transforms/universal directory for which we want to generate test workflows +UNIVERSAL_TRANSFORMS=doc_id ededup fdedup filter html2parquet noop profiler resize tokenization +# Directories in the transforms/code directory for which we want to generate test workflows +CODE_TRANSFORMS=code2parquet code_quality header_cleanser malware proglang_select repo_level_ordering +# Directories in the transforms/language directory for which we want to generate test workflows +LANG_TRANSFORMS=doc_chunk doc_quality lang_id pdf2parquet pii_redactor text_encoder + transform-tests: + for i in $(CODE_TRANSFORMS); do \ + cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/code/$$i?g" > test-code-$$i.yml; \ + done + for i in $(LANG_TRANSFORMS); do \ + cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/language/$$i?g" > test-language-$$i.yml; \ + done for i in $(UNIVERSAL_TRANSFORMS); do \ cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/universal/$$i?g" > test-universal-$$i.yml; \ done diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml new file mode 100644 index 000000000..23df47188 --- /dev/null +++ b/.github/workflows/test-code-code2parquet.yml @@ -0,0 +1,98 @@ +name: Test - transforms/code/code2parquet + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/code2parquet/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/code2parquet/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/code/code2parquet + run: | + make -C transforms/code/code2parquet DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/code/code2parquet + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/code2parquet DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/code/code2parquet publish diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml new file mode 100644 index 000000000..15cb5883c --- /dev/null +++ b/.github/workflows/test-code-code_quality.yml @@ -0,0 +1,98 @@ +name: Test - transforms/code/code_quality + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/code_quality/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/code_quality/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/code/code_quality + run: | + make -C transforms/code/code_quality DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/code/code_quality + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/code_quality DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/code/code_quality publish diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml new file mode 100644 index 000000000..191e19a21 --- /dev/null +++ b/.github/workflows/test-code-header_cleanser.yml @@ -0,0 +1,98 @@ +name: Test - transforms/code/header_cleanser + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/header_cleanser/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/header_cleanser/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/code/header_cleanser + run: | + make -C transforms/code/header_cleanser DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/code/header_cleanser + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/header_cleanser DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/code/header_cleanser publish diff --git a/.github/workflows/test-universal.yml b/.github/workflows/test-code-malware.yml similarity index 87% rename from .github/workflows/test-universal.yml rename to .github/workflows/test-code-malware.yml index 829835e90..d998b4613 100644 --- a/.github/workflows/test-universal.yml +++ b/.github/workflows/test-code-malware.yml @@ -1,4 +1,4 @@ -name: Test - Universal transforms +name: Test - transforms/code/malware on: workflow_dispatch: @@ -9,8 +9,8 @@ on: tags: - "*" paths: - - "!**" - - "transforms/universal/**" + - "transforms/code/malware/**" + - "data-processing-lib/**" - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" @@ -19,8 +19,8 @@ on: - "dev" - "releases/**" paths: - - "!**" - - "transforms/universal/**" + - "transforms/code/malware/**" + - "data-processing-lib/**" - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" @@ -61,9 +61,9 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test Universal Transforms + - name: Test transform source in transforms/code/malware run: | - make -C transforms/universal DOCKER=docker test-src + make -C transforms/code/malware DOCKER=docker test-src test-images: needs: [check_if_push_images] runs-on: ubuntu-22.04 @@ -84,10 +84,10 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test Universal Transform Images + - name: Test transform image in transforms/code/malware run: | make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal DOCKER=docker test-image + make -C transforms/code/malware DOCKER=docker test-image - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -95,4 +95,4 @@ jobs: docker images - name: Publish images if: needs.check_if_push_images.outputs.publish_images == 'true' - run: make -C transforms/universal publish + run: make -C transforms/code/malware publish diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml new file mode 100644 index 000000000..7a81b4704 --- /dev/null +++ b/.github/workflows/test-code-proglang_select.yml @@ -0,0 +1,98 @@ +name: Test - transforms/code/proglang_select + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/proglang_select/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/proglang_select/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/code/proglang_select + run: | + make -C transforms/code/proglang_select DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/code/proglang_select + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/proglang_select DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/code/proglang_select publish diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml new file mode 100644 index 000000000..fac8998f2 --- /dev/null +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -0,0 +1,98 @@ +name: Test - transforms/code/repo_level_ordering + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/repo_level_ordering/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/repo_level_ordering/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/code/repo_level_ordering + run: | + make -C transforms/code/repo_level_ordering DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/code/repo_level_ordering + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/repo_level_ordering DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/code/repo_level_ordering publish diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml new file mode 100644 index 000000000..35364a5e6 --- /dev/null +++ b/.github/workflows/test-language-doc_chunk.yml @@ -0,0 +1,98 @@ +name: Test - transforms/language/doc_chunk + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/language/doc_chunk/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/language/doc_chunk/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/language/doc_chunk + run: | + make -C transforms/language/doc_chunk DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/language/doc_chunk + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/doc_chunk DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/language/doc_chunk publish diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml new file mode 100644 index 000000000..dcccf2682 --- /dev/null +++ b/.github/workflows/test-language-doc_quality.yml @@ -0,0 +1,98 @@ +name: Test - transforms/language/doc_quality + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/language/doc_quality/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/language/doc_quality/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/language/doc_quality + run: | + make -C transforms/language/doc_quality DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/language/doc_quality + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/doc_quality DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/language/doc_quality publish diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml new file mode 100644 index 000000000..43c3ed9ad --- /dev/null +++ b/.github/workflows/test-language-lang_id.yml @@ -0,0 +1,98 @@ +name: Test - transforms/language/lang_id + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/language/lang_id/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/language/lang_id/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/language/lang_id + run: | + make -C transforms/language/lang_id DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/language/lang_id + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/lang_id DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/language/lang_id publish diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml new file mode 100644 index 000000000..826796810 --- /dev/null +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -0,0 +1,98 @@ +name: Test - transforms/language/pdf2parquet + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/language/pdf2parquet/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/language/pdf2parquet/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/language/pdf2parquet + run: | + make -C transforms/language/pdf2parquet DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/language/pdf2parquet + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/pdf2parquet DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/language/pdf2parquet publish diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml new file mode 100644 index 000000000..bc5a6df55 --- /dev/null +++ b/.github/workflows/test-language-pii_redactor.yml @@ -0,0 +1,98 @@ +name: Test - transforms/language/pii_redactor + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/language/pii_redactor/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/language/pii_redactor/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/language/pii_redactor + run: | + make -C transforms/language/pii_redactor DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/language/pii_redactor + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/pii_redactor DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/language/pii_redactor publish diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml new file mode 100644 index 000000000..f43ebaac9 --- /dev/null +++ b/.github/workflows/test-language-text_encoder.yml @@ -0,0 +1,98 @@ +name: Test - transforms/language/text_encoder + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/language/text_encoder/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/language/text_encoder/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/language/text_encoder + run: | + make -C transforms/language/text_encoder DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/language/text_encoder + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/text_encoder DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/language/text_encoder publish diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index c6f42ffd8..8dc3b1f11 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -10,6 +10,7 @@ on: - "*" paths: - "@TARGET_TRANSFORM_DIR@/**" + - "data-processing-lib/**" - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" @@ -19,6 +20,7 @@ on: - "releases/**" paths: - "@TARGET_TRANSFORM_DIR@/**" + - "data-processing-lib/**" - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" @@ -59,7 +61,7 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test Universal Transforms + - name: Test transform source in @TARGET_TRANSFORM_DIR@ run: | make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-src test-images: @@ -82,7 +84,7 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test Universal Transform Images + - name: Test transform image in @TARGET_TRANSFORM_DIR@ run: | make -C data-processing-lib/spark DOCKER=docker image make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-image diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 20ee6eccd..bd4416dec 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -10,6 +10,7 @@ on: - "*" paths: - "transforms/universal/doc_id/**" + - "data-processing-lib/**" - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" @@ -19,6 +20,7 @@ on: - "releases/**" paths: - "transforms/universal/doc_id/**" + - "data-processing-lib/**" - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" @@ -59,7 +61,7 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test Universal Transforms + - name: Test transform source in transforms/universal/doc_id run: | make -C transforms/universal/doc_id DOCKER=docker test-src test-images: @@ -82,7 +84,7 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test Universal Transform Images + - name: Test transform image in transforms/universal/doc_id run: | make -C data-processing-lib/spark DOCKER=docker image make -C transforms/universal/doc_id DOCKER=docker test-image diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml new file mode 100644 index 000000000..b82cf24d0 --- /dev/null +++ b/.github/workflows/test-universal-ededup.yml @@ -0,0 +1,98 @@ +name: Test - transforms/universal/ededup + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/ededup/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/ededup/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/universal/ededup + run: | + make -C transforms/universal/ededup DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/universal/ededup + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/ededup DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/universal/ededup publish diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml new file mode 100644 index 000000000..f0e6b6cc9 --- /dev/null +++ b/.github/workflows/test-universal-fdedup.yml @@ -0,0 +1,98 @@ +name: Test - transforms/universal/fdedup + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/fdedup/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/fdedup/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/universal/fdedup + run: | + make -C transforms/universal/fdedup DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/universal/fdedup + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/fdedup DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/universal/fdedup publish diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml new file mode 100644 index 000000000..94be39ad3 --- /dev/null +++ b/.github/workflows/test-universal-filter.yml @@ -0,0 +1,98 @@ +name: Test - transforms/universal/filter + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/filter/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/filter/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/universal/filter + run: | + make -C transforms/universal/filter DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/universal/filter + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/filter DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/universal/filter publish diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml new file mode 100644 index 000000000..df49c573b --- /dev/null +++ b/.github/workflows/test-universal-html2parquet.yml @@ -0,0 +1,98 @@ +name: Test - transforms/universal/html2parquet + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/html2parquet/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/html2parquet/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/universal/html2parquet + run: | + make -C transforms/universal/html2parquet DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/universal/html2parquet + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/html2parquet DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/universal/html2parquet publish diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 29de62d8e..63a03e4cd 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -10,6 +10,7 @@ on: - "*" paths: - "transforms/universal/noop/**" + - "data-processing-lib/**" - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" @@ -19,6 +20,7 @@ on: - "releases/**" paths: - "transforms/universal/noop/**" + - "data-processing-lib/**" - "!**/**.md" - "!**/doc/**" - "!**/.gitignore" @@ -59,7 +61,7 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test Universal Transforms + - name: Test transform source in transforms/universal/noop run: | make -C transforms/universal/noop DOCKER=docker test-src test-images: @@ -82,7 +84,7 @@ jobs: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true df -h - - name: Test Universal Transform Images + - name: Test transform image in transforms/universal/noop run: | make -C data-processing-lib/spark DOCKER=docker image make -C transforms/universal/noop DOCKER=docker test-image diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml new file mode 100644 index 000000000..c05de63b0 --- /dev/null +++ b/.github/workflows/test-universal-profiler.yml @@ -0,0 +1,98 @@ +name: Test - transforms/universal/profiler + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/profiler/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/profiler/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/universal/profiler + run: | + make -C transforms/universal/profiler DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/universal/profiler + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/profiler DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/universal/profiler publish diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml new file mode 100644 index 000000000..ab5fa44d4 --- /dev/null +++ b/.github/workflows/test-universal-resize.yml @@ -0,0 +1,98 @@ +name: Test - transforms/universal/resize + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/resize/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/resize/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/universal/resize + run: | + make -C transforms/universal/resize DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/universal/resize + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/resize DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/universal/resize publish diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml new file mode 100644 index 000000000..289dc65dd --- /dev/null +++ b/.github/workflows/test-universal-tokenization.yml @@ -0,0 +1,98 @@ +name: Test - transforms/universal/tokenization + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/tokenization/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/tokenization/**" + - "data-processing-lib/**" + - "!**/**.md" + - "!**/doc/**" + - "!**/.gitignore" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/universal/tokenization + run: | + make -C transforms/universal/tokenization DOCKER=docker test-src + test-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/universal/tokenization + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/tokenization DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/universal/tokenization publish diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9bc9f9992..b58484f4d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -83,40 +83,40 @@ jobs: - name: Test data-processing-lib/spark run: | make -C data-processing-lib/spark DOCKER=docker venv test - test-code: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test Code Transforms - run: | - make -C transforms/code DOCKER=docker test-src - test-language: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test Language Transforms - run: | - make -C transforms/language DOCKER=docker test-src + # test-code: + # runs-on: ubuntu-22.04 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Free up space in github runner + # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + # run: | + # df -h + # sudo rm -rf "/usr/local/share/boost" + # sudo rm -rf "$AGENT_TOOLSDIRECTORY" + # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + # df -h + # - name: Test Code Transforms + # run: | + # make -C transforms/code DOCKER=docker test-src + # test-language: + # runs-on: ubuntu-22.04 + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Free up space in github runner + # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + # run: | + # df -h + # sudo rm -rf "/usr/local/share/boost" + # sudo rm -rf "$AGENT_TOOLSDIRECTORY" + # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + # df -h + # - name: Test Language Transforms + # run: | + # make -C transforms/language DOCKER=docker test-src # test-universal: # runs-on: ubuntu-22.04 # steps: @@ -273,73 +273,73 @@ jobs: if: needs.check_if_push_images.outputs.publish_images == 'true' run: | make -C data-processing-lib/spark publish-image - test-code-images: - needs: [check_if_push_images] - runs-on: ubuntu-22.04 - timeout-minutes: 30 - env: - DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test Code Transform Images - run: | - make -C data-processing-lib DOCKER=docker image - make -C transforms/code DOCKER=docker test-image - - name: - Print space - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - docker images - - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' - run: | - make -C transforms/code publish - test-language-images: - needs: [check_if_push_images] - runs-on: ubuntu-22.04 - timeout-minutes: 120 - env: - DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test language Transform Images - run: | - make -C data-processing-lib DOCKER=docker image - make -C transforms/language DOCKER=docker test-image - - name: Print space - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: df -h - - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' - run: make -C transforms/language publish + # test-code-images: + # needs: [check_if_push_images] + # runs-on: ubuntu-22.04 + # timeout-minutes: 30 + # env: + # DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + # DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Free up space in github runner + # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + # run: | + # df -h + # sudo rm -rf "/usr/local/share/boost" + # sudo rm -rf "$AGENT_TOOLSDIRECTORY" + # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + # df -h + # - name: Test Code Transform Images + # run: | + # make -C data-processing-lib DOCKER=docker image + # make -C transforms/code DOCKER=docker test-image + # - name: + # Print space + # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + # run: | + # df -h + # docker images + # - name: Publish images + # if: needs.check_if_push_images.outputs.publish_images == 'true' + # run: | + # make -C transforms/code publish + # test-language-images: + # needs: [check_if_push_images] + # runs-on: ubuntu-22.04 + # timeout-minutes: 120 + # env: + # DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + # DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + # steps: + # - name: Checkout + # uses: actions/checkout@v4 + # - name: Free up space in github runner + # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + # run: | + # df -h + # sudo rm -rf "/usr/local/share/boost" + # sudo rm -rf "$AGENT_TOOLSDIRECTORY" + # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + # df -h + # - name: Test language Transform Images + # run: | + # make -C data-processing-lib DOCKER=docker image + # make -C transforms/language DOCKER=docker test-image + # - name: Print space + # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + # run: df -h + # - name: Publish images + # if: needs.check_if_push_images.outputs.publish_images == 'true' + # run: make -C transforms/language publish # test-universal-images: # needs: [check_if_push_images] # runs-on: ubuntu-22.04 - # timeout-minutes: 120 + # timeout-minutes:kk 120 # env: # DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} # DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} From db18c7466b730a405390daa7c833a5f46a56021e Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 12:57:45 -0400 Subject: [PATCH 19/82] noop README change Signed-off-by: David Wood --- transforms/universal/noop/python/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transforms/universal/noop/python/README.md b/transforms/universal/noop/python/README.md index db76c07b5..361279edd 100644 --- a/transforms/universal/noop/python/README.md +++ b/transforms/universal/noop/python/README.md @@ -51,9 +51,11 @@ make run-cli-sample ... ``` Then + ```shell ls output ``` + To see results of the transform. ### Transforming data using the transform image From 42e18fca418884ea23f1ac6994c6a6de92aec0fc Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 13:07:19 -0400 Subject: [PATCH 20/82] better ignore of .md on test transform workflows Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 4 ++-- .github/workflows/test-code-code_quality.yml | 4 ++-- .github/workflows/test-code-header_cleanser.yml | 4 ++-- .github/workflows/test-code-malware.yml | 4 ++-- .github/workflows/test-code-proglang_select.yml | 4 ++-- .github/workflows/test-code-repo_level_ordering.yml | 4 ++-- .github/workflows/test-language-doc_chunk.yml | 4 ++-- .github/workflows/test-language-doc_quality.yml | 4 ++-- .github/workflows/test-language-lang_id.yml | 4 ++-- .github/workflows/test-language-pdf2parquet.yml | 4 ++-- .github/workflows/test-language-pii_redactor.yml | 4 ++-- .github/workflows/test-language-text_encoder.yml | 4 ++-- .github/workflows/test-transform.template | 4 ++-- .github/workflows/test-universal-doc_id.yml | 4 ++-- .github/workflows/test-universal-ededup.yml | 4 ++-- .github/workflows/test-universal-fdedup.yml | 4 ++-- .github/workflows/test-universal-filter.yml | 4 ++-- .github/workflows/test-universal-html2parquet.yml | 4 ++-- .github/workflows/test-universal-noop.yml | 4 ++-- .github/workflows/test-universal-profiler.yml | 4 ++-- .github/workflows/test-universal-resize.yml | 4 ++-- .github/workflows/test-universal-tokenization.yml | 4 ++-- 22 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index 23df47188..23266554e 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/code/code2parquet/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/code/code2parquet/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 15cb5883c..ca9af6f9c 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/code/code_quality/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/code/code_quality/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 191e19a21..d06b64e26 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/code/header_cleanser/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/code/header_cleanser/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index d998b4613..ed5124bac 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/code/malware/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/code/malware/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index 7a81b4704..6d1710238 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/code/proglang_select/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/code/proglang_select/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index fac8998f2..c73326553 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index 35364a5e6..20a4417d6 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/language/doc_chunk/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/language/doc_chunk/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index dcccf2682..2db9a86d6 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/language/doc_quality/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/language/doc_quality/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 43c3ed9ad..5f368d16e 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/language/lang_id/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/language/lang_id/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 826796810..0b2cfb4fe 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index bc5a6df55..34befd4c2 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/language/pii_redactor/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/language/pii_redactor/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index f43ebaac9..75d29862c 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/language/text_encoder/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/language/text_encoder/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 8dc3b1f11..5ddf5f936 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -11,7 +11,7 @@ on: paths: - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index bd4416dec..f3885e32f 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/universal/doc_id/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/universal/doc_id/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index b82cf24d0..3619ced22 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/universal/ededup/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/universal/ededup/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index f0e6b6cc9..1401b7770 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/universal/fdedup/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/universal/fdedup/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 94be39ad3..de64425fd 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/universal/filter/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/universal/filter/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index df49c573b..70c8f5d85 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/universal/html2parquet/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/universal/html2parquet/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 63a03e4cd..c2f96be81 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/universal/noop/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/universal/noop/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index c05de63b0..af47f10d8 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/universal/profiler/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/universal/profiler/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index ab5fa44d4..b4a3bfa3d 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/universal/resize/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/universal/resize/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 289dc65dd..bd3d096fc 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -11,7 +11,7 @@ on: paths: - "transforms/universal/tokenization/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" pull_request: @@ -21,7 +21,7 @@ on: paths: - "transforms/universal/tokenization/**" - "data-processing-lib/**" - - "!**/**.md" + - "!**.md" - "!**/doc/**" - "!**/.gitignore" From ca09d75967d01fb0302508cdf8adb72917683411 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 13:08:46 -0400 Subject: [PATCH 21/82] noop readme change Signed-off-by: David Wood --- transforms/universal/noop/python/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/transforms/universal/noop/python/README.md b/transforms/universal/noop/python/README.md index 361279edd..f62ff5d5e 100644 --- a/transforms/universal/noop/python/README.md +++ b/transforms/universal/noop/python/README.md @@ -55,12 +55,10 @@ Then ```shell ls output ``` - To see results of the transform. ### Transforming data using the transform image To use the transform image to transform your data, please refer to the [running images quickstart](../../../../doc/quick-start/run-transform-image.md), -substituting the name of this transform image and runtime as appropriate. - +substituting the name of this transform image and runtime as appropriate. \ No newline at end of file From f16780bc236533341bb07930b13a4f9c1c506149 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 13:26:39 -0400 Subject: [PATCH 22/82] noop test transform worklow 1 ignore Signed-off-by: David Wood --- .github/workflows/test-universal-noop.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index c2f96be81..a36c9656b 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -12,8 +12,6 @@ on: - "transforms/universal/noop/**" - "data-processing-lib/**" - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -22,8 +20,6 @@ on: - "transforms/universal/noop/**" - "data-processing-lib/**" - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_images: From 189bdd7f4fa6dfdc3e52bd80a3d436d2e471c3b1 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 13:27:11 -0400 Subject: [PATCH 23/82] noop readme Signed-off-by: David Wood --- transforms/universal/noop/python/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transforms/universal/noop/python/README.md b/transforms/universal/noop/python/README.md index f62ff5d5e..04f71beac 100644 --- a/transforms/universal/noop/python/README.md +++ b/transforms/universal/noop/python/README.md @@ -61,4 +61,5 @@ To see results of the transform. To use the transform image to transform your data, please refer to the [running images quickstart](../../../../doc/quick-start/run-transform-image.md), -substituting the name of this transform image and runtime as appropriate. \ No newline at end of file +substituting the name of this transform image and runtime as appropriate. + From 54b3c69ce495683a9bdafceef0e0645556fc3bf9 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 14:11:31 -0400 Subject: [PATCH 24/82] split out the tests into test-kfp/lib/misc and remove test.yml, add readme Signed-off-by: David Wood --- .github/workflows/README.md | 27 +++ .github/workflows/test-kfp.yml | 175 ++++++++++++++ .github/workflows/test-lib.yml | 100 ++++++++ .github/workflows/test-misc.yml | 53 +++++ .github/workflows/test.yml | 406 -------------------------------- 5 files changed, 355 insertions(+), 406 deletions(-) create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/test-kfp.yml create mode 100644 .github/workflows/test-lib.yml create mode 100644 .github/workflows/test-misc.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 000000000..cf0e40965 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,27 @@ +# Workflow Management + +Here we have the start of a sstem to automatically generated workflows. +In general, the design is to use template ans make to generate/update the workflows. + +``` +make +git commit -a -s -m "update workflows" +git push +``` + +should be sufficient. + +## Transforms +For transforms, we define a unique test workflow for each transform, based on a template [test-transform.template](test-transform.template). +The Makefile is used to (re)generate all workflows a necessary. By design, workflows for a given transform should run when + +* anything of substance effecting operation is modified in the transform's directory tree. +* anything in the core libraries in this repo supporting the transform change (e.g., data-processing/lib). +* Help! the workflow should NOT run when documentation (e.g., !**.md) is changed, however disabling this case does not seem to be working atm. + +When a new transform is added to the repository, + +1. add it to the corresponding macro in the Makefile +1. Run `make` to create the new test .yml for the transform +1. commit and push the change. + diff --git a/.github/workflows/test-kfp.yml b/.github/workflows/test-kfp.yml new file mode 100644 index 000000000..c95f8fbbb --- /dev/null +++ b/.github/workflows/test-kfp.yml @@ -0,0 +1,175 @@ +name: Build and test KFP + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "**/.gitignore" + - "**/.dockerignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "**/.gitignore" + - "**/.dockerignore" +env: + KFP_BLACK_LIST: "doc_chunk-ray,pdf2parquet-ray,pii_redactor" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + while : + do + dir=("code" "universal" "language") && index=$(($RANDOM % ${#dir[@]})) && subdirs=${dir[$index]} && transforms=($(find transforms/$subdirs -type d -maxdepth 1 -mindepth 1 )) + set -- "${transforms[@]}" && transforms=("$@") && size=${#transforms[@]} && index=$(($RANDOM % $size)) + transform=$(basename "${transforms[$index]}") + if [ -d ${transforms[$index]}/kfp_ray ] && echo ${KFP_BLACK_LIST} | grep -qv ${transform} ; then + header_text "Running ${transforms[$index]} workflow test" + break + fi + done + make -C ${transforms[$index]} workflow-test + echo "Run ${transforms[$index]} completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + while : + do + dir=("code" "universal" "language") && index=$(($RANDOM % ${#dir[@]})) && subdirs=${dir[$index]} && transforms=($(find transforms/$subdirs -type d -maxdepth 1 -mindepth 1 )) + set -- "${transforms[@]}" && transforms=("$@") && size=${#transforms[@]} && index=$(($RANDOM % $size)) + transform=$(basename "${transforms[$index]}") + if [ -d ${transforms[$index]}/kfp_ray ] && echo ${KFP_BLACK_LIST} | grep -qv ${transform} ; then + header_text "Running ${transforms[$index]} workflow test" + break + fi + done + make -C ${transforms[$index]} workflow-test + header_text "Run ${transforms[$index]} completed" + build-kfp-components: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Build + run: | + make -C kfp/kfp_ray_components DOCKER=docker image + make KFPv2=1 -C kfp/kfp_ray_components DOCKER=docker image + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C kfp/kfp_ray_components publish diff --git a/.github/workflows/test-lib.yml b/.github/workflows/test-lib.yml new file mode 100644 index 000000000..fe6eff0f1 --- /dev/null +++ b/.github/workflows/test-lib.yml @@ -0,0 +1,100 @@ +name: Test DPK libraries and (Optionally) Push base DPK images + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + # Note: the transform workflows are expected to trigger when data-processing-lib/** changes + - "data-processing-lib/**" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + # Note: the transform workflows are expected to trigger when data-processing-lib/** changes + - "data-processing-lib/**" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-python-lib: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test data-processing-lib/python + run: | + make -C data-processing-lib/python DOCKER=docker venv test + test-ray-lib: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test data-processing-lib/ray + run: | + make -C data-processing-lib/ray DOCKER=docker venv test + test-spark-lib: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test data-processing-lib/spark + run: | + make -C data-processing-lib/spark DOCKER=docker venv test + test-data-processing-lib-images: + needs: [check_if_push_images] + if: needs.check_if_push_images.outputs.publish_images == 'true' + runs-on: ubuntu-22.04 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Code Transform Images + run: | + make -C data-processing-lib/spark DOCKER=docker image + - name: + Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: | + make -C data-processing-lib/spark publish-image diff --git a/.github/workflows/test-misc.yml b/.github/workflows/test-misc.yml new file mode 100644 index 000000000..d28217499 --- /dev/null +++ b/.github/workflows/test-misc.yml @@ -0,0 +1,53 @@ +name: Miscellaneous tests + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "**/.gitignore" + - "**/.dockerignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "**/.gitignore" + - "**/.dockerignore" + +jobs: + test-make: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test top-level recursive make targets. + run: | + make -n clean test build publish set-versions + test-tools: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test tools + run: | + make -C tools DOCKER=docker venv test + test-tool-images: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test Tool images + run: | + make -C tools/ingest2parquet DOCKER=docker test-image diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index b58484f4d..000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,406 +0,0 @@ -name: Build, Test and (Optionally) Push images - -on: - workflow_dispatch: - push: - branches: - - "dev" - - "releases/**" - tags: - - "*" - paths-ignore: - - "**" # temporary disable - - "**.md" - - "examples/**" - - "**/doc/**" - - "**/.gitignore" - - "**/.dockerignore" - pull_request: - branches: - - "dev" - - "releases/**" - paths-ignore: - - "**" # temporary disable - - "**.md" - - "examples/**" - - "**/doc/**" - - "**/.gitignore" - - "**/.dockerignore" -env: - KFP_BLACK_LIST: "doc_chunk-ray,pdf2parquet-ray,pii_redactor" - -jobs: - check_if_push_images: - # check whether the Docker images should be pushed to the remote repository - # The images are pushed if it is a merge to dev branch or a new tag is created. - # The latter being part of the release process. - # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 - outputs: - publish_images: ${{ steps.version.outputs.publish_images }} - steps: - - id: version - run: | - publish_images='false' - if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; - then - publish_images='true' - fi - if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; - then - publish_images='true' - fi - echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" - test-make: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Test top-level recursive make targets. - run: | - make -n clean test build publish set-versions - test-python-lib: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Test data-processing-lib/python - run: | - make -C data-processing-lib/python DOCKER=docker venv test - test-ray-lib: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Test data-processing-lib/ray - run: | - make -C data-processing-lib/ray DOCKER=docker venv test - test-spark-lib: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Test data-processing-lib/spark - run: | - make -C data-processing-lib/spark DOCKER=docker venv test - # test-code: - # runs-on: ubuntu-22.04 - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # - name: Free up space in github runner - # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - # run: | - # df -h - # sudo rm -rf "/usr/local/share/boost" - # sudo rm -rf "$AGENT_TOOLSDIRECTORY" - # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - # df -h - # - name: Test Code Transforms - # run: | - # make -C transforms/code DOCKER=docker test-src - # test-language: - # runs-on: ubuntu-22.04 - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # - name: Free up space in github runner - # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - # run: | - # df -h - # sudo rm -rf "/usr/local/share/boost" - # sudo rm -rf "$AGENT_TOOLSDIRECTORY" - # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - # df -h - # - name: Test Language Transforms - # run: | - # make -C transforms/language DOCKER=docker test-src - # test-universal: - # runs-on: ubuntu-22.04 - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # - name: Free up space in github runner - # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - # run: | - # df -h - # sudo rm -rf "/usr/local/share/boost" - # sudo rm -rf "$AGENT_TOOLSDIRECTORY" - # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup - # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - # df -h - # - name: Test Universal Transforms - # run: | - # make -C transforms/universal DOCKER=docker test-src - test-tools: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Test tools - run: | - make -C tools DOCKER=docker venv test - test-kfp-v1: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test KFP libs (shared and v1) and run a workflow - timeout-minutes: 120 - run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - while : - do - dir=("code" "universal" "language") && index=$(($RANDOM % ${#dir[@]})) && subdirs=${dir[$index]} && transforms=($(find transforms/$subdirs -type d -maxdepth 1 -mindepth 1 )) - set -- "${transforms[@]}" && transforms=("$@") && size=${#transforms[@]} && index=$(($RANDOM % $size)) - transform=$(basename "${transforms[$index]}") - if [ -d ${transforms[$index]}/kfp_ray ] && echo ${KFP_BLACK_LIST} | grep -qv ${transform} ; then - header_text "Running ${transforms[$index]} workflow test" - break - fi - done - make -C ${transforms[$index]} workflow-test - echo "Run ${transforms[$index]} completed" - - test-kfp-v2: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test KFP libs (shared and v2) and run a workflow - timeout-minutes: 120 - run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - while : - do - dir=("code" "universal" "language") && index=$(($RANDOM % ${#dir[@]})) && subdirs=${dir[$index]} && transforms=($(find transforms/$subdirs -type d -maxdepth 1 -mindepth 1 )) - set -- "${transforms[@]}" && transforms=("$@") && size=${#transforms[@]} && index=$(($RANDOM % $size)) - transform=$(basename "${transforms[$index]}") - if [ -d ${transforms[$index]}/kfp_ray ] && echo ${KFP_BLACK_LIST} | grep -qv ${transform} ; then - header_text "Running ${transforms[$index]} workflow test" - break - fi - done - make -C ${transforms[$index]} workflow-test - header_text "Run ${transforms[$index]} completed" - test-data-processing-lib-images: - needs: [check_if_push_images] - if: needs.check_if_push_images.outputs.publish_images == 'true' - runs-on: ubuntu-22.04 - env: - DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - timeout-minutes: 30 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test Code Transform Images - run: | - make -C data-processing-lib/spark DOCKER=docker image - - name: - Print space - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - docker images - - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' - run: | - make -C data-processing-lib/spark publish-image - # test-code-images: - # needs: [check_if_push_images] - # runs-on: ubuntu-22.04 - # timeout-minutes: 30 - # env: - # DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - # DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # - name: Free up space in github runner - # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - # run: | - # df -h - # sudo rm -rf "/usr/local/share/boost" - # sudo rm -rf "$AGENT_TOOLSDIRECTORY" - # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - # df -h - # - name: Test Code Transform Images - # run: | - # make -C data-processing-lib DOCKER=docker image - # make -C transforms/code DOCKER=docker test-image - # - name: - # Print space - # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - # run: | - # df -h - # docker images - # - name: Publish images - # if: needs.check_if_push_images.outputs.publish_images == 'true' - # run: | - # make -C transforms/code publish - # test-language-images: - # needs: [check_if_push_images] - # runs-on: ubuntu-22.04 - # timeout-minutes: 120 - # env: - # DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - # DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # - name: Free up space in github runner - # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - # run: | - # df -h - # sudo rm -rf "/usr/local/share/boost" - # sudo rm -rf "$AGENT_TOOLSDIRECTORY" - # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - # df -h - # - name: Test language Transform Images - # run: | - # make -C data-processing-lib DOCKER=docker image - # make -C transforms/language DOCKER=docker test-image - # - name: Print space - # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - # run: df -h - # - name: Publish images - # if: needs.check_if_push_images.outputs.publish_images == 'true' - # run: make -C transforms/language publish - - # test-universal-images: - # needs: [check_if_push_images] - # runs-on: ubuntu-22.04 - # timeout-minutes:kk 120 - # env: - # DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - # DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # - name: Free up space in github runner - # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - # run: | - # df -h - # sudo rm -rf /opt/ghc - # sudo rm -rf "/usr/local/share/boost" - # sudo rm -rf "$AGENT_TOOLSDIRECTORY" - # sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - # sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - # df -h - # - name: Test Universal Transform Images - # run: | - # make -C data-processing-lib/spark DOCKER=docker image - # make -C transforms/universal DOCKER=docker test-image - # - name: - # Print space - # # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - # run: | - # df -h - # docker images - # - name: Publish images - # if: needs.check_if_push_images.outputs.publish_images == 'true' - # run: make -C transforms/universal publish - build-kfp-components: - needs: [check_if_push_images] - runs-on: ubuntu-22.04 - timeout-minutes: 30 - env: - DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Build - run: | - make -C kfp/kfp_ray_components DOCKER=docker image - make KFPv2=1 -C kfp/kfp_ray_components DOCKER=docker image - - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' - run: make -C kfp/kfp_ray_components publish - test-tool-images: - runs-on: ubuntu-22.04 - timeout-minutes: 30 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test Tool images - run: | - make -C tools/ingest2parquet DOCKER=docker test-image From d530ac95dd96befba32675932c23b21d337cd8c0 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 14:18:34 -0400 Subject: [PATCH 25/82] test-kfp only on kfp/** Signed-off-by: David Wood --- .github/workflows/test-kfp.yml | 17 +++++------------ .github/workflows/test-transform.template | 3 +++ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-kfp.yml b/.github/workflows/test-kfp.yml index c95f8fbbb..34c745ee5 100644 --- a/.github/workflows/test-kfp.yml +++ b/.github/workflows/test-kfp.yml @@ -8,22 +8,15 @@ on: - "releases/**" tags: - "*" - paths-ignore: - - "**.md" - - "examples/**" - - "**/doc/**" - - "**/.gitignore" - - "**/.dockerignore" + paths: + - "kfp/**" pull_request: branches: - "dev" - "releases/**" - paths-ignore: - - "**.md" - - "examples/**" - - "**/doc/**" - - "**/.gitignore" - - "**/.dockerignore" + paths: + - "kfp/**" + env: KFP_BLACK_LIST: "doc_chunk-ray,pdf2parquet-ray,pii_redactor" diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 5ddf5f936..42c2a6974 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - @TARGET_TRANSFORM_DIR@ on: From 781113f00f4ac1dc80856a160cba2a42911239ad Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 14:19:34 -0400 Subject: [PATCH 26/82] noop code change to trigger build Signed-off-by: David Wood --- transforms/universal/noop/python/src/noop_transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/universal/noop/python/src/noop_transform.py b/transforms/universal/noop/python/src/noop_transform.py index dc16e9b53..22e595a5b 100644 --- a/transforms/universal/noop/python/src/noop_transform.py +++ b/transforms/universal/noop/python/src/noop_transform.py @@ -115,5 +115,5 @@ def apply_input_params(self, args: Namespace) -> bool: return False self.params = self.params | captured - self.logger.info(f"noop parameters are:: {self.params}") + self.logger.info(f"noop parameters are::: {self.params}") return True From f09fbeec89d3826dc2589cb0212a6a5ca164197e Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 14:23:21 -0400 Subject: [PATCH 27/82] comments in workflows Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 3 +++ .github/workflows/test-code-code_quality.yml | 3 +++ .github/workflows/test-code-header_cleanser.yml | 3 +++ .github/workflows/test-code-malware.yml | 3 +++ .github/workflows/test-code-proglang_select.yml | 3 +++ .github/workflows/test-code-repo_level_ordering.yml | 3 +++ .github/workflows/test-language-doc_chunk.yml | 3 +++ .github/workflows/test-language-doc_quality.yml | 3 +++ .github/workflows/test-language-lang_id.yml | 3 +++ .github/workflows/test-language-pdf2parquet.yml | 3 +++ .github/workflows/test-language-pii_redactor.yml | 3 +++ .github/workflows/test-language-text_encoder.yml | 3 +++ .github/workflows/test-universal-doc_id.yml | 3 +++ .github/workflows/test-universal-ededup.yml | 3 +++ .github/workflows/test-universal-fdedup.yml | 3 +++ .github/workflows/test-universal-filter.yml | 3 +++ .github/workflows/test-universal-html2parquet.yml | 3 +++ .github/workflows/test-universal-noop.yml | 7 +++++++ .github/workflows/test-universal-profiler.yml | 3 +++ .github/workflows/test-universal-resize.yml | 3 +++ .github/workflows/test-universal-tokenization.yml | 3 +++ 21 files changed, 67 insertions(+) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index 23266554e..e64f1fcf8 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/code/code2parquet on: diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index ca9af6f9c..5b2543d9d 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/code/code_quality on: diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index d06b64e26..c52b8a382 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/code/header_cleanser on: diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index ed5124bac..2ddf94ce1 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/code/malware on: diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index 6d1710238..d689a02ed 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/code/proglang_select on: diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index c73326553..699f6e590 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/code/repo_level_ordering on: diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index 20a4417d6..f1b68c6f5 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/language/doc_chunk on: diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index 2db9a86d6..1b812f3af 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/language/doc_quality on: diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 5f368d16e..da78316f9 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/language/lang_id on: diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 0b2cfb4fe..11b650da8 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/language/pdf2parquet on: diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 34befd4c2..00a8b4bd7 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/language/pii_redactor on: diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index 75d29862c..35297eb3d 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/language/text_encoder on: diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index f3885e32f..64facc011 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/universal/doc_id on: diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 3619ced22..6be50c5fd 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/universal/ededup on: diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 1401b7770..ea4a362c0 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/universal/fdedup on: diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index de64425fd..7dd9198d1 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/universal/filter on: diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index 70c8f5d85..5148adeaf 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/universal/html2parquet on: diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index a36c9656b..2da9f344e 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/universal/noop on: @@ -12,6 +15,8 @@ on: - "transforms/universal/noop/**" - "data-processing-lib/**" - "!**.md" + - "!**/doc/**" + - "!**/.gitignore" pull_request: branches: - "dev" @@ -20,6 +25,8 @@ on: - "transforms/universal/noop/**" - "data-processing-lib/**" - "!**.md" + - "!**/doc/**" + - "!**/.gitignore" jobs: check_if_push_images: diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index af47f10d8..d7aae1ac5 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/universal/profiler on: diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index b4a3bfa3d..f7013561d 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/universal/resize on: diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index bd3d096fc..f10a02631 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -1,3 +1,6 @@ +# +# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# name: Test - transforms/universal/tokenization on: From 193f439a7ece7ac43548a6bef680b95d6736938e Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 14:44:47 -0400 Subject: [PATCH 28/82] updated workflow readme Signed-off-by: David Wood --- .github/workflows/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index cf0e40965..dc97c7330 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,7 +1,7 @@ # Workflow Management -Here we have the start of a sstem to automatically generated workflows. -In general, the design is to use template ans make to generate/update the workflows. +Here we have the start of a system to automatically generated workflows (currently only for transforms). +In general, the design is to use templates and `make` to generate/update the workflows. ``` make @@ -12,16 +12,16 @@ git push should be sufficient. ## Transforms -For transforms, we define a unique test workflow for each transform, based on a template [test-transform.template](test-transform.template). +We define a unique test workflow for each transform, based on a common template [test-transform.template](test-transform.template). The Makefile is used to (re)generate all workflows a necessary. By design, workflows for a given transform should run when * anything of substance effecting operation is modified in the transform's directory tree. -* anything in the core libraries in this repo supporting the transform change (e.g., data-processing/lib). -* Help! the workflow should NOT run when documentation (e.g., !**.md) is changed, however disabling this case does not seem to be working atm. +* anything in the core libraries in this repo (e.g., data-processing/lib) assuming the transform depends on these. +* Help! the workflow should NOT run when documentation (e.g., !**.md) is changed, however this case does not seem to be working atm. When a new transform is added to the repository, -1. add it to the corresponding macro in the Makefile +1. add it to the corresponding macro in the Makefile (in alphabetical order please). 1. Run `make` to create the new test .yml for the transform -1. commit and push the change. +1. commit and push the change to your branch with the new transform. From 3ed0fbdf0c2b5fae3c6eeeb7e4f47a2202f5ffac Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 14:47:34 -0400 Subject: [PATCH 29/82] only run build-library workflow on data-processing-lib changes Signed-off-by: David Wood --- .github/workflows/build-library.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-library.yml b/.github/workflows/build-library.yml index ce4ca7f04..28f744cbd 100644 --- a/.github/workflows/build-library.yml +++ b/.github/workflows/build-library.yml @@ -6,10 +6,14 @@ on: branches: - "dev" - "releases/**" + paths: + - "data-processing-lib/**" pull_request: branches: - "dev" - "releases/**" + paths: + - "data-processing-lib/**" jobs: build-python-lib: runs-on: ubuntu-22.04 From 60343d32999c63fb237d7de87535d11ee2295cdb Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 14:55:08 -0400 Subject: [PATCH 30/82] try and ignore docs in build-library, test-kfp/lib Signed-off-by: David Wood --- .github/workflows/build-library.yml | 6 ++++++ .github/workflows/test-kfp.yml | 6 ++++++ .github/workflows/test-lib.yml | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/.github/workflows/build-library.yml b/.github/workflows/build-library.yml index 28f744cbd..d7219ea34 100644 --- a/.github/workflows/build-library.yml +++ b/.github/workflows/build-library.yml @@ -8,12 +8,18 @@ on: - "releases/**" paths: - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/.gitignore" pull_request: branches: - "dev" - "releases/**" paths: - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/.gitignore" jobs: build-python-lib: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-kfp.yml b/.github/workflows/test-kfp.yml index 34c745ee5..549c0d37d 100644 --- a/.github/workflows/test-kfp.yml +++ b/.github/workflows/test-kfp.yml @@ -10,12 +10,18 @@ on: - "*" paths: - "kfp/**" + - "!**.md" + - "!**/doc/**" + - "!**/.gitignore" pull_request: branches: - "dev" - "releases/**" paths: - "kfp/**" + - "!**.md" + - "!**/doc/**" + - "!**/.gitignore" env: KFP_BLACK_LIST: "doc_chunk-ray,pdf2parquet-ray,pii_redactor" diff --git a/.github/workflows/test-lib.yml b/.github/workflows/test-lib.yml index fe6eff0f1..bb5a39531 100644 --- a/.github/workflows/test-lib.yml +++ b/.github/workflows/test-lib.yml @@ -11,6 +11,9 @@ on: paths: # Note: the transform workflows are expected to trigger when data-processing-lib/** changes - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/.gitignore" pull_request: branches: - "dev" @@ -18,6 +21,9 @@ on: paths: # Note: the transform workflows are expected to trigger when data-processing-lib/** changes - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/.gitignore" jobs: check_if_push_images: From dec3aa9e55900a4798cd1bc182b43aeb20f5db0c Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 15:03:24 -0400 Subject: [PATCH 31/82] workflow title changes for consistency Signed-off-by: David Wood --- .github/workflows/test-kfp.yml | 2 +- .github/workflows/test-lib.yml | 2 +- .github/workflows/test-misc.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-kfp.yml b/.github/workflows/test-kfp.yml index 549c0d37d..1f96cc6da 100644 --- a/.github/workflows/test-kfp.yml +++ b/.github/workflows/test-kfp.yml @@ -1,4 +1,4 @@ -name: Build and test KFP +name: Test/build KFP on: workflow_dispatch: diff --git a/.github/workflows/test-lib.yml b/.github/workflows/test-lib.yml index bb5a39531..1ed6a3f9f 100644 --- a/.github/workflows/test-lib.yml +++ b/.github/workflows/test-lib.yml @@ -1,4 +1,4 @@ -name: Test DPK libraries and (Optionally) Push base DPK images +name: Test DPK libs and (Optionally) Push base DPK images on: workflow_dispatch: diff --git a/.github/workflows/test-misc.yml b/.github/workflows/test-misc.yml index d28217499..50f3edbc1 100644 --- a/.github/workflows/test-misc.yml +++ b/.github/workflows/test-misc.yml @@ -1,4 +1,4 @@ -name: Miscellaneous tests +name: Test - miscellaneous on: workflow_dispatch: From e6b1d62d86158acf1d88710c5374f9fc969f9be7 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 15:15:47 -0400 Subject: [PATCH 32/82] test change on filter source Signed-off-by: David Wood --- transforms/universal/filter/python/src/filter_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/universal/filter/python/src/filter_local.py b/transforms/universal/filter/python/src/filter_local.py index 771cdad0f..e5c8e94a8 100644 --- a/transforms/universal/filter/python/src/filter_local.py +++ b/transforms/universal/filter/python/src/filter_local.py @@ -55,4 +55,4 @@ # Transform the table table_list, metadata = transform.transform(table) print(f"\noutput table has {table_list[0].num_rows} rows") - print(f"output metadata : {metadata}") + print(f"output metadata :: {metadata}") From c40cf1fc56bddc6d4d1c9bb4001ff29dae9af48a Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 15:17:18 -0400 Subject: [PATCH 33/82] change to lib readme Signed-off-by: David Wood --- data-processing-lib/python/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/data-processing-lib/python/README.md b/data-processing-lib/python/README.md index 8fc9ca277..170b0275e 100644 --- a/data-processing-lib/python/README.md +++ b/data-processing-lib/python/README.md @@ -22,6 +22,7 @@ To test, build and publish the library ```shell make test build publish ``` + To up the version number, edit the Makefile to change VERSION and rerun the above. This will require committing both the `Makefile` and the autotmatically updated `pyproject.toml` file. From c4853d1c74a967fad99f5361e59255e62c23134f Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 15:19:12 -0400 Subject: [PATCH 34/82] change to lib source Signed-off-by: David Wood --- .../python/test/data_processing_tests/transform/test_noop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-processing-lib/python/test/data_processing_tests/transform/test_noop.py b/data-processing-lib/python/test/data_processing_tests/transform/test_noop.py index 1eb85fe48..ed0f08872 100644 --- a/data-processing-lib/python/test/data_processing_tests/transform/test_noop.py +++ b/data-processing-lib/python/test/data_processing_tests/transform/test_noop.py @@ -19,7 +19,7 @@ ) -table = pa.Table.from_pydict({"name": pa.array(["Tom", "Dick", "Harry"]), "age": pa.array([0, 1, 2])}) +table = pa.Table.from_pydict({"name": pa.array(["Tom", "Dick", "Jane"]), "age": pa.array([0, 1, 2])}) expected_table = table # We're a noop after all. expected_metadata_list = [{"nfiles": 1, "nrows": 3}, {}] # transform() result # flush() result From c4417fc006a085c5c7eb5081a6afbab060c249af Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 15:39:06 -0400 Subject: [PATCH 35/82] minor job name changes in transform workflows Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 4 ++-- .github/workflows/test-code-code_quality.yml | 4 ++-- .github/workflows/test-code-header_cleanser.yml | 4 ++-- .github/workflows/test-code-malware.yml | 4 ++-- .github/workflows/test-code-proglang_select.yml | 4 ++-- .github/workflows/test-code-repo_level_ordering.yml | 4 ++-- .github/workflows/test-language-doc_chunk.yml | 4 ++-- .github/workflows/test-language-doc_quality.yml | 4 ++-- .github/workflows/test-language-lang_id.yml | 4 ++-- .github/workflows/test-language-pdf2parquet.yml | 4 ++-- .github/workflows/test-language-pii_redactor.yml | 4 ++-- .github/workflows/test-language-text_encoder.yml | 4 ++-- .github/workflows/test-transform.template | 4 ++-- .github/workflows/test-universal-doc_id.yml | 4 ++-- .github/workflows/test-universal-ededup.yml | 4 ++-- .github/workflows/test-universal-fdedup.yml | 4 ++-- .github/workflows/test-universal-filter.yml | 4 ++-- .github/workflows/test-universal-html2parquet.yml | 4 ++-- .github/workflows/test-universal-noop.yml | 4 ++-- .github/workflows/test-universal-profiler.yml | 4 ++-- .github/workflows/test-universal-resize.yml | 4 ++-- .github/workflows/test-universal-tokenization.yml | 4 ++-- 22 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index e64f1fcf8..a191db7f3 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/code/code2parquet run: | make -C transforms/code/code2parquet DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 5b2543d9d..8342e60e8 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/code/code_quality run: | make -C transforms/code/code_quality DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index c52b8a382..89a72ad78 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/code/header_cleanser run: | make -C transforms/code/header_cleanser DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index 2ddf94ce1..9bfcd5527 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/code/malware run: | make -C transforms/code/malware DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index d689a02ed..4049005d7 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/code/proglang_select run: | make -C transforms/code/proglang_select DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index 699f6e590..e2f055624 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/code/repo_level_ordering run: | make -C transforms/code/repo_level_ordering DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index f1b68c6f5..279139d36 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/language/doc_chunk run: | make -C transforms/language/doc_chunk DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index 1b812f3af..d830b04fc 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/language/doc_quality run: | make -C transforms/language/doc_quality DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index da78316f9..e843e8b75 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/language/lang_id run: | make -C transforms/language/lang_id DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 11b650da8..32636a2cd 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/language/pdf2parquet run: | make -C transforms/language/pdf2parquet DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 00a8b4bd7..66667b7a4 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/language/pii_redactor run: | make -C transforms/language/pii_redactor DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index 35297eb3d..cfade57b2 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/language/text_encoder run: | make -C transforms/language/text_encoder DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 42c2a6974..9c0f36a13 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in @TARGET_TRANSFORM_DIR@ run: | make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 64facc011..cc2e8e4c2 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/universal/doc_id run: | make -C transforms/universal/doc_id DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 6be50c5fd..c18e1d1b8 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/universal/ededup run: | make -C transforms/universal/ededup DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index ea4a362c0..a7d47b8cf 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/universal/fdedup run: | make -C transforms/universal/fdedup DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 7dd9198d1..e47703479 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/universal/filter run: | make -C transforms/universal/filter DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index 5148adeaf..612c61b6e 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/universal/html2parquet run: | make -C transforms/universal/html2parquet DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 2da9f344e..e2dcee4f1 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/universal/noop run: | make -C transforms/universal/noop DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index d7aae1ac5..e23f348a5 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/universal/profiler run: | make -C transforms/universal/profiler DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index f7013561d..ec8e6008f 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/universal/resize run: | make -C transforms/universal/resize DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index f10a02631..c096dfb7d 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -29,7 +29,7 @@ on: - "!**/.gitignore" jobs: - check_if_push_images: + check_if_push_image: # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -67,7 +67,7 @@ jobs: - name: Test transform source in transforms/universal/tokenization run: | make -C transforms/universal/tokenization DOCKER=docker test-src - test-images: + test-image: needs: [check_if_push_images] runs-on: ubuntu-22.04 timeout-minutes: 120 From 8270c6c58c542f560842ff2b63b983a2c50bb878 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 16:38:04 -0400 Subject: [PATCH 36/82] noop readme Signed-off-by: David Wood --- data-processing-lib/python/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/data-processing-lib/python/README.md b/data-processing-lib/python/README.md index 170b0275e..e3821397f 100644 --- a/data-processing-lib/python/README.md +++ b/data-processing-lib/python/README.md @@ -28,3 +28,4 @@ the above. This will require committing both the `Makefile` and the autotmatically updated `pyproject.toml` file. + From dc06b944cc95eec6f27b0b34c26ae577a20affa5 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 16:41:04 -0400 Subject: [PATCH 37/82] test-lib workflow ignores Signed-off-by: David Wood --- .github/workflows/test-lib.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-lib.yml b/.github/workflows/test-lib.yml index 1ed6a3f9f..be00c2076 100644 --- a/.github/workflows/test-lib.yml +++ b/.github/workflows/test-lib.yml @@ -11,9 +11,9 @@ on: paths: # Note: the transform workflows are expected to trigger when data-processing-lib/** changes - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" + - "!data-processing-lib/**.md" + - "!data-processing-lib/**/doc/**" + - "!data-processing-lib/**/.gitignore" pull_request: branches: - "dev" @@ -21,9 +21,9 @@ on: paths: # Note: the transform workflows are expected to trigger when data-processing-lib/** changes - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" + - "!data-processing-lib/**.md" + - "!data-processing-lib/**/doc/**" + - "!data-processing-lib/**/.gitignore" jobs: check_if_push_images: From 428833e14d745da5aa1ae238e97e4ef46da83bf9 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 16:44:06 -0400 Subject: [PATCH 38/82] top level readme Signed-off-by: David Wood --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index d85c6fefe..0677e3aba 100644 --- a/README.md +++ b/README.md @@ -213,7 +213,3 @@ You can run transforms via docker image or using virtual environments. This [doc 4. Talk on "Hands on session for fine tuning LLMs" [Video](https://www.youtube.com/watch?v=VEHIA3E64DM) 5. Talk on "Build your own data preparation module using data-prep-kit" [Video](https://www.youtube.com/watch?v=0WUMG6HIgMg) - - - - From 0bd7992f1df239a7e5e043d7a684be6f7994eb80 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 16:45:11 -0400 Subject: [PATCH 39/82] noop test source Signed-off-by: David Wood --- .../python/test/data_processing_tests/transform/test_noop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-processing-lib/python/test/data_processing_tests/transform/test_noop.py b/data-processing-lib/python/test/data_processing_tests/transform/test_noop.py index ed0f08872..1843d1af9 100644 --- a/data-processing-lib/python/test/data_processing_tests/transform/test_noop.py +++ b/data-processing-lib/python/test/data_processing_tests/transform/test_noop.py @@ -19,7 +19,7 @@ ) -table = pa.Table.from_pydict({"name": pa.array(["Tom", "Dick", "Jane"]), "age": pa.array([0, 1, 2])}) +table = pa.Table.from_pydict({"name": pa.array(["Tom", "Dick", "John"]), "age": pa.array([0, 1, 2])}) expected_table = table # We're a noop after all. expected_metadata_list = [{"nfiles": 1, "nrows": 3}, {}] # transform() result # flush() result From 31b7f530879abe4b7301cf530f97d15c30867615 Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 16:51:01 -0400 Subject: [PATCH 40/82] filter source change' Signed-off-by: David Wood --- transforms/universal/filter/python/src/filter_transform.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transforms/universal/filter/python/src/filter_transform.py b/transforms/universal/filter/python/src/filter_transform.py index fe002e497..4ae5e2984 100644 --- a/transforms/universal/filter/python/src/filter_transform.py +++ b/transforms/universal/filter/python/src/filter_transform.py @@ -23,6 +23,8 @@ short_name = "filter" cli_prefix = short_name + "_" +# Misc comment to trigger build + filter_criteria_key = "criteria_list" """ AST Key holds the list of filter criteria (in SQL WHERE clause format)""" filter_logical_operator_key = "logical_operator" From e78df7bf97037f4936cff229a58c54647d4406eb Mon Sep 17 00:00:00 2001 From: David Wood Date: Tue, 17 Sep 2024 16:53:58 -0400 Subject: [PATCH 41/82] updated all transform tets workflows Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 6 ------ .github/workflows/test-code-code_quality.yml | 6 ------ .github/workflows/test-code-header_cleanser.yml | 6 ------ .github/workflows/test-code-malware.yml | 6 ------ .github/workflows/test-code-proglang_select.yml | 6 ------ .github/workflows/test-code-repo_level_ordering.yml | 6 ------ .github/workflows/test-language-doc_chunk.yml | 6 ------ .github/workflows/test-language-doc_quality.yml | 6 ------ .github/workflows/test-language-lang_id.yml | 6 ------ .github/workflows/test-language-pdf2parquet.yml | 6 ------ .github/workflows/test-language-pii_redactor.yml | 6 ------ .github/workflows/test-language-text_encoder.yml | 6 ------ .github/workflows/test-transform.template | 6 ------ .github/workflows/test-universal-doc_id.yml | 6 ------ .github/workflows/test-universal-ededup.yml | 6 ------ .github/workflows/test-universal-fdedup.yml | 6 ------ .github/workflows/test-universal-filter.yml | 6 ------ .github/workflows/test-universal-html2parquet.yml | 6 ------ .github/workflows/test-universal-noop.yml | 6 ------ .github/workflows/test-universal-profiler.yml | 6 ------ .github/workflows/test-universal-resize.yml | 6 ------ .github/workflows/test-universal-tokenization.yml | 6 ------ 22 files changed, 132 deletions(-) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index a191db7f3..ebdd23feb 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/code/code2parquet/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/code/code2parquet/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 8342e60e8..4f27a50c9 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/code/code_quality/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/code/code_quality/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 89a72ad78..e14e12b2f 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/code/header_cleanser/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/code/header_cleanser/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index 9bfcd5527..792633d52 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/code/malware/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/code/malware/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index 4049005d7..37f1101ef 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/code/proglang_select/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/code/proglang_select/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index e2f055624..258b54fcc 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index 279139d36..e441cea38 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/language/doc_chunk/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/language/doc_chunk/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index d830b04fc..2b930c6f2 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/language/doc_quality/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/language/doc_quality/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index e843e8b75..02d3ca6e9 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/language/lang_id/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/language/lang_id/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 32636a2cd..c27ab194a 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 66667b7a4..b1a5c4a1c 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/language/pii_redactor/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/language/pii_redactor/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index cfade57b2..58aff72ec 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/language/text_encoder/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/language/text_encoder/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 9c0f36a13..d64c7a8b7 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -14,9 +14,6 @@ on: paths: - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index cc2e8e4c2..7cdd47b79 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/universal/doc_id/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/universal/doc_id/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index c18e1d1b8..75c938b88 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/universal/ededup/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/universal/ededup/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index a7d47b8cf..6c3eee7c5 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/universal/fdedup/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/universal/fdedup/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index e47703479..546335adf 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/universal/filter/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/universal/filter/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index 612c61b6e..6818990fd 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/universal/html2parquet/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/universal/html2parquet/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index e2dcee4f1..033a6cfed 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/universal/noop/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/universal/noop/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index e23f348a5..2171e7437 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/universal/profiler/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/universal/profiler/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index ec8e6008f..f105ff9ad 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/universal/resize/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/universal/resize/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index c096dfb7d..056d33e51 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -14,9 +14,6 @@ on: paths: - "transforms/universal/tokenization/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" pull_request: branches: - "dev" @@ -24,9 +21,6 @@ on: paths: - "transforms/universal/tokenization/**" - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" jobs: check_if_push_image: From 684ae7ab14f8fbb7a13fb5056aab8e01cbe4df72 Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 18 Sep 2024 10:15:03 -0400 Subject: [PATCH 42/82] fix typo in test template on check_images Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 4 ++-- .github/workflows/test-code-code_quality.yml | 4 ++-- .github/workflows/test-code-header_cleanser.yml | 4 ++-- .github/workflows/test-code-malware.yml | 4 ++-- .github/workflows/test-code-proglang_select.yml | 4 ++-- .github/workflows/test-code-repo_level_ordering.yml | 4 ++-- .github/workflows/test-language-doc_chunk.yml | 4 ++-- .github/workflows/test-language-doc_quality.yml | 4 ++-- .github/workflows/test-language-lang_id.yml | 4 ++-- .github/workflows/test-language-pdf2parquet.yml | 4 ++-- .github/workflows/test-language-pii_redactor.yml | 4 ++-- .github/workflows/test-language-text_encoder.yml | 4 ++-- .github/workflows/test-transform.template | 4 ++-- .github/workflows/test-universal-doc_id.yml | 4 ++-- .github/workflows/test-universal-ededup.yml | 4 ++-- .github/workflows/test-universal-fdedup.yml | 4 ++-- .github/workflows/test-universal-filter.yml | 4 ++-- .github/workflows/test-universal-html2parquet.yml | 4 ++-- .github/workflows/test-universal-noop.yml | 4 ++-- .github/workflows/test-universal-profiler.yml | 4 ++-- .github/workflows/test-universal-resize.yml | 4 ++-- .github/workflows/test-universal-tokenization.yml | 4 ++-- 22 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index ebdd23feb..25f45e0a9 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/code/code2parquet DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/code/code2parquet publish diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 4f27a50c9..0dd33d6e0 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/code/code_quality DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/code/code_quality publish diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index e14e12b2f..38de3d1f5 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/code/header_cleanser DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/code/header_cleanser publish diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index 792633d52..4c2c3caee 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/code/malware DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/code/malware publish diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index 37f1101ef..abc45af8c 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/code/proglang_select DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/code/proglang_select publish diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index 258b54fcc..af43684e7 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/code/repo_level_ordering DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/code/repo_level_ordering publish diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index e441cea38..481322aba 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/language/doc_chunk DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/language/doc_chunk publish diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index 2b930c6f2..6e7cbcfe8 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/language/doc_quality DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/language/doc_quality publish diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 02d3ca6e9..f297da4b7 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/language/lang_id DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/language/lang_id publish diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index c27ab194a..62f666b1a 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/language/pdf2parquet DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/language/pdf2parquet publish diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index b1a5c4a1c..146c3620b 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/language/pii_redactor DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/language/pii_redactor publish diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index 58aff72ec..70549089b 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/language/text_encoder DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/language/text_encoder publish diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index d64c7a8b7..d0206e705 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -62,7 +62,7 @@ jobs: run: | make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C @TARGET_TRANSFORM_DIR@ publish diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 7cdd47b79..b2c2c8224 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/universal/doc_id DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/universal/doc_id publish diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 75c938b88..9e6195034 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/universal/ededup DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/universal/ededup publish diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 6c3eee7c5..0ddc4d43f 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/universal/fdedup DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/universal/fdedup publish diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 546335adf..127bba7a5 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/universal/filter DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/universal/filter publish diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index 6818990fd..c113e7800 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/universal/html2parquet DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/universal/html2parquet publish diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 033a6cfed..0f6f52b2d 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/universal/noop DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/universal/noop publish diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 2171e7437..0cdffae3b 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/universal/profiler DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/universal/profiler publish diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index f105ff9ad..f5a4f37f8 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/universal/resize DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/universal/resize publish diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 056d33e51..4411876f1 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -62,7 +62,7 @@ jobs: run: | make -C transforms/universal/tokenization DOCKER=docker test-src test-image: - needs: [check_if_push_images] + needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 env: @@ -91,5 +91,5 @@ jobs: df -h docker images - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' + if: needs.check_if_push_image.outputs.publish_images == 'true' run: make -C transforms/universal/tokenization publish From f419059616e175ca120cc22c0295c175006cd698 Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 18 Sep 2024 10:18:48 -0400 Subject: [PATCH 43/82] noop src change Signed-off-by: David Wood --- transforms/universal/noop/python/src/noop_transform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/universal/noop/python/src/noop_transform.py b/transforms/universal/noop/python/src/noop_transform.py index 22e595a5b..92a64975d 100644 --- a/transforms/universal/noop/python/src/noop_transform.py +++ b/transforms/universal/noop/python/src/noop_transform.py @@ -55,7 +55,7 @@ def transform(self, table: pa.Table, file_name: str = None) -> tuple[list[pa.Tab if self.sleep is not None: self.logger.info(f"Sleep for {self.sleep} seconds") time.sleep(self.sleep) - self.logger.info("Sleep completed - continue") + self.logger.info("Sleep completed - continuing") # Add some sample metadata. self.logger.debug(f"Transformed one table with {len(table)} rows") metadata = {"nfiles": 1, "nrows": len(table)} From a887bf9f19f585593cf179f2f9297a1ae961ffd5 Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 18 Sep 2024 10:53:41 -0400 Subject: [PATCH 44/82] check for makefile in test transform workflow Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 15 ++++++++++++--- .github/workflows/test-code-code_quality.yml | 15 ++++++++++++--- .github/workflows/test-code-header_cleanser.yml | 15 ++++++++++++--- .github/workflows/test-code-malware.yml | 15 ++++++++++++--- .github/workflows/test-code-proglang_select.yml | 15 ++++++++++++--- .../workflows/test-code-repo_level_ordering.yml | 15 ++++++++++++--- .github/workflows/test-language-doc_chunk.yml | 15 ++++++++++++--- .github/workflows/test-language-doc_quality.yml | 15 ++++++++++++--- .github/workflows/test-language-lang_id.yml | 15 ++++++++++++--- .github/workflows/test-language-pdf2parquet.yml | 15 ++++++++++++--- .../workflows/test-language-pii_redactor.yml | 15 ++++++++++++--- .../workflows/test-language-text_encoder.yml | 15 ++++++++++++--- .github/workflows/test-transform.template | 17 +++++++++++++---- .github/workflows/test-universal-doc_id.yml | 15 ++++++++++++--- .github/workflows/test-universal-ededup.yml | 15 ++++++++++++--- .github/workflows/test-universal-fdedup.yml | 15 ++++++++++++--- .github/workflows/test-universal-filter.yml | 15 ++++++++++++--- .../workflows/test-universal-html2parquet.yml | 15 ++++++++++++--- .github/workflows/test-universal-noop.yml | 15 ++++++++++++--- .github/workflows/test-universal-profiler.yml | 15 ++++++++++++--- .github/workflows/test-universal-resize.yml | 15 ++++++++++++--- .../workflows/test-universal-tokenization.yml | 15 ++++++++++++--- 22 files changed, 265 insertions(+), 67 deletions(-) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index 25f45e0a9..ee487ad87 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/code/code2parquet run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/code/code2parquet DOCKER=docker test-image + if [ -e "transforms/code/code2parquet/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/code2parquet DOCKER=docker test-image + else + echo "transforms/code/code2parquet/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/code/code2parquet publish + run: | + if [ -e "transforms/code/code2parquet/Makefile" ]; then + make -C transforms/code/code2parquet publish + else + echo "transforms/code/code2parquet/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 0dd33d6e0..f8a702360 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/code/code_quality run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/code/code_quality DOCKER=docker test-image + if [ -e "transforms/code/code_quality/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/code_quality DOCKER=docker test-image + else + echo "transforms/code/code_quality/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/code/code_quality publish + run: | + if [ -e "transforms/code/code_quality/Makefile" ]; then + make -C transforms/code/code_quality publish + else + echo "transforms/code/code_quality/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 38de3d1f5..9216ef5c4 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/code/header_cleanser run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/code/header_cleanser DOCKER=docker test-image + if [ -e "transforms/code/header_cleanser/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/header_cleanser DOCKER=docker test-image + else + echo "transforms/code/header_cleanser/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/code/header_cleanser publish + run: | + if [ -e "transforms/code/header_cleanser/Makefile" ]; then + make -C transforms/code/header_cleanser publish + else + echo "transforms/code/header_cleanser/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index 4c2c3caee..0dcb8c413 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/code/malware run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/code/malware DOCKER=docker test-image + if [ -e "transforms/code/malware/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/malware DOCKER=docker test-image + else + echo "transforms/code/malware/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/code/malware publish + run: | + if [ -e "transforms/code/malware/Makefile" ]; then + make -C transforms/code/malware publish + else + echo "transforms/code/malware/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index abc45af8c..e33974dc0 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/code/proglang_select run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/code/proglang_select DOCKER=docker test-image + if [ -e "transforms/code/proglang_select/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/proglang_select DOCKER=docker test-image + else + echo "transforms/code/proglang_select/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/code/proglang_select publish + run: | + if [ -e "transforms/code/proglang_select/Makefile" ]; then + make -C transforms/code/proglang_select publish + else + echo "transforms/code/proglang_select/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index af43684e7..43f741376 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/code/repo_level_ordering run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/code/repo_level_ordering DOCKER=docker test-image + if [ -e "transforms/code/repo_level_ordering/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/code/repo_level_ordering DOCKER=docker test-image + else + echo "transforms/code/repo_level_ordering/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/code/repo_level_ordering publish + run: | + if [ -e "transforms/code/repo_level_ordering/Makefile" ]; then + make -C transforms/code/repo_level_ordering publish + else + echo "transforms/code/repo_level_ordering/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index 481322aba..fb9c6aa56 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/language/doc_chunk run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/language/doc_chunk DOCKER=docker test-image + if [ -e "transforms/language/doc_chunk/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/doc_chunk DOCKER=docker test-image + else + echo "transforms/language/doc_chunk/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/language/doc_chunk publish + run: | + if [ -e "transforms/language/doc_chunk/Makefile" ]; then + make -C transforms/language/doc_chunk publish + else + echo "transforms/language/doc_chunk/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index 6e7cbcfe8..920fb029b 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/language/doc_quality run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/language/doc_quality DOCKER=docker test-image + if [ -e "transforms/language/doc_quality/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/doc_quality DOCKER=docker test-image + else + echo "transforms/language/doc_quality/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/language/doc_quality publish + run: | + if [ -e "transforms/language/doc_quality/Makefile" ]; then + make -C transforms/language/doc_quality publish + else + echo "transforms/language/doc_quality/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index f297da4b7..0b4cde6cc 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/language/lang_id run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/language/lang_id DOCKER=docker test-image + if [ -e "transforms/language/lang_id/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/lang_id DOCKER=docker test-image + else + echo "transforms/language/lang_id/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/language/lang_id publish + run: | + if [ -e "transforms/language/lang_id/Makefile" ]; then + make -C transforms/language/lang_id publish + else + echo "transforms/language/lang_id/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 62f666b1a..cae8c4a77 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/language/pdf2parquet run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/language/pdf2parquet DOCKER=docker test-image + if [ -e "transforms/language/pdf2parquet/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/pdf2parquet DOCKER=docker test-image + else + echo "transforms/language/pdf2parquet/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/language/pdf2parquet publish + run: | + if [ -e "transforms/language/pdf2parquet/Makefile" ]; then + make -C transforms/language/pdf2parquet publish + else + echo "transforms/language/pdf2parquet/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 146c3620b..55a7ebe41 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/language/pii_redactor run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/language/pii_redactor DOCKER=docker test-image + if [ -e "transforms/language/pii_redactor/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/pii_redactor DOCKER=docker test-image + else + echo "transforms/language/pii_redactor/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/language/pii_redactor publish + run: | + if [ -e "transforms/language/pii_redactor/Makefile" ]; then + make -C transforms/language/pii_redactor publish + else + echo "transforms/language/pii_redactor/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index 70549089b..379de4660 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/language/text_encoder run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/language/text_encoder DOCKER=docker test-image + if [ -e "transforms/language/text_encoder/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/language/text_encoder DOCKER=docker test-image + else + echo "transforms/language/text_encoder/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/language/text_encoder publish + run: | + if [ -e "transforms/language/text_encoder/Makefile" ]; then + make -C transforms/language/text_encoder publish + else + echo "transforms/language/text_encoder/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index d0206e705..af80abfc4 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -1,7 +1,7 @@ # # DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files # -name: Test - @TARGET_TRANSFORM_DIR@ +name: Test - @TARGET_TRANSFORM_DIR@ on: workflow_dispatch: @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in @TARGET_TRANSFORM_DIR@ run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-image + if [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-image + else + echo "@TARGET_TRANSFORM_DIR@/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C @TARGET_TRANSFORM_DIR@ publish + run: | + if [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ]; then + make -C @TARGET_TRANSFORM_DIR@ publish + else + echo "@TARGET_TRANSFORM_DIR@/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index b2c2c8224..85e14cb04 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/doc_id run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/doc_id DOCKER=docker test-image + if [ -e "transforms/universal/doc_id/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/doc_id DOCKER=docker test-image + else + echo "transforms/universal/doc_id/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/doc_id publish + run: | + if [ -e "transforms/universal/doc_id/Makefile" ]; then + make -C transforms/universal/doc_id publish + else + echo "transforms/universal/doc_id/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 9e6195034..92ed63829 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/ededup run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/ededup DOCKER=docker test-image + if [ -e "transforms/universal/ededup/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/ededup DOCKER=docker test-image + else + echo "transforms/universal/ededup/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/ededup publish + run: | + if [ -e "transforms/universal/ededup/Makefile" ]; then + make -C transforms/universal/ededup publish + else + echo "transforms/universal/ededup/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 0ddc4d43f..c6a83a2cf 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/fdedup run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/fdedup DOCKER=docker test-image + if [ -e "transforms/universal/fdedup/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/fdedup DOCKER=docker test-image + else + echo "transforms/universal/fdedup/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/fdedup publish + run: | + if [ -e "transforms/universal/fdedup/Makefile" ]; then + make -C transforms/universal/fdedup publish + else + echo "transforms/universal/fdedup/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 127bba7a5..ab285b88f 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/filter run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/filter DOCKER=docker test-image + if [ -e "transforms/universal/filter/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/filter DOCKER=docker test-image + else + echo "transforms/universal/filter/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/filter publish + run: | + if [ -e "transforms/universal/filter/Makefile" ]; then + make -C transforms/universal/filter publish + else + echo "transforms/universal/filter/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index c113e7800..d42f013ad 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/html2parquet run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/html2parquet DOCKER=docker test-image + if [ -e "transforms/universal/html2parquet/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/html2parquet DOCKER=docker test-image + else + echo "transforms/universal/html2parquet/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/html2parquet publish + run: | + if [ -e "transforms/universal/html2parquet/Makefile" ]; then + make -C transforms/universal/html2parquet publish + else + echo "transforms/universal/html2parquet/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 0f6f52b2d..04d7280f9 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/noop run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/noop DOCKER=docker test-image + if [ -e "transforms/universal/noop/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/noop DOCKER=docker test-image + else + echo "transforms/universal/noop/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/noop publish + run: | + if [ -e "transforms/universal/noop/Makefile" ]; then + make -C transforms/universal/noop publish + else + echo "transforms/universal/noop/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 0cdffae3b..4a8774a02 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/profiler run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/profiler DOCKER=docker test-image + if [ -e "transforms/universal/profiler/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/profiler DOCKER=docker test-image + else + echo "transforms/universal/profiler/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/profiler publish + run: | + if [ -e "transforms/universal/profiler/Makefile" ]; then + make -C transforms/universal/profiler publish + else + echo "transforms/universal/profiler/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index f5a4f37f8..ab4775ec0 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/resize run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/resize DOCKER=docker test-image + if [ -e "transforms/universal/resize/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/resize DOCKER=docker test-image + else + echo "transforms/universal/resize/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/resize publish + run: | + if [ -e "transforms/universal/resize/Makefile" ]; then + make -C transforms/universal/resize publish + else + echo "transforms/universal/resize/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 4411876f1..b6e5019fc 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -83,8 +83,12 @@ jobs: df -h - name: Test transform image in transforms/universal/tokenization run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal/tokenization DOCKER=docker test-image + if [ -e "transforms/universal/tokenization/Makefile" ]; then + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal/tokenization DOCKER=docker test-image + else + echo "transforms/universal/tokenization/Makefile not found - testing disabled for this transform." + fi - name: Print space # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 run: | @@ -92,4 +96,9 @@ jobs: docker images - name: Publish images if: needs.check_if_push_image.outputs.publish_images == 'true' - run: make -C transforms/universal/tokenization publish + run: | + if [ -e "transforms/universal/tokenization/Makefile" ]; then + make -C transforms/universal/tokenization publish + else + echo "transforms/universal/tokenization/Makefile not found - publishing disabled for this transform." + fi From f9399c1c27d219c64fd640c65d9d86b2b9b9e278 Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 18 Sep 2024 11:15:30 -0400 Subject: [PATCH 45/82] automatically determine transforms in transforms directory for which to generate test workflows Signed-off-by: David Wood --- .github/workflows/Makefile | 22 +- .github/workflows/README.md | 5 +- .github/workflows/test-code-code2parquet.yml | 2 +- .github/workflows/test-code-code_quality.yml | 2 +- .../workflows/test-code-header_cleanser.yml | 2 +- .github/workflows/test-code-malware.yml | 2 +- .../workflows/test-code-proglang_select.yml | 2 +- .../test-code-repo_level_ordering.yml | 2 +- .github/workflows/test-language-doc_chunk.yml | 2 +- .../workflows/test-language-doc_quality.yml | 2 +- .github/workflows/test-language-lang_id.yml | 2 +- .../workflows/test-language-pdf2parquet.yml | 2 +- .../workflows/test-language-pii_redactor.yml | 2 +- .../workflows/test-language-text_encoder.yml | 2 +- .github/workflows/test-transform.template | 2 +- .github/workflows/test-universal-doc_id.yml | 2 +- .github/workflows/test-universal-ededup.yml | 2 +- .github/workflows/test-universal-fdedup.yml | 2 +- .github/workflows/test-universal-filter.yml | 2 +- .../workflows/test-universal-html2parquet.yml | 2 +- .github/workflows/test-universal-noop.yml | 2 +- .github/workflows/test-universal-profiler.yml | 2 +- .github/workflows/test-universal-resize.yml | 2 +- .../workflows/test-universal-tokenization.yml | 2 +- .github/workflows/test.yml.old | 405 ++++++++++++++++++ 25 files changed, 443 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/test.yml.old diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile index cf6a9b96d..d8da3e720 100644 --- a/.github/workflows/Makefile +++ b/.github/workflows/Makefile @@ -8,13 +8,19 @@ LANG_TRANSFORMS=doc_chunk doc_quality lang_id pdf2parquet pii_redactor text_enco transform-tests: - for i in $(CODE_TRANSFORMS); do \ - cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/code/$$i?g" > test-code-$$i.yml; \ - done - for i in $(LANG_TRANSFORMS); do \ - cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/language/$$i?g" > test-language-$$i.yml; \ - done - for i in $(UNIVERSAL_TRANSFORMS); do \ - cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/universal/$$i?g" > test-universal-$$i.yml; \ + $(MAKE) TRANSFORM_SUBDIR=universal .transform-tests + $(MAKE) TRANSFORM_SUBDIR=language .transform-tests + $(MAKE) TRANSFORM_SUBDIR=code .transform-tests + +# Expects +# TRANSFORM_SUBDIR transforms subdirectory (such as universal) +.transform-tests: + @for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -depth 1 -type d); do \ + dir=$$(basename $$i); \ + yml=test-$(TRANSFORM_SUBDIR)-$$dir.yml; \ + echo Generating $$yml; \ + cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml; \ done + + diff --git a/.github/workflows/README.md b/.github/workflows/README.md index dc97c7330..843ca5cf0 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,6 +1,6 @@ # Workflow Management -Here we have the start of a system to automatically generated workflows (currently only for transforms). +Here we have the start of a system to automatically generated github workflows (currently only for transforms). In general, the design is to use templates and `make` to generate/update the workflows. ``` @@ -21,7 +21,6 @@ The Makefile is used to (re)generate all workflows a necessary. By design, work When a new transform is added to the repository, -1. add it to the corresponding macro in the Makefile (in alphabetical order please). -1. Run `make` to create the new test .yml for the transform +1. Run `make` in this directory to create the new test .yml for all transforms found in transforms/{universal,code,language} directories 1. commit and push the change to your branch with the new transform. diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index ee487ad87..a9e701631 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/code/code2parquet diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index f8a702360..4847e292a 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/code/code_quality diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 9216ef5c4..b8b0d751f 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/code/header_cleanser diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index 0dcb8c413..dd13cbe74 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/code/malware diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index e33974dc0..16e1ba330 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/code/proglang_select diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index 43f741376..26144e57b 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/code/repo_level_ordering diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index fb9c6aa56..cc969c66a 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/language/doc_chunk diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index 920fb029b..c3da25938 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/language/doc_quality diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 0b4cde6cc..d07f2e0a6 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/language/lang_id diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index cae8c4a77..e5284b4a5 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/language/pdf2parquet diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 55a7ebe41..3eded38f5 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/language/pii_redactor diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index 379de4660..cf9a0604a 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/language/text_encoder diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index af80abfc4..1ed7fc89b 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - @TARGET_TRANSFORM_DIR@ diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 85e14cb04..3110ba0eb 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/universal/doc_id diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 92ed63829..7da3e6569 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/universal/ededup diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index c6a83a2cf..1fb9ead05 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/universal/fdedup diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index ab285b88f..6eefa1c05 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/universal/filter diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index d42f013ad..3a4763682 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/universal/html2parquet diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 04d7280f9..703f70baf 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/universal/noop diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 4a8774a02..6fcd60642 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/universal/profiler diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index ab4775ec0..b27f0bf9c 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/universal/resize diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index b6e5019fc..1d4af4eaa 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -1,5 +1,5 @@ # -# DO NOT EDIT THIS: it is generated from test-transform.template, Edit there and run make to change these files +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files # name: Test - transforms/universal/tokenization diff --git a/.github/workflows/test.yml.old b/.github/workflows/test.yml.old new file mode 100644 index 000000000..552ac012b --- /dev/null +++ b/.github/workflows/test.yml.old @@ -0,0 +1,405 @@ +name: Build, Test and (Optionally) Push images + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "**/.gitignore" + - "**/.dockerignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "**/.gitignore" + - "**/.dockerignore" +env: + KFP_BLACK_LIST: "doc_chunk-ray,pdf2parquet-ray,pii_redactor" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-make: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test top-level recursive make targets. + run: | + make -n clean test build publish set-versions + test-python-lib: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test data-processing-lib/python + run: | + make -C data-processing-lib/python DOCKER=docker venv test + test-ray-lib: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test data-processing-lib/ray + run: | + make -C data-processing-lib/ray DOCKER=docker venv test + test-spark-lib: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test data-processing-lib/spark + run: | + make -C data-processing-lib/spark DOCKER=docker venv test + test-code: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Code Transforms + run: | + make -C transforms/code DOCKER=docker test-src + test-language: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Language Transforms + run: | + make -C transforms/language DOCKER=docker test-src + test-universal: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Universal Transforms + run: | + make -C transforms/universal DOCKER=docker test-src + test-tools: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test tools + run: | + make -C tools DOCKER=docker venv test + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + while : + do + dir=("code" "universal" "language") && index=$(($RANDOM % ${#dir[@]})) && subdirs=${dir[$index]} && transforms=($(find transforms/$subdirs -type d -maxdepth 1 -mindepth 1 )) + set -- "${transforms[@]}" && transforms=("$@") && size=${#transforms[@]} && index=$(($RANDOM % $size)) + transform=$(basename "${transforms[$index]}") + if [ -d ${transforms[$index]}/kfp_ray ] && echo ${KFP_BLACK_LIST} | grep -qv ${transform} ; then + header_text "Running ${transforms[$index]} workflow test" + break + fi + done + make -C ${transforms[$index]} workflow-test + echo "Run ${transforms[$index]} completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + while : + do + dir=("code" "universal" "language") && index=$(($RANDOM % ${#dir[@]})) && subdirs=${dir[$index]} && transforms=($(find transforms/$subdirs -type d -maxdepth 1 -mindepth 1 )) + set -- "${transforms[@]}" && transforms=("$@") && size=${#transforms[@]} && index=$(($RANDOM % $size)) + transform=$(basename "${transforms[$index]}") + if [ -d ${transforms[$index]}/kfp_ray ] && echo ${KFP_BLACK_LIST} | grep -qv ${transform} ; then + header_text "Running ${transforms[$index]} workflow test" + break + fi + done + make -C ${transforms[$index]} workflow-test + header_text "Run ${transforms[$index]} completed" + test-data-processing-lib-images: + needs: [check_if_push_images] + if: needs.check_if_push_images.outputs.publish_images == 'true' + runs-on: ubuntu-22.04 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Code Transform Images + run: | + make -C data-processing-lib/spark DOCKER=docker image + - name: + Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: | + make -C data-processing-lib/spark publish-image + test-code-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Code Transform Images + run: | + make -C data-processing-lib DOCKER=docker image + make -C transforms/code DOCKER=docker test-image + - name: + Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: | + make -C transforms/code publish + test-language-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test language Transform Images + run: | + make -C data-processing-lib DOCKER=docker image + make -C transforms/language DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: df -h + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/language publish + + test-universal-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Universal Transform Images + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal DOCKER=docker test-image + - name: + Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/universal publish + build-kfp-components: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Build + run: | + make -C kfp/kfp_ray_components DOCKER=docker image + make KFPv2=1 -C kfp/kfp_ray_components DOCKER=docker image + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C kfp/kfp_ray_components publish + test-tool-images: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test Tool images + run: | + make -C tools/ingest2parquet DOCKER=docker test-image + From e064e00cca1a40238610842e541c74b57dbad0d8 Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 18 Sep 2024 16:07:24 -0400 Subject: [PATCH 46/82] worklow readme, transform existence verification, disable build-library, and tools tests Signed-off-by: David Wood --- .github/workflows/README.md | 22 ++++++++++++++++--- ...uild-library.yml => build-library.yml.old} | 0 .github/workflows/test-code-code2parquet.yml | 10 ++++++++- .github/workflows/test-code-code_quality.yml | 10 ++++++++- .../workflows/test-code-header_cleanser.yml | 10 ++++++++- .github/workflows/test-code-malware.yml | 10 ++++++++- .../workflows/test-code-proglang_select.yml | 10 ++++++++- .../test-code-repo_level_ordering.yml | 10 ++++++++- .github/workflows/test-kfp.yml | 2 ++ .github/workflows/test-language-doc_chunk.yml | 10 ++++++++- .../workflows/test-language-doc_quality.yml | 10 ++++++++- .github/workflows/test-language-lang_id.yml | 10 ++++++++- .../workflows/test-language-pdf2parquet.yml | 10 ++++++++- .../workflows/test-language-pii_redactor.yml | 10 ++++++++- .../workflows/test-language-text_encoder.yml | 10 ++++++++- .github/workflows/test-misc.yml | 18 +++++---------- .github/workflows/test-transform.template | 10 ++++++++- .github/workflows/test-universal-doc_id.yml | 10 ++++++++- .github/workflows/test-universal-ededup.yml | 10 ++++++++- .github/workflows/test-universal-fdedup.yml | 10 ++++++++- .github/workflows/test-universal-filter.yml | 10 ++++++++- .../workflows/test-universal-html2parquet.yml | 10 ++++++++- .github/workflows/test-universal-noop.yml | 10 ++++++++- .github/workflows/test-universal-profiler.yml | 10 ++++++++- .github/workflows/test-universal-resize.yml | 10 ++++++++- .../workflows/test-universal-tokenization.yml | 10 ++++++++- scripts/check-workflows.sh | 20 +++++++++++++++++ 27 files changed, 244 insertions(+), 38 deletions(-) rename .github/workflows/{build-library.yml => build-library.yml.old} (100%) create mode 100755 scripts/check-workflows.sh diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 843ca5cf0..c357ff33a 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -3,12 +3,25 @@ Here we have the start of a system to automatically generated github workflows (currently only for transforms). In general, the design is to use templates and `make` to generate/update the workflows. +Goals +1. Run only tests for a given transform when only the transform changes. +Includes python, ray, spark and kfp_ray as available. +2. When the core dpk lib components files changes, test all transforms +3. When the shared kfp components changes, test a randomly selected transform test + (We would like to avoid running all transform kfp tests in one PR) +4. Extra credit: If .md or other non-code changes are made, run no tests. + +Assumptions: +1. All transforms will have test workflows. A transform can disable its tests locally +(temporarily?) by renaming its transforms/universal/noop/Makefile.disabled. ``` -make +git clone .... +... +git checkout -b new-branch +make # Creates new test*.yml workflows git commit -a -s -m "update workflows" -git push +git push --set-upstream origin new-branch ``` - should be sufficient. ## Transforms @@ -24,3 +37,6 @@ When a new transform is added to the repository, 1. Run `make` in this directory to create the new test .yml for all transforms found in transforms/{universal,code,language} directories 1. commit and push the change to your branch with the new transform. +## KFP + +## DPK libraries \ No newline at end of file diff --git a/.github/workflows/build-library.yml b/.github/workflows/build-library.yml.old similarity index 100% rename from .github/workflows/build-library.yml rename to .github/workflows/build-library.yml.old diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index a9e701631..5bddc137f 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/code/code2parquet/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/code/code2parquet/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/code/code2parquet run: | if [ -e "transforms/code/code2parquet/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/code/code2parquet DOCKER=docker test-image else echo "transforms/code/code2parquet/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 4847e292a..faccf10d0 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/code/code_quality/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/code/code_quality/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/code/code_quality run: | if [ -e "transforms/code/code_quality/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/code/code_quality DOCKER=docker test-image else echo "transforms/code/code_quality/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index b8b0d751f..3adce163d 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/code/header_cleanser/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/code/header_cleanser/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/code/header_cleanser run: | if [ -e "transforms/code/header_cleanser/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/code/header_cleanser DOCKER=docker test-image else echo "transforms/code/header_cleanser/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index dd13cbe74..90074b77f 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/code/malware/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/code/malware/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/code/malware run: | if [ -e "transforms/code/malware/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/code/malware DOCKER=docker test-image else echo "transforms/code/malware/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index 16e1ba330..5f3146d93 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/code/proglang_select/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/code/proglang_select/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/code/proglang_select run: | if [ -e "transforms/code/proglang_select/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/code/proglang_select DOCKER=docker test-image else echo "transforms/code/proglang_select/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index 26144e57b..50eefc1d3 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/code/repo_level_ordering run: | if [ -e "transforms/code/repo_level_ordering/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/code/repo_level_ordering DOCKER=docker test-image else echo "transforms/code/repo_level_ordering/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-kfp.yml b/.github/workflows/test-kfp.yml index 1f96cc6da..f0984c21b 100644 --- a/.github/workflows/test-kfp.yml +++ b/.github/workflows/test-kfp.yml @@ -12,6 +12,7 @@ on: - "kfp/**" - "!**.md" - "!**/doc/**" + - "!**/images/**" - "!**/.gitignore" pull_request: branches: @@ -21,6 +22,7 @@ on: - "kfp/**" - "!**.md" - "!**/doc/**" + - "!**/images/**" - "!**/.gitignore" env: diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index cc969c66a..a6b203032 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/language/doc_chunk/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/language/doc_chunk/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/language/doc_chunk run: | if [ -e "transforms/language/doc_chunk/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/language/doc_chunk DOCKER=docker test-image else echo "transforms/language/doc_chunk/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index c3da25938..0c6972e7b 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/language/doc_quality/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/language/doc_quality/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/language/doc_quality run: | if [ -e "transforms/language/doc_quality/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/language/doc_quality DOCKER=docker test-image else echo "transforms/language/doc_quality/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index d07f2e0a6..78b755738 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/language/lang_id/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/language/lang_id/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/language/lang_id run: | if [ -e "transforms/language/lang_id/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/language/lang_id DOCKER=docker test-image else echo "transforms/language/lang_id/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index e5284b4a5..0d4ca89e0 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/language/pdf2parquet run: | if [ -e "transforms/language/pdf2parquet/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/language/pdf2parquet DOCKER=docker test-image else echo "transforms/language/pdf2parquet/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 3eded38f5..90508be12 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/language/pii_redactor/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/language/pii_redactor/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/language/pii_redactor run: | if [ -e "transforms/language/pii_redactor/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/language/pii_redactor DOCKER=docker test-image else echo "transforms/language/pii_redactor/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index cf9a0604a..b971199bb 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/language/text_encoder/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/language/text_encoder/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/language/text_encoder run: | if [ -e "transforms/language/text_encoder/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/language/text_encoder DOCKER=docker test-image else echo "transforms/language/text_encoder/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-misc.yml b/.github/workflows/test-misc.yml index 50f3edbc1..2c601bbd5 100644 --- a/.github/workflows/test-misc.yml +++ b/.github/workflows/test-misc.yml @@ -12,6 +12,7 @@ on: - "**.md" - "examples/**" - "**/doc/**" + - "**/images/**" - "**/.gitignore" - "**/.dockerignore" pull_request: @@ -22,6 +23,7 @@ on: - "**.md" - "examples/**" - "**/doc/**" + - "**/images/**" - "**/.gitignore" - "**/.dockerignore" @@ -34,20 +36,10 @@ jobs: - name: Test top-level recursive make targets. run: | make -n clean test build publish set-versions - test-tools: + check-transform-test-workflows: runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4 - - name: Test tools - run: | - make -C tools DOCKER=docker venv test - test-tool-images: - runs-on: ubuntu-22.04 - timeout-minutes: 30 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test Tool images - run: | - make -C tools/ingest2parquet DOCKER=docker test-image + - name: Make sure all transforms have a test workflow + run: bash scripts/check-workflows.sh diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 1ed7fc89b..57750a032 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -14,6 +14,10 @@ on: paths: - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in @TARGET_TRANSFORM_DIR@ run: | if [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-image else echo "@TARGET_TRANSFORM_DIR@/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 3110ba0eb..0f75c469c 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/universal/doc_id/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/universal/doc_id/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/universal/doc_id run: | if [ -e "transforms/universal/doc_id/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/universal/doc_id DOCKER=docker test-image else echo "transforms/universal/doc_id/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 7da3e6569..ce3988434 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/universal/ededup/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/universal/ededup/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/universal/ededup run: | if [ -e "transforms/universal/ededup/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/universal/ededup DOCKER=docker test-image else echo "transforms/universal/ededup/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 1fb9ead05..7262a25fb 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/universal/fdedup/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/universal/fdedup/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/universal/fdedup run: | if [ -e "transforms/universal/fdedup/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/universal/fdedup DOCKER=docker test-image else echo "transforms/universal/fdedup/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 6eefa1c05..40f6e2f96 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/universal/filter/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/universal/filter/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/universal/filter run: | if [ -e "transforms/universal/filter/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/universal/filter DOCKER=docker test-image else echo "transforms/universal/filter/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index 3a4763682..466cdb0ed 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/universal/html2parquet/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/universal/html2parquet/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/universal/html2parquet run: | if [ -e "transforms/universal/html2parquet/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/universal/html2parquet DOCKER=docker test-image else echo "transforms/universal/html2parquet/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 703f70baf..0a46d249e 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/universal/noop/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/universal/noop/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/universal/noop run: | if [ -e "transforms/universal/noop/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/universal/noop DOCKER=docker test-image else echo "transforms/universal/noop/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 6fcd60642..2d49adca6 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/universal/profiler/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/universal/profiler/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/universal/profiler run: | if [ -e "transforms/universal/profiler/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/universal/profiler DOCKER=docker test-image else echo "transforms/universal/profiler/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index b27f0bf9c..2a69e323c 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/universal/resize/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/universal/resize/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/universal/resize run: | if [ -e "transforms/universal/resize/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/universal/resize DOCKER=docker test-image else echo "transforms/universal/resize/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 1d4af4eaa..0641b7e2b 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -14,6 +14,10 @@ on: paths: - "transforms/universal/tokenization/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" pull_request: branches: - "dev" @@ -21,6 +25,10 @@ on: paths: - "transforms/universal/tokenization/**" - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" jobs: check_if_push_image: @@ -84,7 +92,7 @@ jobs: - name: Test transform image in transforms/universal/tokenization run: | if [ -e "transforms/universal/tokenization/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + make -C data-processing-lib/spark DOCKER=docker image make -C transforms/universal/tokenization DOCKER=docker test-image else echo "transforms/universal/tokenization/Makefile not found - testing disabled for this transform." diff --git a/scripts/check-workflows.sh b/scripts/check-workflows.sh new file mode 100755 index 000000000..40f4e2615 --- /dev/null +++ b/scripts/check-workflows.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Check that each transform in transforms// has a corresponding +# .github/workflows/test--.yml file. +if [ ! -d transforms ]; then + echo Please run this script from the top of the repository + exit 1 +fi +for i in $(find transforms -maxdepth 2 -mindepth 2 -type d | grep -v venv); do + transform=$(basename $i) + category=$(dirname $i) + category=$(basename $category) + workflow=.github/workflows/test-$category-$transform.yml + if [ ! -e $workflow ]; then + echo Missing $workflow for transform $category/$transform + echo Fix this by running make in the .github/workflows directory + exit 1 + else + echo Verified existence of $workflow + fi +done From 3a81d216623fb93b5d062f51d41a1cbab3abbed8 Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 18 Sep 2024 17:25:50 -0400 Subject: [PATCH 47/82] workflow readme details on kfp and misc tests Signed-off-by: David Wood --- .github/workflows/README.md | 55 +++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index c357ff33a..a923a96cf 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -3,7 +3,7 @@ Here we have the start of a system to automatically generated github workflows (currently only for transforms). In general, the design is to use templates and `make` to generate/update the workflows. -Goals +#### Goals 1. Run only tests for a given transform when only the transform changes. Includes python, ray, spark and kfp_ray as available. 2. When the core dpk lib components files changes, test all transforms @@ -11,20 +11,21 @@ Includes python, ray, spark and kfp_ray as available. (We would like to avoid running all transform kfp tests in one PR) 4. Extra credit: If .md or other non-code changes are made, run no tests. -Assumptions: +#### Assumptions 1. All transforms will have test workflows. A transform can disable its tests locally -(temporarily?) by renaming its transforms/universal/noop/Makefile.disabled. -``` -git clone .... -... -git checkout -b new-branch -make # Creates new test*.yml workflows -git commit -a -s -m "update workflows" -git push --set-upstream origin new-branch -``` -should be sufficient. +(temporarily?) by renaming its Makefile. For example, +`cp transforms/universal/noop/Makefile transforms/universal/noop/Makefile.disabled`. + +## DPK libraries (`data-processing-lib` directory) +The DPK libraries, in data-processing-lib/{python,ray,spark}, are tested +via the fixed +[test-lib.yml](test-lib.yml) +file and is triggered when any code files in that tree change. + +The transforms test workflows also depend on this directory tree and so +changes made here will trigger transform tests. -## Transforms +## Transforms (`transforms` directory tree) We define a unique test workflow for each transform, based on a common template [test-transform.template](test-transform.template). The Makefile is used to (re)generate all workflows a necessary. By design, workflows for a given transform should run when @@ -37,6 +38,30 @@ When a new transform is added to the repository, 1. Run `make` in this directory to create the new test .yml for all transforms found in transforms/{universal,code,language} directories 1. commit and push the change to your branch with the new transform. -## KFP +Something like the following: +``` +git clone .... +... +git checkout -b new-branch +make # Creates new test*.yml workflows +git commit -a -s -m "update workflows" +git push --set-upstream origin new-branch +``` + +## KFP (`kfp` directory tree) + +Like DPK core libs, kfp tests are defined in +[test-kfp.yml](test-kfp.yml) and run whenever changes are made in +the `kfp` directory tree. Tests currently include + +1. test kfp on randomly selected transform. + +Eventually we would like to enable the transform-specific kfp test +when only the transform code is modified or maybe when only +the `kfp_ray` directory contents is modified. + +## Miscellaneous +[test-misc.yml](test-misc.yml) defines some repo consistency tests including -## DPK libraries \ No newline at end of file +1. Make sure `set-versions` make target can be run recursively throughout the repo +2. Makes sure there is a test workflow for each transform in the repo. \ No newline at end of file From 5d957ed21abe40e188bb7026cab59cc855d3d41a Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 18 Sep 2024 21:01:28 -0400 Subject: [PATCH 48/82] backing out change to dpk lib code Signed-off-by: David Wood --- .../python/test/data_processing_tests/transform/test_noop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-processing-lib/python/test/data_processing_tests/transform/test_noop.py b/data-processing-lib/python/test/data_processing_tests/transform/test_noop.py index 1843d1af9..1eb85fe48 100644 --- a/data-processing-lib/python/test/data_processing_tests/transform/test_noop.py +++ b/data-processing-lib/python/test/data_processing_tests/transform/test_noop.py @@ -19,7 +19,7 @@ ) -table = pa.Table.from_pydict({"name": pa.array(["Tom", "Dick", "John"]), "age": pa.array([0, 1, 2])}) +table = pa.Table.from_pydict({"name": pa.array(["Tom", "Dick", "Harry"]), "age": pa.array([0, 1, 2])}) expected_table = table # We're a noop after all. expected_metadata_list = [{"nfiles": 1, "nrows": 3}, {}] # transform() result # flush() result From b02b52c555877bdf480774219b1d52eeebebaec7 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 07:50:35 -0400 Subject: [PATCH 49/82] restore filter code Signed-off-by: David Wood --- transforms/universal/filter/python/src/filter_local.py | 2 +- transforms/universal/filter/python/src/filter_transform.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/transforms/universal/filter/python/src/filter_local.py b/transforms/universal/filter/python/src/filter_local.py index e5c8e94a8..771cdad0f 100644 --- a/transforms/universal/filter/python/src/filter_local.py +++ b/transforms/universal/filter/python/src/filter_local.py @@ -55,4 +55,4 @@ # Transform the table table_list, metadata = transform.transform(table) print(f"\noutput table has {table_list[0].num_rows} rows") - print(f"output metadata :: {metadata}") + print(f"output metadata : {metadata}") diff --git a/transforms/universal/filter/python/src/filter_transform.py b/transforms/universal/filter/python/src/filter_transform.py index 4ae5e2984..fe002e497 100644 --- a/transforms/universal/filter/python/src/filter_transform.py +++ b/transforms/universal/filter/python/src/filter_transform.py @@ -23,8 +23,6 @@ short_name = "filter" cli_prefix = short_name + "_" -# Misc comment to trigger build - filter_criteria_key = "criteria_list" """ AST Key holds the list of filter criteria (in SQL WHERE clause format)""" filter_logical_operator_key = "logical_operator" From 7cb439006dc2e478df31791f885680a4c6490bf9 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 07:55:26 -0400 Subject: [PATCH 50/82] restore noop code Signed-off-by: David Wood --- transforms/universal/noop/python/src/noop_transform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transforms/universal/noop/python/src/noop_transform.py b/transforms/universal/noop/python/src/noop_transform.py index 92a64975d..89e589cd7 100644 --- a/transforms/universal/noop/python/src/noop_transform.py +++ b/transforms/universal/noop/python/src/noop_transform.py @@ -55,7 +55,7 @@ def transform(self, table: pa.Table, file_name: str = None) -> tuple[list[pa.Tab if self.sleep is not None: self.logger.info(f"Sleep for {self.sleep} seconds") time.sleep(self.sleep) - self.logger.info("Sleep completed - continuing") + self.logger.info("Sleep completed - continue") # Add some sample metadata. self.logger.debug(f"Transformed one table with {len(table)} rows") metadata = {"nfiles": 1, "nrows": len(table)} @@ -115,5 +115,5 @@ def apply_input_params(self, args: Namespace) -> bool: return False self.params = self.params | captured - self.logger.info(f"noop parameters are::: {self.params}") + self.logger.info(f"noop parameters are : {self.params}") return True From b780cf5a3bbf9305603e1e1300cde893ec7a2d1f Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 08:17:38 -0400 Subject: [PATCH 51/82] workflow readme Signed-off-by: David Wood --- .github/workflows/README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index a923a96cf..faf8139b4 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -26,12 +26,18 @@ The transforms test workflows also depend on this directory tree and so changes made here will trigger transform tests. ## Transforms (`transforms` directory tree) -We define a unique test workflow for each transform, based on a common template [test-transform.template](test-transform.template). -The Makefile is used to (re)generate all workflows a necessary. By design, workflows for a given transform should run when +We define a unique test workflow for each transform, based on a common +template [test-transform.template](test-transform.template). +The [Makefile](Makefile) is used to (re)generate all workflows a necessary. +By design, workflows for a given transform should run when * anything of substance effecting operation is modified in the transform's directory tree. * anything in the core libraries in this repo (e.g., data-processing/lib) assuming the transform depends on these. -* Help! the workflow should NOT run when documentation (e.g., !**.md) is changed, however this case does not seem to be working atm. + +Note that the kfp tests (in kfp_ray/Makefile workflow-test) for a given transform are +**not** currently being run when the transform's tests are run. +Currently these are run randomly via the [test-kfp.yml](test-kfp.yml). +We expect to fix this is in the future. When a new transform is added to the repository, From 9a5ccdc42de4e04aef415c74b5ae0bc65f5ce81a Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 08:19:56 -0400 Subject: [PATCH 52/82] really restore noop code Signed-off-by: David Wood --- transforms/universal/noop/python/src/noop_transform.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transforms/universal/noop/python/src/noop_transform.py b/transforms/universal/noop/python/src/noop_transform.py index 89e589cd7..a4bf5bb5c 100644 --- a/transforms/universal/noop/python/src/noop_transform.py +++ b/transforms/universal/noop/python/src/noop_transform.py @@ -64,7 +64,6 @@ def transform(self, table: pa.Table, file_name: str = None) -> tuple[list[pa.Tab class NOOPTransformConfiguration(TransformConfiguration): - # Adding yet another test comment for cicd-opt branch testing - daw3rd 9/17/2024. """ Provides support for configuring and using the associated Transform class include configuration with CLI args. From 93c56c2b614602b5882cf8d6f633a8b4bce73114 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 14:53:05 -0400 Subject: [PATCH 53/82] check for makefile in transform test-src testing Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 6 +++++- .github/workflows/test-code-code_quality.yml | 6 +++++- .github/workflows/test-code-header_cleanser.yml | 6 +++++- .github/workflows/test-code-malware.yml | 6 +++++- .github/workflows/test-code-proglang_select.yml | 6 +++++- .github/workflows/test-code-repo_level_ordering.yml | 6 +++++- .github/workflows/test-language-doc_chunk.yml | 6 +++++- .github/workflows/test-language-doc_quality.yml | 6 +++++- .github/workflows/test-language-lang_id.yml | 6 +++++- .github/workflows/test-language-pdf2parquet.yml | 6 +++++- .github/workflows/test-language-pii_redactor.yml | 6 +++++- .github/workflows/test-language-text_encoder.yml | 6 +++++- .github/workflows/test-transform.template | 6 +++++- .github/workflows/test-universal-doc_id.yml | 6 +++++- .github/workflows/test-universal-ededup.yml | 6 +++++- .github/workflows/test-universal-fdedup.yml | 6 +++++- .github/workflows/test-universal-filter.yml | 6 +++++- .github/workflows/test-universal-html2parquet.yml | 6 +++++- .github/workflows/test-universal-noop.yml | 6 +++++- .github/workflows/test-universal-profiler.yml | 6 +++++- .github/workflows/test-universal-resize.yml | 6 +++++- .github/workflows/test-universal-tokenization.yml | 6 +++++- 22 files changed, 110 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index 5bddc137f..683898ed0 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/code/code2parquet run: | - make -C transforms/code/code2parquet DOCKER=docker test-src + if [ -e "transforms/code/code2parquet/Makefile" ]; then + make -C transforms/code/code2parquet DOCKER=docker test-src + else + echo "transforms/code/code2parquet/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index faccf10d0..902fff8f8 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/code/code_quality run: | - make -C transforms/code/code_quality DOCKER=docker test-src + if [ -e "transforms/code/code_quality/Makefile" ]; then + make -C transforms/code/code_quality DOCKER=docker test-src + else + echo "transforms/code/code_quality/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 3adce163d..34741f024 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/code/header_cleanser run: | - make -C transforms/code/header_cleanser DOCKER=docker test-src + if [ -e "transforms/code/header_cleanser/Makefile" ]; then + make -C transforms/code/header_cleanser DOCKER=docker test-src + else + echo "transforms/code/header_cleanser/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index 90074b77f..51faf6da2 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/code/malware run: | - make -C transforms/code/malware DOCKER=docker test-src + if [ -e "transforms/code/malware/Makefile" ]; then + make -C transforms/code/malware DOCKER=docker test-src + else + echo "transforms/code/malware/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index 5f3146d93..f087df795 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/code/proglang_select run: | - make -C transforms/code/proglang_select DOCKER=docker test-src + if [ -e "transforms/code/proglang_select/Makefile" ]; then + make -C transforms/code/proglang_select DOCKER=docker test-src + else + echo "transforms/code/proglang_select/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index 50eefc1d3..82dbff679 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/code/repo_level_ordering run: | - make -C transforms/code/repo_level_ordering DOCKER=docker test-src + if [ -e "transforms/code/repo_level_ordering/Makefile" ]; then + make -C transforms/code/repo_level_ordering DOCKER=docker test-src + else + echo "transforms/code/repo_level_ordering/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index a6b203032..0e21ec51c 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/language/doc_chunk run: | - make -C transforms/language/doc_chunk DOCKER=docker test-src + if [ -e "transforms/language/doc_chunk/Makefile" ]; then + make -C transforms/language/doc_chunk DOCKER=docker test-src + else + echo "transforms/language/doc_chunk/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index 0c6972e7b..53e183945 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/language/doc_quality run: | - make -C transforms/language/doc_quality DOCKER=docker test-src + if [ -e "transforms/language/doc_quality/Makefile" ]; then + make -C transforms/language/doc_quality DOCKER=docker test-src + else + echo "transforms/language/doc_quality/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 78b755738..d9480988d 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/language/lang_id run: | - make -C transforms/language/lang_id DOCKER=docker test-src + if [ -e "transforms/language/lang_id/Makefile" ]; then + make -C transforms/language/lang_id DOCKER=docker test-src + else + echo "transforms/language/lang_id/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 0d4ca89e0..1e0825036 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/language/pdf2parquet run: | - make -C transforms/language/pdf2parquet DOCKER=docker test-src + if [ -e "transforms/language/pdf2parquet/Makefile" ]; then + make -C transforms/language/pdf2parquet DOCKER=docker test-src + else + echo "transforms/language/pdf2parquet/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 90508be12..9c9eaa543 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/language/pii_redactor run: | - make -C transforms/language/pii_redactor DOCKER=docker test-src + if [ -e "transforms/language/pii_redactor/Makefile" ]; then + make -C transforms/language/pii_redactor DOCKER=docker test-src + else + echo "transforms/language/pii_redactor/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index b971199bb..4ac7ff929 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/language/text_encoder run: | - make -C transforms/language/text_encoder DOCKER=docker test-src + if [ -e "transforms/language/text_encoder/Makefile" ]; then + make -C transforms/language/text_encoder DOCKER=docker test-src + else + echo "transforms/language/text_encoder/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 57750a032..34d4c4e5e 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in @TARGET_TRANSFORM_DIR@ run: | - make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-src + if [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ]; then + make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-src + else + echo "@TARGET_TRANSFORM_DIR@/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 0f75c469c..215fe718c 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/universal/doc_id run: | - make -C transforms/universal/doc_id DOCKER=docker test-src + if [ -e "transforms/universal/doc_id/Makefile" ]; then + make -C transforms/universal/doc_id DOCKER=docker test-src + else + echo "transforms/universal/doc_id/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index ce3988434..1b22c523d 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/universal/ededup run: | - make -C transforms/universal/ededup DOCKER=docker test-src + if [ -e "transforms/universal/ededup/Makefile" ]; then + make -C transforms/universal/ededup DOCKER=docker test-src + else + echo "transforms/universal/ededup/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 7262a25fb..01f8cfb0a 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/universal/fdedup run: | - make -C transforms/universal/fdedup DOCKER=docker test-src + if [ -e "transforms/universal/fdedup/Makefile" ]; then + make -C transforms/universal/fdedup DOCKER=docker test-src + else + echo "transforms/universal/fdedup/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 40f6e2f96..17d53accb 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/universal/filter run: | - make -C transforms/universal/filter DOCKER=docker test-src + if [ -e "transforms/universal/filter/Makefile" ]; then + make -C transforms/universal/filter DOCKER=docker test-src + else + echo "transforms/universal/filter/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index 466cdb0ed..1496fa4e9 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/universal/html2parquet run: | - make -C transforms/universal/html2parquet DOCKER=docker test-src + if [ -e "transforms/universal/html2parquet/Makefile" ]; then + make -C transforms/universal/html2parquet DOCKER=docker test-src + else + echo "transforms/universal/html2parquet/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 0a46d249e..9df33abbd 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/universal/noop run: | - make -C transforms/universal/noop DOCKER=docker test-src + if [ -e "transforms/universal/noop/Makefile" ]; then + make -C transforms/universal/noop DOCKER=docker test-src + else + echo "transforms/universal/noop/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 2d49adca6..390cc491f 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/universal/profiler run: | - make -C transforms/universal/profiler DOCKER=docker test-src + if [ -e "transforms/universal/profiler/Makefile" ]; then + make -C transforms/universal/profiler DOCKER=docker test-src + else + echo "transforms/universal/profiler/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index 2a69e323c..5ec4739e3 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/universal/resize run: | - make -C transforms/universal/resize DOCKER=docker test-src + if [ -e "transforms/universal/resize/Makefile" ]; then + make -C transforms/universal/resize DOCKER=docker test-src + else + echo "transforms/universal/resize/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 0641b7e2b..8fc9f7918 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -68,7 +68,11 @@ jobs: df -h - name: Test transform source in transforms/universal/tokenization run: | - make -C transforms/universal/tokenization DOCKER=docker test-src + if [ -e "transforms/universal/tokenization/Makefile" ]; then + make -C transforms/universal/tokenization DOCKER=docker test-src + else + echo "transforms/universal/tokenization/Makefile not found - source testing disabled for this transform." + fi test-image: needs: [check_if_push_image] runs-on: ubuntu-22.04 From 8f8592d85df0f6f60f29bb2b4db5a8594a77043f Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 14:55:42 -0400 Subject: [PATCH 54/82] don't include lib test dependencies in transform test workflows Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 2 ++ .github/workflows/test-code-code_quality.yml | 2 ++ .github/workflows/test-code-header_cleanser.yml | 2 ++ .github/workflows/test-code-malware.yml | 2 ++ .github/workflows/test-code-proglang_select.yml | 2 ++ .github/workflows/test-code-repo_level_ordering.yml | 2 ++ .github/workflows/test-language-doc_chunk.yml | 2 ++ .github/workflows/test-language-doc_quality.yml | 2 ++ .github/workflows/test-language-lang_id.yml | 2 ++ .github/workflows/test-language-pdf2parquet.yml | 2 ++ .github/workflows/test-language-pii_redactor.yml | 2 ++ .github/workflows/test-language-text_encoder.yml | 2 ++ .github/workflows/test-transform.template | 2 ++ .github/workflows/test-universal-doc_id.yml | 2 ++ .github/workflows/test-universal-ededup.yml | 2 ++ .github/workflows/test-universal-fdedup.yml | 2 ++ .github/workflows/test-universal-filter.yml | 2 ++ .github/workflows/test-universal-html2parquet.yml | 2 ++ .github/workflows/test-universal-noop.yml | 2 ++ .github/workflows/test-universal-profiler.yml | 2 ++ .github/workflows/test-universal-resize.yml | 2 ++ .github/workflows/test-universal-tokenization.yml | 2 ++ 22 files changed, 44 insertions(+) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index 683898ed0..a2b050bff 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/code/code2parquet/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/code/code2parquet/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 902fff8f8..37448a551 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/code/code_quality/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/code/code_quality/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 34741f024..d2d6eb18f 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/code/header_cleanser/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/code/header_cleanser/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index 51faf6da2..5c5524eb4 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/code/malware/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/code/malware/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index f087df795..240154174 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/code/proglang_select/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/code/proglang_select/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index 82dbff679..6df16d1db 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index 0e21ec51c..8e923a379 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/language/doc_chunk/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/language/doc_chunk/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index 53e183945..a847197d1 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/language/doc_quality/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/language/doc_quality/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index d9480988d..2842b1e2f 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/language/lang_id/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/language/lang_id/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 1e0825036..1b32d4f7a 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 9c9eaa543..c219d1d99 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/language/pii_redactor/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/language/pii_redactor/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index 4ac7ff929..5acb540fd 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/language/text_encoder/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/language/text_encoder/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 34d4c4e5e..30b06ab65 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -14,6 +14,7 @@ on: paths: - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 215fe718c..7804a4a03 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/doc_id/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/universal/doc_id/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 1b22c523d..b1852fd61 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/ededup/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/universal/ededup/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 01f8cfb0a..5e67b03c8 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/fdedup/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/universal/fdedup/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 17d53accb..78e3c7816 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/filter/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/universal/filter/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index 1496fa4e9..ca7b1c7d7 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/html2parquet/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/universal/html2parquet/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 9df33abbd..e749f89f0 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/noop/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/universal/noop/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 390cc491f..ce0cd8ead 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/profiler/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/universal/profiler/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index 5ec4739e3..00d852f4b 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/resize/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/universal/resize/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 8fc9f7918..ce94fc2d2 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/tokenization/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -25,6 +26,7 @@ on: paths: - "transforms/universal/tokenization/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" - "!**.md" - "!**/doc/**" - "!**/images/**" From 40ce888dd3fae8e8daa6e4a4db5fcfc4fe56a1fb Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 14:56:17 -0400 Subject: [PATCH 55/82] noop code change Signed-off-by: David Wood --- transforms/universal/noop/python/src/noop_transform_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/universal/noop/python/src/noop_transform_python.py b/transforms/universal/noop/python/src/noop_transform_python.py index 54668523c..a3313121b 100644 --- a/transforms/universal/noop/python/src/noop_transform_python.py +++ b/transforms/universal/noop/python/src/noop_transform_python.py @@ -41,5 +41,5 @@ def __init__(self): if __name__ == "__main__": # launcher = NOOPRayLauncher() launcher = PythonTransformLauncher(NOOPPythonTransformConfiguration()) - logger.info("Launching noop transform") + logger.info("Launching noop transform...") launcher.launch() From cb0d1f9f027f0d1ad60e9e4727d597cc037362e0 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 15:03:51 -0400 Subject: [PATCH 56/82] disable noop, don't include lib test-data in transform dependencies Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 2 ++ .github/workflows/test-code-code_quality.yml | 2 ++ .github/workflows/test-code-header_cleanser.yml | 2 ++ .github/workflows/test-code-malware.yml | 2 ++ .github/workflows/test-code-proglang_select.yml | 2 ++ .github/workflows/test-code-repo_level_ordering.yml | 2 ++ .github/workflows/test-language-doc_chunk.yml | 2 ++ .github/workflows/test-language-doc_quality.yml | 2 ++ .github/workflows/test-language-lang_id.yml | 2 ++ .github/workflows/test-language-pdf2parquet.yml | 2 ++ .github/workflows/test-language-pii_redactor.yml | 2 ++ .github/workflows/test-language-text_encoder.yml | 2 ++ .github/workflows/test-transform.template | 2 ++ .github/workflows/test-universal-doc_id.yml | 2 ++ .github/workflows/test-universal-ededup.yml | 2 ++ .github/workflows/test-universal-fdedup.yml | 2 ++ .github/workflows/test-universal-filter.yml | 2 ++ .github/workflows/test-universal-html2parquet.yml | 2 ++ .github/workflows/test-universal-noop.yml | 2 ++ .github/workflows/test-universal-profiler.yml | 2 ++ .github/workflows/test-universal-resize.yml | 2 ++ .github/workflows/test-universal-tokenization.yml | 2 ++ transforms/universal/noop/{Makefile => Makefile.disabled} | 0 23 files changed, 44 insertions(+) rename transforms/universal/noop/{Makefile => Makefile.disabled} (100%) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index a2b050bff..69e05a880 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -15,6 +15,7 @@ on: - "transforms/code/code2parquet/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/code/code2parquet/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 37448a551..560059650 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -15,6 +15,7 @@ on: - "transforms/code/code_quality/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/code/code_quality/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index d2d6eb18f..d87764c3e 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -15,6 +15,7 @@ on: - "transforms/code/header_cleanser/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/code/header_cleanser/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index 5c5524eb4..5b187b3e0 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -15,6 +15,7 @@ on: - "transforms/code/malware/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/code/malware/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index 240154174..27e9a8ae5 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -15,6 +15,7 @@ on: - "transforms/code/proglang_select/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/code/proglang_select/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index 6df16d1db..7628f664f 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -15,6 +15,7 @@ on: - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index 8e923a379..d81962ec8 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -15,6 +15,7 @@ on: - "transforms/language/doc_chunk/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/language/doc_chunk/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index a847197d1..3c9a21d2f 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -15,6 +15,7 @@ on: - "transforms/language/doc_quality/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/language/doc_quality/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 2842b1e2f..cb4710d2b 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -15,6 +15,7 @@ on: - "transforms/language/lang_id/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/language/lang_id/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 1b32d4f7a..3ba8aac9e 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -15,6 +15,7 @@ on: - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index c219d1d99..30d805dc2 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -15,6 +15,7 @@ on: - "transforms/language/pii_redactor/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/language/pii_redactor/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index 5acb540fd..dbd013721 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -15,6 +15,7 @@ on: - "transforms/language/text_encoder/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/language/text_encoder/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 30b06ab65..3ec8c3d66 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -15,6 +15,7 @@ on: - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 7804a4a03..a6785332b 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -15,6 +15,7 @@ on: - "transforms/universal/doc_id/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/universal/doc_id/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index b1852fd61..9cc56842e 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -15,6 +15,7 @@ on: - "transforms/universal/ededup/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/universal/ededup/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 5e67b03c8..3426eb485 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -15,6 +15,7 @@ on: - "transforms/universal/fdedup/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/universal/fdedup/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 78e3c7816..0fdc7ae5a 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -15,6 +15,7 @@ on: - "transforms/universal/filter/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/universal/filter/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index ca7b1c7d7..1735b4aaa 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -15,6 +15,7 @@ on: - "transforms/universal/html2parquet/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/universal/html2parquet/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index e749f89f0..5e74fb1b1 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -15,6 +15,7 @@ on: - "transforms/universal/noop/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/universal/noop/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index ce0cd8ead..c8e020ab6 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -15,6 +15,7 @@ on: - "transforms/universal/profiler/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/universal/profiler/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index 00d852f4b..078082bfb 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -15,6 +15,7 @@ on: - "transforms/universal/resize/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/universal/resize/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index ce94fc2d2..18d8ed3a3 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -15,6 +15,7 @@ on: - "transforms/universal/tokenization/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -27,6 +28,7 @@ on: - "transforms/universal/tokenization/**" - "data-processing-lib/**" - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/transforms/universal/noop/Makefile b/transforms/universal/noop/Makefile.disabled similarity index 100% rename from transforms/universal/noop/Makefile rename to transforms/universal/noop/Makefile.disabled From d6c578468220e52638740a40954a4e060341c4ae Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 15:19:10 -0400 Subject: [PATCH 57/82] use job.id.if on Makefile to enable transform test job Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 3 +++ .github/workflows/test-code-code_quality.yml | 3 +++ .github/workflows/test-code-header_cleanser.yml | 3 +++ .github/workflows/test-code-malware.yml | 3 +++ .github/workflows/test-code-proglang_select.yml | 3 +++ .github/workflows/test-code-repo_level_ordering.yml | 3 +++ .github/workflows/test-language-doc_chunk.yml | 3 +++ .github/workflows/test-language-doc_quality.yml | 3 +++ .github/workflows/test-language-lang_id.yml | 3 +++ .github/workflows/test-language-pdf2parquet.yml | 3 +++ .github/workflows/test-language-pii_redactor.yml | 3 +++ .github/workflows/test-language-text_encoder.yml | 3 +++ .github/workflows/test-transform.template | 3 +++ .github/workflows/test-universal-doc_id.yml | 3 +++ .github/workflows/test-universal-ededup.yml | 3 +++ .github/workflows/test-universal-fdedup.yml | 3 +++ .github/workflows/test-universal-filter.yml | 3 +++ .github/workflows/test-universal-html2parquet.yml | 3 +++ .github/workflows/test-universal-noop.yml | 3 +++ .github/workflows/test-universal-profiler.yml | 3 +++ .github/workflows/test-universal-resize.yml | 3 +++ .github/workflows/test-universal-tokenization.yml | 3 +++ 22 files changed, 66 insertions(+) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index 69e05a880..36ccf15b0 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/code/code2parquet/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/code/code2parquet/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/code/code2parquet/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/code/code2parquet/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 560059650..a96fc892a 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/code/code_quality/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/code/code_quality/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/code/code_quality/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/code/code_quality/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index d87764c3e..911a2a3dd 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/code/header_cleanser/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/code/header_cleanser/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/code/header_cleanser/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/code/header_cleanser/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index 5b187b3e0..c5fdab760 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/code/malware/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/code/malware/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/code/malware/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/code/malware/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index 27e9a8ae5..cb0e15e1c 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/code/proglang_select/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/code/proglang_select/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/code/proglang_select/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/code/proglang_select/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index 7628f664f..694ffc39f 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/code/repo_level_ordering/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/code/repo_level_ordering/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/code/repo_level_ordering/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/code/repo_level_ordering/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index d81962ec8..e7d4d4dfd 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/language/doc_chunk/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/language/doc_chunk/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/language/doc_chunk/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/language/doc_chunk/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index 3c9a21d2f..958da73aa 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/language/doc_quality/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/language/doc_quality/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/language/doc_quality/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/language/doc_quality/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index cb4710d2b..b936122c1 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/language/lang_id/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/language/lang_id/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/language/lang_id/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/language/lang_id/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 3ba8aac9e..309f4cb50 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/language/pdf2parquet/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/language/pdf2parquet/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/language/pdf2parquet/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/language/pdf2parquet/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 30d805dc2..814cfc75e 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/language/pii_redactor/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/language/pii_redactor/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/language/pii_redactor/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/language/pii_redactor/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index dbd013721..33606d6c7 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/language/text_encoder/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/language/text_encoder/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/language/text_encoder/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/language/text_encoder/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 3ec8c3d66..de1371e78 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "@TARGET_TRANSFORM_DIR@/Makefile not found - source testing disabled for this transform." fi test-image: + if: [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index a6785332b..2236a4856 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/universal/doc_id/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/universal/doc_id/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/universal/doc_id/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/universal/doc_id/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 9cc56842e..93c829848 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/universal/ededup/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/universal/ededup/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/universal/ededup/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/universal/ededup/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 3426eb485..3d57f3312 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/universal/fdedup/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/universal/fdedup/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/universal/fdedup/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/universal/fdedup/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 0fdc7ae5a..b8a2698cd 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/universal/filter/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/universal/filter/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/universal/filter/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/universal/filter/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index 1735b4aaa..b4e5c1dcf 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/universal/html2parquet/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/universal/html2parquet/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/universal/html2parquet/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/universal/html2parquet/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 5e74fb1b1..3106266e4 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/universal/noop/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/universal/noop/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/universal/noop/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/universal/noop/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index c8e020ab6..05517e60f 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/universal/profiler/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/universal/profiler/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/universal/profiler/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/universal/profiler/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index 078082bfb..e534d761a 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/universal/resize/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/universal/resize/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/universal/resize/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/universal/resize/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 18d8ed3a3..3f9ba009a 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -36,6 +36,7 @@ on: jobs: check_if_push_image: + if: [-e "transforms/universal/tokenization/Makefile"] # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -57,6 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: + if: [-e "transforms/universal/tokenization/Makefile"] runs-on: ubuntu-22.04 steps: - name: Checkout @@ -78,6 +80,7 @@ jobs: echo "transforms/universal/tokenization/Makefile not found - source testing disabled for this transform." fi test-image: + if: [-e "transforms/universal/tokenization/Makefile"] needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 From 4f6b2f2b4e97a92c5e64f73db31ea76f14955d23 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 15:21:41 -0400 Subject: [PATCH 58/82] use job.id.if on Makefile to enable transform test job Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 6 +++--- .github/workflows/test-code-code_quality.yml | 6 +++--- .github/workflows/test-code-header_cleanser.yml | 6 +++--- .github/workflows/test-code-malware.yml | 6 +++--- .github/workflows/test-code-proglang_select.yml | 6 +++--- .github/workflows/test-code-repo_level_ordering.yml | 6 +++--- .github/workflows/test-language-doc_chunk.yml | 6 +++--- .github/workflows/test-language-doc_quality.yml | 6 +++--- .github/workflows/test-language-lang_id.yml | 6 +++--- .github/workflows/test-language-pdf2parquet.yml | 6 +++--- .github/workflows/test-language-pii_redactor.yml | 6 +++--- .github/workflows/test-language-text_encoder.yml | 6 +++--- .github/workflows/test-transform.template | 6 +++--- .github/workflows/test-universal-doc_id.yml | 6 +++--- .github/workflows/test-universal-ededup.yml | 6 +++--- .github/workflows/test-universal-fdedup.yml | 6 +++--- .github/workflows/test-universal-filter.yml | 6 +++--- .github/workflows/test-universal-html2parquet.yml | 6 +++--- .github/workflows/test-universal-noop.yml | 6 +++--- .github/workflows/test-universal-profiler.yml | 6 +++--- .github/workflows/test-universal-resize.yml | 6 +++--- .github/workflows/test-universal-tokenization.yml | 6 +++--- 22 files changed, 66 insertions(+), 66 deletions(-) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index 36ccf15b0..73d471db2 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/code/code2parquet/Makefile"] + if: test -e "transforms/code/code2parquet/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/code/code2parquet/Makefile"] + if: test -e "transforms/code/code2parquet/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/code/code2parquet/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/code/code2parquet/Makefile"] + if: test -e "transforms/code/code2parquet/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index a96fc892a..f40db492a 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/code/code_quality/Makefile"] + if: test -e "transforms/code/code_quality/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/code/code_quality/Makefile"] + if: test -e "transforms/code/code_quality/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/code/code_quality/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/code/code_quality/Makefile"] + if: test -e "transforms/code/code_quality/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 911a2a3dd..801c2928e 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/code/header_cleanser/Makefile"] + if: test -e "transforms/code/header_cleanser/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/code/header_cleanser/Makefile"] + if: test -e "transforms/code/header_cleanser/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/code/header_cleanser/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/code/header_cleanser/Makefile"] + if: test -e "transforms/code/header_cleanser/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index c5fdab760..6c95110d4 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/code/malware/Makefile"] + if: test -e "transforms/code/malware/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/code/malware/Makefile"] + if: test -e "transforms/code/malware/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/code/malware/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/code/malware/Makefile"] + if: test -e "transforms/code/malware/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index cb0e15e1c..a87bbfb69 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/code/proglang_select/Makefile"] + if: test -e "transforms/code/proglang_select/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/code/proglang_select/Makefile"] + if: test -e "transforms/code/proglang_select/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/code/proglang_select/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/code/proglang_select/Makefile"] + if: test -e "transforms/code/proglang_select/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index 694ffc39f..498c8d5f1 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/code/repo_level_ordering/Makefile"] + if: test -e "transforms/code/repo_level_ordering/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/code/repo_level_ordering/Makefile"] + if: test -e "transforms/code/repo_level_ordering/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/code/repo_level_ordering/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/code/repo_level_ordering/Makefile"] + if: test -e "transforms/code/repo_level_ordering/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index e7d4d4dfd..be6083ef1 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/language/doc_chunk/Makefile"] + if: test -e "transforms/language/doc_chunk/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/language/doc_chunk/Makefile"] + if: test -e "transforms/language/doc_chunk/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/language/doc_chunk/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/language/doc_chunk/Makefile"] + if: test -e "transforms/language/doc_chunk/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index 958da73aa..dd4a31d5c 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/language/doc_quality/Makefile"] + if: test -e "transforms/language/doc_quality/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/language/doc_quality/Makefile"] + if: test -e "transforms/language/doc_quality/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/language/doc_quality/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/language/doc_quality/Makefile"] + if: test -e "transforms/language/doc_quality/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index b936122c1..377b79c39 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/language/lang_id/Makefile"] + if: test -e "transforms/language/lang_id/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/language/lang_id/Makefile"] + if: test -e "transforms/language/lang_id/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/language/lang_id/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/language/lang_id/Makefile"] + if: test -e "transforms/language/lang_id/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 309f4cb50..040d7f25f 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/language/pdf2parquet/Makefile"] + if: test -e "transforms/language/pdf2parquet/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/language/pdf2parquet/Makefile"] + if: test -e "transforms/language/pdf2parquet/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/language/pdf2parquet/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/language/pdf2parquet/Makefile"] + if: test -e "transforms/language/pdf2parquet/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 814cfc75e..c8e7b2d57 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/language/pii_redactor/Makefile"] + if: test -e "transforms/language/pii_redactor/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/language/pii_redactor/Makefile"] + if: test -e "transforms/language/pii_redactor/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/language/pii_redactor/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/language/pii_redactor/Makefile"] + if: test -e "transforms/language/pii_redactor/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index 33606d6c7..23f12b4cf 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/language/text_encoder/Makefile"] + if: test -e "transforms/language/text_encoder/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/language/text_encoder/Makefile"] + if: test -e "transforms/language/text_encoder/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/language/text_encoder/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/language/text_encoder/Makefile"] + if: test -e "transforms/language/text_encoder/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index de1371e78..4eef4025c 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ] + if: test -e "@TARGET_TRANSFORM_DIR@/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ] + if: test -e "@TARGET_TRANSFORM_DIR@/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "@TARGET_TRANSFORM_DIR@/Makefile not found - source testing disabled for this transform." fi test-image: - if: [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ] + if: test -e "@TARGET_TRANSFORM_DIR@/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 2236a4856..0d453c5a7 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/universal/doc_id/Makefile"] + if: test -e "transforms/universal/doc_id/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/universal/doc_id/Makefile"] + if: test -e "transforms/universal/doc_id/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/universal/doc_id/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/universal/doc_id/Makefile"] + if: test -e "transforms/universal/doc_id/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 93c829848..a0759eca1 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/universal/ededup/Makefile"] + if: test -e "transforms/universal/ededup/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/universal/ededup/Makefile"] + if: test -e "transforms/universal/ededup/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/universal/ededup/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/universal/ededup/Makefile"] + if: test -e "transforms/universal/ededup/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 3d57f3312..2ffb444fd 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/universal/fdedup/Makefile"] + if: test -e "transforms/universal/fdedup/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/universal/fdedup/Makefile"] + if: test -e "transforms/universal/fdedup/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/universal/fdedup/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/universal/fdedup/Makefile"] + if: test -e "transforms/universal/fdedup/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index b8a2698cd..8f3311a4c 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/universal/filter/Makefile"] + if: test -e "transforms/universal/filter/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/universal/filter/Makefile"] + if: test -e "transforms/universal/filter/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/universal/filter/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/universal/filter/Makefile"] + if: test -e "transforms/universal/filter/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index b4e5c1dcf..25a67cb68 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/universal/html2parquet/Makefile"] + if: test -e "transforms/universal/html2parquet/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/universal/html2parquet/Makefile"] + if: test -e "transforms/universal/html2parquet/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/universal/html2parquet/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/universal/html2parquet/Makefile"] + if: test -e "transforms/universal/html2parquet/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 3106266e4..55391f24e 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/universal/noop/Makefile"] + if: test -e "transforms/universal/noop/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/universal/noop/Makefile"] + if: test -e "transforms/universal/noop/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/universal/noop/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/universal/noop/Makefile"] + if: test -e "transforms/universal/noop/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 05517e60f..88c01a6d0 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/universal/profiler/Makefile"] + if: test -e "transforms/universal/profiler/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/universal/profiler/Makefile"] + if: test -e "transforms/universal/profiler/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/universal/profiler/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/universal/profiler/Makefile"] + if: test -e "transforms/universal/profiler/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index e534d761a..9cdcf795b 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/universal/resize/Makefile"] + if: test -e "transforms/universal/resize/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/universal/resize/Makefile"] + if: test -e "transforms/universal/resize/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/universal/resize/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/universal/resize/Makefile"] + if: test -e "transforms/universal/resize/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 3f9ba009a..8c3b1b6aa 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -36,7 +36,7 @@ on: jobs: check_if_push_image: - if: [-e "transforms/universal/tokenization/Makefile"] + if: test -e "transforms/universal/tokenization/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -58,7 +58,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: [-e "transforms/universal/tokenization/Makefile"] + if: test -e "transforms/universal/tokenization/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -80,7 +80,7 @@ jobs: echo "transforms/universal/tokenization/Makefile not found - source testing disabled for this transform." fi test-image: - if: [-e "transforms/universal/tokenization/Makefile"] + if: test -e "transforms/universal/tokenization/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 From c0f4935f4c5ff82f654ecc6e4c57e480d59340c1 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 15:22:43 -0400 Subject: [PATCH 59/82] restore noop Makefile Signed-off-by: David Wood --- transforms/universal/noop/{Makefile.disabled => Makefile} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename transforms/universal/noop/{Makefile.disabled => Makefile} (100%) diff --git a/transforms/universal/noop/Makefile.disabled b/transforms/universal/noop/Makefile similarity index 100% rename from transforms/universal/noop/Makefile.disabled rename to transforms/universal/noop/Makefile From 0a09daa4461f0bfb4c4b95410d1e96e919591382 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 15:44:42 -0400 Subject: [PATCH 60/82] exclude kfp_ray from transfor test workflow and change noop code Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 2 ++ .github/workflows/test-code-code_quality.yml | 2 ++ .github/workflows/test-code-header_cleanser.yml | 2 ++ .github/workflows/test-code-malware.yml | 2 ++ .github/workflows/test-code-proglang_select.yml | 2 ++ .github/workflows/test-code-repo_level_ordering.yml | 2 ++ .github/workflows/test-language-doc_chunk.yml | 2 ++ .github/workflows/test-language-doc_quality.yml | 2 ++ .github/workflows/test-language-lang_id.yml | 2 ++ .github/workflows/test-language-pdf2parquet.yml | 2 ++ .github/workflows/test-language-pii_redactor.yml | 2 ++ .github/workflows/test-language-text_encoder.yml | 2 ++ .github/workflows/test-transform.template | 2 ++ .github/workflows/test-universal-doc_id.yml | 2 ++ .github/workflows/test-universal-ededup.yml | 2 ++ .github/workflows/test-universal-fdedup.yml | 2 ++ .github/workflows/test-universal-filter.yml | 2 ++ .github/workflows/test-universal-html2parquet.yml | 2 ++ .github/workflows/test-universal-noop.yml | 2 ++ .github/workflows/test-universal-profiler.yml | 2 ++ .github/workflows/test-universal-resize.yml | 2 ++ .github/workflows/test-universal-tokenization.yml | 2 ++ transforms/universal/noop/python/src/noop_transform_python.py | 2 +- 23 files changed, 45 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index 73d471db2..43bbd9213 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/code/code2parquet/**" - "data-processing-lib/**" + - "!transforms/code/code2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/code/code2parquet/**" - "data-processing-lib/**" + - "!transforms/code/code2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index f40db492a..93e08917e 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/code/code_quality/**" - "data-processing-lib/**" + - "!transforms/code/code_quality/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/code/code_quality/**" - "data-processing-lib/**" + - "!transforms/code/code_quality/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 801c2928e..73e921b56 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/code/header_cleanser/**" - "data-processing-lib/**" + - "!transforms/code/header_cleanser/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/code/header_cleanser/**" - "data-processing-lib/**" + - "!transforms/code/header_cleanser/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index 6c95110d4..c0c1dd619 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/code/malware/**" - "data-processing-lib/**" + - "!transforms/code/malware/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/code/malware/**" - "data-processing-lib/**" + - "!transforms/code/malware/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index a87bbfb69..947bd8c5a 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/code/proglang_select/**" - "data-processing-lib/**" + - "!transforms/code/proglang_select/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/code/proglang_select/**" - "data-processing-lib/**" + - "!transforms/code/proglang_select/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index 498c8d5f1..24c1deb48 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" + - "!transforms/code/repo_level_ordering/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" + - "!transforms/code/repo_level_ordering/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index be6083ef1..72f6d0b8d 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/language/doc_chunk/**" - "data-processing-lib/**" + - "!transforms/language/doc_chunk/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/language/doc_chunk/**" - "data-processing-lib/**" + - "!transforms/language/doc_chunk/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index dd4a31d5c..4af2f4a64 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/language/doc_quality/**" - "data-processing-lib/**" + - "!transforms/language/doc_quality/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/language/doc_quality/**" - "data-processing-lib/**" + - "!transforms/language/doc_quality/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 377b79c39..6f29d324d 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/language/lang_id/**" - "data-processing-lib/**" + - "!transforms/language/lang_id/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/language/lang_id/**" - "data-processing-lib/**" + - "!transforms/language/lang_id/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 040d7f25f..9e1c1d1c6 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" + - "!transforms/language/pdf2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" + - "!transforms/language/pdf2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index c8e7b2d57..fc52cf4c2 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/language/pii_redactor/**" - "data-processing-lib/**" + - "!transforms/language/pii_redactor/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/language/pii_redactor/**" - "data-processing-lib/**" + - "!transforms/language/pii_redactor/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index 23f12b4cf..06f0549da 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/language/text_encoder/**" - "data-processing-lib/**" + - "!transforms/language/text_encoder/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/language/text_encoder/**" - "data-processing-lib/**" + - "!transforms/language/text_encoder/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 4eef4025c..0cd7cad83 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -14,6 +14,7 @@ on: paths: - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" + - "!@TARGET_TRANSFORM_DIR@/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" + - "!@TARGET_TRANSFORM_DIR@/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 0d453c5a7..f0b644247 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/doc_id/**" - "data-processing-lib/**" + - "!transforms/universal/doc_id/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/universal/doc_id/**" - "data-processing-lib/**" + - "!transforms/universal/doc_id/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index a0759eca1..043693fc7 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/ededup/**" - "data-processing-lib/**" + - "!transforms/universal/ededup/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/universal/ededup/**" - "data-processing-lib/**" + - "!transforms/universal/ededup/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 2ffb444fd..e16779780 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/fdedup/**" - "data-processing-lib/**" + - "!transforms/universal/fdedup/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/universal/fdedup/**" - "data-processing-lib/**" + - "!transforms/universal/fdedup/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 8f3311a4c..be7b18a6e 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/filter/**" - "data-processing-lib/**" + - "!transforms/universal/filter/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/universal/filter/**" - "data-processing-lib/**" + - "!transforms/universal/filter/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index 25a67cb68..a05c41c05 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/html2parquet/**" - "data-processing-lib/**" + - "!transforms/universal/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/universal/html2parquet/**" - "data-processing-lib/**" + - "!transforms/universal/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 55391f24e..47460100c 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/noop/**" - "data-processing-lib/**" + - "!transforms/universal/noop/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/universal/noop/**" - "data-processing-lib/**" + - "!transforms/universal/noop/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 88c01a6d0..714d20ac3 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/profiler/**" - "data-processing-lib/**" + - "!transforms/universal/profiler/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/universal/profiler/**" - "data-processing-lib/**" + - "!transforms/universal/profiler/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index 9cdcf795b..561f6449a 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/resize/**" - "data-processing-lib/**" + - "!transforms/universal/resize/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/universal/resize/**" - "data-processing-lib/**" + - "!transforms/universal/resize/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 8c3b1b6aa..aa38b7191 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -14,6 +14,7 @@ on: paths: - "transforms/universal/tokenization/**" - "data-processing-lib/**" + - "!transforms/universal/tokenization/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" @@ -27,6 +28,7 @@ on: paths: - "transforms/universal/tokenization/**" - "data-processing-lib/**" + - "!transforms/universal/tokenization/**/kfp_ray/**" # This is/will be tested in separate workflow - "!data-processing-lib/**/test/**" - "!data-processing-lib/**/test-data/**" - "!**.md" diff --git a/transforms/universal/noop/python/src/noop_transform_python.py b/transforms/universal/noop/python/src/noop_transform_python.py index a3313121b..25a5ff512 100644 --- a/transforms/universal/noop/python/src/noop_transform_python.py +++ b/transforms/universal/noop/python/src/noop_transform_python.py @@ -41,5 +41,5 @@ def __init__(self): if __name__ == "__main__": # launcher = NOOPRayLauncher() launcher = PythonTransformLauncher(NOOPPythonTransformConfiguration()) - logger.info("Launching noop transform...") + logger.info("Launching noop transform..") launcher.launch() From 1bfb33da84a5b949d2300ffd9b6c06f643831961 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 15:47:39 -0400 Subject: [PATCH 61/82] remove if: from test workflows Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 3 --- .github/workflows/test-code-code_quality.yml | 3 --- .github/workflows/test-code-header_cleanser.yml | 3 --- .github/workflows/test-code-malware.yml | 3 --- .github/workflows/test-code-proglang_select.yml | 3 --- .github/workflows/test-code-repo_level_ordering.yml | 3 --- .github/workflows/test-language-doc_chunk.yml | 3 --- .github/workflows/test-language-doc_quality.yml | 3 --- .github/workflows/test-language-lang_id.yml | 3 --- .github/workflows/test-language-pdf2parquet.yml | 3 --- .github/workflows/test-language-pii_redactor.yml | 3 --- .github/workflows/test-language-text_encoder.yml | 3 --- .github/workflows/test-transform.template | 3 --- .github/workflows/test-universal-doc_id.yml | 3 --- .github/workflows/test-universal-ededup.yml | 3 --- .github/workflows/test-universal-fdedup.yml | 3 --- .github/workflows/test-universal-filter.yml | 3 --- .github/workflows/test-universal-html2parquet.yml | 3 --- .github/workflows/test-universal-noop.yml | 3 --- .github/workflows/test-universal-profiler.yml | 3 --- .github/workflows/test-universal-resize.yml | 3 --- .github/workflows/test-universal-tokenization.yml | 3 --- 22 files changed, 66 deletions(-) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index 43bbd9213..996610e53 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/code/code2parquet/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/code/code2parquet/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/code/code2parquet/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/code/code2parquet/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 93e08917e..e855962ab 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/code/code_quality/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/code/code_quality/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/code/code_quality/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/code/code_quality/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 73e921b56..74b713cda 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/code/header_cleanser/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/code/header_cleanser/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/code/header_cleanser/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/code/header_cleanser/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index c0c1dd619..cef5746a2 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/code/malware/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/code/malware/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/code/malware/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/code/malware/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index 947bd8c5a..86ea55f55 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/code/proglang_select/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/code/proglang_select/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/code/proglang_select/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/code/proglang_select/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index 24c1deb48..8451b174b 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/code/repo_level_ordering/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/code/repo_level_ordering/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/code/repo_level_ordering/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/code/repo_level_ordering/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index 72f6d0b8d..98341903b 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/language/doc_chunk/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/language/doc_chunk/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/language/doc_chunk/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/language/doc_chunk/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index 4af2f4a64..540c2490c 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/language/doc_quality/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/language/doc_quality/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/language/doc_quality/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/language/doc_quality/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 6f29d324d..1c310270a 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/language/lang_id/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/language/lang_id/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/language/lang_id/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/language/lang_id/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 9e1c1d1c6..85e6e15e4 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/language/pdf2parquet/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/language/pdf2parquet/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/language/pdf2parquet/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/language/pdf2parquet/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index fc52cf4c2..c162a3322 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/language/pii_redactor/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/language/pii_redactor/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/language/pii_redactor/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/language/pii_redactor/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index 06f0549da..cc4cdf0f5 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/language/text_encoder/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/language/text_encoder/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/language/text_encoder/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/language/text_encoder/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 0cd7cad83..bf8a56534 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "@TARGET_TRANSFORM_DIR@/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "@TARGET_TRANSFORM_DIR@/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "@TARGET_TRANSFORM_DIR@/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "@TARGET_TRANSFORM_DIR@/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index f0b644247..056dae834 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/universal/doc_id/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/universal/doc_id/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/universal/doc_id/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/universal/doc_id/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 043693fc7..9a9e3d174 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/universal/ededup/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/universal/ededup/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/universal/ededup/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/universal/ededup/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index e16779780..4814c7c7b 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/universal/fdedup/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/universal/fdedup/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/universal/fdedup/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/universal/fdedup/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index be7b18a6e..4ce46c874 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/universal/filter/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/universal/filter/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/universal/filter/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/universal/filter/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index a05c41c05..46608c3d0 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/universal/html2parquet/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/universal/html2parquet/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/universal/html2parquet/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/universal/html2parquet/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 47460100c..44aa72c76 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/universal/noop/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/universal/noop/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/universal/noop/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/universal/noop/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 714d20ac3..0b34a5bcf 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/universal/profiler/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/universal/profiler/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/universal/profiler/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/universal/profiler/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index 561f6449a..9c8f65c6f 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/universal/resize/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/universal/resize/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/universal/resize/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/universal/resize/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index aa38b7191..7e78fa6e2 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -38,7 +38,6 @@ on: jobs: check_if_push_image: - if: test -e "transforms/universal/tokenization/Makefile" # check whether the Docker images should be pushed to the remote repository # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. @@ -60,7 +59,6 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - if: test -e "transforms/universal/tokenization/Makefile" runs-on: ubuntu-22.04 steps: - name: Checkout @@ -82,7 +80,6 @@ jobs: echo "transforms/universal/tokenization/Makefile not found - source testing disabled for this transform." fi test-image: - if: test -e "transforms/universal/tokenization/Makefile" needs: [check_if_push_image] runs-on: ubuntu-22.04 timeout-minutes: 120 From a80c0fdd9f03f07c2d1e4de9e741d7e734283ea9 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 16:21:55 -0400 Subject: [PATCH 62/82] backout noop code change Signed-off-by: David Wood --- transforms/universal/noop/python/src/noop_transform_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/universal/noop/python/src/noop_transform_python.py b/transforms/universal/noop/python/src/noop_transform_python.py index 25a5ff512..b5c49e8d7 100644 --- a/transforms/universal/noop/python/src/noop_transform_python.py +++ b/transforms/universal/noop/python/src/noop_transform_python.py @@ -41,5 +41,5 @@ def __init__(self): if __name__ == "__main__": # launcher = NOOPRayLauncher() launcher = PythonTransformLauncher(NOOPPythonTransformConfiguration()) - logger.info("Launching noop transform..") + logger.info("Launching noop transform.") launcher.launch() From ee080338c1f44affc1e055f6896ba1c2ce96ff5b Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 16:22:51 -0400 Subject: [PATCH 63/82] backout noop code change Signed-off-by: David Wood --- transforms/universal/noop/python/src/noop_transform_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/universal/noop/python/src/noop_transform_python.py b/transforms/universal/noop/python/src/noop_transform_python.py index b5c49e8d7..54668523c 100644 --- a/transforms/universal/noop/python/src/noop_transform_python.py +++ b/transforms/universal/noop/python/src/noop_transform_python.py @@ -41,5 +41,5 @@ def __init__(self): if __name__ == "__main__": # launcher = NOOPRayLauncher() launcher = PythonTransformLauncher(NOOPPythonTransformConfiguration()) - logger.info("Launching noop transform.") + logger.info("Launching noop transform") launcher.launch() From 30a152459c7122b24274d88e66e6ccfcbb4f5751 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 21:13:03 -0400 Subject: [PATCH 64/82] only build spark image for transform image tests that need it Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet.yml | 4 +++- .github/workflows/test-code-code_quality.yml | 4 +++- .github/workflows/test-code-header_cleanser.yml | 4 +++- .github/workflows/test-code-malware.yml | 4 +++- .github/workflows/test-code-proglang_select.yml | 4 +++- .github/workflows/test-code-repo_level_ordering.yml | 4 +++- .github/workflows/test-language-doc_chunk.yml | 4 +++- .github/workflows/test-language-doc_quality.yml | 4 +++- .github/workflows/test-language-lang_id.yml | 4 +++- .github/workflows/test-language-pdf2parquet.yml | 4 +++- .github/workflows/test-language-pii_redactor.yml | 4 +++- .github/workflows/test-language-text_encoder.yml | 4 +++- .github/workflows/test-transform.template | 4 +++- .github/workflows/test-universal-doc_id.yml | 4 +++- .github/workflows/test-universal-ededup.yml | 4 +++- .github/workflows/test-universal-fdedup.yml | 4 +++- .github/workflows/test-universal-filter.yml | 4 +++- .github/workflows/test-universal-html2parquet.yml | 4 +++- .github/workflows/test-universal-noop.yml | 4 +++- .github/workflows/test-universal-profiler.yml | 4 +++- .github/workflows/test-universal-resize.yml | 4 +++- .github/workflows/test-universal-tokenization.yml | 4 +++- 22 files changed, 66 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index 996610e53..f8f1654e7 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/code/code2parquet run: | if [ -e "transforms/code/code2parquet/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/code/code2parquet/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/code/code2parquet DOCKER=docker test-image else echo "transforms/code/code2parquet/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index e855962ab..d53c81c61 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/code/code_quality run: | if [ -e "transforms/code/code_quality/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/code/code_quality/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/code/code_quality DOCKER=docker test-image else echo "transforms/code/code_quality/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 74b713cda..1834f4983 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/code/header_cleanser run: | if [ -e "transforms/code/header_cleanser/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/code/header_cleanser/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/code/header_cleanser DOCKER=docker test-image else echo "transforms/code/header_cleanser/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index cef5746a2..debc779d1 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/code/malware run: | if [ -e "transforms/code/malware/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/code/malware/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/code/malware DOCKER=docker test-image else echo "transforms/code/malware/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index 86ea55f55..36bf6a869 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/code/proglang_select run: | if [ -e "transforms/code/proglang_select/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/code/proglang_select/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/code/proglang_select DOCKER=docker test-image else echo "transforms/code/proglang_select/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index 8451b174b..fe0ee23bb 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/code/repo_level_ordering run: | if [ -e "transforms/code/repo_level_ordering/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/code/repo_level_ordering/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/code/repo_level_ordering DOCKER=docker test-image else echo "transforms/code/repo_level_ordering/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index 98341903b..fa3ea58ca 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/language/doc_chunk run: | if [ -e "transforms/language/doc_chunk/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/language/doc_chunk/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/language/doc_chunk DOCKER=docker test-image else echo "transforms/language/doc_chunk/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index 540c2490c..dde61e1fa 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/language/doc_quality run: | if [ -e "transforms/language/doc_quality/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/language/doc_quality/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/language/doc_quality DOCKER=docker test-image else echo "transforms/language/doc_quality/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 1c310270a..3b39358c9 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/language/lang_id run: | if [ -e "transforms/language/lang_id/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/language/lang_id/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/language/lang_id DOCKER=docker test-image else echo "transforms/language/lang_id/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 85e6e15e4..bb523c57e 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/language/pdf2parquet run: | if [ -e "transforms/language/pdf2parquet/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/language/pdf2parquet/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/language/pdf2parquet DOCKER=docker test-image else echo "transforms/language/pdf2parquet/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index c162a3322..9656a2f24 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/language/pii_redactor run: | if [ -e "transforms/language/pii_redactor/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/language/pii_redactor/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/language/pii_redactor DOCKER=docker test-image else echo "transforms/language/pii_redactor/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index cc4cdf0f5..f7622f8e0 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/language/text_encoder run: | if [ -e "transforms/language/text_encoder/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/language/text_encoder/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/language/text_encoder DOCKER=docker test-image else echo "transforms/language/text_encoder/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index bf8a56534..e0966717e 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -102,7 +102,9 @@ jobs: - name: Test transform image in @TARGET_TRANSFORM_DIR@ run: | if [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "@TARGET_TRANSFORM_DIR@/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-image else echo "@TARGET_TRANSFORM_DIR@/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 056dae834..66d0283ca 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/doc_id run: | if [ -e "transforms/universal/doc_id/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/doc_id/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/doc_id DOCKER=docker test-image else echo "transforms/universal/doc_id/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 9a9e3d174..225c27cc3 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/ededup run: | if [ -e "transforms/universal/ededup/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/ededup/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/ededup DOCKER=docker test-image else echo "transforms/universal/ededup/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 4814c7c7b..356736fca 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/fdedup run: | if [ -e "transforms/universal/fdedup/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/fdedup/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/fdedup DOCKER=docker test-image else echo "transforms/universal/fdedup/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 4ce46c874..44858feff 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/filter run: | if [ -e "transforms/universal/filter/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/filter/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/filter DOCKER=docker test-image else echo "transforms/universal/filter/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index 46608c3d0..7d3f83a61 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/html2parquet run: | if [ -e "transforms/universal/html2parquet/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/html2parquet/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/html2parquet DOCKER=docker test-image else echo "transforms/universal/html2parquet/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 44aa72c76..cd72703d1 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/noop run: | if [ -e "transforms/universal/noop/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/noop/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/noop DOCKER=docker test-image else echo "transforms/universal/noop/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 0b34a5bcf..50cd8cd26 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/profiler run: | if [ -e "transforms/universal/profiler/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/profiler/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/profiler DOCKER=docker test-image else echo "transforms/universal/profiler/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index 9c8f65c6f..99e14b1b8 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/resize run: | if [ -e "transforms/universal/resize/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/resize/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/resize DOCKER=docker test-image else echo "transforms/universal/resize/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 7e78fa6e2..e7a620882 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/tokenization run: | if [ -e "transforms/universal/tokenization/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/tokenization/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/tokenization DOCKER=docker test-image else echo "transforms/universal/tokenization/Makefile not found - testing disabled for this transform." From 6d6a233c1576785ead93d50e2762e6216e4f418e Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 21:14:00 -0400 Subject: [PATCH 65/82] header_cleanser code change Signed-off-by: David Wood --- .../python/src/header_cleanser_transform_python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transforms/code/header_cleanser/python/src/header_cleanser_transform_python.py b/transforms/code/header_cleanser/python/src/header_cleanser_transform_python.py index 9d4938b65..21c07d4ad 100644 --- a/transforms/code/header_cleanser/python/src/header_cleanser_transform_python.py +++ b/transforms/code/header_cleanser/python/src/header_cleanser_transform_python.py @@ -27,5 +27,5 @@ def __init__(self): if __name__ == "__main__": launcher = PythonTransformLauncher(HeaderCleanserPythonTransformConfiguration()) - logger.info("Launching license copyright header removal") + logger.info("Launching license copyright header removal.") launcher.launch() From f495f8226cebc2c839cb199b48e4da06cf7812f0 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 19 Sep 2024 21:18:38 -0400 Subject: [PATCH 66/82] only build spark image for transform image tests that need it Signed-off-by: David Wood --- .github/workflows/test-language-pdf2parquet.yml | 4 +++- .github/workflows/test-language-pii_redactor.yml | 4 +++- .github/workflows/test-language-text_encoder.yml | 4 +++- .github/workflows/test-transform.template | 4 +++- .github/workflows/test-universal-doc_id.yml | 4 +++- .github/workflows/test-universal-filter.yml | 4 +++- .github/workflows/test-universal-html2parquet.yml | 4 +++- .github/workflows/test-universal-noop.yml | 4 +++- .github/workflows/test-universal-profiler.yml | 4 +++- .github/workflows/test-universal-resize.yml | 4 +++- .github/workflows/test-universal-tokenization.yml | 4 +++- 11 files changed, 33 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 85e6e15e4..bb523c57e 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/language/pdf2parquet run: | if [ -e "transforms/language/pdf2parquet/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/language/pdf2parquet/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/language/pdf2parquet DOCKER=docker test-image else echo "transforms/language/pdf2parquet/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index c162a3322..9656a2f24 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/language/pii_redactor run: | if [ -e "transforms/language/pii_redactor/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/language/pii_redactor/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/language/pii_redactor DOCKER=docker test-image else echo "transforms/language/pii_redactor/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index cc4cdf0f5..f7622f8e0 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/language/text_encoder run: | if [ -e "transforms/language/text_encoder/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/language/text_encoder/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/language/text_encoder DOCKER=docker test-image else echo "transforms/language/text_encoder/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index bf8a56534..e0966717e 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -102,7 +102,9 @@ jobs: - name: Test transform image in @TARGET_TRANSFORM_DIR@ run: | if [ -e "@TARGET_TRANSFORM_DIR@/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "@TARGET_TRANSFORM_DIR@/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C @TARGET_TRANSFORM_DIR@ DOCKER=docker test-image else echo "@TARGET_TRANSFORM_DIR@/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 056dae834..66d0283ca 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/doc_id run: | if [ -e "transforms/universal/doc_id/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/doc_id/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/doc_id DOCKER=docker test-image else echo "transforms/universal/doc_id/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 4ce46c874..44858feff 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/filter run: | if [ -e "transforms/universal/filter/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/filter/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/filter DOCKER=docker test-image else echo "transforms/universal/filter/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml index 46608c3d0..7d3f83a61 100644 --- a/.github/workflows/test-universal-html2parquet.yml +++ b/.github/workflows/test-universal-html2parquet.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/html2parquet run: | if [ -e "transforms/universal/html2parquet/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/html2parquet/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/html2parquet DOCKER=docker test-image else echo "transforms/universal/html2parquet/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 44aa72c76..cd72703d1 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/noop run: | if [ -e "transforms/universal/noop/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/noop/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/noop DOCKER=docker test-image else echo "transforms/universal/noop/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 0b34a5bcf..50cd8cd26 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/profiler run: | if [ -e "transforms/universal/profiler/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/profiler/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/profiler DOCKER=docker test-image else echo "transforms/universal/profiler/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index 9c8f65c6f..99e14b1b8 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/resize run: | if [ -e "transforms/universal/resize/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/resize/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/resize DOCKER=docker test-image else echo "transforms/universal/resize/Makefile not found - testing disabled for this transform." diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 7e78fa6e2..e7a620882 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -102,7 +102,9 @@ jobs: - name: Test transform image in transforms/universal/tokenization run: | if [ -e "transforms/universal/tokenization/Makefile" ]; then - make -C data-processing-lib/spark DOCKER=docker image + if [ -d "transforms/universal/tokenization/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi make -C transforms/universal/tokenization DOCKER=docker test-image else echo "transforms/universal/tokenization/Makefile not found - testing disabled for this transform." From 85f9bed6a1a624f333fc39cca0648b714fcb215c Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 2 Oct 2024 15:19:10 -0400 Subject: [PATCH 67/82] update workflows to trigger on .make.* changes Signed-off-by: David Wood --- .../workflows/test-code-code2parquet-kfp.yml | 5 +- .github/workflows/test-code-code2parquet.yml | 4 + .../workflows/test-code-code_quality-kfp.yml | 5 +- .github/workflows/test-code-code_quality.yml | 4 + .../test-code-header_cleanser-kfp.yml | 5 +- .../workflows/test-code-header_cleanser.yml | 4 + .../test-code-license_select-kfp.yml | 117 ++++++++++++++++ .../workflows/test-code-license_select.yml | 4 + .github/workflows/test-code-malware-kfp.yml | 5 +- .github/workflows/test-code-malware.yml | 4 + .../test-code-proglang_select-kfp.yml | 5 +- .../workflows/test-code-proglang_select.yml | 4 + .../test-code-repo_level_ordering-kfp.yml | 5 +- .../test-code-repo_level_ordering.yml | 4 + .github/workflows/test-kfp-transform.template | 5 +- .github/workflows/test-language-doc_chunk.yml | 4 + .../test-language-doc_quality-kfp.yml | 5 +- .../workflows/test-language-doc_quality.yml | 4 + .../workflows/test-language-html2parquet.yml | 4 + .../workflows/test-language-lang_id-kfp.yml | 5 +- .github/workflows/test-language-lang_id.yml | 4 + .../workflows/test-language-pdf2parquet.yml | 4 + .../workflows/test-language-pii_redactor.yml | 4 + .../test-language-text_encoder-kfp.yml | 5 +- .../workflows/test-language-text_encoder.yml | 4 + .github/workflows/test-transform.template | 4 + .../workflows/test-universal-doc_id-kfp.yml | 5 +- .github/workflows/test-universal-doc_id.yml | 4 + .../workflows/test-universal-ededup-kfp.yml | 5 +- .github/workflows/test-universal-ededup.yml | 4 + .../workflows/test-universal-fdedup-kfp.yml | 5 +- .github/workflows/test-universal-fdedup.yml | 4 + .../workflows/test-universal-filter-kfp.yml | 5 +- .github/workflows/test-universal-filter.yml | 4 + .github/workflows/test-universal-hap.yml | 4 + .../workflows/test-universal-html2parquet.yml | 128 ++++++++++++++++++ .github/workflows/test-universal-noop-kfp.yml | 5 +- .github/workflows/test-universal-noop.yml | 4 + .../workflows/test-universal-profiler-kfp.yml | 5 +- .github/workflows/test-universal-profiler.yml | 4 + .../workflows/test-universal-resize-kfp.yml | 5 +- .github/workflows/test-universal-resize.yml | 4 + .../test-universal-tokenization-kfp.yml | 5 +- .../workflows/test-universal-tokenization.yml | 4 + 44 files changed, 413 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/test-code-license_select-kfp.yml create mode 100644 .github/workflows/test-universal-html2parquet.yml diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml index 41f58e0cb..be0502e41 100644 --- a/.github/workflows/test-code-code2parquet-kfp.yml +++ b/.github/workflows/test-code-code2parquet-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/code2parquet/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/code2parquet/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index f8f1654e7..dd4de1df5 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/code2parquet/**" - "data-processing-lib/**" - "!transforms/code/code2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/code2parquet/**" - "data-processing-lib/**" - "!transforms/code/code2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml index 21fa63296..6aea20218 100644 --- a/.github/workflows/test-code-code_quality-kfp.yml +++ b/.github/workflows/test-code-code_quality-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/code_quality/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/code_quality/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index d53c81c61..4cdadc937 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/code_quality/**" - "data-processing-lib/**" - "!transforms/code/code_quality/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/code_quality/**" - "data-processing-lib/**" - "!transforms/code/code_quality/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml index 25f54b528..b542e55a5 100644 --- a/.github/workflows/test-code-header_cleanser-kfp.yml +++ b/.github/workflows/test-code-header_cleanser-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/header_cleanser/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/header_cleanser/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 1834f4983..fe0c70169 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/header_cleanser/**" - "data-processing-lib/**" - "!transforms/code/header_cleanser/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/header_cleanser/**" - "data-processing-lib/**" - "!transforms/code/header_cleanser/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-code-license_select-kfp.yml b/.github/workflows/test-code-license_select-kfp.yml new file mode 100644 index 000000000..0b7f58b61 --- /dev/null +++ b/.github/workflows/test-code-license_select-kfp.yml @@ -0,0 +1,117 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/code/license_select + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - ".make.*" + - "transforms/.make.workflow" + - "transforms/code/license_select/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - ".make.*" + - "transforms/.make.workflow" + - "transforms/code/license_select/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/license_select workflow-test + echo "Run transforms/code/license_select completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/license_select workflow-test + header_text "Run transforms/code/license_select completed" diff --git a/.github/workflows/test-code-license_select.yml b/.github/workflows/test-code-license_select.yml index ab24b582b..ca6870a21 100644 --- a/.github/workflows/test-code-license_select.yml +++ b/.github/workflows/test-code-license_select.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/license_select/**" - "data-processing-lib/**" - "!transforms/code/license_select/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/license_select/**" - "data-processing-lib/**" - "!transforms/code/license_select/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-code-malware-kfp.yml b/.github/workflows/test-code-malware-kfp.yml index 9bd937f46..f49298a8d 100644 --- a/.github/workflows/test-code-malware-kfp.yml +++ b/.github/workflows/test-code-malware-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/malware/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/malware/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index debc779d1..d51695af1 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/malware/**" - "data-processing-lib/**" - "!transforms/code/malware/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/malware/**" - "data-processing-lib/**" - "!transforms/code/malware/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-code-proglang_select-kfp.yml b/.github/workflows/test-code-proglang_select-kfp.yml index bbe257964..8b1673603 100644 --- a/.github/workflows/test-code-proglang_select-kfp.yml +++ b/.github/workflows/test-code-proglang_select-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/proglang_select/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/proglang_select/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index 36bf6a869..eca3aa2f8 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/proglang_select/**" - "data-processing-lib/**" - "!transforms/code/proglang_select/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/proglang_select/**" - "data-processing-lib/**" - "!transforms/code/proglang_select/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-code-repo_level_ordering-kfp.yml b/.github/workflows/test-code-repo_level_ordering-kfp.yml index c26ecda52..af4e2ccb0 100644 --- a/.github/workflows/test-code-repo_level_ordering-kfp.yml +++ b/.github/workflows/test-code-repo_level_ordering-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/repo_level_ordering/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/code/repo_level_ordering/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index fe0ee23bb..d6fd5db97 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" - "!transforms/code/repo_level_ordering/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/code/repo_level_ordering/**" - "data-processing-lib/**" - "!transforms/code/repo_level_ordering/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-kfp-transform.template b/.github/workflows/test-kfp-transform.template index 434a57238..5711f5b76 100644 --- a/.github/workflows/test-kfp-transform.template +++ b/.github/workflows/test-kfp-transform.template @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "@TARGET_TRANSFORM_DIR@/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "@TARGET_TRANSFORM_DIR@/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index fa3ea58ca..fd2569278 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/doc_chunk/**" - "data-processing-lib/**" - "!transforms/language/doc_chunk/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/doc_chunk/**" - "data-processing-lib/**" - "!transforms/language/doc_chunk/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-language-doc_quality-kfp.yml b/.github/workflows/test-language-doc_quality-kfp.yml index e9f678595..9a2af2980 100644 --- a/.github/workflows/test-language-doc_quality-kfp.yml +++ b/.github/workflows/test-language-doc_quality-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/language/doc_quality/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/language/doc_quality/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index dde61e1fa..c552be473 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/doc_quality/**" - "data-processing-lib/**" - "!transforms/language/doc_quality/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/doc_quality/**" - "data-processing-lib/**" - "!transforms/language/doc_quality/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-language-html2parquet.yml b/.github/workflows/test-language-html2parquet.yml index 8caf4efd9..d2464cf93 100644 --- a/.github/workflows/test-language-html2parquet.yml +++ b/.github/workflows/test-language-html2parquet.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/html2parquet/**" - "data-processing-lib/**" - "!transforms/language/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/html2parquet/**" - "data-processing-lib/**" - "!transforms/language/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-language-lang_id-kfp.yml b/.github/workflows/test-language-lang_id-kfp.yml index cf3dec397..232bc3c03 100644 --- a/.github/workflows/test-language-lang_id-kfp.yml +++ b/.github/workflows/test-language-lang_id-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/language/lang_id/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/language/lang_id/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 3b39358c9..c9cd2a4ac 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/lang_id/**" - "data-processing-lib/**" - "!transforms/language/lang_id/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/lang_id/**" - "data-processing-lib/**" - "!transforms/language/lang_id/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index bb523c57e..11353bea3 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" - "!transforms/language/pdf2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/pdf2parquet/**" - "data-processing-lib/**" - "!transforms/language/pdf2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 9656a2f24..4c82779d9 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/pii_redactor/**" - "data-processing-lib/**" - "!transforms/language/pii_redactor/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/pii_redactor/**" - "data-processing-lib/**" - "!transforms/language/pii_redactor/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-language-text_encoder-kfp.yml b/.github/workflows/test-language-text_encoder-kfp.yml index d90b76820..c89faad31 100644 --- a/.github/workflows/test-language-text_encoder-kfp.yml +++ b/.github/workflows/test-language-text_encoder-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/language/text_encoder/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/language/text_encoder/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index f7622f8e0..5d2ee4936 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/text_encoder/**" - "data-processing-lib/**" - "!transforms/language/text_encoder/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/language/text_encoder/**" - "data-processing-lib/**" - "!transforms/language/text_encoder/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index e0966717e..6f5b4112c 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" - "!@TARGET_TRANSFORM_DIR@/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "@TARGET_TRANSFORM_DIR@/**" - "data-processing-lib/**" - "!@TARGET_TRANSFORM_DIR@/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-universal-doc_id-kfp.yml b/.github/workflows/test-universal-doc_id-kfp.yml index 28c1d8717..393e48470 100644 --- a/.github/workflows/test-universal-doc_id-kfp.yml +++ b/.github/workflows/test-universal-doc_id-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/doc_id/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/doc_id/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 66d0283ca..671a8545e 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/doc_id/**" - "data-processing-lib/**" - "!transforms/universal/doc_id/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/doc_id/**" - "data-processing-lib/**" - "!transforms/universal/doc_id/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-universal-ededup-kfp.yml b/.github/workflows/test-universal-ededup-kfp.yml index 5d3481e30..6f52e0572 100644 --- a/.github/workflows/test-universal-ededup-kfp.yml +++ b/.github/workflows/test-universal-ededup-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/ededup/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/ededup/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 225c27cc3..1384df79c 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/ededup/**" - "data-processing-lib/**" - "!transforms/universal/ededup/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/ededup/**" - "data-processing-lib/**" - "!transforms/universal/ededup/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-universal-fdedup-kfp.yml b/.github/workflows/test-universal-fdedup-kfp.yml index 9d331ed47..5f21bbc9a 100644 --- a/.github/workflows/test-universal-fdedup-kfp.yml +++ b/.github/workflows/test-universal-fdedup-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/fdedup/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/fdedup/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index 356736fca..c5df0abdd 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/fdedup/**" - "data-processing-lib/**" - "!transforms/universal/fdedup/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/fdedup/**" - "data-processing-lib/**" - "!transforms/universal/fdedup/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-universal-filter-kfp.yml b/.github/workflows/test-universal-filter-kfp.yml index 59ebbde3c..fafb76bf8 100644 --- a/.github/workflows/test-universal-filter-kfp.yml +++ b/.github/workflows/test-universal-filter-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/filter/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/filter/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 44858feff..6b627c73b 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/filter/**" - "data-processing-lib/**" - "!transforms/universal/filter/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/filter/**" - "data-processing-lib/**" - "!transforms/universal/filter/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-universal-hap.yml b/.github/workflows/test-universal-hap.yml index b92e5867c..3ceaaf5c2 100644 --- a/.github/workflows/test-universal-hap.yml +++ b/.github/workflows/test-universal-hap.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/hap/**" - "data-processing-lib/**" - "!transforms/universal/hap/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/hap/**" - "data-processing-lib/**" - "!transforms/universal/hap/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml new file mode 100644 index 000000000..0c1e9e709 --- /dev/null +++ b/.github/workflows/test-universal-html2parquet.yml @@ -0,0 +1,128 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test - transforms/universal/html2parquet + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - ".make.*" + - "transforms/.make.transforms" + - "transforms/universal/html2parquet/**" + - "data-processing-lib/**" + - "!transforms/universal/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow + - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - ".make.*" + - "transforms/.make.transforms" + - "transforms/universal/html2parquet/**" + - "data-processing-lib/**" + - "!transforms/universal/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow + - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + +jobs: + check_if_push_image: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-src: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform source in transforms/universal/html2parquet + run: | + if [ -e "transforms/universal/html2parquet/Makefile" ]; then + make -C transforms/universal/html2parquet DOCKER=docker test-src + else + echo "transforms/universal/html2parquet/Makefile not found - source testing disabled for this transform." + fi + test-image: + needs: [check_if_push_image] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test transform image in transforms/universal/html2parquet + run: | + if [ -e "transforms/universal/html2parquet/Makefile" ]; then + if [ -d "transforms/universal/html2parquet/spark" ]; then + make -C data-processing-lib/spark DOCKER=docker image + fi + make -C transforms/universal/html2parquet DOCKER=docker test-image + else + echo "transforms/universal/html2parquet/Makefile not found - testing disabled for this transform." + fi + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_image.outputs.publish_images == 'true' + run: | + if [ -e "transforms/universal/html2parquet/Makefile" ]; then + make -C transforms/universal/html2parquet publish + else + echo "transforms/universal/html2parquet/Makefile not found - publishing disabled for this transform." + fi diff --git a/.github/workflows/test-universal-noop-kfp.yml b/.github/workflows/test-universal-noop-kfp.yml index 19c62ab49..021c53a10 100644 --- a/.github/workflows/test-universal-noop-kfp.yml +++ b/.github/workflows/test-universal-noop-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/noop/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/noop/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index cd72703d1..91b6a3c79 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/noop/**" - "data-processing-lib/**" - "!transforms/universal/noop/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/noop/**" - "data-processing-lib/**" - "!transforms/universal/noop/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-universal-profiler-kfp.yml b/.github/workflows/test-universal-profiler-kfp.yml index 3d377922d..209a84565 100644 --- a/.github/workflows/test-universal-profiler-kfp.yml +++ b/.github/workflows/test-universal-profiler-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/profiler/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/profiler/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 50cd8cd26..93b0615af 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/profiler/**" - "data-processing-lib/**" - "!transforms/universal/profiler/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/profiler/**" - "data-processing-lib/**" - "!transforms/universal/profiler/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-universal-resize-kfp.yml b/.github/workflows/test-universal-resize-kfp.yml index fe7377178..6e5d61ef5 100644 --- a/.github/workflows/test-universal-resize-kfp.yml +++ b/.github/workflows/test-universal-resize-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/resize/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/resize/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index 99e14b1b8..1098b1bdd 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/resize/**" - "data-processing-lib/**" - "!transforms/universal/resize/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/resize/**" - "data-processing-lib/**" - "!transforms/universal/resize/**/kfp_ray/**" # This is/will be tested in separate workflow diff --git a/.github/workflows/test-universal-tokenization-kfp.yml b/.github/workflows/test-universal-tokenization-kfp.yml index f127db59b..19b9d3836 100644 --- a/.github/workflows/test-universal-tokenization-kfp.yml +++ b/.github/workflows/test-universal-tokenization-kfp.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/tokenization/**" - "!kfp/**" # This is tested in separate workflow - "!data-processing-lib/**" # This is tested in separate workflow @@ -24,6 +26,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.workflow" - "transforms/universal/tokenization/**" - "!data-processing-lib/**" # This is tested in separate workflow - "!kfp/**" # This is tested in separate workflow @@ -32,7 +36,6 @@ on: - "!**/images/**" - "!**.gitignore" - jobs: test-kfp-v1: runs-on: ubuntu-22.04 diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index e7a620882..9b3908d9e 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -12,6 +12,8 @@ on: tags: - "*" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/tokenization/**" - "data-processing-lib/**" - "!transforms/universal/tokenization/**/kfp_ray/**" # This is/will be tested in separate workflow @@ -26,6 +28,8 @@ on: - "dev" - "releases/**" paths: + - ".make.*" + - "transforms/.make.transforms" - "transforms/universal/tokenization/**" - "data-processing-lib/**" - "!transforms/universal/tokenization/**/kfp_ray/**" # This is/will be tested in separate workflow From 19d3edf38def123be81be599397845a5ef5fd433 Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 2 Oct 2024 15:23:10 -0400 Subject: [PATCH 68/82] blank line added to .make.versions to test new workflows Signed-off-by: David Wood --- .make.defaults | 1 + 1 file changed, 1 insertion(+) diff --git a/.make.defaults b/.make.defaults index e55006125..033d4a8c2 100644 --- a/.make.defaults +++ b/.make.defaults @@ -24,6 +24,7 @@ # rm -rf other-stuff # ####################################################################################### + SHELL=/bin/bash include $(REPOROOT)/.make.versions From 1086fc83dc637e568b4030abe8c8c602686148a7 Mon Sep 17 00:00:00 2001 From: David Wood Date: Wed, 2 Oct 2024 15:23:54 -0400 Subject: [PATCH 69/82] remove change to .make.versions Signed-off-by: David Wood --- .make.defaults | 1 - 1 file changed, 1 deletion(-) diff --git a/.make.defaults b/.make.defaults index 033d4a8c2..e55006125 100644 --- a/.make.defaults +++ b/.make.defaults @@ -24,7 +24,6 @@ # rm -rf other-stuff # ####################################################################################### - SHELL=/bin/bash include $(REPOROOT)/.make.versions From 5a444321a3f13bfafa6fca46310d1bd18354942c Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 08:13:12 -0400 Subject: [PATCH 70/82] remove test-universal-html2parquet.yml since it is now moved to language Signed-off-by: David Wood --- .../workflows/test-universal-html2parquet.yml | 128 ------------------ 1 file changed, 128 deletions(-) delete mode 100644 .github/workflows/test-universal-html2parquet.yml diff --git a/.github/workflows/test-universal-html2parquet.yml b/.github/workflows/test-universal-html2parquet.yml deleted file mode 100644 index 0c1e9e709..000000000 --- a/.github/workflows/test-universal-html2parquet.yml +++ /dev/null @@ -1,128 +0,0 @@ -# -# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files -# -name: Test - transforms/universal/html2parquet - -on: - workflow_dispatch: - push: - branches: - - "dev" - - "releases/**" - tags: - - "*" - paths: - - ".make.*" - - "transforms/.make.transforms" - - "transforms/universal/html2parquet/**" - - "data-processing-lib/**" - - "!transforms/universal/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow - - "!data-processing-lib/**/test/**" - - "!data-processing-lib/**/test-data/**" - - "!**.md" - - "!**/doc/**" - - "!**/images/**" - - "!**.gitignore" - pull_request: - branches: - - "dev" - - "releases/**" - paths: - - ".make.*" - - "transforms/.make.transforms" - - "transforms/universal/html2parquet/**" - - "data-processing-lib/**" - - "!transforms/universal/html2parquet/**/kfp_ray/**" # This is/will be tested in separate workflow - - "!data-processing-lib/**/test/**" - - "!data-processing-lib/**/test-data/**" - - "!**.md" - - "!**/doc/**" - - "!**/images/**" - - "!**.gitignore" - -jobs: - check_if_push_image: - # check whether the Docker images should be pushed to the remote repository - # The images are pushed if it is a merge to dev branch or a new tag is created. - # The latter being part of the release process. - # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 - outputs: - publish_images: ${{ steps.version.outputs.publish_images }} - steps: - - id: version - run: | - publish_images='false' - if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; - then - publish_images='true' - fi - if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; - then - publish_images='true' - fi - echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" - test-src: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test transform source in transforms/universal/html2parquet - run: | - if [ -e "transforms/universal/html2parquet/Makefile" ]; then - make -C transforms/universal/html2parquet DOCKER=docker test-src - else - echo "transforms/universal/html2parquet/Makefile not found - source testing disabled for this transform." - fi - test-image: - needs: [check_if_push_image] - runs-on: ubuntu-22.04 - timeout-minutes: 120 - env: - DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test transform image in transforms/universal/html2parquet - run: | - if [ -e "transforms/universal/html2parquet/Makefile" ]; then - if [ -d "transforms/universal/html2parquet/spark" ]; then - make -C data-processing-lib/spark DOCKER=docker image - fi - make -C transforms/universal/html2parquet DOCKER=docker test-image - else - echo "transforms/universal/html2parquet/Makefile not found - testing disabled for this transform." - fi - - name: Print space - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - docker images - - name: Publish images - if: needs.check_if_push_image.outputs.publish_images == 'true' - run: | - if [ -e "transforms/universal/html2parquet/Makefile" ]; then - make -C transforms/universal/html2parquet publish - else - echo "transforms/universal/html2parquet/Makefile not found - publishing disabled for this transform." - fi From ad3347cf33b562431e0ab00433ac1bd6b1244f6f Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 09:38:24 -0400 Subject: [PATCH 71/82] mv kfp blacklist definition to check-workflows.sh script and have it check for kfp workflows Signed-off-by: David Wood --- .github/workflows/Makefile | 4 +-- scripts/check-workflows.sh | 50 +++++++++++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile index 275fd4688..6015d64e5 100644 --- a/.github/workflows/Makefile +++ b/.github/workflows/Makefile @@ -8,7 +8,6 @@ LANG_TRANSFORMS=doc_chunk doc_quality lang_id pdf2parquet pii_redactor text_enco # A list that holds transforms that should not be tested with KFP -KFP_BLACK_LIST="doc_chunk,pdf2parquet,pii_redactor" transform-tests: $(MAKE) TRANSFORM_SUBDIR=universal .transform-tests @@ -29,7 +28,8 @@ transform-tests: done .transform-kfp-tests: - @for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do \ + @KFP_BLACK_LIST=$$(cd ../..; bash scripts/check-workflows.sh -show-kfp-black-list); \ + for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do \ dir=$$(basename $$i); \ z=$$(echo ${KFP_BLACK_LIST} | grep -v $$dir); \ if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ] || [ -z "$$z" ]; then \ diff --git a/scripts/check-workflows.sh b/scripts/check-workflows.sh index 40f4e2615..afc73a886 100755 --- a/scripts/check-workflows.sh +++ b/scripts/check-workflows.sh @@ -1,20 +1,52 @@ #!/bin/bash -# Check that each transform in transforms// has a corresponding -# .github/workflows/test--.yml file. +usage() { +cat << EOF +Check that each transform in transforms// has a corresponding + .github/workflows/test--.yml file and, + .github/workflows/test---kfp.yml file if + there is a kfp_ray directory for the transform, and + the transform is not in the kfp black list. +Options: + -show-kfp-black-list: prints the space separate list of transform + directories (base names) and exits. + -help: show this message. +EOF +} + if [ ! -d transforms ]; then echo Please run this script from the top of the repository exit 1 fi +KFP_BLACK_LIST="doc_chunk pdf2parquet pii_redactor" +while [ $# -ne 0 ]; do + case $1 in + -show-kfp-black-list) echo $KFP_BLACK_LIST; exit 0; + ;; + *help) usage; exit 0; + ;; + *) echo Unrecognized option $1. exit 1 + ;; + esac + shift; +done for i in $(find transforms -maxdepth 2 -mindepth 2 -type d | grep -v venv); do transform=$(basename $i) category=$(dirname $i) category=$(basename $category) - workflow=.github/workflows/test-$category-$transform.yml - if [ ! -e $workflow ]; then - echo Missing $workflow for transform $category/$transform - echo Fix this by running make in the .github/workflows directory - exit 1 + workflows=.github/workflows/test-$category-$transform.yml + is_blacklisted=$(echo $KFP_BLACK_LIST | grep $transform) + if [ -d $i/kfp_ray -a -z "$is_blacklisted" ]; then + workflows="$workflows .github/workflows/test-$category-$transform-kfp.yml" else - echo Verified existence of $workflow - fi + echo KFP workflow for $transform is not expected. + fi + for workflow in $workflows; do + if [ ! -e $workflow ]; then + echo Missing $workflow for transform $category/$transform + echo Fix this by running make in the .github/workflows directory + exit 1 + else + echo Verified existence of $workflow + fi + done done From e8611ebf699361dbd4367c642aab907867d8c678 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 09:42:34 -0400 Subject: [PATCH 72/82] license_select_wf.py comment to trigger kfp tests here Signed-off-by: David Wood --- transforms/code/license_select/kfp_ray/license_select_wf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transforms/code/license_select/kfp_ray/license_select_wf.py b/transforms/code/license_select/kfp_ray/license_select_wf.py index 1938decfe..e97986a92 100644 --- a/transforms/code/license_select/kfp_ray/license_select_wf.py +++ b/transforms/code/license_select/kfp_ray/license_select_wf.py @@ -24,6 +24,7 @@ task_image = "quay.io/dataprep1/data-prep-kit/license_select-ray:latest" +# # components base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" From e369340d98b1e1110200f547e01c12728bb72969 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 10:23:04 -0400 Subject: [PATCH 73/82] have transform top level makefile check for kfp_ray directory before recursing into Signed-off-by: David Wood --- transforms/code/code2parquet/Makefile | 19 ++++++++++++++----- transforms/code/code_quality/Makefile | 19 ++++++++++++++----- transforms/code/header_cleanser/Makefile | 19 ++++++++++++++----- transforms/code/license_select/Makefile | 19 ++++++++++++++----- transforms/code/malware/Makefile | 19 ++++++++++++++----- transforms/code/proglang_select/Makefile | 19 ++++++++++++++----- transforms/code/repo_level_ordering/Makefile | 16 ++++++++++++++-- transforms/language/doc_chunk/Makefile | 19 ++++++++++++++----- transforms/language/doc_quality/Makefile | 15 --------------- transforms/language/html2parquet/Makefile | 15 ++++++++++++++- transforms/language/lang_id/Makefile | 19 ++++++++++++++----- transforms/language/pdf2parquet/Makefile | 19 ++++++++++++++----- transforms/language/pii_redactor/Makefile | 19 ++++++++++++++----- transforms/language/text_encoder/Makefile | 19 ++++++++++++++----- transforms/universal/doc_id/Makefile | 19 ++++++++++++++----- transforms/universal/ededup/Makefile | 19 ++++++++++++++----- transforms/universal/fdedup/Makefile | 19 ++++++++++++++----- transforms/universal/filter/Makefile | 19 ++++++++++++++----- transforms/universal/hap/Makefile | 16 ++++++++++++---- transforms/universal/noop/Makefile | 19 ++++++++++++++----- transforms/universal/profiler/Makefile | 19 ++++++++++++++----- transforms/universal/resize/Makefile | 19 ++++++++++++++----- transforms/universal/tokenization/Makefile | 19 ++++++++++++++----- 23 files changed, 306 insertions(+), 117 deletions(-) diff --git a/transforms/code/code2parquet/Makefile b/transforms/code/code2parquet/Makefile index bc4077099..f4abce22c 100644 --- a/transforms/code/code2parquet/Makefile +++ b/transforms/code/code2parquet/Makefile @@ -55,16 +55,25 @@ set-versions: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build \ No newline at end of file + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/code_quality/Makefile b/transforms/code/code_quality/Makefile index 204ea8856..79ac7a32a 100644 --- a/transforms/code/code_quality/Makefile +++ b/transforms/code/code_quality/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/header_cleanser/Makefile b/transforms/code/header_cleanser/Makefile index 204ea8856..79ac7a32a 100644 --- a/transforms/code/header_cleanser/Makefile +++ b/transforms/code/header_cleanser/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/license_select/Makefile b/transforms/code/license_select/Makefile index b19f5c963..3227b4ead 100644 --- a/transforms/code/license_select/Makefile +++ b/transforms/code/license_select/Makefile @@ -47,16 +47,25 @@ load-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/malware/Makefile b/transforms/code/malware/Makefile index 05d3c3111..79ac7a32a 100644 --- a/transforms/code/malware/Makefile +++ b/transforms/code/malware/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/proglang_select/Makefile b/transforms/code/proglang_select/Makefile index 9c7c898e4..9ff00bad4 100644 --- a/transforms/code/proglang_select/Makefile +++ b/transforms/code/proglang_select/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/code/repo_level_ordering/Makefile b/transforms/code/repo_level_ordering/Makefile index cebfb4848..3227b4ead 100644 --- a/transforms/code/repo_level_ordering/Makefile +++ b/transforms/code/repo_level_ordering/Makefile @@ -45,15 +45,27 @@ load-image:: @# Help: Recursively make $@ in all subdirs $(MAKE) RULE=$@ .recurse -# kfp implementation is not yet added, so below targets don't do anything. .PHONY: workflow-venv workflow-venv: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/language/doc_chunk/Makefile b/transforms/language/doc_chunk/Makefile index 05d3c3111..79ac7a32a 100644 --- a/transforms/language/doc_chunk/Makefile +++ b/transforms/language/doc_chunk/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/language/doc_quality/Makefile b/transforms/language/doc_quality/Makefile index 5cded280a..612199d63 100644 --- a/transforms/language/doc_quality/Makefile +++ b/transforms/language/doc_quality/Makefile @@ -45,18 +45,3 @@ load-image:: @# Help: Recursively make $@ in all subdirs $(MAKE) RULE=$@ .recurse -.PHONY: workflow-venv -workflow-venv: - $(MAKE) -C kfp_ray workflow-venv - -.PHONY: workflow-test -workflow-test: - $(MAKE) -C kfp_ray workflow-test - -.PHONY: workflow-upload -workflow-upload: - $(MAKE) -C kfp_ray workflow-upload - -.PHONY: workflow-build -workflow-build: - $(MAKE) -C kfp_ray workflow-build \ No newline at end of file diff --git a/transforms/language/html2parquet/Makefile b/transforms/language/html2parquet/Makefile index 017eb23b4..79ac7a32a 100644 --- a/transforms/language/html2parquet/Makefile +++ b/transforms/language/html2parquet/Makefile @@ -55,12 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/language/lang_id/Makefile b/transforms/language/lang_id/Makefile index 2967ceb67..0334ed721 100644 --- a/transforms/language/lang_id/Makefile +++ b/transforms/language/lang_id/Makefile @@ -60,16 +60,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/language/pdf2parquet/Makefile b/transforms/language/pdf2parquet/Makefile index 05d3c3111..79ac7a32a 100644 --- a/transforms/language/pdf2parquet/Makefile +++ b/transforms/language/pdf2parquet/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/language/pii_redactor/Makefile b/transforms/language/pii_redactor/Makefile index f9f635f07..317e3b1cc 100644 --- a/transforms/language/pii_redactor/Makefile +++ b/transforms/language/pii_redactor/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/language/text_encoder/Makefile b/transforms/language/text_encoder/Makefile index 05d3c3111..79ac7a32a 100644 --- a/transforms/language/text_encoder/Makefile +++ b/transforms/language/text_encoder/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/doc_id/Makefile b/transforms/universal/doc_id/Makefile index 7ebb1b8e1..6d8eff8e4 100644 --- a/transforms/universal/doc_id/Makefile +++ b/transforms/universal/doc_id/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/ededup/Makefile b/transforms/universal/ededup/Makefile index 05d3c3111..79ac7a32a 100644 --- a/transforms/universal/ededup/Makefile +++ b/transforms/universal/ededup/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/fdedup/Makefile b/transforms/universal/fdedup/Makefile index 05d3c3111..79ac7a32a 100644 --- a/transforms/universal/fdedup/Makefile +++ b/transforms/universal/fdedup/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/filter/Makefile b/transforms/universal/filter/Makefile index 9c7c898e4..9ff00bad4 100644 --- a/transforms/universal/filter/Makefile +++ b/transforms/universal/filter/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/hap/Makefile b/transforms/universal/hap/Makefile index 05d3c3111..cdb34d72e 100644 --- a/transforms/universal/hap/Makefile +++ b/transforms/universal/hap/Makefile @@ -55,16 +55,24 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/universal/noop/Makefile b/transforms/universal/noop/Makefile index 05d3c3111..79ac7a32a 100644 --- a/transforms/universal/noop/Makefile +++ b/transforms/universal/noop/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/profiler/Makefile b/transforms/universal/profiler/Makefile index 05d3c3111..79ac7a32a 100644 --- a/transforms/universal/profiler/Makefile +++ b/transforms/universal/profiler/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/resize/Makefile b/transforms/universal/resize/Makefile index 05d3c3111..79ac7a32a 100644 --- a/transforms/universal/resize/Makefile +++ b/transforms/universal/resize/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + diff --git a/transforms/universal/tokenization/Makefile b/transforms/universal/tokenization/Makefile index 05d3c3111..79ac7a32a 100644 --- a/transforms/universal/tokenization/Makefile +++ b/transforms/universal/tokenization/Makefile @@ -55,16 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - $(MAKE) -C kfp_ray workflow-venv + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - $(MAKE) -C kfp_ray workflow-test - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - $(MAKE) -C kfp_ray workflow-upload + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - $(MAKE) -C kfp_ray workflow-build + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + From 602719a41a6181c42fba93c95f1c623ad95a91f9 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 10:26:33 -0400 Subject: [PATCH 74/82] fix doc_quality Makefile kfp rules Signed-off-by: David Wood --- transforms/language/doc_quality/Makefile | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/transforms/language/doc_quality/Makefile b/transforms/language/doc_quality/Makefile index 612199d63..e6b2b7535 100644 --- a/transforms/language/doc_quality/Makefile +++ b/transforms/language/doc_quality/Makefile @@ -45,3 +45,27 @@ load-image:: @# Help: Recursively make $@ in all subdirs $(MAKE) RULE=$@ .recurse +.PHONY: workflow-venv +workflow-venv: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi + +.PHONY: workflow-test +workflow-test: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + +.PHONY: workflow-upload +workflow-upload: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi + +.PHONY: workflow-build +workflow-build: + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi + From 5dcc0926d91de618994e75d6bc5d18f9ff45eccf Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 10:34:15 -0400 Subject: [PATCH 75/82] fix tabbing in recent Makefile updates Signed-off-by: David Wood --- transforms/code/code2parquet/Makefile | 26 ++++++++++---------- transforms/code/code_quality/Makefile | 26 ++++++++++---------- transforms/code/header_cleanser/Makefile | 26 ++++++++++---------- transforms/code/license_select/Makefile | 26 ++++++++++---------- transforms/code/malware/Makefile | 26 ++++++++++---------- transforms/code/proglang_select/Makefile | 26 ++++++++++---------- transforms/code/repo_level_ordering/Makefile | 26 ++++++++++---------- transforms/language/doc_chunk/Makefile | 26 ++++++++++---------- transforms/language/doc_quality/Makefile | 26 ++++++++++---------- transforms/language/html2parquet/Makefile | 26 ++++++++++---------- transforms/language/lang_id/Makefile | 26 ++++++++++---------- transforms/language/pdf2parquet/Makefile | 26 ++++++++++---------- transforms/language/pii_redactor/Makefile | 26 ++++++++++---------- transforms/language/text_encoder/Makefile | 26 ++++++++++---------- transforms/universal/doc_id/Makefile | 26 ++++++++++---------- transforms/universal/ededup/Makefile | 26 ++++++++++---------- transforms/universal/fdedup/Makefile | 26 ++++++++++---------- transforms/universal/filter/Makefile | 26 ++++++++++---------- transforms/universal/noop/Makefile | 26 ++++++++++---------- transforms/universal/profiler/Makefile | 26 ++++++++++---------- transforms/universal/resize/Makefile | 26 ++++++++++---------- transforms/universal/tokenization/Makefile | 26 ++++++++++---------- 22 files changed, 286 insertions(+), 286 deletions(-) diff --git a/transforms/code/code2parquet/Makefile b/transforms/code/code2parquet/Makefile index f4abce22c..027d29644 100644 --- a/transforms/code/code2parquet/Makefile +++ b/transforms/code/code2parquet/Makefile @@ -55,25 +55,25 @@ set-versions: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/code/code_quality/Makefile b/transforms/code/code_quality/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/code/code_quality/Makefile +++ b/transforms/code/code_quality/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/code/header_cleanser/Makefile b/transforms/code/header_cleanser/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/code/header_cleanser/Makefile +++ b/transforms/code/header_cleanser/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/code/license_select/Makefile b/transforms/code/license_select/Makefile index 3227b4ead..04b1cc451 100644 --- a/transforms/code/license_select/Makefile +++ b/transforms/code/license_select/Makefile @@ -47,25 +47,25 @@ load-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/code/malware/Makefile b/transforms/code/malware/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/code/malware/Makefile +++ b/transforms/code/malware/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/code/proglang_select/Makefile b/transforms/code/proglang_select/Makefile index 9ff00bad4..9e222ee79 100644 --- a/transforms/code/proglang_select/Makefile +++ b/transforms/code/proglang_select/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/code/repo_level_ordering/Makefile b/transforms/code/repo_level_ordering/Makefile index 3227b4ead..04b1cc451 100644 --- a/transforms/code/repo_level_ordering/Makefile +++ b/transforms/code/repo_level_ordering/Makefile @@ -47,25 +47,25 @@ load-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/language/doc_chunk/Makefile b/transforms/language/doc_chunk/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/language/doc_chunk/Makefile +++ b/transforms/language/doc_chunk/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/language/doc_quality/Makefile b/transforms/language/doc_quality/Makefile index e6b2b7535..a3f1865be 100644 --- a/transforms/language/doc_quality/Makefile +++ b/transforms/language/doc_quality/Makefile @@ -47,25 +47,25 @@ load-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/language/html2parquet/Makefile b/transforms/language/html2parquet/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/language/html2parquet/Makefile +++ b/transforms/language/html2parquet/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/language/lang_id/Makefile b/transforms/language/lang_id/Makefile index 0334ed721..af4a86873 100644 --- a/transforms/language/lang_id/Makefile +++ b/transforms/language/lang_id/Makefile @@ -60,25 +60,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/language/pdf2parquet/Makefile b/transforms/language/pdf2parquet/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/language/pdf2parquet/Makefile +++ b/transforms/language/pdf2parquet/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/language/pii_redactor/Makefile b/transforms/language/pii_redactor/Makefile index 317e3b1cc..8764d0dc2 100644 --- a/transforms/language/pii_redactor/Makefile +++ b/transforms/language/pii_redactor/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/language/text_encoder/Makefile b/transforms/language/text_encoder/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/language/text_encoder/Makefile +++ b/transforms/language/text_encoder/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/universal/doc_id/Makefile b/transforms/universal/doc_id/Makefile index 6d8eff8e4..be26d3bf4 100644 --- a/transforms/universal/doc_id/Makefile +++ b/transforms/universal/doc_id/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/universal/ededup/Makefile b/transforms/universal/ededup/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/universal/ededup/Makefile +++ b/transforms/universal/ededup/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/universal/fdedup/Makefile b/transforms/universal/fdedup/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/universal/fdedup/Makefile +++ b/transforms/universal/fdedup/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/universal/filter/Makefile b/transforms/universal/filter/Makefile index 9ff00bad4..9e222ee79 100644 --- a/transforms/universal/filter/Makefile +++ b/transforms/universal/filter/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/universal/noop/Makefile b/transforms/universal/noop/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/universal/noop/Makefile +++ b/transforms/universal/noop/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/universal/profiler/Makefile b/transforms/universal/profiler/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/universal/profiler/Makefile +++ b/transforms/universal/profiler/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/universal/resize/Makefile b/transforms/universal/resize/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/universal/resize/Makefile +++ b/transforms/universal/resize/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi diff --git a/transforms/universal/tokenization/Makefile b/transforms/universal/tokenization/Makefile index 79ac7a32a..bca6f7e85 100644 --- a/transforms/universal/tokenization/Makefile +++ b/transforms/universal/tokenization/Makefile @@ -55,25 +55,25 @@ docker-save-image:: .PHONY: workflow-venv workflow-venv: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-venv; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-venv; \ + fi .PHONY: workflow-test workflow-test: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-test; \ - fi - + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-test; \ + fi + .PHONY: workflow-upload workflow-upload: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-upload; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-upload; \ + fi .PHONY: workflow-build workflow-build: - if [ -e kfp_ray ]; then \ - $(MAKE) -C kfp_ray workflow-build; \ - fi + if [ -e kfp_ray ]; then \ + $(MAKE) -C kfp_ray workflow-build; \ + fi From 7947986fdcedd3ce287040cff8736972606b46a5 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 11:50:16 -0400 Subject: [PATCH 76/82] fix kfp workflows to only build the target transform Signed-off-by: David Wood --- .github/workflows/Makefile | 11 ++++++++--- .github/workflows/test-code-code2parquet-kfp.yml | 2 +- .github/workflows/test-code-code_quality-kfp.yml | 2 +- .github/workflows/test-code-header_cleanser-kfp.yml | 2 +- .github/workflows/test-code-license_select-kfp.yml | 2 +- .github/workflows/test-code-malware-kfp.yml | 2 +- .github/workflows/test-code-proglang_select-kfp.yml | 2 +- .../workflows/test-code-repo_level_ordering-kfp.yml | 2 +- .github/workflows/test-kfp-transform.template | 2 +- .github/workflows/test-language-doc_quality-kfp.yml | 2 +- .github/workflows/test-language-lang_id-kfp.yml | 2 +- .github/workflows/test-language-text_encoder-kfp.yml | 2 +- .github/workflows/test-universal-doc_id-kfp.yml | 2 +- .github/workflows/test-universal-ededup-kfp.yml | 2 +- .github/workflows/test-universal-fdedup-kfp.yml | 2 +- .github/workflows/test-universal-filter-kfp.yml | 2 +- .github/workflows/test-universal-noop-kfp.yml | 2 +- .github/workflows/test-universal-profiler-kfp.yml | 2 +- .github/workflows/test-universal-resize-kfp.yml | 2 +- .github/workflows/test-universal-tokenization-kfp.yml | 2 +- 20 files changed, 27 insertions(+), 22 deletions(-) diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile index 6015d64e5..751b3201f 100644 --- a/.github/workflows/Makefile +++ b/.github/workflows/Makefile @@ -31,11 +31,16 @@ transform-tests: @KFP_BLACK_LIST=$$(cd ../..; bash scripts/check-workflows.sh -show-kfp-black-list); \ for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do \ dir=$$(basename $$i); \ - z=$$(echo ${KFP_BLACK_LIST} | grep -v $$dir); \ - if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ] || [ -z "$$z" ]; then \ + yml=test-$(TRANSFORM_SUBDIR)-$$dir-kfp.yml; \ + if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ]; then \ + echo No kfp_ray directory for $$dir. Skipping generation of $$yml; \ + continue; \ + fi; \ + z=$$(echo $${KFP_BLACK_LIST} | grep $$dir); \ + if [ ! -z "$$z" ]; then \ + echo $$dir is black listed. Skipping generation of $$yml; \ continue; \ fi; \ - yml=test-$(TRANSFORM_SUBDIR)-$$dir-kfp.yml; \ echo Generating $$yml; \ cat test-kfp-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml; \ done diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml index be0502e41..4b46dce56 100644 --- a/.github/workflows/test-code-code2parquet-kfp.yml +++ b/.github/workflows/test-code-code2parquet-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/code2parquet workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/code2parquet workflow-test header_text "Run transforms/code/code2parquet completed" diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml index 6aea20218..5e431fd63 100644 --- a/.github/workflows/test-code-code_quality-kfp.yml +++ b/.github/workflows/test-code-code_quality-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/code_quality workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/code_quality workflow-test header_text "Run transforms/code/code_quality completed" diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml index b542e55a5..d0037dfdc 100644 --- a/.github/workflows/test-code-header_cleanser-kfp.yml +++ b/.github/workflows/test-code-header_cleanser-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/header_cleanser workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/header_cleanser workflow-test header_text "Run transforms/code/header_cleanser completed" diff --git a/.github/workflows/test-code-license_select-kfp.yml b/.github/workflows/test-code-license_select-kfp.yml index 0b7f58b61..62aaaef71 100644 --- a/.github/workflows/test-code-license_select-kfp.yml +++ b/.github/workflows/test-code-license_select-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/license_select workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/license_select workflow-test header_text "Run transforms/code/license_select completed" diff --git a/.github/workflows/test-code-malware-kfp.yml b/.github/workflows/test-code-malware-kfp.yml index f49298a8d..cba1df550 100644 --- a/.github/workflows/test-code-malware-kfp.yml +++ b/.github/workflows/test-code-malware-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/malware workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/malware workflow-test header_text "Run transforms/code/malware completed" diff --git a/.github/workflows/test-code-proglang_select-kfp.yml b/.github/workflows/test-code-proglang_select-kfp.yml index 8b1673603..ce085ece6 100644 --- a/.github/workflows/test-code-proglang_select-kfp.yml +++ b/.github/workflows/test-code-proglang_select-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/proglang_select workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/proglang_select workflow-test header_text "Run transforms/code/proglang_select completed" diff --git a/.github/workflows/test-code-repo_level_ordering-kfp.yml b/.github/workflows/test-code-repo_level_ordering-kfp.yml index af4e2ccb0..98e139d41 100644 --- a/.github/workflows/test-code-repo_level_ordering-kfp.yml +++ b/.github/workflows/test-code-repo_level_ordering-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/repo_level_ordering workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/repo_level_ordering workflow-test header_text "Run transforms/code/repo_level_ordering completed" diff --git a/.github/workflows/test-kfp-transform.template b/.github/workflows/test-kfp-transform.template index 5711f5b76..36b94de45 100644 --- a/.github/workflows/test-kfp-transform.template +++ b/.github/workflows/test-kfp-transform.template @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C @TARGET_TRANSFORM_DIR@ workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C @TARGET_TRANSFORM_DIR@ workflow-test header_text "Run @TARGET_TRANSFORM_DIR@ completed" diff --git a/.github/workflows/test-language-doc_quality-kfp.yml b/.github/workflows/test-language-doc_quality-kfp.yml index 9a2af2980..87767dfdf 100644 --- a/.github/workflows/test-language-doc_quality-kfp.yml +++ b/.github/workflows/test-language-doc_quality-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/language/doc_quality workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/language/doc_quality workflow-test header_text "Run transforms/language/doc_quality completed" diff --git a/.github/workflows/test-language-lang_id-kfp.yml b/.github/workflows/test-language-lang_id-kfp.yml index 232bc3c03..4b4b87675 100644 --- a/.github/workflows/test-language-lang_id-kfp.yml +++ b/.github/workflows/test-language-lang_id-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/language/lang_id workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/language/lang_id workflow-test header_text "Run transforms/language/lang_id completed" diff --git a/.github/workflows/test-language-text_encoder-kfp.yml b/.github/workflows/test-language-text_encoder-kfp.yml index c89faad31..5b24270d3 100644 --- a/.github/workflows/test-language-text_encoder-kfp.yml +++ b/.github/workflows/test-language-text_encoder-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/language/text_encoder workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/language/text_encoder workflow-test header_text "Run transforms/language/text_encoder completed" diff --git a/.github/workflows/test-universal-doc_id-kfp.yml b/.github/workflows/test-universal-doc_id-kfp.yml index 393e48470..3d2286fe5 100644 --- a/.github/workflows/test-universal-doc_id-kfp.yml +++ b/.github/workflows/test-universal-doc_id-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/doc_id workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/doc_id workflow-test header_text "Run transforms/universal/doc_id completed" diff --git a/.github/workflows/test-universal-ededup-kfp.yml b/.github/workflows/test-universal-ededup-kfp.yml index 6f52e0572..20a32c946 100644 --- a/.github/workflows/test-universal-ededup-kfp.yml +++ b/.github/workflows/test-universal-ededup-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/ededup workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/ededup workflow-test header_text "Run transforms/universal/ededup completed" diff --git a/.github/workflows/test-universal-fdedup-kfp.yml b/.github/workflows/test-universal-fdedup-kfp.yml index 5f21bbc9a..bbe1615ac 100644 --- a/.github/workflows/test-universal-fdedup-kfp.yml +++ b/.github/workflows/test-universal-fdedup-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/fdedup workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/fdedup workflow-test header_text "Run transforms/universal/fdedup completed" diff --git a/.github/workflows/test-universal-filter-kfp.yml b/.github/workflows/test-universal-filter-kfp.yml index fafb76bf8..5e331bc1b 100644 --- a/.github/workflows/test-universal-filter-kfp.yml +++ b/.github/workflows/test-universal-filter-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/filter workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/filter workflow-test header_text "Run transforms/universal/filter completed" diff --git a/.github/workflows/test-universal-noop-kfp.yml b/.github/workflows/test-universal-noop-kfp.yml index 021c53a10..a87b09d9d 100644 --- a/.github/workflows/test-universal-noop-kfp.yml +++ b/.github/workflows/test-universal-noop-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/noop workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/noop workflow-test header_text "Run transforms/universal/noop completed" diff --git a/.github/workflows/test-universal-profiler-kfp.yml b/.github/workflows/test-universal-profiler-kfp.yml index 209a84565..b08373433 100644 --- a/.github/workflows/test-universal-profiler-kfp.yml +++ b/.github/workflows/test-universal-profiler-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/profiler workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/profiler workflow-test header_text "Run transforms/universal/profiler completed" diff --git a/.github/workflows/test-universal-resize-kfp.yml b/.github/workflows/test-universal-resize-kfp.yml index 6e5d61ef5..8771851d5 100644 --- a/.github/workflows/test-universal-resize-kfp.yml +++ b/.github/workflows/test-universal-resize-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/resize workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/resize workflow-test header_text "Run transforms/universal/resize completed" diff --git a/.github/workflows/test-universal-tokenization-kfp.yml b/.github/workflows/test-universal-tokenization-kfp.yml index 19b9d3836..1055fb0fa 100644 --- a/.github/workflows/test-universal-tokenization-kfp.yml +++ b/.github/workflows/test-universal-tokenization-kfp.yml @@ -111,7 +111,7 @@ jobs: export KFPv2=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/tokenization workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/tokenization workflow-test header_text "Run transforms/universal/tokenization completed" From 0fbb40e39e51a41ce0be2e090d22f2b344d9ae11 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 12:05:30 -0400 Subject: [PATCH 77/82] switch workflows from ubuntu-22.04 to ubuntu-latest to try and avoid excessive job queuing Signed-off-by: David Wood --- .github/workflows/deploy-docs.yml | 2 +- .github/workflows/deploy-library.yml | 6 +++--- .github/workflows/deploy-transforms.yml | 4 ++-- .github/workflows/test-code-code2parquet-kfp.yml | 4 ++-- .github/workflows/test-code-code2parquet.yml | 6 +++--- .github/workflows/test-code-code_quality-kfp.yml | 4 ++-- .github/workflows/test-code-code_quality.yml | 6 +++--- .github/workflows/test-code-header_cleanser-kfp.yml | 4 ++-- .github/workflows/test-code-header_cleanser.yml | 6 +++--- .github/workflows/test-code-license_select-kfp.yml | 4 ++-- .github/workflows/test-code-license_select.yml | 6 +++--- .github/workflows/test-code-malware-kfp.yml | 4 ++-- .github/workflows/test-code-malware.yml | 6 +++--- .github/workflows/test-code-proglang_select-kfp.yml | 4 ++-- .github/workflows/test-code-proglang_select.yml | 6 +++--- .../workflows/test-code-repo_level_ordering-kfp.yml | 4 ++-- .github/workflows/test-code-repo_level_ordering.yml | 6 +++--- .github/workflows/test-kfp-transform.template | 4 ++-- .github/workflows/test-kfp.yml | 8 ++++---- .github/workflows/test-language-doc_chunk.yml | 6 +++--- .github/workflows/test-language-doc_quality-kfp.yml | 4 ++-- .github/workflows/test-language-doc_quality.yml | 6 +++--- .github/workflows/test-language-html2parquet.yml | 6 +++--- .github/workflows/test-language-lang_id-kfp.yml | 4 ++-- .github/workflows/test-language-lang_id.yml | 6 +++--- .github/workflows/test-language-pdf2parquet.yml | 6 +++--- .github/workflows/test-language-pii_redactor.yml | 6 +++--- .github/workflows/test-language-text_encoder-kfp.yml | 4 ++-- .github/workflows/test-language-text_encoder.yml | 6 +++--- .github/workflows/test-lib.yml | 10 +++++----- .github/workflows/test-misc.yml | 4 ++-- .github/workflows/test-packaging-python.yml | 2 +- .github/workflows/test-packaging-ray.yml | 2 +- .github/workflows/test-transform.template | 6 +++--- .github/workflows/test-universal-doc_id-kfp.yml | 4 ++-- .github/workflows/test-universal-doc_id.yml | 6 +++--- .github/workflows/test-universal-ededup-kfp.yml | 4 ++-- .github/workflows/test-universal-ededup.yml | 6 +++--- .github/workflows/test-universal-fdedup-kfp.yml | 4 ++-- .github/workflows/test-universal-fdedup.yml | 6 +++--- .github/workflows/test-universal-filter-kfp.yml | 4 ++-- .github/workflows/test-universal-filter.yml | 6 +++--- .github/workflows/test-universal-hap.yml | 6 +++--- .github/workflows/test-universal-noop-kfp.yml | 4 ++-- .github/workflows/test-universal-noop.yml | 6 +++--- .github/workflows/test-universal-profiler-kfp.yml | 4 ++-- .github/workflows/test-universal-profiler.yml | 6 +++--- .github/workflows/test-universal-resize-kfp.yml | 4 ++-- .github/workflows/test-universal-resize.yml | 6 +++--- .github/workflows/test-universal-tokenization-kfp.yml | 4 ++-- .github/workflows/test-universal-tokenization.yml | 6 +++--- .github/workflows/workflow-manual-run.yml | 2 +- 52 files changed, 130 insertions(+), 130 deletions(-) diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index a2909c55d..09678e937 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -8,7 +8,7 @@ on: - "releases/**" jobs: deploy: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest env: REPO_URL: "https://github.com/${{ github.repository }}" REPO_BRANCH: "dev" diff --git a/.github/workflows/deploy-library.yml b/.github/workflows/deploy-library.yml index 8ec97ed9e..0c2473175 100644 --- a/.github/workflows/deploy-library.yml +++ b/.github/workflows/deploy-library.yml @@ -14,7 +14,7 @@ permissions: jobs: build-package: name: Build Ray data processing libraries - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -30,7 +30,7 @@ jobs: name: Publish packages to test.pypi.org # disabled if: false - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest needs: build-package steps: @@ -47,7 +47,7 @@ jobs: publish-pypi: name: Publish release to pypi.org - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest needs: build-package # disabled as of now if: false diff --git a/.github/workflows/deploy-transforms.yml b/.github/workflows/deploy-transforms.yml index 7fe5c8b4d..0f002187d 100644 --- a/.github/workflows/deploy-transforms.yml +++ b/.github/workflows/deploy-transforms.yml @@ -9,7 +9,7 @@ on: jobs: build-images: name: Build and check images - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -23,7 +23,7 @@ jobs: name: Publish packages to quay.io # disabled if: false - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest needs: build-images steps: diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml index 4b46dce56..a157523e6 100644 --- a/.github/workflows/test-code-code2parquet-kfp.yml +++ b/.github/workflows/test-code-code2parquet-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/code/code2parquet completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index dd4de1df5..4de4e0ebb 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml index 5e431fd63..facb954be 100644 --- a/.github/workflows/test-code-code_quality-kfp.yml +++ b/.github/workflows/test-code-code_quality-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/code/code_quality completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 4cdadc937..6986abfa7 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml index d0037dfdc..b662e0064 100644 --- a/.github/workflows/test-code-header_cleanser-kfp.yml +++ b/.github/workflows/test-code-header_cleanser-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/code/header_cleanser completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index fe0c70169..1e54d6421 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-code-license_select-kfp.yml b/.github/workflows/test-code-license_select-kfp.yml index 62aaaef71..0982daca1 100644 --- a/.github/workflows/test-code-license_select-kfp.yml +++ b/.github/workflows/test-code-license_select-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/code/license_select completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-code-license_select.yml b/.github/workflows/test-code-license_select.yml index ca6870a21..62286e0e4 100644 --- a/.github/workflows/test-code-license_select.yml +++ b/.github/workflows/test-code-license_select.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-code-malware-kfp.yml b/.github/workflows/test-code-malware-kfp.yml index cba1df550..e07e0e688 100644 --- a/.github/workflows/test-code-malware-kfp.yml +++ b/.github/workflows/test-code-malware-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/code/malware completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index d51695af1..f39bdb07e 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-code-proglang_select-kfp.yml b/.github/workflows/test-code-proglang_select-kfp.yml index ce085ece6..635e4d87d 100644 --- a/.github/workflows/test-code-proglang_select-kfp.yml +++ b/.github/workflows/test-code-proglang_select-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/code/proglang_select completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index eca3aa2f8..f06e5ad94 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-code-repo_level_ordering-kfp.yml b/.github/workflows/test-code-repo_level_ordering-kfp.yml index 98e139d41..ae41eecb4 100644 --- a/.github/workflows/test-code-repo_level_ordering-kfp.yml +++ b/.github/workflows/test-code-repo_level_ordering-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/code/repo_level_ordering completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index d6fd5db97..fb05194d3 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-kfp-transform.template b/.github/workflows/test-kfp-transform.template index 36b94de45..27e66ede7 100644 --- a/.github/workflows/test-kfp-transform.template +++ b/.github/workflows/test-kfp-transform.template @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run @TARGET_TRANSFORM_DIR@ completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-kfp.yml b/.github/workflows/test-kfp.yml index 6719c322e..f05ce0290 100644 --- a/.github/workflows/test-kfp.yml +++ b/.github/workflows/test-kfp.yml @@ -51,7 +51,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -68,7 +68,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -117,7 +117,7 @@ jobs: echo "Run ${transforms[$index]} completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -167,7 +167,7 @@ jobs: header_text "Run ${transforms[$index]} completed" build-kfp-components: needs: [check_if_push_images] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 30 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index fd2569278..bb49dedaf 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-doc_quality-kfp.yml b/.github/workflows/test-language-doc_quality-kfp.yml index 87767dfdf..6b548cbe0 100644 --- a/.github/workflows/test-language-doc_quality-kfp.yml +++ b/.github/workflows/test-language-doc_quality-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/language/doc_quality completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index c552be473..bcca97ec5 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-html2parquet.yml b/.github/workflows/test-language-html2parquet.yml index d2464cf93..0d74b5ed2 100644 --- a/.github/workflows/test-language-html2parquet.yml +++ b/.github/workflows/test-language-html2parquet.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-lang_id-kfp.yml b/.github/workflows/test-language-lang_id-kfp.yml index 4b4b87675..b2d8d149c 100644 --- a/.github/workflows/test-language-lang_id-kfp.yml +++ b/.github/workflows/test-language-lang_id-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/language/lang_id completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index c9cd2a4ac..5bd81023b 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 11353bea3..85ad89ce9 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 4c82779d9..6b0b6b795 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-language-text_encoder-kfp.yml b/.github/workflows/test-language-text_encoder-kfp.yml index 5b24270d3..b00660419 100644 --- a/.github/workflows/test-language-text_encoder-kfp.yml +++ b/.github/workflows/test-language-text_encoder-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/language/text_encoder completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index 5d2ee4936..c84601340 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-lib.yml b/.github/workflows/test-lib.yml index be00c2076..ed4893b5e 100644 --- a/.github/workflows/test-lib.yml +++ b/.github/workflows/test-lib.yml @@ -31,7 +31,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -48,7 +48,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-python-lib: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -56,7 +56,7 @@ jobs: run: | make -C data-processing-lib/python DOCKER=docker venv test test-ray-lib: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -64,7 +64,7 @@ jobs: run: | make -C data-processing-lib/ray DOCKER=docker venv test test-spark-lib: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -74,7 +74,7 @@ jobs: test-data-processing-lib-images: needs: [check_if_push_images] if: needs.check_if_push_images.outputs.publish_images == 'true' - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} diff --git a/.github/workflows/test-misc.yml b/.github/workflows/test-misc.yml index 2c601bbd5..62c1a187a 100644 --- a/.github/workflows/test-misc.yml +++ b/.github/workflows/test-misc.yml @@ -29,7 +29,7 @@ on: jobs: test-make: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -37,7 +37,7 @@ jobs: run: | make -n clean test build publish set-versions check-transform-test-workflows: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-packaging-python.yml b/.github/workflows/test-packaging-python.yml index 4ee491c8e..e88eeeae2 100644 --- a/.github/workflows/test-packaging-python.yml +++ b/.github/workflows/test-packaging-python.yml @@ -27,7 +27,7 @@ on: jobs: test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-packaging-ray.yml b/.github/workflows/test-packaging-ray.yml index 4b812540c..9dbce3110 100644 --- a/.github/workflows/test-packaging-ray.yml +++ b/.github/workflows/test-packaging-ray.yml @@ -27,7 +27,7 @@ on: jobs: test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 6f5b4112c..712a91490 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-doc_id-kfp.yml b/.github/workflows/test-universal-doc_id-kfp.yml index 3d2286fe5..a8ca1aca9 100644 --- a/.github/workflows/test-universal-doc_id-kfp.yml +++ b/.github/workflows/test-universal-doc_id-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/universal/doc_id completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 671a8545e..8dbb11fd4 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-ededup-kfp.yml b/.github/workflows/test-universal-ededup-kfp.yml index 20a32c946..a6a35149d 100644 --- a/.github/workflows/test-universal-ededup-kfp.yml +++ b/.github/workflows/test-universal-ededup-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/universal/ededup completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index 1384df79c..aace9621d 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-fdedup-kfp.yml b/.github/workflows/test-universal-fdedup-kfp.yml index bbe1615ac..6e3c6da90 100644 --- a/.github/workflows/test-universal-fdedup-kfp.yml +++ b/.github/workflows/test-universal-fdedup-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/universal/fdedup completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index c5df0abdd..b6eb684c8 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-filter-kfp.yml b/.github/workflows/test-universal-filter-kfp.yml index 5e331bc1b..47e864029 100644 --- a/.github/workflows/test-universal-filter-kfp.yml +++ b/.github/workflows/test-universal-filter-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/universal/filter completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 6b627c73b..7d78ac455 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-hap.yml b/.github/workflows/test-universal-hap.yml index 3ceaaf5c2..8512e1fb9 100644 --- a/.github/workflows/test-universal-hap.yml +++ b/.github/workflows/test-universal-hap.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-noop-kfp.yml b/.github/workflows/test-universal-noop-kfp.yml index a87b09d9d..ba6f5acb2 100644 --- a/.github/workflows/test-universal-noop-kfp.yml +++ b/.github/workflows/test-universal-noop-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/universal/noop completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 91b6a3c79..594c7a424 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-profiler-kfp.yml b/.github/workflows/test-universal-profiler-kfp.yml index b08373433..ffac3be18 100644 --- a/.github/workflows/test-universal-profiler-kfp.yml +++ b/.github/workflows/test-universal-profiler-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/universal/profiler completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index 93b0615af..c9db499bf 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-resize-kfp.yml b/.github/workflows/test-universal-resize-kfp.yml index 8771851d5..233aa02d0 100644 --- a/.github/workflows/test-universal-resize-kfp.yml +++ b/.github/workflows/test-universal-resize-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/universal/resize completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index 1098b1bdd..408420a2b 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/test-universal-tokenization-kfp.yml b/.github/workflows/test-universal-tokenization-kfp.yml index 1055fb0fa..1a1fb7902 100644 --- a/.github/workflows/test-universal-tokenization-kfp.yml +++ b/.github/workflows/test-universal-tokenization-kfp.yml @@ -38,7 +38,7 @@ on: jobs: test-kfp-v1: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -77,7 +77,7 @@ jobs: echo "Run transforms/universal/tokenization completed" test-kfp-v2: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 9b3908d9e..475019f00 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -46,7 +46,7 @@ jobs: # The images are pushed if it is a merge to dev branch or a new tag is created. # The latter being part of the release process. # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest outputs: publish_images: ${{ steps.version.outputs.publish_images }} steps: @@ -63,7 +63,7 @@ jobs: fi echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" test-src: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 @@ -85,7 +85,7 @@ jobs: fi test-image: needs: [check_if_push_image] - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest timeout-minutes: 120 env: DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} diff --git a/.github/workflows/workflow-manual-run.yml b/.github/workflows/workflow-manual-run.yml index 3c0f37d47..f0f7028b6 100644 --- a/.github/workflows/workflow-manual-run.yml +++ b/.github/workflows/workflow-manual-run.yml @@ -22,7 +22,7 @@ jobs: KFPv2: ${{ github.event.inputs.kfp_v2 }} WORKFLOW_PATH: ${{ github.event.inputs.workflow-path }} DEBUG_MODE: ${{ github.event.inputs.debug }} - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 From a81b65aa54da3adef5855952ba880ede71ee0858 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 12:48:36 -0400 Subject: [PATCH 78/82] remove non-change from license_select_wf.py to trigger new ci/cd Signed-off-by: David Wood --- transforms/code/license_select/kfp_ray/license_select_wf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transforms/code/license_select/kfp_ray/license_select_wf.py b/transforms/code/license_select/kfp_ray/license_select_wf.py index e97986a92..1938decfe 100644 --- a/transforms/code/license_select/kfp_ray/license_select_wf.py +++ b/transforms/code/license_select/kfp_ray/license_select_wf.py @@ -24,7 +24,6 @@ task_image = "quay.io/dataprep1/data-prep-kit/license_select-ray:latest" -# # components base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest" From 6fd07b3bec51450635ffb641eda4aba5998ec34b Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 16:03:51 -0400 Subject: [PATCH 79/82] one more fix to kfp transform workflows to not workflow-build in all transforms Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet-kfp.yml | 2 +- .github/workflows/test-code-code_quality-kfp.yml | 2 +- .github/workflows/test-code-header_cleanser-kfp.yml | 2 +- .github/workflows/test-code-license_select-kfp.yml | 2 +- .github/workflows/test-code-malware-kfp.yml | 2 +- .github/workflows/test-code-proglang_select-kfp.yml | 2 +- .github/workflows/test-code-repo_level_ordering-kfp.yml | 2 +- .github/workflows/test-kfp-transform.template | 2 +- .github/workflows/test-language-doc_quality-kfp.yml | 2 +- .github/workflows/test-language-lang_id-kfp.yml | 2 +- .github/workflows/test-language-text_encoder-kfp.yml | 2 +- .github/workflows/test-universal-doc_id-kfp.yml | 2 +- .github/workflows/test-universal-ededup-kfp.yml | 2 +- .github/workflows/test-universal-fdedup-kfp.yml | 2 +- .github/workflows/test-universal-filter-kfp.yml | 2 +- .github/workflows/test-universal-noop-kfp.yml | 2 +- .github/workflows/test-universal-profiler-kfp.yml | 2 +- .github/workflows/test-universal-resize-kfp.yml | 2 +- .github/workflows/test-universal-tokenization-kfp.yml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml index a157523e6..bc6a085d0 100644 --- a/.github/workflows/test-code-code2parquet-kfp.yml +++ b/.github/workflows/test-code-code2parquet-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/code2parquet workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/code2parquet workflow-test echo "Run transforms/code/code2parquet completed" diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml index facb954be..099e55e60 100644 --- a/.github/workflows/test-code-code_quality-kfp.yml +++ b/.github/workflows/test-code-code_quality-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/code_quality workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/code_quality workflow-test echo "Run transforms/code/code_quality completed" diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml index b662e0064..348b15604 100644 --- a/.github/workflows/test-code-header_cleanser-kfp.yml +++ b/.github/workflows/test-code-header_cleanser-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/header_cleanser workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/header_cleanser workflow-test echo "Run transforms/code/header_cleanser completed" diff --git a/.github/workflows/test-code-license_select-kfp.yml b/.github/workflows/test-code-license_select-kfp.yml index 0982daca1..c4611f82d 100644 --- a/.github/workflows/test-code-license_select-kfp.yml +++ b/.github/workflows/test-code-license_select-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/license_select workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/license_select workflow-test echo "Run transforms/code/license_select completed" diff --git a/.github/workflows/test-code-malware-kfp.yml b/.github/workflows/test-code-malware-kfp.yml index e07e0e688..3f6471cc3 100644 --- a/.github/workflows/test-code-malware-kfp.yml +++ b/.github/workflows/test-code-malware-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/malware workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/malware workflow-test echo "Run transforms/code/malware completed" diff --git a/.github/workflows/test-code-proglang_select-kfp.yml b/.github/workflows/test-code-proglang_select-kfp.yml index 635e4d87d..d10de0fcc 100644 --- a/.github/workflows/test-code-proglang_select-kfp.yml +++ b/.github/workflows/test-code-proglang_select-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/proglang_select workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/proglang_select workflow-test echo "Run transforms/code/proglang_select completed" diff --git a/.github/workflows/test-code-repo_level_ordering-kfp.yml b/.github/workflows/test-code-repo_level_ordering-kfp.yml index ae41eecb4..a1ef2b62e 100644 --- a/.github/workflows/test-code-repo_level_ordering-kfp.yml +++ b/.github/workflows/test-code-repo_level_ordering-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/code/repo_level_ordering workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/code/repo_level_ordering workflow-test echo "Run transforms/code/repo_level_ordering completed" diff --git a/.github/workflows/test-kfp-transform.template b/.github/workflows/test-kfp-transform.template index 27e66ede7..bd39463d4 100644 --- a/.github/workflows/test-kfp-transform.template +++ b/.github/workflows/test-kfp-transform.template @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C @TARGET_TRANSFORM_DIR@ workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C @TARGET_TRANSFORM_DIR@ workflow-test echo "Run @TARGET_TRANSFORM_DIR@ completed" diff --git a/.github/workflows/test-language-doc_quality-kfp.yml b/.github/workflows/test-language-doc_quality-kfp.yml index 6b548cbe0..c0f61105d 100644 --- a/.github/workflows/test-language-doc_quality-kfp.yml +++ b/.github/workflows/test-language-doc_quality-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/language/doc_quality workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/language/doc_quality workflow-test echo "Run transforms/language/doc_quality completed" diff --git a/.github/workflows/test-language-lang_id-kfp.yml b/.github/workflows/test-language-lang_id-kfp.yml index b2d8d149c..040ae3a1e 100644 --- a/.github/workflows/test-language-lang_id-kfp.yml +++ b/.github/workflows/test-language-lang_id-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/language/lang_id workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/language/lang_id workflow-test echo "Run transforms/language/lang_id completed" diff --git a/.github/workflows/test-language-text_encoder-kfp.yml b/.github/workflows/test-language-text_encoder-kfp.yml index b00660419..9e8c9dd18 100644 --- a/.github/workflows/test-language-text_encoder-kfp.yml +++ b/.github/workflows/test-language-text_encoder-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/language/text_encoder workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/language/text_encoder workflow-test echo "Run transforms/language/text_encoder completed" diff --git a/.github/workflows/test-universal-doc_id-kfp.yml b/.github/workflows/test-universal-doc_id-kfp.yml index a8ca1aca9..cef5196f9 100644 --- a/.github/workflows/test-universal-doc_id-kfp.yml +++ b/.github/workflows/test-universal-doc_id-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/doc_id workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/doc_id workflow-test echo "Run transforms/universal/doc_id completed" diff --git a/.github/workflows/test-universal-ededup-kfp.yml b/.github/workflows/test-universal-ededup-kfp.yml index a6a35149d..b2e66a7c3 100644 --- a/.github/workflows/test-universal-ededup-kfp.yml +++ b/.github/workflows/test-universal-ededup-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/ededup workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/ededup workflow-test echo "Run transforms/universal/ededup completed" diff --git a/.github/workflows/test-universal-fdedup-kfp.yml b/.github/workflows/test-universal-fdedup-kfp.yml index 6e3c6da90..a0f43391a 100644 --- a/.github/workflows/test-universal-fdedup-kfp.yml +++ b/.github/workflows/test-universal-fdedup-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/fdedup workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/fdedup workflow-test echo "Run transforms/universal/fdedup completed" diff --git a/.github/workflows/test-universal-filter-kfp.yml b/.github/workflows/test-universal-filter-kfp.yml index 47e864029..7deea3cbc 100644 --- a/.github/workflows/test-universal-filter-kfp.yml +++ b/.github/workflows/test-universal-filter-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/filter workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/filter workflow-test echo "Run transforms/universal/filter completed" diff --git a/.github/workflows/test-universal-noop-kfp.yml b/.github/workflows/test-universal-noop-kfp.yml index ba6f5acb2..cd30b90a8 100644 --- a/.github/workflows/test-universal-noop-kfp.yml +++ b/.github/workflows/test-universal-noop-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/noop workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/noop workflow-test echo "Run transforms/universal/noop completed" diff --git a/.github/workflows/test-universal-profiler-kfp.yml b/.github/workflows/test-universal-profiler-kfp.yml index ffac3be18..0935ac058 100644 --- a/.github/workflows/test-universal-profiler-kfp.yml +++ b/.github/workflows/test-universal-profiler-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/profiler workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/profiler workflow-test echo "Run transforms/universal/profiler completed" diff --git a/.github/workflows/test-universal-resize-kfp.yml b/.github/workflows/test-universal-resize-kfp.yml index 233aa02d0..9890220cc 100644 --- a/.github/workflows/test-universal-resize-kfp.yml +++ b/.github/workflows/test-universal-resize-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/resize workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/resize workflow-test echo "Run transforms/universal/resize completed" diff --git a/.github/workflows/test-universal-tokenization-kfp.yml b/.github/workflows/test-universal-tokenization-kfp.yml index 1a1fb7902..45962f6ea 100644 --- a/.github/workflows/test-universal-tokenization-kfp.yml +++ b/.github/workflows/test-universal-tokenization-kfp.yml @@ -71,7 +71,7 @@ jobs: export DEPLOY_KUBEFLOW=1 make -C $K8S_SETUP_SCRIPTS setup make -C kfp/kfp_support_lib test - make -C transforms workflow-build + make -C transforms/universal/tokenization workflow-build source $K8S_SETUP_SCRIPTS/common.sh make -C transforms/universal/tokenization workflow-test echo "Run transforms/universal/tokenization completed" From 148b762c993a8bfc4c49b3e2f10f21be78545009 Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 16:53:19 -0400 Subject: [PATCH 80/82] fix repo_level_ordering/ray/Makefile to support kind/kfp testing targets Signed-off-by: David Wood --- transforms/code/repo_level_ordering/ray/Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/transforms/code/repo_level_ordering/ray/Makefile b/transforms/code/repo_level_ordering/ray/Makefile index 771ed9240..83f8692de 100644 --- a/transforms/code/repo_level_ordering/ray/Makefile +++ b/transforms/code/repo_level_ordering/ray/Makefile @@ -50,3 +50,10 @@ run-s3-sample: .transforms.run-s3-ray-sample minio-start: .minio-start load-image:: .transforms.load-image + +kind-load-image:: .transforms.kind-load-image + +docker-load-image: .defaults.docker-load-image + +docker-save-image: .defaults.docker-save-image + From ed0556dd5549c591f23d81f4099f8839e2df38dc Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 17:03:26 -0400 Subject: [PATCH 81/82] change kfp test workflows to not test if transform's Makefile or kfp_ray dir is not present Signed-off-by: David Wood --- .../workflows/test-code-code2parquet-kfp.yml | 94 ++++++++++--------- .../workflows/test-code-code_quality-kfp.yml | 94 ++++++++++--------- .../test-code-header_cleanser-kfp.yml | 94 ++++++++++--------- .../test-code-license_select-kfp.yml | 94 ++++++++++--------- .github/workflows/test-code-malware-kfp.yml | 94 ++++++++++--------- .../test-code-proglang_select-kfp.yml | 94 ++++++++++--------- .../test-code-repo_level_ordering-kfp.yml | 94 ++++++++++--------- .github/workflows/test-kfp-transform.template | 94 ++++++++++--------- .../test-language-doc_quality-kfp.yml | 94 ++++++++++--------- .../workflows/test-language-lang_id-kfp.yml | 94 ++++++++++--------- .../test-language-text_encoder-kfp.yml | 94 ++++++++++--------- .../workflows/test-universal-doc_id-kfp.yml | 94 ++++++++++--------- .../workflows/test-universal-ededup-kfp.yml | 94 ++++++++++--------- .../workflows/test-universal-fdedup-kfp.yml | 94 ++++++++++--------- .../workflows/test-universal-filter-kfp.yml | 94 ++++++++++--------- .github/workflows/test-universal-noop-kfp.yml | 94 ++++++++++--------- .../workflows/test-universal-profiler-kfp.yml | 94 ++++++++++--------- .../workflows/test-universal-resize-kfp.yml | 94 ++++++++++--------- .../test-universal-tokenization-kfp.yml | 94 ++++++++++--------- 19 files changed, 969 insertions(+), 817 deletions(-) diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml index bc6a085d0..85c635e51 100644 --- a/.github/workflows/test-code-code2parquet-kfp.yml +++ b/.github/workflows/test-code-code2parquet-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/code2parquet workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/code2parquet workflow-test - echo "Run transforms/code/code2parquet completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/code2parquet workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/code2parquet workflow-test + echo "Run transforms/code/code2parquet completed" + else + echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/code2parquet workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/code2parquet workflow-test - header_text "Run transforms/code/code2parquet completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/code2parquet workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/code2parquet workflow-test + echo "Run transforms/code/code2parquet completed" + else + echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml index 099e55e60..6228038ed 100644 --- a/.github/workflows/test-code-code_quality-kfp.yml +++ b/.github/workflows/test-code-code_quality-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/code_quality workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/code_quality workflow-test - echo "Run transforms/code/code_quality completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/code_quality workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/code_quality workflow-test + echo "Run transforms/code/code_quality completed" + else + echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/code_quality workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/code_quality workflow-test - header_text "Run transforms/code/code_quality completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/code_quality workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/code_quality workflow-test + echo "Run transforms/code/code_quality completed" + else + echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml index 348b15604..0c0c1de2f 100644 --- a/.github/workflows/test-code-header_cleanser-kfp.yml +++ b/.github/workflows/test-code-header_cleanser-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/header_cleanser workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/header_cleanser workflow-test - echo "Run transforms/code/header_cleanser completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/header_cleanser workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/header_cleanser workflow-test + echo "Run transforms/code/header_cleanser completed" + else + echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/header_cleanser workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/header_cleanser workflow-test - header_text "Run transforms/code/header_cleanser completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/header_cleanser workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/header_cleanser workflow-test + echo "Run transforms/code/header_cleanser completed" + else + echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-license_select-kfp.yml b/.github/workflows/test-code-license_select-kfp.yml index c4611f82d..1aaabbeec 100644 --- a/.github/workflows/test-code-license_select-kfp.yml +++ b/.github/workflows/test-code-license_select-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/license_select workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/license_select workflow-test - echo "Run transforms/code/license_select completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/license_select workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/license_select workflow-test + echo "Run transforms/code/license_select completed" + else + echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/license_select workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/license_select workflow-test - header_text "Run transforms/code/license_select completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/license_select workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/license_select workflow-test + echo "Run transforms/code/license_select completed" + else + echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-malware-kfp.yml b/.github/workflows/test-code-malware-kfp.yml index 3f6471cc3..0b683acca 100644 --- a/.github/workflows/test-code-malware-kfp.yml +++ b/.github/workflows/test-code-malware-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/malware workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/malware workflow-test - echo "Run transforms/code/malware completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/malware workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/malware workflow-test + echo "Run transforms/code/malware completed" + else + echo "Skipping transforms/code/malware kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/malware workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/malware workflow-test - header_text "Run transforms/code/malware completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/malware workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/malware workflow-test + echo "Run transforms/code/malware completed" + else + echo "Skipping transforms/code/malware kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-proglang_select-kfp.yml b/.github/workflows/test-code-proglang_select-kfp.yml index d10de0fcc..88af8e004 100644 --- a/.github/workflows/test-code-proglang_select-kfp.yml +++ b/.github/workflows/test-code-proglang_select-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/proglang_select workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/proglang_select workflow-test - echo "Run transforms/code/proglang_select completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/proglang_select workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/proglang_select workflow-test + echo "Run transforms/code/proglang_select completed" + else + echo "Skipping transforms/code/proglang_select kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/proglang_select workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/proglang_select workflow-test - header_text "Run transforms/code/proglang_select completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/proglang_select workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/proglang_select workflow-test + echo "Run transforms/code/proglang_select completed" + else + echo "Skipping transforms/code/proglang_select kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-code-repo_level_ordering-kfp.yml b/.github/workflows/test-code-repo_level_ordering-kfp.yml index a1ef2b62e..c741faaaa 100644 --- a/.github/workflows/test-code-repo_level_ordering-kfp.yml +++ b/.github/workflows/test-code-repo_level_ordering-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/repo_level_ordering workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/repo_level_ordering workflow-test - echo "Run transforms/code/repo_level_ordering completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/repo_level_ordering workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/repo_level_ordering workflow-test + echo "Run transforms/code/repo_level_ordering completed" + else + echo "Skipping transforms/code/repo_level_ordering kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/code/repo_level_ordering workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/repo_level_ordering workflow-test - header_text "Run transforms/code/repo_level_ordering completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/code/repo_level_ordering workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/repo_level_ordering workflow-test + echo "Run transforms/code/repo_level_ordering completed" + else + echo "Skipping transforms/code/repo_level_ordering kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-kfp-transform.template b/.github/workflows/test-kfp-transform.template index bd39463d4..1df901367 100644 --- a/.github/workflows/test-kfp-transform.template +++ b/.github/workflows/test-kfp-transform.template @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C @TARGET_TRANSFORM_DIR@ workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C @TARGET_TRANSFORM_DIR@ workflow-test - echo "Run @TARGET_TRANSFORM_DIR@ completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C @TARGET_TRANSFORM_DIR@ workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C @TARGET_TRANSFORM_DIR@ workflow-test + echo "Run @TARGET_TRANSFORM_DIR@ completed" + else + echo "Skipping @TARGET_TRANSFORM_DIR@ kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C @TARGET_TRANSFORM_DIR@ workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C @TARGET_TRANSFORM_DIR@ workflow-test - header_text "Run @TARGET_TRANSFORM_DIR@ completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C @TARGET_TRANSFORM_DIR@ workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C @TARGET_TRANSFORM_DIR@ workflow-test + echo "Run @TARGET_TRANSFORM_DIR@ completed" + else + echo "Skipping @TARGET_TRANSFORM_DIR@ kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-language-doc_quality-kfp.yml b/.github/workflows/test-language-doc_quality-kfp.yml index c0f61105d..f9e299067 100644 --- a/.github/workflows/test-language-doc_quality-kfp.yml +++ b/.github/workflows/test-language-doc_quality-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/language/doc_quality workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/language/doc_quality workflow-test - echo "Run transforms/language/doc_quality completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/language/doc_quality workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/doc_quality workflow-test + echo "Run transforms/language/doc_quality completed" + else + echo "Skipping transforms/language/doc_quality kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/language/doc_quality workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/language/doc_quality workflow-test - header_text "Run transforms/language/doc_quality completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/language/doc_quality workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/doc_quality workflow-test + echo "Run transforms/language/doc_quality completed" + else + echo "Skipping transforms/language/doc_quality kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-language-lang_id-kfp.yml b/.github/workflows/test-language-lang_id-kfp.yml index 040ae3a1e..a1a80bb94 100644 --- a/.github/workflows/test-language-lang_id-kfp.yml +++ b/.github/workflows/test-language-lang_id-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/language/lang_id workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/language/lang_id workflow-test - echo "Run transforms/language/lang_id completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/language/lang_id workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/lang_id workflow-test + echo "Run transforms/language/lang_id completed" + else + echo "Skipping transforms/language/lang_id kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/language/lang_id workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/language/lang_id workflow-test - header_text "Run transforms/language/lang_id completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/language/lang_id workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/lang_id workflow-test + echo "Run transforms/language/lang_id completed" + else + echo "Skipping transforms/language/lang_id kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-language-text_encoder-kfp.yml b/.github/workflows/test-language-text_encoder-kfp.yml index 9e8c9dd18..4ac236ce8 100644 --- a/.github/workflows/test-language-text_encoder-kfp.yml +++ b/.github/workflows/test-language-text_encoder-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/language/text_encoder workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/language/text_encoder workflow-test - echo "Run transforms/language/text_encoder completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/language/text_encoder workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/text_encoder workflow-test + echo "Run transforms/language/text_encoder completed" + else + echo "Skipping transforms/language/text_encoder kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/language/text_encoder workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/language/text_encoder workflow-test - header_text "Run transforms/language/text_encoder completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/language/text_encoder workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/text_encoder workflow-test + echo "Run transforms/language/text_encoder completed" + else + echo "Skipping transforms/language/text_encoder kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-doc_id-kfp.yml b/.github/workflows/test-universal-doc_id-kfp.yml index cef5196f9..e59589420 100644 --- a/.github/workflows/test-universal-doc_id-kfp.yml +++ b/.github/workflows/test-universal-doc_id-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/doc_id workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/doc_id workflow-test - echo "Run transforms/universal/doc_id completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/doc_id workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/doc_id workflow-test + echo "Run transforms/universal/doc_id completed" + else + echo "Skipping transforms/universal/doc_id kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/doc_id workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/doc_id workflow-test - header_text "Run transforms/universal/doc_id completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/doc_id workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/doc_id workflow-test + echo "Run transforms/universal/doc_id completed" + else + echo "Skipping transforms/universal/doc_id kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-ededup-kfp.yml b/.github/workflows/test-universal-ededup-kfp.yml index b2e66a7c3..09a67edc8 100644 --- a/.github/workflows/test-universal-ededup-kfp.yml +++ b/.github/workflows/test-universal-ededup-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/ededup workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/ededup workflow-test - echo "Run transforms/universal/ededup completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/ededup workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/ededup workflow-test + echo "Run transforms/universal/ededup completed" + else + echo "Skipping transforms/universal/ededup kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/ededup workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/ededup workflow-test - header_text "Run transforms/universal/ededup completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/ededup workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/ededup workflow-test + echo "Run transforms/universal/ededup completed" + else + echo "Skipping transforms/universal/ededup kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-fdedup-kfp.yml b/.github/workflows/test-universal-fdedup-kfp.yml index a0f43391a..43da527f6 100644 --- a/.github/workflows/test-universal-fdedup-kfp.yml +++ b/.github/workflows/test-universal-fdedup-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/fdedup workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/fdedup workflow-test - echo "Run transforms/universal/fdedup completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/fdedup workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/fdedup workflow-test + echo "Run transforms/universal/fdedup completed" + else + echo "Skipping transforms/universal/fdedup kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/fdedup workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/fdedup workflow-test - header_text "Run transforms/universal/fdedup completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/fdedup workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/fdedup workflow-test + echo "Run transforms/universal/fdedup completed" + else + echo "Skipping transforms/universal/fdedup kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-filter-kfp.yml b/.github/workflows/test-universal-filter-kfp.yml index 7deea3cbc..3d0593713 100644 --- a/.github/workflows/test-universal-filter-kfp.yml +++ b/.github/workflows/test-universal-filter-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/filter workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/filter workflow-test - echo "Run transforms/universal/filter completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/filter workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/filter workflow-test + echo "Run transforms/universal/filter completed" + else + echo "Skipping transforms/universal/filter kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/filter workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/filter workflow-test - header_text "Run transforms/universal/filter completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/filter workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/filter workflow-test + echo "Run transforms/universal/filter completed" + else + echo "Skipping transforms/universal/filter kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-noop-kfp.yml b/.github/workflows/test-universal-noop-kfp.yml index cd30b90a8..fa0786bcf 100644 --- a/.github/workflows/test-universal-noop-kfp.yml +++ b/.github/workflows/test-universal-noop-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/noop workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/noop workflow-test - echo "Run transforms/universal/noop completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/noop workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/noop workflow-test + echo "Run transforms/universal/noop completed" + else + echo "Skipping transforms/universal/noop kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/noop workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/noop workflow-test - header_text "Run transforms/universal/noop completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/noop workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/noop workflow-test + echo "Run transforms/universal/noop completed" + else + echo "Skipping transforms/universal/noop kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-profiler-kfp.yml b/.github/workflows/test-universal-profiler-kfp.yml index 0935ac058..faf414235 100644 --- a/.github/workflows/test-universal-profiler-kfp.yml +++ b/.github/workflows/test-universal-profiler-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/profiler workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/profiler workflow-test - echo "Run transforms/universal/profiler completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/profiler workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/profiler workflow-test + echo "Run transforms/universal/profiler completed" + else + echo "Skipping transforms/universal/profiler kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/profiler workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/profiler workflow-test - header_text "Run transforms/universal/profiler completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/profiler workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/profiler workflow-test + echo "Run transforms/universal/profiler completed" + else + echo "Skipping transforms/universal/profiler kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-resize-kfp.yml b/.github/workflows/test-universal-resize-kfp.yml index 9890220cc..f55a82821 100644 --- a/.github/workflows/test-universal-resize-kfp.yml +++ b/.github/workflows/test-universal-resize-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/resize workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/resize workflow-test - echo "Run transforms/universal/resize completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/resize workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/resize workflow-test + echo "Run transforms/universal/resize completed" + else + echo "Skipping transforms/universal/resize kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/resize workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/resize workflow-test - header_text "Run transforms/universal/resize completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/resize workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/resize workflow-test + echo "Run transforms/universal/resize completed" + else + echo "Skipping transforms/universal/resize kfp test for lack of Makefile and/or kfp_ray" + fi diff --git a/.github/workflows/test-universal-tokenization-kfp.yml b/.github/workflows/test-universal-tokenization-kfp.yml index 45962f6ea..20741386d 100644 --- a/.github/workflows/test-universal-tokenization-kfp.yml +++ b/.github/workflows/test-universal-tokenization-kfp.yml @@ -54,27 +54,31 @@ jobs: - name: Test KFP libs (shared and v1) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/tokenization workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/tokenization workflow-test - echo "Run transforms/universal/tokenization completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/tokenization workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/tokenization workflow-test + echo "Run transforms/universal/tokenization completed" + else + echo "Skipping transforms/universal/tokenization kfp test for lack of Makefile and/or kfp_ray" + fi test-kfp-v2: runs-on: ubuntu-latest @@ -93,25 +97,29 @@ jobs: - name: Test KFP libs (shared and v2) and run a workflow timeout-minutes: 120 run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms/universal/tokenization workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/universal/tokenization workflow-test - header_text "Run transforms/universal/tokenization completed" + if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms/universal/tokenization workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/tokenization workflow-test + echo "Run transforms/universal/tokenization completed" + else + echo "Skipping transforms/universal/tokenization kfp test for lack of Makefile and/or kfp_ray" + fi From eadce898542f00493b7c0cc6cdde04d86dc7be6b Mon Sep 17 00:00:00 2001 From: David Wood Date: Thu, 3 Oct 2024 17:31:50 -0400 Subject: [PATCH 82/82] set cancel-in-progress=true in workflows Signed-off-by: David Wood --- .github/workflows/test-code-code2parquet-kfp.yml | 5 +++++ .github/workflows/test-code-code2parquet.yml | 5 +++++ .github/workflows/test-code-code_quality-kfp.yml | 5 +++++ .github/workflows/test-code-code_quality.yml | 5 +++++ .github/workflows/test-code-header_cleanser-kfp.yml | 5 +++++ .github/workflows/test-code-header_cleanser.yml | 5 +++++ .github/workflows/test-code-license_select-kfp.yml | 5 +++++ .github/workflows/test-code-license_select.yml | 5 +++++ .github/workflows/test-code-malware-kfp.yml | 5 +++++ .github/workflows/test-code-malware.yml | 5 +++++ .github/workflows/test-code-proglang_select-kfp.yml | 5 +++++ .github/workflows/test-code-proglang_select.yml | 5 +++++ .github/workflows/test-code-repo_level_ordering-kfp.yml | 5 +++++ .github/workflows/test-code-repo_level_ordering.yml | 5 +++++ .github/workflows/test-kfp-transform.template | 5 +++++ .github/workflows/test-kfp.yml | 5 +++++ .github/workflows/test-language-doc_chunk.yml | 5 +++++ .github/workflows/test-language-doc_quality-kfp.yml | 5 +++++ .github/workflows/test-language-doc_quality.yml | 5 +++++ .github/workflows/test-language-html2parquet.yml | 5 +++++ .github/workflows/test-language-lang_id-kfp.yml | 5 +++++ .github/workflows/test-language-lang_id.yml | 5 +++++ .github/workflows/test-language-pdf2parquet.yml | 5 +++++ .github/workflows/test-language-pii_redactor.yml | 5 +++++ .github/workflows/test-language-text_encoder-kfp.yml | 5 +++++ .github/workflows/test-language-text_encoder.yml | 5 +++++ .github/workflows/test-lib.yml | 5 +++++ .github/workflows/test-transform.template | 5 +++++ .github/workflows/test-universal-doc_id-kfp.yml | 5 +++++ .github/workflows/test-universal-doc_id.yml | 5 +++++ .github/workflows/test-universal-ededup-kfp.yml | 5 +++++ .github/workflows/test-universal-ededup.yml | 5 +++++ .github/workflows/test-universal-fdedup-kfp.yml | 5 +++++ .github/workflows/test-universal-fdedup.yml | 5 +++++ .github/workflows/test-universal-filter-kfp.yml | 5 +++++ .github/workflows/test-universal-filter.yml | 5 +++++ .github/workflows/test-universal-hap.yml | 5 +++++ .github/workflows/test-universal-noop-kfp.yml | 5 +++++ .github/workflows/test-universal-noop.yml | 5 +++++ .github/workflows/test-universal-profiler-kfp.yml | 5 +++++ .github/workflows/test-universal-profiler.yml | 5 +++++ .github/workflows/test-universal-resize-kfp.yml | 5 +++++ .github/workflows/test-universal-resize.yml | 5 +++++ .github/workflows/test-universal-tokenization-kfp.yml | 5 +++++ .github/workflows/test-universal-tokenization.yml | 5 +++++ 45 files changed, 225 insertions(+) diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml index 85c635e51..6de24d4b0 100644 --- a/.github/workflows/test-code-code2parquet-kfp.yml +++ b/.github/workflows/test-code-code2parquet-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-code-code2parquet.yml b/.github/workflows/test-code-code2parquet.yml index 4de4e0ebb..3f83e9856 100644 --- a/.github/workflows/test-code-code2parquet.yml +++ b/.github/workflows/test-code-code2parquet.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml index 6228038ed..2e22c04a9 100644 --- a/.github/workflows/test-code-code_quality-kfp.yml +++ b/.github/workflows/test-code-code_quality-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-code-code_quality.yml b/.github/workflows/test-code-code_quality.yml index 6986abfa7..5a901edbb 100644 --- a/.github/workflows/test-code-code_quality.yml +++ b/.github/workflows/test-code-code_quality.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml index 0c0c1de2f..6cc4727aa 100644 --- a/.github/workflows/test-code-header_cleanser-kfp.yml +++ b/.github/workflows/test-code-header_cleanser-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-code-header_cleanser.yml b/.github/workflows/test-code-header_cleanser.yml index 1e54d6421..05f09a8c5 100644 --- a/.github/workflows/test-code-header_cleanser.yml +++ b/.github/workflows/test-code-header_cleanser.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-code-license_select-kfp.yml b/.github/workflows/test-code-license_select-kfp.yml index 1aaabbeec..94d662d1d 100644 --- a/.github/workflows/test-code-license_select-kfp.yml +++ b/.github/workflows/test-code-license_select-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-code-license_select.yml b/.github/workflows/test-code-license_select.yml index 62286e0e4..59592c82f 100644 --- a/.github/workflows/test-code-license_select.yml +++ b/.github/workflows/test-code-license_select.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-code-malware-kfp.yml b/.github/workflows/test-code-malware-kfp.yml index 0b683acca..2c9e3186c 100644 --- a/.github/workflows/test-code-malware-kfp.yml +++ b/.github/workflows/test-code-malware-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-code-malware.yml b/.github/workflows/test-code-malware.yml index f39bdb07e..44196c62c 100644 --- a/.github/workflows/test-code-malware.yml +++ b/.github/workflows/test-code-malware.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-code-proglang_select-kfp.yml b/.github/workflows/test-code-proglang_select-kfp.yml index 88af8e004..c23e0f1ff 100644 --- a/.github/workflows/test-code-proglang_select-kfp.yml +++ b/.github/workflows/test-code-proglang_select-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-code-proglang_select.yml b/.github/workflows/test-code-proglang_select.yml index f06e5ad94..4723e5d3a 100644 --- a/.github/workflows/test-code-proglang_select.yml +++ b/.github/workflows/test-code-proglang_select.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-code-repo_level_ordering-kfp.yml b/.github/workflows/test-code-repo_level_ordering-kfp.yml index c741faaaa..57b39f313 100644 --- a/.github/workflows/test-code-repo_level_ordering-kfp.yml +++ b/.github/workflows/test-code-repo_level_ordering-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-code-repo_level_ordering.yml b/.github/workflows/test-code-repo_level_ordering.yml index fb05194d3..19ec8daf5 100644 --- a/.github/workflows/test-code-repo_level_ordering.yml +++ b/.github/workflows/test-code-repo_level_ordering.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-kfp-transform.template b/.github/workflows/test-kfp-transform.template index 1df901367..1003ba643 100644 --- a/.github/workflows/test-kfp-transform.template +++ b/.github/workflows/test-kfp-transform.template @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-kfp.yml b/.github/workflows/test-kfp.yml index f05ce0290..01deebcfa 100644 --- a/.github/workflows/test-kfp.yml +++ b/.github/workflows/test-kfp.yml @@ -42,6 +42,11 @@ on: - "!**/images/**" - "!**/.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + env: KFP_BLACK_LIST: "doc_chunk-ray,pdf2parquet-ray,pii_redactor" diff --git a/.github/workflows/test-language-doc_chunk.yml b/.github/workflows/test-language-doc_chunk.yml index bb49dedaf..ec78512e5 100644 --- a/.github/workflows/test-language-doc_chunk.yml +++ b/.github/workflows/test-language-doc_chunk.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-language-doc_quality-kfp.yml b/.github/workflows/test-language-doc_quality-kfp.yml index f9e299067..1c5237b20 100644 --- a/.github/workflows/test-language-doc_quality-kfp.yml +++ b/.github/workflows/test-language-doc_quality-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-language-doc_quality.yml b/.github/workflows/test-language-doc_quality.yml index bcca97ec5..443c22152 100644 --- a/.github/workflows/test-language-doc_quality.yml +++ b/.github/workflows/test-language-doc_quality.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-language-html2parquet.yml b/.github/workflows/test-language-html2parquet.yml index 0d74b5ed2..e5ef8e510 100644 --- a/.github/workflows/test-language-html2parquet.yml +++ b/.github/workflows/test-language-html2parquet.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-language-lang_id-kfp.yml b/.github/workflows/test-language-lang_id-kfp.yml index a1a80bb94..c6eb179b8 100644 --- a/.github/workflows/test-language-lang_id-kfp.yml +++ b/.github/workflows/test-language-lang_id-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-language-lang_id.yml b/.github/workflows/test-language-lang_id.yml index 5bd81023b..7c318a3a1 100644 --- a/.github/workflows/test-language-lang_id.yml +++ b/.github/workflows/test-language-lang_id.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-language-pdf2parquet.yml b/.github/workflows/test-language-pdf2parquet.yml index 85ad89ce9..fbdd81b8e 100644 --- a/.github/workflows/test-language-pdf2parquet.yml +++ b/.github/workflows/test-language-pdf2parquet.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-language-pii_redactor.yml b/.github/workflows/test-language-pii_redactor.yml index 6b0b6b795..5ecc80b08 100644 --- a/.github/workflows/test-language-pii_redactor.yml +++ b/.github/workflows/test-language-pii_redactor.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-language-text_encoder-kfp.yml b/.github/workflows/test-language-text_encoder-kfp.yml index 4ac236ce8..8e238dfcf 100644 --- a/.github/workflows/test-language-text_encoder-kfp.yml +++ b/.github/workflows/test-language-text_encoder-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-language-text_encoder.yml b/.github/workflows/test-language-text_encoder.yml index c84601340..d49c1193d 100644 --- a/.github/workflows/test-language-text_encoder.yml +++ b/.github/workflows/test-language-text_encoder.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-lib.yml b/.github/workflows/test-lib.yml index ed4893b5e..5a1cff872 100644 --- a/.github/workflows/test-lib.yml +++ b/.github/workflows/test-lib.yml @@ -25,6 +25,11 @@ on: - "!data-processing-lib/**/doc/**" - "!data-processing-lib/**/.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_images: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-transform.template b/.github/workflows/test-transform.template index 712a91490..f3907d56a 100644 --- a/.github/workflows/test-transform.template +++ b/.github/workflows/test-transform.template @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-universal-doc_id-kfp.yml b/.github/workflows/test-universal-doc_id-kfp.yml index e59589420..8ed1df919 100644 --- a/.github/workflows/test-universal-doc_id-kfp.yml +++ b/.github/workflows/test-universal-doc_id-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-universal-doc_id.yml b/.github/workflows/test-universal-doc_id.yml index 8dbb11fd4..d314f3b25 100644 --- a/.github/workflows/test-universal-doc_id.yml +++ b/.github/workflows/test-universal-doc_id.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-universal-ededup-kfp.yml b/.github/workflows/test-universal-ededup-kfp.yml index 09a67edc8..93408a260 100644 --- a/.github/workflows/test-universal-ededup-kfp.yml +++ b/.github/workflows/test-universal-ededup-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-universal-ededup.yml b/.github/workflows/test-universal-ededup.yml index aace9621d..8b4034570 100644 --- a/.github/workflows/test-universal-ededup.yml +++ b/.github/workflows/test-universal-ededup.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-universal-fdedup-kfp.yml b/.github/workflows/test-universal-fdedup-kfp.yml index 43da527f6..27dfc5493 100644 --- a/.github/workflows/test-universal-fdedup-kfp.yml +++ b/.github/workflows/test-universal-fdedup-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-universal-fdedup.yml b/.github/workflows/test-universal-fdedup.yml index b6eb684c8..5f68d4799 100644 --- a/.github/workflows/test-universal-fdedup.yml +++ b/.github/workflows/test-universal-fdedup.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-universal-filter-kfp.yml b/.github/workflows/test-universal-filter-kfp.yml index 3d0593713..bd2f57229 100644 --- a/.github/workflows/test-universal-filter-kfp.yml +++ b/.github/workflows/test-universal-filter-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-universal-filter.yml b/.github/workflows/test-universal-filter.yml index 7d78ac455..43e936166 100644 --- a/.github/workflows/test-universal-filter.yml +++ b/.github/workflows/test-universal-filter.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-universal-hap.yml b/.github/workflows/test-universal-hap.yml index 8512e1fb9..c845506c1 100644 --- a/.github/workflows/test-universal-hap.yml +++ b/.github/workflows/test-universal-hap.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-universal-noop-kfp.yml b/.github/workflows/test-universal-noop-kfp.yml index fa0786bcf..01b14e51b 100644 --- a/.github/workflows/test-universal-noop-kfp.yml +++ b/.github/workflows/test-universal-noop-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-universal-noop.yml b/.github/workflows/test-universal-noop.yml index 594c7a424..13e066d58 100644 --- a/.github/workflows/test-universal-noop.yml +++ b/.github/workflows/test-universal-noop.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-universal-profiler-kfp.yml b/.github/workflows/test-universal-profiler-kfp.yml index faf414235..e30f7bafa 100644 --- a/.github/workflows/test-universal-profiler-kfp.yml +++ b/.github/workflows/test-universal-profiler-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-universal-profiler.yml b/.github/workflows/test-universal-profiler.yml index c9db499bf..e018e0ed3 100644 --- a/.github/workflows/test-universal-profiler.yml +++ b/.github/workflows/test-universal-profiler.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-universal-resize-kfp.yml b/.github/workflows/test-universal-resize-kfp.yml index f55a82821..630de3c05 100644 --- a/.github/workflows/test-universal-resize-kfp.yml +++ b/.github/workflows/test-universal-resize-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-universal-resize.yml b/.github/workflows/test-universal-resize.yml index 408420a2b..b3399e5ec 100644 --- a/.github/workflows/test-universal-resize.yml +++ b/.github/workflows/test-universal-resize.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository diff --git a/.github/workflows/test-universal-tokenization-kfp.yml b/.github/workflows/test-universal-tokenization-kfp.yml index 20741386d..ff13a444c 100644 --- a/.github/workflows/test-universal-tokenization-kfp.yml +++ b/.github/workflows/test-universal-tokenization-kfp.yml @@ -36,6 +36,11 @@ on: - "!**/images/**" - "!**.gitignore" +# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: test-kfp-v1: runs-on: ubuntu-latest diff --git a/.github/workflows/test-universal-tokenization.yml b/.github/workflows/test-universal-tokenization.yml index 475019f00..ae547c396 100644 --- a/.github/workflows/test-universal-tokenization.yml +++ b/.github/workflows/test-universal-tokenization.yml @@ -40,6 +40,11 @@ on: - "!**/images/**" - "!**.gitignore" +# Taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: check_if_push_image: # check whether the Docker images should be pushed to the remote repository