-
Notifications
You must be signed in to change notification settings - Fork 122
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Signed-off-by: Constantin M Adam <cmadam@us.ibm.com>
- Loading branch information
Showing
761 changed files
with
13,177 additions
and
1,130 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| name: Build, Test, and Upload PyPI package of data-connector-lib | ||
|
|
||
| on: | ||
| release: | ||
| types: | ||
| - published | ||
| workflow_dispatch: | ||
|
|
||
| permissions: | ||
| contents: read | ||
| # see https://docs.pypi.org/trusted-publishers/ | ||
| id-token: write | ||
|
|
||
| jobs: | ||
| build-package: | ||
| name: Build data connector library | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| # for setuptools-scm | ||
| fetch-depth: 0 | ||
| - name: Build Packages for pypi | ||
| run: | | ||
| make -C data-connector-lib build | ||
| publish-test-pypi: | ||
| name: Publish packages to test.pypi.org | ||
| # disabled | ||
| if: false | ||
| runs-on: ubuntu-22.04 | ||
| needs: build-package | ||
|
|
||
| steps: | ||
| - name: Fetch build artifacts | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: Packages | ||
| path: dist | ||
|
|
||
| - name: Upload to Test PyPI | ||
| uses: pypa/gh-action-pypi-publish@release/v1 | ||
| with: | ||
| repository-url: https://test.pypi.org/legacy/ | ||
|
|
||
| publish-pypi: | ||
| name: Publish release to pypi.org | ||
| runs-on: ubuntu-22.04 | ||
| needs: build-package | ||
| # disabled as of now | ||
| if: false | ||
| steps: | ||
| - name: Fetch build artifacts | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: Packages | ||
| path: dist | ||
| - name: Upload to PyPI | ||
| uses: pypa/gh-action-pypi-publish@release/v1 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| # | ||
| # DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files | ||
| # | ||
| name: Test - transforms/code/code_profiler | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| push: | ||
| branches: | ||
| - "dev" | ||
| - "releases/**" | ||
| tags: | ||
| - "*" | ||
| paths: | ||
| - "transforms/code/code_profiler/**" | ||
| - "data-processing-lib/**" | ||
| - "!transforms/code/code_profiler/**/kfp_ray/**" # This is/will be tested in separate workflow | ||
| - "!data-processing-lib/**/test/**" | ||
| - "!data-processing-lib/**/test-data/**" | ||
| - "!**.md" | ||
| - "!**/doc/**" | ||
| - "!**/images/**" | ||
| - "!**.gitignore" | ||
| pull_request: | ||
| branches: | ||
| - "dev" | ||
| - "releases/**" | ||
| paths: | ||
| - "transforms/code/code_profiler/**" | ||
| - "data-processing-lib/**" | ||
| - "!transforms/code/code_profiler/**/kfp_ray/**" # This is/will be tested in separate workflow | ||
| - "!data-processing-lib/**/test/**" | ||
| - "!data-processing-lib/**/test-data/**" | ||
| - "!**.md" | ||
| - "!**/doc/**" | ||
| - "!**/images/**" | ||
| - "!**.gitignore" | ||
|
|
||
| jobs: | ||
| check_if_push_image: | ||
| # check whether the Docker images should be pushed to the remote repository | ||
| # The images are pushed if it is a merge to dev branch or a new tag is created. | ||
| # The latter being part of the release process. | ||
| # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. | ||
| runs-on: ubuntu-22.04 | ||
| outputs: | ||
| publish_images: ${{ steps.version.outputs.publish_images }} | ||
| steps: | ||
| - id: version | ||
| run: | | ||
| publish_images='false' | ||
| if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; | ||
| then | ||
| publish_images='true' | ||
| fi | ||
| if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; | ||
| then | ||
| publish_images='true' | ||
| fi | ||
| echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" | ||
| test-src: | ||
| runs-on: ubuntu-22.04 | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Free up space in github runner | ||
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | ||
| run: | | ||
| df -h | ||
| sudo rm -rf "/usr/local/share/boost" | ||
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | ||
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup | ||
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | ||
| df -h | ||
| - name: Test transform source in transforms/code/code_profiler | ||
| run: | | ||
| if [ -e "transforms/code/code_profiler/Makefile" ]; then | ||
| make -C transforms/code/code_profiler DOCKER=docker test-src | ||
| else | ||
| echo "transforms/code/code_profiler/Makefile not found - source testing disabled for this transform." | ||
| fi | ||
| test-image: | ||
| needs: [check_if_push_image] | ||
| runs-on: ubuntu-22.04 | ||
| timeout-minutes: 120 | ||
| env: | ||
| DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} | ||
| DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} | ||
| steps: | ||
| - name: Checkout | ||
| uses: actions/checkout@v4 | ||
| - name: Free up space in github runner | ||
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | ||
| run: | | ||
| df -h | ||
| sudo rm -rf /opt/ghc | ||
| sudo rm -rf "/usr/local/share/boost" | ||
| sudo rm -rf "$AGENT_TOOLSDIRECTORY" | ||
| sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup | ||
| sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true | ||
| df -h | ||
| - name: Test transform image in transforms/code/code_profiler | ||
| run: | | ||
| if [ -e "transforms/code/code_profiler/Makefile" ]; then | ||
| if [ -d "transforms/code/code_profiler/spark" ]; then | ||
| make -C data-processing-lib/spark DOCKER=docker image | ||
| fi | ||
| make -C transforms/code/code_profiler DOCKER=docker test-image | ||
| else | ||
| echo "transforms/code/code_profiler/Makefile not found - testing disabled for this transform." | ||
| fi | ||
| - name: Print space | ||
| # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | ||
| run: | | ||
| df -h | ||
| docker images | ||
| - name: Publish images | ||
| if: needs.check_if_push_image.outputs.publish_images == 'true' | ||
| run: | | ||
| if [ -e "transforms/code/code_profiler/Makefile" ]; then | ||
| make -C transforms/code/code_profiler publish | ||
| else | ||
| echo "transforms/code/code_profiler/Makefile not found - publishing disabled for this transform." | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.