Updated github actions and the ome-converter plugin (#509)

* feat: updated and fixed ome-converter build: bumped version 0.3.0 -> 0.3.1 on ome converter plugin build: updated ome-converter dep on bfio to 2.3.3 bumped version fix: fp usage fix: returning after preview feat: using preadator feat: using preadator fix: new pre-commit hooks ci: updated pre-commit config to check test files fix: fixed all tests * Bump version: 0.3.0 → 0.3.1-dev0 * ci: filter action now feetches from forks * ci: making test and release workflows callable * ci: testing base ref in git actions * ci: improved package-filter * ci: updated tests and release workflows * ci: fixed tests workflow * ci: fixed release workflow
PolusAI · Jan 29, 2024 · 4fcd563 · 4fcd563
1 parent ec63b2d
commit 4fcd563
Show file tree

Hide file tree

Showing 17 changed files with 375 additions and 278 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -15,7 +15,7 @@ on:
       DOCKER_USERNAME:
         description: 'Docker Hub username'
         required: true
-      DOCKER_PASSWORD:
+      DOCKER_TOKEN:
         description: 'Docker Hub password'
         required: true
 
@@ -44,7 +44,7 @@ jobs:
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
+          password: ${{ secrets.DOCKER_TOKEN }}
       - name: Check | Image exists
         run: |
           tag=${{ steps.docker_tag.outputs.tag }}

diff --git a/.github/workflows/package-filter.yml b/.github/workflows/package-filter.yml
@@ -2,6 +2,12 @@ name: Package Filter
 
 on:
   workflow_call:
+    inputs:
+      num-commits:
+        description: "The of commits to check for updated packages. If 0, the action will assume that it is running on a non-master branch and will check all commits on the current branch against the master branch. For any larger value, the action will check the last n commits for any updated packages."
+        required: true
+        default: 0
+        type: number
     outputs:
       matrix:
         description: "The directories containing the updated packages"
@@ -21,13 +27,34 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: ${{ github.head_ref }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          persist-credentials: false
       - name: Find Updated Packages
         id: package-filter
         run: |
           PACKAGE_DIRS=""
+          COMPANION_FILES="VERSION Dockerfile .bumpversion.cfg plugin.json"
 
-          for changed_file in $(git diff --name-only origin/${{ github.base_ref }}...)
+          # echo the base ref
+          base_ref=${{ github.base_ref }}
+          if [[ -z "$base_ref" ]]
+          then
+            base_ref="master"
+            echo "::warning::Action not running on PR, defaulting to base branch to master"
+          fi
+          echo "The base ref is $base_ref"
+
+          # Get the comparison point in the repo
+          if [[ ${{ inputs.num-commits }} == 0 ]]
+          then
+            comparison_point="origin/${base_ref}"
+          else
+            comparison_point="HEAD~${{ inputs.num-commits }}"
+          fi
+          echo "The comparison point is ${comparison_point}"
+
+          for changed_file in $(git diff --name-only ${comparison_point}...)
           do
             pkg_dir=$(dirname ${changed_file})
 
@@ -40,24 +67,29 @@ jobs:
             # Check if the changed file is a pyproject.toml file
             if [[ "$(basename ${changed_file})" == *"pyproject.toml"* ]]
             then
+              pkg_dir=$(dirname ${changed_file})
 
-              # Check that the package has a VERSION file
-              if [[ ! -f "$(dirname ${changed_file})/VERSION" ]]
-              then
-                echo "::error::$(dirname ${changed_file}) does not have a VERSION file" && exit 1
-              fi
+              # Check that the package has all the necessary companion files
+              for companion_file in $COMPANION_FILES
+              do
+                if [[ ! -f "$pkg_dir/$companion_file" ]]
+                then
+                  echo "::error::${pkg_dir} does not have a $companion_file file" && exit 1
+                fi
+              done
 
               # Check that the version is a dev version
-              if [[ "$(cat $(dirname ${changed_file})/VERSION)" != *"dev"* ]]
+              if [[ "$(cat ${pkg_dir}/VERSION)" != *"dev"* ]]
               then
-                echo "::error::$(dirname ${changed_file}) does not have a dev version" && exit 1
+                echo "::error::${pkg_dir} does not have a dev version" && exit 1
               fi
 
-              PACKAGE_DIRS="$PACKAGE_DIRS $(dirname ${changed_file})"
+              PACKAGE_DIRS="$PACKAGE_DIRS ${pkg_dir}"
             fi
           done
 
           # Check if any packages were found
+          echo "The updated packages are $PACKAGE_DIRS"
           if [[ -z "$PACKAGE_DIRS" ]]
           then
             echo "::error::No updated packages were found" && exit 1

diff --git a/.github/workflows/package-release.yml b/.github/workflows/package-release.yml
@@ -1,33 +1,80 @@
 name: Package Release
 
+env:
+  DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+  DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
+
 on:
-  push:
-    branches:
-      - master
-      - main
-      - dev
+  workflow_dispatch:
+    inputs:
+      num-commits:
+        description: "The of commits to check for updated packages. If 0, the action will check all commits on the branch. For any larger value, the action will check the last n commits for any updated packages."
+        required: true
+        default: 1
+        type: number
+      repo_name:
+        description: 'Name of the base repository. The user can ignore this input if the action is triggered from the base repository.'
+        required: true
+        type: string
+        default: 'image-tools'
+  workflow_call:
+    inputs:
+      repo_name:
+        description: 'Name of the base repository'
+        required: true
+        type: string
+      num-commits:
+        description: "The of commits to check for updated packages. If 0, the action will check all commits on the master branch. For any larger value, the action will check the last n commits for any updated packages."
+        required: true
+        default: 1
+        type: number
+    secrets:
+      DOCKER_USERNAME:
+        description: 'Docker Hub username'
+        required: true
+      DOCKER_TOKEN:
+        description: 'Docker Hub password'
+        required: true
 
 permissions:
   contents: write
 
 jobs:
   package-filter:
     name: Filter for updated package
-    if: github.repository == 'polusai/polus-plugins'
+    if: github.repository == 'polusai/${{ github.event.inputs.repo_name }}'
     uses: ./.github/workflows/package-filter.yml
+    with:
+      num-commits: ${{ fromJson(github.event.inputs.num-commits) }}
 
   package-release:
     name: Release "${{ matrix.package_name }}"
-    if: github.repository == 'polusai/polus-plugins'
+    if: github.repository == 'polusai/${{ github.event.inputs.repo_name }}'
     needs: package-filter
     strategy:
+      fail-fast: false
       matrix: ${{fromJson(needs.package-filter.outputs.matrix)}}
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.inputs.branch }}
+          persist-credentials: false
+      - name: Generate a token
+        id: generate_token
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.APP_PRIVATE_KEY }}
+      - name: Use the token
+        env:
+          GH_TOKEN: ${{ steps.generate_token.outputs.token }}
+        run: |
+          gh api octocat
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
       - name: Install bump2version
@@ -38,27 +85,28 @@ jobs:
         run: |
           cd "${{ matrix.package_dir }}"
           bump2version release --no-commit
-      - name: Commit and push all changed files
+      - name: Commit all changed files
         env:
-          CI_COMMIT_AUTHOR: Continuous Integration
+          CI_COMMIT_AUTHOR: polusai-auth-helper[bot]
+          CI_COMMIT_EMAIL: ${{ secrets.APP_ID }}+polusai-auth-helper[bot]@users.noreply.github.com
         run: |
           git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}"
-          git config --global user.email "username@users.noreply.github.com"
+          git config --global user.email "${{ env.CI_COMMIT_EMAIL }}"
           git commit -a -m "build: Bumped release version for ${{ matrix.package_name }}"
       - name: Push changes
         uses: ad-m/github-push-action@master
         with:
-          github_token: ${{ secrets.CI_TOKEN }}
-          branch: ${{ github.ref }}
+          force: true
+          github_token: ${{ steps.generate_token.outputs.token }}
 
   docker:
     name: Build Docker images
-    if: github.repository == 'polusai/polus-plugins'
+    if: github.repository == 'polusai/${{ github.event.inputs.repo_name }}'
     needs: [package-filter, package-release]
     uses: ./.github/workflows/docker.yml
     with:
       matrix: ${{ needs.package-filter.outputs.matrix }}
       push: true
     secrets:
       DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_PASSWORD: ${{ secrets.DOCKER_TOKEN }}
+      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -1,5 +1,9 @@
 name: Package tests
 
+env:
+  DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+  DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
+
 on:
   pull_request:
     branches:
@@ -11,19 +15,14 @@ on:
       - main
       - master
       - dev
-  # workflow_call:
-  #   secrets:
-  #     DOCKER_USERNAME:
-  #       description: 'Docker Hub username'
-  #       required: true
-  #     DOCKER_TOKEN:
-  #       description: 'Docker Hub password'
-  #       required: true
-  #       description: 'Docker Hub username'
-  #       required: true
-  #     DOCKER_TOKEN:
-  #       description: 'Docker Hub password'
-  #       required: true
+  workflow_call:
+    secrets:
+      DOCKER_USERNAME:
+        description: 'Docker Hub username'
+        required: true
+      DOCKER_TOKEN:
+        description: 'Docker Hub password'
+        required: true
 
 permissions:
   contents: read
@@ -32,6 +31,8 @@ jobs:
   package-filter:
     name: Filter for updated package
     uses: ./.github/workflows/package-filter.yml
+    with:
+      num-commits: 0
 
   tests:
     name: Test "${{ matrix.package_name }}"
@@ -69,51 +70,13 @@ jobs:
           poetry install
           poetry run pytest -v
 
-  # docker:
-  #   name: Build Docker images
-  #   needs: [package-filter, tests]
-  #   uses: ./.github/workflows/docker.yml
-  #   with:
-  #     matrix: ${{ needs.package-filter.outputs.matrix }}
-  #     push: true
-  #   secrets:
-  #     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-  #     DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
-
   docker:
-    name: Docker "${{ matrix.package_name }}"
+    name: Build Docker images
     needs: [package-filter, tests]
-    strategy:
-      fail-fast: false
-      matrix: ${{fromJson(needs.package-filter.outputs.matrix)}}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Get | Docker Tag
-        id: docker_tag
-        run: |
-          package_dir="${{ matrix.package_dir }}"
-          version=$(cat ${package_dir}/VERSION)
-          tag=polusai/${{ matrix.package_name }}:${version}
-          echo "tag=${tag}" >> $GITHUB_OUTPUT
-      - name: Setup | Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Login | DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      - name: Check | Image exists
-        run: |
-          tag=${{ steps.docker_tag.outputs.tag }}
-          docker pull ${tag} > /dev/null \
-              && $(echo "::error::${tag} already exists on DockerHub" && exit 1) \
-              || echo "success"
-      - name: Publish | Docker Image
-        uses: docker/build-push-action@v5
-        with:
-          context: "{{defaultContext}}:${{ matrix.package_dir }}"
-          platforms: linux/amd64,linux/arm64
-          push: false
-          tags: ${{ steps.docker_tag.outputs.tag }}
+    uses: ./.github/workflows/docker.yml
+    with:
+      matrix: ${{ needs.package-filter.outputs.matrix }}
+      push: false
+    secrets:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -41,20 +41,12 @@ repos:
     rev: 'v0.0.274'
     hooks:
       - id: ruff
-        exclude: |
-          (?x)(
-            test_[a-zA-Z0-9]+.py$|
-            ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$
-          )
+        exclude: ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$
         args: [--fix]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: 'v1.4.0'
     hooks:
       - id: mypy
-        exclude: |
-          (?x)(
-            test_[a-zA-Z0-9]+.py$|
-            ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$
-          )
+        exclude: ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$
         additional_dependencies: [types-requests==2.31.0.1]
diff --git a/formats/ome-converter-plugin/.bumpversion.cfg b/formats/ome-converter-plugin/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.0
+current_version = 0.3.1-dev0
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<dev>\d+))?
@@ -24,4 +24,6 @@ replace = version = "{new_version}"
 
 [bumpversion:file:VERSION]
 
+[bumpversion:file:README.md]
+
 [bumpversion:file:src/polus/plugins/formats/ome_converter/__init__.py]
diff --git a/formats/ome-converter-plugin/Dockerfile b/formats/ome-converter-plugin/Dockerfile
@@ -1,4 +1,4 @@
-FROM polusai/bfio:2.1.9
+FROM polusai/bfio:2.3.3
 
 # environment variables defined in polusai/bfio
 ENV EXEC_DIR="/opt/executables"
@@ -12,7 +12,6 @@ WORKDIR ${EXEC_DIR}
 COPY pyproject.toml ${EXEC_DIR}
 COPY VERSION ${EXEC_DIR}
 COPY README.md ${EXEC_DIR}
-RUN pip3 install --index-url https://test.pypi.org/simple/ filepattern==2.2.7
 COPY src ${EXEC_DIR}/src
 
 RUN pip3 install ${EXEC_DIR} --no-cache-dir