From 4fcd5639b34de253756f51b224a57862bb02a93c Mon Sep 17 00:00:00 2001 From: Najib <32004868+nishaq503@users.noreply.github.com> Date: Mon, 29 Jan 2024 15:32:35 -0500 Subject: [PATCH] Updated github actions and the ome-converter plugin (#509) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: updated and fixed ome-converter build: bumped version 0.3.0 -> 0.3.1 on ome converter plugin build: updated ome-converter dep on bfio to 2.3.3 bumped version fix: fp usage fix: returning after preview feat: using preadator feat: using preadator fix: new pre-commit hooks ci: updated pre-commit config to check test files fix: fixed all tests * Bump version: 0.3.0 → 0.3.1-dev0 * ci: filter action now feetches from forks * ci: making test and release workflows callable * ci: testing base ref in git actions * ci: improved package-filter * ci: updated tests and release workflows * ci: fixed tests workflow * ci: fixed release workflow --- .github/workflows/docker.yml | 4 +- .github/workflows/package-filter.yml | 52 ++++++-- .github/workflows/package-release.yml | 78 ++++++++--- .github/workflows/tests.yml | 81 +++--------- .pre-commit-config.yaml | 12 +- formats/ome-converter-plugin/.bumpversion.cfg | 4 +- formats/ome-converter-plugin/Dockerfile | 3 +- formats/ome-converter-plugin/README.md | 4 +- formats/ome-converter-plugin/VERSION | 2 +- formats/ome-converter-plugin/plugin.json | 4 +- formats/ome-converter-plugin/pyproject.toml | 29 ++-- .../plugins/formats/ome_converter/__init__.py | 11 +- .../plugins/formats/ome_converter/__main__.py | 68 ++++++---- .../formats/ome_converter/image_converter.py | 69 ++++++---- .../ome-converter-plugin/tests/__init__.py | 2 +- .../ome-converter-plugin/tests/conftest.py | 105 +++++++++++++++ .../ome-converter-plugin/tests/test_main.py | 125 ++++-------------- 17 files changed, 375 insertions(+), 278 deletions(-) create mode 100644 formats/ome-converter-plugin/tests/conftest.py diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 503ca0501..cf9bfa35f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -15,7 +15,7 @@ on: DOCKER_USERNAME: description: 'Docker Hub username' required: true - DOCKER_PASSWORD: + DOCKER_TOKEN: description: 'Docker Hub password' required: true @@ -44,7 +44,7 @@ jobs: uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + password: ${{ secrets.DOCKER_TOKEN }} - name: Check | Image exists run: | tag=${{ steps.docker_tag.outputs.tag }} diff --git a/.github/workflows/package-filter.yml b/.github/workflows/package-filter.yml index ee43ba18f..427bc9b8e 100644 --- a/.github/workflows/package-filter.yml +++ b/.github/workflows/package-filter.yml @@ -2,6 +2,12 @@ name: Package Filter on: workflow_call: + inputs: + num-commits: + description: "The of commits to check for updated packages. If 0, the action will assume that it is running on a non-master branch and will check all commits on the current branch against the master branch. For any larger value, the action will check the last n commits for any updated packages." + required: true + default: 0 + type: number outputs: matrix: description: "The directories containing the updated packages" @@ -21,13 +27,34 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ github.head_ref }} + ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + persist-credentials: false - name: Find Updated Packages id: package-filter run: | PACKAGE_DIRS="" + COMPANION_FILES="VERSION Dockerfile .bumpversion.cfg plugin.json" - for changed_file in $(git diff --name-only origin/${{ github.base_ref }}...) + # echo the base ref + base_ref=${{ github.base_ref }} + if [[ -z "$base_ref" ]] + then + base_ref="master" + echo "::warning::Action not running on PR, defaulting to base branch to master" + fi + echo "The base ref is $base_ref" + + # Get the comparison point in the repo + if [[ ${{ inputs.num-commits }} == 0 ]] + then + comparison_point="origin/${base_ref}" + else + comparison_point="HEAD~${{ inputs.num-commits }}" + fi + echo "The comparison point is ${comparison_point}" + + for changed_file in $(git diff --name-only ${comparison_point}...) do pkg_dir=$(dirname ${changed_file}) @@ -40,24 +67,29 @@ jobs: # Check if the changed file is a pyproject.toml file if [[ "$(basename ${changed_file})" == *"pyproject.toml"* ]] then + pkg_dir=$(dirname ${changed_file}) - # Check that the package has a VERSION file - if [[ ! -f "$(dirname ${changed_file})/VERSION" ]] - then - echo "::error::$(dirname ${changed_file}) does not have a VERSION file" && exit 1 - fi + # Check that the package has all the necessary companion files + for companion_file in $COMPANION_FILES + do + if [[ ! -f "$pkg_dir/$companion_file" ]] + then + echo "::error::${pkg_dir} does not have a $companion_file file" && exit 1 + fi + done # Check that the version is a dev version - if [[ "$(cat $(dirname ${changed_file})/VERSION)" != *"dev"* ]] + if [[ "$(cat ${pkg_dir}/VERSION)" != *"dev"* ]] then - echo "::error::$(dirname ${changed_file}) does not have a dev version" && exit 1 + echo "::error::${pkg_dir} does not have a dev version" && exit 1 fi - PACKAGE_DIRS="$PACKAGE_DIRS $(dirname ${changed_file})" + PACKAGE_DIRS="$PACKAGE_DIRS ${pkg_dir}" fi done # Check if any packages were found + echo "The updated packages are $PACKAGE_DIRS" if [[ -z "$PACKAGE_DIRS" ]] then echo "::error::No updated packages were found" && exit 1 diff --git a/.github/workflows/package-release.yml b/.github/workflows/package-release.yml index c2be4f984..9ee5bc802 100644 --- a/.github/workflows/package-release.yml +++ b/.github/workflows/package-release.yml @@ -1,11 +1,40 @@ name: Package Release +env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} + on: - push: - branches: - - master - - main - - dev + workflow_dispatch: + inputs: + num-commits: + description: "The of commits to check for updated packages. If 0, the action will check all commits on the branch. For any larger value, the action will check the last n commits for any updated packages." + required: true + default: 1 + type: number + repo_name: + description: 'Name of the base repository. The user can ignore this input if the action is triggered from the base repository.' + required: true + type: string + default: 'image-tools' + workflow_call: + inputs: + repo_name: + description: 'Name of the base repository' + required: true + type: string + num-commits: + description: "The of commits to check for updated packages. If 0, the action will check all commits on the master branch. For any larger value, the action will check the last n commits for any updated packages." + required: true + default: 1 + type: number + secrets: + DOCKER_USERNAME: + description: 'Docker Hub username' + required: true + DOCKER_TOKEN: + description: 'Docker Hub password' + required: true permissions: contents: write @@ -13,21 +42,39 @@ permissions: jobs: package-filter: name: Filter for updated package - if: github.repository == 'polusai/polus-plugins' + if: github.repository == 'polusai/${{ github.event.inputs.repo_name }}' uses: ./.github/workflows/package-filter.yml + with: + num-commits: ${{ fromJson(github.event.inputs.num-commits) }} package-release: name: Release "${{ matrix.package_name }}" - if: github.repository == 'polusai/polus-plugins' + if: github.repository == 'polusai/${{ github.event.inputs.repo_name }}' needs: package-filter strategy: + fail-fast: false matrix: ${{fromJson(needs.package-filter.outputs.matrix)}} runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{ github.event.inputs.branch }} + persist-credentials: false + - name: Generate a token + id: generate_token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + - name: Use the token + env: + GH_TOKEN: ${{ steps.generate_token.outputs.token }} + run: | + gh api octocat - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.9' - name: Install bump2version @@ -38,22 +85,23 @@ jobs: run: | cd "${{ matrix.package_dir }}" bump2version release --no-commit - - name: Commit and push all changed files + - name: Commit all changed files env: - CI_COMMIT_AUTHOR: Continuous Integration + CI_COMMIT_AUTHOR: polusai-auth-helper[bot] + CI_COMMIT_EMAIL: ${{ secrets.APP_ID }}+polusai-auth-helper[bot]@users.noreply.github.com run: | git config --global user.name "${{ env.CI_COMMIT_AUTHOR }}" - git config --global user.email "username@users.noreply.github.com" + git config --global user.email "${{ env.CI_COMMIT_EMAIL }}" git commit -a -m "build: Bumped release version for ${{ matrix.package_name }}" - name: Push changes uses: ad-m/github-push-action@master with: - github_token: ${{ secrets.CI_TOKEN }} - branch: ${{ github.ref }} + force: true + github_token: ${{ steps.generate_token.outputs.token }} docker: name: Build Docker images - if: github.repository == 'polusai/polus-plugins' + if: github.repository == 'polusai/${{ github.event.inputs.repo_name }}' needs: [package-filter, package-release] uses: ./.github/workflows/docker.yml with: @@ -61,4 +109,4 @@ jobs: push: true secrets: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_TOKEN }} + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 25acc72a5..d0505ba28 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,5 +1,9 @@ name: Package tests +env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} + on: pull_request: branches: @@ -11,19 +15,14 @@ on: - main - master - dev - # workflow_call: - # secrets: - # DOCKER_USERNAME: - # description: 'Docker Hub username' - # required: true - # DOCKER_TOKEN: - # description: 'Docker Hub password' - # required: true - # description: 'Docker Hub username' - # required: true - # DOCKER_TOKEN: - # description: 'Docker Hub password' - # required: true + workflow_call: + secrets: + DOCKER_USERNAME: + description: 'Docker Hub username' + required: true + DOCKER_TOKEN: + description: 'Docker Hub password' + required: true permissions: contents: read @@ -32,6 +31,8 @@ jobs: package-filter: name: Filter for updated package uses: ./.github/workflows/package-filter.yml + with: + num-commits: 0 tests: name: Test "${{ matrix.package_name }}" @@ -69,51 +70,13 @@ jobs: poetry install poetry run pytest -v - # docker: - # name: Build Docker images - # needs: [package-filter, tests] - # uses: ./.github/workflows/docker.yml - # with: - # matrix: ${{ needs.package-filter.outputs.matrix }} - # push: true - # secrets: - # DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - # DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} - docker: - name: Docker "${{ matrix.package_name }}" + name: Build Docker images needs: [package-filter, tests] - strategy: - fail-fast: false - matrix: ${{fromJson(needs.package-filter.outputs.matrix)}} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Get | Docker Tag - id: docker_tag - run: | - package_dir="${{ matrix.package_dir }}" - version=$(cat ${package_dir}/VERSION) - tag=polusai/${{ matrix.package_name }}:${version} - echo "tag=${tag}" >> $GITHUB_OUTPUT - - name: Setup | Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login | DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Check | Image exists - run: | - tag=${{ steps.docker_tag.outputs.tag }} - docker pull ${tag} > /dev/null \ - && $(echo "::error::${tag} already exists on DockerHub" && exit 1) \ - || echo "success" - - name: Publish | Docker Image - uses: docker/build-push-action@v5 - with: - context: "{{defaultContext}}:${{ matrix.package_dir }}" - platforms: linux/amd64,linux/arm64 - push: false - tags: ${{ steps.docker_tag.outputs.tag }} + uses: ./.github/workflows/docker.yml + with: + matrix: ${{ needs.package-filter.outputs.matrix }} + push: false + secrets: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b27f4d530..652e5375b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,20 +41,12 @@ repos: rev: 'v0.0.274' hooks: - id: ruff - exclude: | - (?x)( - test_[a-zA-Z0-9]+.py$| - ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$ - ) + exclude: ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$ args: [--fix] - repo: https://github.com/pre-commit/mirrors-mypy rev: 'v1.4.0' hooks: - id: mypy - exclude: | - (?x)( - test_[a-zA-Z0-9]+.py$| - ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$ - ) + exclude: ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$ additional_dependencies: [types-requests==2.31.0.1] diff --git a/formats/ome-converter-plugin/.bumpversion.cfg b/formats/ome-converter-plugin/.bumpversion.cfg index 167ea3ee2..d4c76c4c3 100644 --- a/formats/ome-converter-plugin/.bumpversion.cfg +++ b/formats/ome-converter-plugin/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.0 +current_version = 0.3.1-dev0 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? @@ -24,4 +24,6 @@ replace = version = "{new_version}" [bumpversion:file:VERSION] +[bumpversion:file:README.md] + [bumpversion:file:src/polus/plugins/formats/ome_converter/__init__.py] diff --git a/formats/ome-converter-plugin/Dockerfile b/formats/ome-converter-plugin/Dockerfile index 26f1e6bd6..344e17193 100644 --- a/formats/ome-converter-plugin/Dockerfile +++ b/formats/ome-converter-plugin/Dockerfile @@ -1,4 +1,4 @@ -FROM polusai/bfio:2.1.9 +FROM polusai/bfio:2.3.3 # environment variables defined in polusai/bfio ENV EXEC_DIR="/opt/executables" @@ -12,7 +12,6 @@ WORKDIR ${EXEC_DIR} COPY pyproject.toml ${EXEC_DIR} COPY VERSION ${EXEC_DIR} COPY README.md ${EXEC_DIR} -RUN pip3 install --index-url https://test.pypi.org/simple/ filepattern==2.2.7 COPY src ${EXEC_DIR}/src RUN pip3 install ${EXEC_DIR} --no-cache-dir diff --git a/formats/ome-converter-plugin/README.md b/formats/ome-converter-plugin/README.md index b23cd8871..59be5e909 100644 --- a/formats/ome-converter-plugin/README.md +++ b/formats/ome-converter-plugin/README.md @@ -1,4 +1,4 @@ -# OME Converter (0.3.0) +# OME Converter (v0.3.1-dev0) This WIPP plugin converts BioFormats supported data types to the OME Zarr or OME TIF file format. This is not a complete implementation, rather it implements a file @@ -31,4 +31,4 @@ This plugin takes 3 input arguments and 1 output argument: | `--filePattern` | A filepattern, used to select data for conversion | Input | string | | `--fileExtension`| A desired file format for conversion | Input | enum | | `--outDir` | Output collection | Output | genericData | -| `--preview` | Generate a JSON file with outputs | Output | JSON | \ No newline at end of file +| `--preview` | Generate a JSON file with outputs | Output | JSON | diff --git a/formats/ome-converter-plugin/VERSION b/formats/ome-converter-plugin/VERSION index 0d91a54c7..999d0eb66 100644 --- a/formats/ome-converter-plugin/VERSION +++ b/formats/ome-converter-plugin/VERSION @@ -1 +1 @@ -0.3.0 +0.3.1-dev0 diff --git a/formats/ome-converter-plugin/plugin.json b/formats/ome-converter-plugin/plugin.json index 7613dcf3c..02bc92f91 100644 --- a/formats/ome-converter-plugin/plugin.json +++ b/formats/ome-converter-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "OME Converter", - "version": "0.3.0", + "version": "0.3.1-dev0", "title": "OME Converter", "description": "Convert Bioformats supported format to OME Zarr or OME TIF", "author": "Nick Schaub (nick.schaub@nih.gov), Hamdah Shafqat Abbasi (hamdahshafqat.abbasi@nih.gov)", @@ -8,7 +8,7 @@ "repository": "https://github.com/PolusAI/polus-plugins", "website": "https://ncats.nih.gov/preclinical/core/informatics", "citation": "", - "containerId": "polusai/ome-converter-plugin:0.3.0", + "containerId": "polusai/ome-converter-plugin:0.3.1-dev0", "baseCommand": [ "python3", "-m", diff --git a/formats/ome-converter-plugin/pyproject.toml b/formats/ome-converter-plugin/pyproject.toml index 5c702859e..19d89ccf2 100644 --- a/formats/ome-converter-plugin/pyproject.toml +++ b/formats/ome-converter-plugin/pyproject.toml @@ -1,32 +1,26 @@ [tool.poetry] name = "polus-plugins-formats-ome-converter" -version = "0.3.0" +version = "0.3.1-dev0" description = "Convert BioFormats datatypes to ome.tif or ome.zarr file format" authors = [ - "Nick Schaub ", - "Hamdah Shafqat abbasi " - ] + "Nick Schaub ", + "Hamdah Shafqat abbasi ", + "Najib Ishaq " +] readme = "README.md" packages = [{include = "polus", from = "src"}] -[[tool.poetry.source]] -name = "test" -url = "https://test.pypi.org/simple/" -secondary = true - [tool.poetry.dependencies] -python = "^3.9" -bfio = {version = "2.1.9", extras = ["all"]} -filepattern = "^2.2.7" +python = ">=3.9,<3.12" +bfio = {version = "^2.3.3", extras = ["all"]} +filepattern = "^2.0.4" typer = "^0.7.0" tqdm = "^4.64.1" +preadator = "0.4.0-dev2" [tool.poetry.group.dev.dependencies] bump2version = "^1.0.1" pre-commit = "^3.0.4" -black = "^23.1.0" -flake8 = "^6.0.0" -mypy = "^1.0.0" pytest = "^7.2.1" ipykernel = "^6.21.2" requests = "^2.28.2" @@ -35,3 +29,8 @@ scikit-image = "^0.19.3" [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +pythonpath = [ + "." +] diff --git a/formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/__init__.py b/formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/__init__.py index 69cf13ec0..4a64587db 100644 --- a/formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/__init__.py +++ b/formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/__init__.py @@ -1,9 +1,6 @@ """Ome Converter.""" -__version__ = "0.3.0" -from polus.plugins.formats.ome_converter.image_converter import ( # noqa - batch_convert as batch_convert, -) -from polus.plugins.formats.ome_converter.image_converter import ( # noqa - convert_image as convert_image, -) +__version__ = "0.3.1-dev0" + +from polus.plugins.formats.ome_converter.image_converter import batch_convert +from polus.plugins.formats.ome_converter.image_converter import convert_image diff --git a/formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/__main__.py b/formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/__main__.py index a2fdca9bd..506cc2517 100644 --- a/formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/__main__.py +++ b/formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/__main__.py @@ -1,21 +1,20 @@ """Ome Converter.""" import json -import os import logging +import os import pathlib -from concurrent.futures import ProcessPoolExecutor, as_completed -from typing import Any, Optional +from concurrent.futures import as_completed +from typing import Any +from typing import Optional import filepattern as fp +import preadator import typer +from polus.plugins.formats.ome_converter.image_converter import NUM_THREADS +from polus.plugins.formats.ome_converter.image_converter import Extension +from polus.plugins.formats.ome_converter.image_converter import convert_image from tqdm import tqdm -from polus.plugins.formats.ome_converter.image_converter import ( - NUM_THREADS, - Extension, - convert_image, -) - app = typer.Typer() # Initialize the logger @@ -33,51 +32,68 @@ def main( ..., "--inpDir", help="Input generic data collection to be processed by this plugin", + exists=True, + resolve_path=True, + readable=True, + file_okay=False, + dir_okay=True, ), pattern: str = typer.Option( - ".+", "--filePattern", help="A filepattern defining the images to be converted" + ".+", + "--filePattern", + help="A filepattern defining the images to be converted", ), file_extension: Extension = typer.Option( - None, "--fileExtension", help="Type of data conversion" + Extension, + "--fileExtension", + help="Type of data conversion", + ), + out_dir: pathlib.Path = typer.Option( + ..., + "--outDir", + help="Output collection", + exists=True, + resolve_path=True, + writable=True, + file_okay=False, + dir_okay=True, ), - out_dir: pathlib.Path = typer.Option(..., "--outDir", help="Output collection"), preview: Optional[bool] = typer.Option( - False, "--preview", help="Output a JSON preview of files" + False, + "--preview", + help="Output a JSON preview of files", ), ) -> None: - """Convert bioformat supported image datatypes conversion to ome.tif or ome.zarr file format.""" + """Convert bioformat supported image datatypes conversion to ome.tif or ome.zarr.""" logger.info(f"inpDir = {inp_dir}") logger.info(f"outDir = {out_dir}") logger.info(f"filePattern = {pattern}") logger.info(f"fileExtension = {file_extension}") - inp_dir = inp_dir.resolve() - out_dir = out_dir.resolve() - - assert inp_dir.exists(), f"{inp_dir} does not exist!! Please check input path again" - assert ( - out_dir.exists() - ), f"{out_dir} does not exist!! Please check output path again" - fps = fp.FilePattern(inp_dir, pattern) if preview: - with open(pathlib.Path(out_dir, "preview.json"), "w") as jfile: + with out_dir.joinpath("preview.json").open("w") as jfile: out_json: dict[str, Any] = { "filepattern": pattern, "outDir": [], } - for file in fps: + for file in fps(): out_name = str(file[1][0].name.split(".")[0]) + file_extension out_json["outDir"].append(out_name) json.dump(out_json, jfile, indent=2) + return - with ProcessPoolExecutor(max_workers=NUM_THREADS) as executor: + with preadator.ProcessManager( + name="ome_converter", + num_processes=NUM_THREADS, + threads_per_process=2, + ) as executor: threads = [] for files in fps(): file = files[1][0] threads.append( - executor.submit(convert_image, file, file_extension, out_dir) + executor.submit_process(convert_image, file, file_extension, out_dir), ) for f in tqdm( diff --git a/formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/image_converter.py b/formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/image_converter.py index cdad71285..8bc5f174a 100644 --- a/formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/image_converter.py +++ b/formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/image_converter.py @@ -3,13 +3,15 @@ import logging import os import pathlib -from concurrent.futures import ProcessPoolExecutor, as_completed +from concurrent.futures import as_completed from multiprocessing import cpu_count from sys import platform from typing import Optional import filepattern as fp -from bfio import BioReader, BioWriter +import preadator +from bfio import BioReader +from bfio import BioWriter from tqdm import tqdm logger = logging.getLogger(__name__) @@ -19,9 +21,9 @@ if platform == "linux" or platform == "linux2": - NUM_THREADS = len(os.sched_getaffinity(0)) # type: ignore + NUM_THREADS = len(os.sched_getaffinity(0)) // 2 # type: ignore else: - NUM_THREADS = max(cpu_count() // 2, 2) + NUM_THREADS = max(cpu_count() // 2, 1) POLUS_IMG_EXT = os.environ.get("POLUS_IMG_EXT", ".ome.tif") @@ -35,7 +37,9 @@ class Extension(str, enum.Enum): def convert_image( - inp_image: pathlib.Path, file_extension: Extension, out_dir: pathlib.Path + inp_image: pathlib.Path, + file_extension: Extension, + out_dir: pathlib.Path, ) -> None: """Convert bioformats supported datatypes to ome.tif or ome.zarr file format. @@ -44,56 +48,70 @@ def convert_image( file_extension: Type of data conversion. out_dir: Path to output directory. """ - with BioReader(inp_image) as br: + with BioReader(inp_image, max_workers=2) as br: # Loop through timepoints for t in range(br.T): # Loop through channels for c in range(br.C): extension = "".join( - [suffix for suffix in inp_image.suffixes[-2:] if len(suffix) < 6] + [ + suffix + for suffix in inp_image.suffixes[-2:] + if len(suffix) < 6 # noqa: PLR2004 + ], ) out_path = out_dir.joinpath( - inp_image.name.replace(extension, file_extension) + inp_image.name.replace(extension, file_extension), ) if br.C > 1: out_path = out_dir.joinpath( - out_path.name.replace(file_extension, f"_c{c}" + file_extension) + out_path.name.replace( + file_extension, + f"_c{c}" + file_extension, + ), ) if br.T > 1: out_path = out_dir.joinpath( - out_path.name.replace(file_extension, f"_t{t}" + file_extension) + out_path.name.replace( + file_extension, + f"_t{t}" + file_extension, + ), ) with BioWriter( out_path, - max_workers=NUM_THREADS, + max_workers=2, metadata=br.metadata, ) as bw: bw.C = 1 bw.T = 1 # Handling of parsing channels when channels names are not provided. - if not bw.channel_names == [None]: + if bw.channel_names != [None]: bw.channel_names = [br.channel_names[c]] - # Loop through z-slices for z in range(br.Z): # Loop across the length of the image for y in range(0, br.Y, TILE_SIZE): y_max = min([br.Y, y + TILE_SIZE]) - bw.max_workers = NUM_THREADS - br.max_workers = NUM_THREADS - # Loop across the depth of the image for x in range(0, br.X, TILE_SIZE): x_max = min([br.X, x + TILE_SIZE]) bw[ - y:y_max, x:x_max, z : z + 1, 0, 0 # noqa: E203 + y:y_max, + x:x_max, + z : z + 1, + 0, + 0, ] = br[ - y:y_max, x:x_max, z : z + 1, c, t # noqa: E203 + y:y_max, + x:x_max, + z : z + 1, + c, + t, ] @@ -103,7 +121,7 @@ def batch_convert( file_pattern: Optional[str], file_extension: Extension, ) -> None: - """Convert bioformats supported datatypes in batches to ome.tif or ome.zarr file format. + """Convert bioformats supported datatypes in batches to ome.tif or ome.zarr. Args: inp_dir: Path of an input directory. @@ -116,21 +134,20 @@ def batch_convert( logger.info(f"file_pattern = {file_pattern}") logger.info(f"file_extension = {file_extension}") - assert inp_dir.exists(), f"{inp_dir} does not exist!! Please check input path again" - assert ( - out_dir.exists() - ), f"{out_dir} does not exist!! Please check output path again" - file_pattern = ".+" if file_pattern is None else file_pattern fps = fp.FilePattern(inp_dir, file_pattern) - with ProcessPoolExecutor(max_workers=NUM_THREADS) as executor: + with preadator.ProcessManager( + name="ome_converter", + num_processes=NUM_THREADS, + threads_per_process=2, + ) as executor: threads = [] for files in fps(): file = files[1][0] threads.append( - executor.submit(convert_image, file, file_extension, out_dir) + executor.submit(convert_image, file, file_extension, out_dir), ) for f in tqdm( diff --git a/formats/ome-converter-plugin/tests/__init__.py b/formats/ome-converter-plugin/tests/__init__.py index 6d914b30c..c8bf6d98c 100644 --- a/formats/ome-converter-plugin/tests/__init__.py +++ b/formats/ome-converter-plugin/tests/__init__.py @@ -1 +1 @@ -"""ome converter plugin.""" +"""Tests for ome converter plugin.""" diff --git a/formats/ome-converter-plugin/tests/conftest.py b/formats/ome-converter-plugin/tests/conftest.py new file mode 100644 index 000000000..4b1931a57 --- /dev/null +++ b/formats/ome-converter-plugin/tests/conftest.py @@ -0,0 +1,105 @@ +"""Pytest configuration file for the OME-Converter plugin tests.""" + + +import pathlib +import shutil +import tempfile +import typing + +import numpy +import pytest +import requests +import skimage.data +import skimage.io +import skimage.measure + + +def pytest_addoption(parser: pytest.Parser) -> None: + """Add options to pytest.""" + parser.addoption( + "--downloads", + action="store_true", + dest="downloads", + default=False, + help="run tests that download large data files", + ) + + +@pytest.fixture(params=[".ome.tif", ".ome.zarr"]) +def file_extension(request) -> str: # noqa: ANN001 + """To get the parameter of the fixture.""" + return request.param + + +@pytest.fixture( + params=[ + (512, ".png"), + (512, ".tif"), + (2048, ".png"), + (2048, ".tif"), + ], +) +def get_params(request) -> tuple[int, str]: # noqa: ANN001 + """To get the parameter of the fixture.""" + return request.param + + +@pytest.fixture() +def synthetic_images( + get_params: tuple[int, str], +) -> typing.Generator[tuple[list[numpy.ndarray], pathlib.Path], None, None]: + """Generate random synthetic images.""" + size, extension = get_params + + syn_dir = pathlib.Path(tempfile.mkdtemp(suffix="_syn_data")) + images: list[numpy.ndarray] = [] + for i in range(10): + # Create images + blobs: numpy.ndarray = skimage.data.binary_blobs( + length=size, + volume_fraction=0.05, + blob_size_fraction=0.05, + ) + syn_img: numpy.ndarray = skimage.measure.label(blobs) + outname = f"syn_image_{i}{extension}" + + # Save image + out_path = pathlib.Path(syn_dir, outname) + skimage.io.imsave(out_path, syn_img) + images.append(syn_img) + + yield images, syn_dir + + shutil.rmtree(syn_dir) + + +@pytest.fixture() +def output_directory() -> typing.Generator[pathlib.Path, None, None]: + """Generate random synthetic images.""" + out_dir = pathlib.Path(tempfile.mkdtemp(suffix="_out_dir")) + yield out_dir + shutil.rmtree(out_dir) + + +@pytest.fixture() +def download_images() -> typing.Generator[pathlib.Path, None, None]: + """Download test.""" + imagelist = { + ("0.tif", "https://osf.io/j6aer/download/"), + ( + "cameraman.png", + "https://people.math.sc.edu/Burkardt/data/tif/cameraman.png", + ), + ("venus1.png", "https://people.math.sc.edu/Burkardt/data/tif/venus1.png"), + } + inp_dir = pathlib.Path(tempfile.mkdtemp(suffix="_inp_dir")) + for image in imagelist: + file, url = image + outfile = pathlib.Path(inp_dir, file) + + r = requests.get(url, timeout=60) + with outfile.open("wb") as fw: + fw.write(r.content) + + yield outfile + shutil.rmtree(inp_dir) diff --git a/formats/ome-converter-plugin/tests/test_main.py b/formats/ome-converter-plugin/tests/test_main.py index 607fee1bf..b6b22c930 100644 --- a/formats/ome-converter-plugin/tests/test_main.py +++ b/formats/ome-converter-plugin/tests/test_main.py @@ -1,83 +1,23 @@ """Testing of Ome Converter.""" + import pathlib -import shutil -import tempfile -from collections.abc import Generator -from typing import Any, List, Tuple import numpy as np import pytest -import requests -import skimage from bfio import BioReader -from skimage import io +from polus.plugins.formats.ome_converter.__main__ import app +from polus.plugins.formats.ome_converter.image_converter import batch_convert +from polus.plugins.formats.ome_converter.image_converter import convert_image from typer.testing import CliRunner -from polus.plugins.formats.ome_converter.__main__ import app as app -from polus.plugins.formats.ome_converter.image_converter import ( - batch_convert, - convert_image, -) - runner = CliRunner() -EXT = [".ome.tif", ".ome.zarr"] - - -@pytest.fixture(params=EXT) -def file_extension(request): - """To get the parameter of the fixture.""" - yield request.param - - -@pytest.fixture( - params=[ - (256, ".png"), - (512, ".tif"), - (1024, ".png"), - (2048, ".tif"), - (4096, ".tif"), - ] -) -def get_params(request): - """To get the parameter of the fixture.""" - yield request.param - - -@pytest.fixture -def synthetic_images( - get_params, -) -> Generator[Tuple[List[Any], pathlib.Path], None, None]: - """Generate random synthetic images.""" - size, extension = get_params - - syn_dir = pathlib.Path(tempfile.mkdtemp(dir=pathlib.Path.cwd())) - images = [] - for i in range(10): - # Create images - blobs = skimage.data.binary_blobs( - length=size, volume_fraction=0.05, blob_size_fraction=0.05 - ) - syn_img = skimage.measure.label(blobs) - outname = f"syn_image_{i}{extension}" - # Save image - out_path = pathlib.Path(syn_dir, outname) - io.imsave(out_path, syn_img) - images.append(syn_img) - - yield images, syn_dir - - -@pytest.fixture -def output_directory() -> Generator[pathlib.Path, None, None]: - """Generate random synthetic images.""" - out_dir = pathlib.Path(tempfile.mkdtemp(dir=pathlib.Path.cwd())) - yield out_dir - shutil.rmtree(out_dir) - - -def test_batch_converter(synthetic_images, file_extension, output_directory) -> None: +def test_batch_converter( + synthetic_images: tuple[list[np.ndarray], pathlib.Path], + file_extension: str, + output_directory: pathlib.Path, +) -> None: """Create synthetic image. This unit test runs the batch_converter and validates that the converted data is @@ -94,45 +34,33 @@ def test_batch_converter(synthetic_images, file_extension, output_directory) -> for f in output_directory.iterdir(): with BioReader(f) as br: assert np.all(image) == np.all(br[:]) - shutil.rmtree(inp_dir) -@pytest.fixture -def images() -> Generator[pathlib.Path, None, None]: - """Download test /Users/abbasih2/Documents/Polus_Repos/polus-plugins/formats/ome-converter-plugin/data/inputimages.""" - imagelist = { - ("0.tif", "https://osf.io/j6aer/download/"), - ( - "cameraman.png", - "https://people.math.sc.edu/Burkardt/data/tif/cameraman.png", - ), - ("venus1.png", "https://people.math.sc.edu/Burkardt/data/tif/venus1.png"), - } - inp_dir = pathlib.Path(tempfile.mkdtemp(dir=pathlib.Path.cwd())) - for image in imagelist: - file, url = image - outfile = pathlib.Path(inp_dir, file) +@pytest.mark.skipif("not config.getoption('downloads')") +def test_image_converter_omezarr( + download_images: pathlib.Path, + file_extension: str, + output_directory: pathlib.Path, +) -> None: + """Testing of bioformats supported image datatypes conversion. - r = requests.get(url) - with open(outfile, "wb") as fw: - fw.write(r.content) - - yield outfile - shutil.rmtree(inp_dir) - - -def test_image_converter_omezarr(images, file_extension, output_directory) -> None: - """Testing of bioformat supported image datatypes conversion to ome.zarr and ome.tif file format.""" - br_img = BioReader(images) + This test will convert the downloaded images to the specified file extension + and validate that the converted data is the same as the input data. + """ + br_img = BioReader(download_images) image = br_img.read() - convert_image(images, file_extension, output_directory) + convert_image(download_images, file_extension, output_directory) for f in output_directory.iterdir(): with BioReader(f) as br: assert np.all(image) == np.all(br[:]) -def test_cli(synthetic_images, output_directory, file_extension) -> None: +def test_cli( + synthetic_images: tuple[list[np.ndarray], pathlib.Path], + output_directory: pathlib.Path, + file_extension: str, +) -> None: """Test Cli.""" _, inp_dir = synthetic_images @@ -151,4 +79,3 @@ def test_cli(synthetic_images, output_directory, file_extension) -> None: ) assert result.exit_code == 0 - shutil.rmtree(inp_dir)