feat: updated and fixed ome-converter

build: bumped version 0.3.0 -> 0.3.1 on ome converter plugin

build: updated ome-converter dep on bfio to 2.3.3

bumped version

fix: fp usage

fix: returning after preview

feat: using preadator

feat: using preadator

fix: new pre-commit hooks

ci: updated pre-commit config to check test files

fix: fixed all tests

.pre-commit-config.yaml

@@ -41,20 +41,12 @@ repos:
     rev: 'v0.0.274'
     hooks:
       - id: ruff
-        exclude: |
-          (?x)(
-            test_[a-zA-Z0-9]+.py$|
-            ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$
-          )
+        exclude: ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$
         args: [--fix]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: 'v1.4.0'
     hooks:
       - id: mypy
-        exclude: |
-          (?x)(
-            test_[a-zA-Z0-9]+.py$|
-            ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$
-          )
+        exclude: ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$
         additional_dependencies: [types-requests==2.31.0.1]

formats/ome-converter-plugin/.bumpversion.cfg

@@ -24,4 +24,6 @@ replace = version = "{new_version}"
 
 [bumpversion:file:VERSION]
 
+[bumpversion:file:README.md]
+
 [bumpversion:file:src/polus/plugins/formats/ome_converter/__init__.py]

formats/ome-converter-plugin/Dockerfile

@@ -1,4 +1,4 @@
-FROM polusai/bfio:2.1.9
+FROM polusai/bfio:2.3.3
 
 # environment variables defined in polusai/bfio
 ENV EXEC_DIR="/opt/executables"
@@ -12,7 +12,6 @@ WORKDIR ${EXEC_DIR}
 COPY pyproject.toml ${EXEC_DIR}
 COPY VERSION ${EXEC_DIR}
 COPY README.md ${EXEC_DIR}
-RUN pip3 install --index-url https://test.pypi.org/simple/ filepattern==2.2.7
 COPY src ${EXEC_DIR}/src
 
 RUN pip3 install ${EXEC_DIR} --no-cache-dir

formats/ome-converter-plugin/README.md

@@ -1,4 +1,4 @@
-# OME Converter (0.3.0)
+# OME Converter (v0.3.0)
 
 This WIPP plugin converts BioFormats supported data types to the
 OME Zarr or OME TIF file format. This is not a complete implementation, rather it implements a file
@@ -31,4 +31,4 @@ This plugin takes 3 input arguments and 1 output argument:
 | `--filePattern`  | A filepattern, used to select data for conversion            | Input  | string      |
 | `--fileExtension`| A desired file format for conversion                         | Input  | enum        |
 | `--outDir`       | Output collection                                            | Output | genericData |
-| `--preview`      | Generate a JSON file with outputs                            | Output | JSON        |
+| `--preview`      | Generate a JSON file with outputs                            | Output | JSON        |

formats/ome-converter-plugin/pyproject.toml

@@ -3,30 +3,24 @@ name = "polus-plugins-formats-ome-converter"
 version = "0.3.0"
 description = "Convert BioFormats datatypes to ome.tif or ome.zarr file format"
 authors = [
-           "Nick Schaub <nick.schaub@nih.gov>",
-           "Hamdah Shafqat abbasi <hamdahshafqat.abbasi@nih.gov>"
-           ]
+    "Nick Schaub <nick.schaub@nih.gov>",
+    "Hamdah Shafqat abbasi <hamdahshafqat.abbasi@nih.gov>",
+    "Najib Ishaq <najib.ishaq@nih.gov>"
+]
 readme = "README.md"
 packages = [{include = "polus", from = "src"}]
 
-[[tool.poetry.source]]
-name = "test"
-url = "https://test.pypi.org/simple/"
-secondary = true
-
 [tool.poetry.dependencies]
-python = "^3.9"
-bfio = {version = "2.1.9", extras = ["all"]}
-filepattern = "^2.2.7"
+python = ">=3.9,<3.12"
+bfio = {version = "^2.3.3", extras = ["all"]}
+filepattern = "^2.0.4"
 typer = "^0.7.0"
 tqdm = "^4.64.1"
+preadator = "0.4.0-dev2"
 
 [tool.poetry.group.dev.dependencies]
 bump2version = "^1.0.1"
 pre-commit = "^3.0.4"
-black = "^23.1.0"
-flake8 = "^6.0.0"
-mypy = "^1.0.0"
 pytest = "^7.2.1"
 ipykernel = "^6.21.2"
 requests = "^2.28.2"
@@ -35,3 +29,8 @@ scikit-image = "^0.19.3"
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+pythonpath = [
+  "."
+]

formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/__init__.py

@@ -1,9 +1,6 @@
 """Ome Converter."""
-__version__ = "0.3.0"
 
-from polus.plugins.formats.ome_converter.image_converter import (  # noqa
-    batch_convert as batch_convert,
-)
-from polus.plugins.formats.ome_converter.image_converter import (  # noqa
-    convert_image as convert_image,
-)
+__version__ = "0.3.2"
+
+from polus.plugins.formats.ome_converter.image_converter import batch_convert
+from polus.plugins.formats.ome_converter.image_converter import convert_image

formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/__main__.py

@@ -1,21 +1,20 @@
 """Ome Converter."""
 import json
-import os
 import logging
+import os
 import pathlib
-from concurrent.futures import ProcessPoolExecutor, as_completed
-from typing import Any, Optional
+from concurrent.futures import as_completed
+from typing import Any
+from typing import Optional
 
 import filepattern as fp
+import preadator
 import typer
+from polus.plugins.formats.ome_converter.image_converter import NUM_THREADS
+from polus.plugins.formats.ome_converter.image_converter import Extension
+from polus.plugins.formats.ome_converter.image_converter import convert_image
 from tqdm import tqdm
 
-from polus.plugins.formats.ome_converter.image_converter import (
-    NUM_THREADS,
-    Extension,
-    convert_image,
-)
-
 app = typer.Typer()
 
 # Initialize the logger
@@ -33,51 +32,68 @@ def main(
         ...,
         "--inpDir",
         help="Input generic data collection to be processed by this plugin",
+        exists=True,
+        resolve_path=True,
+        readable=True,
+        file_okay=False,
+        dir_okay=True,
     ),
     pattern: str = typer.Option(
-        ".+", "--filePattern", help="A filepattern defining the images to be converted"
+        ".+",
+        "--filePattern",
+        help="A filepattern defining the images to be converted",
     ),
     file_extension: Extension = typer.Option(
-        None, "--fileExtension", help="Type of data conversion"
+        Extension,
+        "--fileExtension",
+        help="Type of data conversion",
+    ),
+    out_dir: pathlib.Path = typer.Option(
+        ...,
+        "--outDir",
+        help="Output collection",
+        exists=True,
+        resolve_path=True,
+        writable=True,
+        file_okay=False,
+        dir_okay=True,
     ),
-    out_dir: pathlib.Path = typer.Option(..., "--outDir", help="Output collection"),
     preview: Optional[bool] = typer.Option(
-        False, "--preview", help="Output a JSON preview of files"
+        False,
+        "--preview",
+        help="Output a JSON preview of files",
     ),
 ) -> None:
-    """Convert bioformat supported image datatypes conversion to ome.tif or ome.zarr file format."""
+    """Convert bioformat supported image datatypes conversion to ome.tif or ome.zarr."""
     logger.info(f"inpDir = {inp_dir}")
     logger.info(f"outDir = {out_dir}")
     logger.info(f"filePattern = {pattern}")
     logger.info(f"fileExtension = {file_extension}")
 
-    inp_dir = inp_dir.resolve()
-    out_dir = out_dir.resolve()
-
-    assert inp_dir.exists(), f"{inp_dir} does not exist!! Please check input path again"
-    assert (
-        out_dir.exists()
-    ), f"{out_dir} does not exist!! Please check output path again"
-
     fps = fp.FilePattern(inp_dir, pattern)
 
     if preview:
-        with open(pathlib.Path(out_dir, "preview.json"), "w") as jfile:
+        with out_dir.joinpath("preview.json").open("w") as jfile:
             out_json: dict[str, Any] = {
                 "filepattern": pattern,
                 "outDir": [],
             }
-            for file in fps:
+            for file in fps():
                 out_name = str(file[1][0].name.split(".")[0]) + file_extension
                 out_json["outDir"].append(out_name)
             json.dump(out_json, jfile, indent=2)
+        return
 
-    with ProcessPoolExecutor(max_workers=NUM_THREADS) as executor:
+    with preadator.ProcessManager(
+        name="ome_converter",
+        num_processes=NUM_THREADS,
+        threads_per_process=2,
+    ) as executor:
         threads = []
         for files in fps():
             file = files[1][0]
             threads.append(
-                executor.submit(convert_image, file, file_extension, out_dir)
+                executor.submit_process(convert_image, file, file_extension, out_dir),
             )
 
         for f in tqdm(

formats/ome-converter-plugin/src/polus/plugins/formats/ome_converter/image_converter.py

@@ -3,13 +3,15 @@
 import logging
 import os
 import pathlib
-from concurrent.futures import ProcessPoolExecutor, as_completed
+from concurrent.futures import as_completed
 from multiprocessing import cpu_count
 from sys import platform
 from typing import Optional
 
 import filepattern as fp
-from bfio import BioReader, BioWriter
+import preadator
+from bfio import BioReader
+from bfio import BioWriter
 from tqdm import tqdm
 
 logger = logging.getLogger(__name__)
@@ -19,9 +21,9 @@
 
 
 if platform == "linux" or platform == "linux2":
-    NUM_THREADS = len(os.sched_getaffinity(0))  # type: ignore
+    NUM_THREADS = len(os.sched_getaffinity(0)) // 2  # type: ignore
 else:
-    NUM_THREADS = max(cpu_count() // 2, 2)
+    NUM_THREADS = max(cpu_count() // 2, 1)
 
 POLUS_IMG_EXT = os.environ.get("POLUS_IMG_EXT", ".ome.tif")
 
@@ -35,7 +37,9 @@ class Extension(str, enum.Enum):
 
 
 def convert_image(
-    inp_image: pathlib.Path, file_extension: Extension, out_dir: pathlib.Path
+    inp_image: pathlib.Path,
+    file_extension: Extension,
+    out_dir: pathlib.Path,
 ) -> None:
     """Convert bioformats supported datatypes to ome.tif or ome.zarr file format.
 
@@ -44,56 +48,70 @@ def convert_image(
         file_extension: Type of data conversion.
         out_dir: Path to output directory.
     """
-    with BioReader(inp_image) as br:
+    with BioReader(inp_image, max_workers=2) as br:
         # Loop through timepoints
         for t in range(br.T):
             # Loop through channels
             for c in range(br.C):
                 extension = "".join(
-                    [suffix for suffix in inp_image.suffixes[-2:] if len(suffix) < 6]
+                    [
+                        suffix
+                        for suffix in inp_image.suffixes[-2:]
+                        if len(suffix) < 6  # noqa: PLR2004
+                    ],
                 )
 
                 out_path = out_dir.joinpath(
-                    inp_image.name.replace(extension, file_extension)
+                    inp_image.name.replace(extension, file_extension),
                 )
                 if br.C > 1:
                     out_path = out_dir.joinpath(
-                        out_path.name.replace(file_extension, f"_c{c}" + file_extension)
+                        out_path.name.replace(
+                            file_extension,
+                            f"_c{c}" + file_extension,
+                        ),
                     )
                 if br.T > 1:
                     out_path = out_dir.joinpath(
-                        out_path.name.replace(file_extension, f"_t{t}" + file_extension)
+                        out_path.name.replace(
+                            file_extension,
+                            f"_t{t}" + file_extension,
+                        ),
                     )
 
                 with BioWriter(
                     out_path,
-                    max_workers=NUM_THREADS,
+                    max_workers=2,
                     metadata=br.metadata,
                 ) as bw:
                     bw.C = 1
                     bw.T = 1
 
                     # Handling of parsing channels when channels names are not provided.
-                    if not bw.channel_names == [None]:
+                    if bw.channel_names != [None]:
                         bw.channel_names = [br.channel_names[c]]
 
-
                     # Loop through z-slices
                     for z in range(br.Z):
                         # Loop across the length of the image
                         for y in range(0, br.Y, TILE_SIZE):
                             y_max = min([br.Y, y + TILE_SIZE])
 
-                            bw.max_workers = NUM_THREADS
-                            br.max_workers = NUM_THREADS
-
                             # Loop across the depth of the image
                             for x in range(0, br.X, TILE_SIZE):
                                 x_max = min([br.X, x + TILE_SIZE])
                                 bw[
-                                    y:y_max, x:x_max, z : z + 1, 0, 0  # noqa: E203
+                                    y:y_max,
+                                    x:x_max,
+                                    z : z + 1,
+                                    0,
+                                    0,
                                 ] = br[
-                                    y:y_max, x:x_max, z : z + 1, c, t  # noqa: E203
+                                    y:y_max,
+                                    x:x_max,
+                                    z : z + 1,
+                                    c,
+                                    t,
                                 ]
 
 
@@ -103,7 +121,7 @@ def batch_convert(
     file_pattern: Optional[str],
     file_extension: Extension,
 ) -> None:
-    """Convert bioformats supported datatypes in batches to ome.tif or ome.zarr file format.
+    """Convert bioformats supported datatypes in batches to ome.tif or ome.zarr.
 
     Args:
         inp_dir: Path of an input directory.
@@ -116,21 +134,20 @@ def batch_convert(
     logger.info(f"file_pattern = {file_pattern}")
     logger.info(f"file_extension = {file_extension}")
 
-    assert inp_dir.exists(), f"{inp_dir} does not exist!! Please check input path again"
-    assert (
-        out_dir.exists()
-    ), f"{out_dir} does not exist!! Please check output path again"
-
     file_pattern = ".+" if file_pattern is None else file_pattern
 
     fps = fp.FilePattern(inp_dir, file_pattern)
 
-    with ProcessPoolExecutor(max_workers=NUM_THREADS) as executor:
+    with preadator.ProcessManager(
+        name="ome_converter",
+        num_processes=NUM_THREADS,
+        threads_per_process=2,
+    ) as executor:
         threads = []
         for files in fps():
             file = files[1][0]
             threads.append(
-                executor.submit(convert_image, file, file_extension, out_dir)
+                executor.submit(convert_image, file, file_extension, out_dir),
             )
 
         for f in tqdm(

formats/ome-converter-plugin/tests/__init__.py

@@ -1 +1 @@
-"""ome converter plugin."""
+"""Tests for ome converter plugin."""