From 70fb877b6f6dbe437f17d3be92a0023217d430ff Mon Sep 17 00:00:00 2001 From: Camilo Velez Date: Thu, 25 Jan 2024 23:34:20 -0500 Subject: [PATCH 1/4] feat(api): add Pydantic V2 support --- .pre-commit-config.yaml | 21 +- pyproject.toml | 7 +- src/polus/plugins/__init__.py | 4 + .../plugins/_plugins/classes/__init__.py | 39 +- .../{plugin_methods.py => plugin_base.py} | 177 +++---- ...plugin_classes.py => plugin_classes_v1.py} | 70 +-- .../_plugins/classes/plugin_classes_v2.py | 429 ++++++++++++++++ src/polus/plugins/_plugins/io/__init__.py | 35 ++ .../plugins/_plugins/{io.py => io/io_v1.py} | 6 +- src/polus/plugins/_plugins/io/io_v2.py | 468 ++++++++++++++++++ .../plugins/_plugins/manifests/__init__.py | 26 + .../_plugins/manifests/manifest_utils.py | 202 -------- .../_plugins/manifests/manifest_utils_v1.py | 206 ++++++++ .../_plugins/manifests/manifest_utils_v2.py | 202 ++++++++ .../_plugins/models/PolusComputeSchema.py | 136 ----- src/polus/plugins/_plugins/models/__init__.py | 33 +- .../models/pydanticv1/PolusComputeSchema.py | 137 +++++ .../{ => pydanticv1}/WIPPPluginSchema.py | 100 ++-- .../_plugins/models/pydanticv1/__init__.py | 0 .../models/{ => pydanticv1}/compute.py | 16 +- .../_plugins/models/pydanticv1/wipp.py | 79 +++ .../models/pydanticv2/PolusComputeSchema.py | 136 +++++ .../models/pydanticv2/WIPPPluginSchema.py | 241 +++++++++ .../_plugins/models/pydanticv2/__init__.py | 0 .../_plugins/models/pydanticv2/compute.py | 28 ++ .../_plugins/models/pydanticv2/wipp.py | 79 +++ src/polus/plugins/_plugins/models/wipp.py | 51 -- src/polus/plugins/_plugins/update/__init__.py | 14 + .../{update.py => update/update_v1.py} | 90 ++-- .../plugins/_plugins/update/update_v2.py | 115 +++++ tests/test_cwl.py | 12 +- tests/test_io.py | 4 +- tests/test_manifests.py | 33 +- tests/test_plugins.py | 20 +- tests/test_version.py | 91 +++- 35 files changed, 2637 insertions(+), 670 deletions(-) rename src/polus/plugins/_plugins/classes/{plugin_methods.py => plugin_base.py} (62%) rename src/polus/plugins/_plugins/classes/{plugin_classes.py => plugin_classes_v1.py} (88%) create mode 100644 src/polus/plugins/_plugins/classes/plugin_classes_v2.py create mode 100644 src/polus/plugins/_plugins/io/__init__.py rename src/polus/plugins/_plugins/{io.py => io/io_v1.py} (98%) create mode 100644 src/polus/plugins/_plugins/io/io_v2.py create mode 100644 src/polus/plugins/_plugins/manifests/__init__.py delete mode 100644 src/polus/plugins/_plugins/manifests/manifest_utils.py create mode 100644 src/polus/plugins/_plugins/manifests/manifest_utils_v1.py create mode 100644 src/polus/plugins/_plugins/manifests/manifest_utils_v2.py delete mode 100644 src/polus/plugins/_plugins/models/PolusComputeSchema.py create mode 100644 src/polus/plugins/_plugins/models/pydanticv1/PolusComputeSchema.py rename src/polus/plugins/_plugins/models/{ => pydanticv1}/WIPPPluginSchema.py (69%) create mode 100644 src/polus/plugins/_plugins/models/pydanticv1/__init__.py rename src/polus/plugins/_plugins/models/{ => pydanticv1}/compute.py (57%) create mode 100644 src/polus/plugins/_plugins/models/pydanticv1/wipp.py create mode 100644 src/polus/plugins/_plugins/models/pydanticv2/PolusComputeSchema.py create mode 100644 src/polus/plugins/_plugins/models/pydanticv2/WIPPPluginSchema.py create mode 100644 src/polus/plugins/_plugins/models/pydanticv2/__init__.py create mode 100644 src/polus/plugins/_plugins/models/pydanticv2/compute.py create mode 100644 src/polus/plugins/_plugins/models/pydanticv2/wipp.py delete mode 100644 src/polus/plugins/_plugins/models/wipp.py create mode 100644 src/polus/plugins/_plugins/update/__init__.py rename src/polus/plugins/_plugins/{update.py => update/update_v1.py} (57%) create mode 100644 src/polus/plugins/_plugins/update/update_v2.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b27f4d530..3433d66e5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,6 @@ fail_fast: true repos: - - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.4.0 hooks: @@ -26,35 +25,41 @@ repos: args: ["--fix=lf"] description: Forces to replace line ending by the UNIX 'lf' character. - id: trailing-whitespace - exclude: '.bumpversion.cfg' + exclude: ".bumpversion.cfg" - id: check-merge-conflict - repo: https://github.com/psf/black - rev: '23.3.0' + rev: "23.3.0" hooks: - id: black language_version: python3.9 - exclude: ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$ + exclude: | + (?x)( + ^src\/polus\/plugins\/_plugins\/models\/pydanticv1\/\w*Schema.py$| + ^src\/polus\/plugins\/_plugins\/models\/pydanticv2\/\w*Schema.py$ + ) - repo: https://github.com/charliermarsh/ruff-pre-commit # Ruff version. - rev: 'v0.0.274' + rev: "v0.0.274" hooks: - id: ruff exclude: | (?x)( test_[a-zA-Z0-9]+.py$| - ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$ + ^src\/polus\/plugins\/_plugins\/models\/pydanticv1\/\w*Schema.py$| + ^src\/polus\/plugins\/_plugins\/models\/pydanticv2\/\w*Schema.py$ ) args: [--fix] - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v1.4.0' + rev: "v1.4.0" hooks: - id: mypy exclude: | (?x)( test_[a-zA-Z0-9]+.py$| - ^src\/polus\/plugins\/_plugins\/models\/\w*Schema.py$ + ^src\/polus\/plugins\/_plugins\/models\/pydanticv1\/\w*Schema.py$| + ^src\/polus\/plugins\/_plugins\/models\/pydanticv2\/\w*Schema.py$ ) additional_dependencies: [types-requests==2.31.0.1] diff --git a/pyproject.toml b/pyproject.toml index 3ad3fa419..67bb0ace1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,11 +15,12 @@ python = ">=3.9, <3.12" click = "^8.1.3" cwltool = "^3.1.20230513155734" fsspec = "^2023.6.0" -pydantic = ">=1.10.9, <2.0" +pydantic = ">=1.10.0" pygithub = "^1.58.2" python-on-whales = "^0.57.0" pyyaml = "^6.0" tqdm = "^4.65.0" +validators = "^0.22.0" xmltodict = "^0.13.0" [tool.poetry.group.dev.dependencies] @@ -27,15 +28,13 @@ python = ">=3.9, <3.12" black = "^23.3.0" bump2version = "^1.0.1" -datamodel-code-generator = "^0.17.1" +datamodel-code-generator = "^0.23.0" flake8 = "^6.0.0" fsspec = "^2023.1.0" mypy = "^1.4.0" nox = "^2022.11.21" poetry = "^1.3.2" pre-commit = "^3.3.3" -pydantic = "^1.10.4" -pydantic-to-typescript = "^1.0.10" pytest = "^7.3.2" pytest-benchmark = "^4.0.0" pytest-cov = "^4.1.0" diff --git a/src/polus/plugins/__init__.py b/src/polus/plugins/__init__.py index 0319cb364..a69b802fa 100644 --- a/src/polus/plugins/__init__.py +++ b/src/polus/plugins/__init__.py @@ -30,6 +30,8 @@ """ logger = logging.getLogger("polus.plugins") +VERSION = "0.1.1" + refresh() # calls the refresh method when library is imported @@ -39,6 +41,8 @@ def __getattr__(name: str) -> Union[Plugin, ComputePlugin, list]: return list_plugins() if name in list_plugins(): return get_plugin(name) + if name == "__version__": + return VERSION msg = f"module '{__name__}' has no attribute '{name}'" raise AttributeError(msg) diff --git a/src/polus/plugins/_plugins/classes/__init__.py b/src/polus/plugins/_plugins/classes/__init__.py index 91200c48e..200b2c821 100644 --- a/src/polus/plugins/_plugins/classes/__init__.py +++ b/src/polus/plugins/_plugins/classes/__init__.py @@ -1,13 +1,32 @@ """Plugin classes and functions.""" -from polus.plugins._plugins.classes.plugin_classes import ComputePlugin -from polus.plugins._plugins.classes.plugin_classes import Plugin -from polus.plugins._plugins.classes.plugin_classes import get_plugin -from polus.plugins._plugins.classes.plugin_classes import list_plugins -from polus.plugins._plugins.classes.plugin_classes import load_config -from polus.plugins._plugins.classes.plugin_classes import refresh -from polus.plugins._plugins.classes.plugin_classes import remove_all -from polus.plugins._plugins.classes.plugin_classes import remove_plugin -from polus.plugins._plugins.classes.plugin_classes import submit_plugin +import pydantic + +PYDANTIC_VERSION = pydantic.__version__ + +if PYDANTIC_VERSION.split(".")[0] == "1": + from polus.plugins._plugins.classes.plugin_classes_v1 import PLUGINS + from polus.plugins._plugins.classes.plugin_classes_v1 import ComputePlugin + from polus.plugins._plugins.classes.plugin_classes_v1 import Plugin + from polus.plugins._plugins.classes.plugin_classes_v1 import _load_plugin + from polus.plugins._plugins.classes.plugin_classes_v1 import get_plugin + from polus.plugins._plugins.classes.plugin_classes_v1 import list_plugins + from polus.plugins._plugins.classes.plugin_classes_v1 import load_config + from polus.plugins._plugins.classes.plugin_classes_v1 import refresh + from polus.plugins._plugins.classes.plugin_classes_v1 import remove_all + from polus.plugins._plugins.classes.plugin_classes_v1 import remove_plugin + from polus.plugins._plugins.classes.plugin_classes_v1 import submit_plugin +elif PYDANTIC_VERSION.split(".")[0] == "2": + from polus.plugins._plugins.classes.plugin_classes_v2 import PLUGINS + from polus.plugins._plugins.classes.plugin_classes_v2 import ComputePlugin + from polus.plugins._plugins.classes.plugin_classes_v2 import Plugin + from polus.plugins._plugins.classes.plugin_classes_v2 import _load_plugin + from polus.plugins._plugins.classes.plugin_classes_v2 import get_plugin + from polus.plugins._plugins.classes.plugin_classes_v2 import list_plugins + from polus.plugins._plugins.classes.plugin_classes_v2 import load_config + from polus.plugins._plugins.classes.plugin_classes_v2 import refresh + from polus.plugins._plugins.classes.plugin_classes_v2 import remove_all + from polus.plugins._plugins.classes.plugin_classes_v2 import remove_plugin + from polus.plugins._plugins.classes.plugin_classes_v2 import submit_plugin __all__ = [ "Plugin", @@ -19,4 +38,6 @@ "remove_plugin", "remove_all", "load_config", + "_load_plugin", + "PLUGINS", ] diff --git a/src/polus/plugins/_plugins/classes/plugin_methods.py b/src/polus/plugins/_plugins/classes/plugin_base.py similarity index 62% rename from src/polus/plugins/_plugins/classes/plugin_methods.py rename to src/polus/plugins/_plugins/classes/plugin_base.py index 91b254a8c..02d142b7f 100644 --- a/src/polus/plugins/_plugins/classes/plugin_methods.py +++ b/src/polus/plugins/_plugins/classes/plugin_base.py @@ -3,58 +3,60 @@ import enum import json import logging -import pathlib import random import signal -import typing -from os.path import relpath +from pathlib import Path +from typing import Any +from typing import Optional +from typing import TypeVar +from typing import Union import fsspec import yaml # type: ignore from cwltool.context import RuntimeContext from cwltool.factory import Factory from cwltool.utils import CWLObjectType -from python_on_whales import docker - from polus.plugins._plugins.cwl import CWL_BASE_DICT -from polus.plugins._plugins.io import ( - input_to_cwl, - io_to_yml, - output_to_cwl, - outputs_cwl, -) +from polus.plugins._plugins.io import input_to_cwl +from polus.plugins._plugins.io import io_to_yml +from polus.plugins._plugins.io import output_to_cwl +from polus.plugins._plugins.io import outputs_cwl from polus.plugins._plugins.utils import name_cleaner +from python_on_whales import docker logger = logging.getLogger("polus.plugins") -StrPath = typing.TypeVar("StrPath", str, pathlib.Path) +StrPath = TypeVar("StrPath", str, Path) class IOKeyError(Exception): """Raised when trying to set invalid I/O parameter.""" -class MissingInputValues(Exception): +class MissingInputValuesError(Exception): """Raised when there are required input values that have not been set.""" -class _PluginMethods: - def _check_inputs(self): +class BasePlugin: + """Base Class for Plugins.""" + + def _check_inputs(self) -> None: """Check if all required inputs have been set.""" _in = [x for x in self.inputs if x.required and not x.value] # type: ignore if len(_in) > 0: - raise MissingInputValues( - f"{[x.name for x in _in]} are required inputs but have not been set" # type: ignore + msg = f"{[x.name for x in _in]} are required inputs but have not been set" + raise MissingInputValuesError( + msg, # type: ignore ) @property - def organization(self): + def organization(self) -> str: """Plugin container's organization.""" return self.containerId.split("/")[0] - def load_config(self, path: StrPath): + def load_config(self, path: StrPath) -> None: """Load configured plugin from file.""" - with open(path, encoding="utf=8") as fw: + with Path(path).open(encoding="utf=8") as fw: config = json.load(fw) inp = config["inputs"] out = config["outputs"] @@ -68,20 +70,13 @@ def load_config(self, path: StrPath): def run( self, - gpus: typing.Union[None, str, int] = "all", - **kwargs, - ): + gpus: Union[None, str, int] = "all", + **kwargs: Union[None, str, int], + ) -> None: + """Run plugin in Docker container.""" self._check_inputs() - inp_dirs = [] - out_dirs = [] - - for i in self.inputs: - if isinstance(i.value, pathlib.Path): - inp_dirs.append(str(i.value)) - - for o in self.outputs: - if isinstance(o.value, pathlib.Path): - out_dirs.append(str(o.value)) + inp_dirs = [x for x in self.inputs if isinstance(x.value, Path)] + out_dirs = [x for x in self.outputs if isinstance(x.value, Path)] inp_dirs_dict = {x: f"/data/inputs/input{n}" for (n, x) in enumerate(inp_dirs)} out_dirs_dict = { @@ -105,7 +100,7 @@ def run( i._validate() args.append(f"--{i.name}") - if isinstance(i.value, pathlib.Path): + if isinstance(i.value, Path): args.append(inp_dirs_dict[str(i.value)]) elif isinstance(i.value, enum.Enum): @@ -119,7 +114,7 @@ def run( o._validate() args.append(f"--{o.name}") - if isinstance(o.value, pathlib.Path): + if isinstance(o.value, Path): args.append(out_dirs_dict[str(o.value)]) elif isinstance(o.value, enum.Enum): @@ -128,20 +123,24 @@ def run( else: args.append(str(o.value)) - container_name = f"polus{random.randint(10, 99)}" + random_int = random.randint(10, 99) # noqa: S311 # only for naming + container_name = f"polus{random_int}" def sig( - signal, frame # pylint: disable=W0613, W0621 - ): # signal handler to kill container when KeyboardInterrupt - print(f"Exiting container {container_name}") + signal, # noqa # pylint: disable=W0613, W0621 + frame, # noqa # pylint: disable=W0613, W0621 + ) -> None: # signal handler to kill container when KeyboardInterrupt + logger.info(f"Exiting container {container_name}") docker.kill(container_name) signal.signal( - signal.SIGINT, sig + signal.SIGINT, + sig, ) # make of sig the handler for KeyboardInterrupt if gpus is None: logger.info( - f"Running container without GPU. {self.__class__.__name__} version {self.version.version}" + f"""Running container without GPU. {self.__class__.__name__} + version {self.version!s}""", ) docker_ = docker.run( self.containerId, @@ -151,10 +150,11 @@ def sig( mounts=mnts, **kwargs, ) - print(docker_) + print(docker_) # noqa else: logger.info( - f"Running container with GPU: --gpus {gpus}. {self.__class__.__name__} version {self.version.version}" + f"""Running container with GPU: --gpus {gpus}. + {self.__class__.__name__} version {self.version!s}""", ) docker_ = docker.run( self.containerId, @@ -165,36 +165,31 @@ def sig( mounts=mnts, **kwargs, ) - print(docker_) + print(docker_) # noqa @property - def _config(self): - model_ = self.dict() - for inp in model_["inputs"]: - inp["value"] = None - return model_ - - @property - def manifest(self): + def manifest(self) -> dict: """Plugin manifest.""" manifest_ = json.loads(self.json(exclude={"_io_keys", "versions", "id"})) manifest_["version"] = manifest_["version"]["version"] return manifest_ - def __getattribute__(self, name): - if name != "_io_keys" and hasattr(self, "_io_keys"): - if name in self._io_keys: - value = self._io_keys[name].value - if isinstance(value, enum.Enum): - value = value.name - return value + def __getattribute__(self, name: str) -> Any: # noqa + if name == "__class__": # pydantic v2 change + return super().__getattribute__(name) + if name != "_io_keys" and hasattr(self, "_io_keys") and name in self._io_keys: + value = self._io_keys[name].value + if isinstance(value, enum.Enum): + value = value.name + return value return super().__getattribute__(name) - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: Any) -> None: # noqa if name == "_fs": if not issubclass(type(value), fsspec.spec.AbstractFileSystem): - raise ValueError("_fs must be an fsspec FileSystem") + msg = "_fs must be an fsspec FileSystem" + raise ValueError(msg) for i in self.inputs: i._fs = value for o in self.outputs: @@ -204,18 +199,22 @@ def __setattr__(self, name, value): if name != "_io_keys" and hasattr(self, "_io_keys"): if name in self._io_keys: logger.debug( - f"Value of {name} in {self.__class__.__name__} set to {value}" + f"Value of {name} in {self.__class__.__name__} set to {value}", ) self._io_keys[name].value = value return + msg = ( + f"Attempting to set {name} in " + "{self.__class__.__name__} but " + "{{name}} is not a valid I/O parameter" + ) raise IOKeyError( - f"Attempting to set {name} in {self.__class__.__name__} but" - "{name} is not a valid I/O parameter" + msg, ) super().__setattr__(name, value) - def _to_cwl(self): + def _to_cwl(self) -> dict: """Return CWL yml as dict.""" cwl_dict = CWL_BASE_DICT cwl_dict["inputs"] = {} @@ -230,12 +229,14 @@ def _to_cwl(self): cwl_dict["requirements"]["DockerRequirement"]["dockerPull"] = self.containerId return cwl_dict - def save_cwl(self, path: StrPath) -> pathlib.Path: + def save_cwl(self, path: StrPath) -> Path: """Save plugin as CWL command line tool.""" - assert str(path).rsplit(".", maxsplit=1)[-1] == "cwl", "Path must end in .cwl" - with open(path, "w", encoding="utf-8") as file: + if str(path).rsplit(".", maxsplit=1)[-1] != "cwl": + msg = "path must end in .cwl" + raise ValueError(msg) + with Path(path).open("w", encoding="utf-8") as file: yaml.dump(self._to_cwl(), file) - return pathlib.Path(path) + return Path(path) @property def _cwl_io(self) -> dict: @@ -244,19 +245,24 @@ def _cwl_io(self) -> dict: x.name: io_to_yml(x) for x in self._io_keys.values() if x.value is not None } - def save_cwl_io(self, path) -> pathlib.Path: - """Save plugin's I/O values to yml file to be used with CWL command line tool.""" + def save_cwl_io(self, path: StrPath) -> Path: + """Save plugin's I/O values to yml file. + + To be used with CWL Command Line Tool. + """ self._check_inputs() - assert str(path).rsplit(".", maxsplit=1)[-1] == "yml", "Path must end in .yml" - with open(path, "w", encoding="utf-8") as file: + if str(path).rsplit(".", maxsplit=1)[-1] != "yml": + msg = "path must end in .yml" + raise ValueError(msg) + with Path(path).open("w", encoding="utf-8") as file: yaml.dump(self._cwl_io, file) - return pathlib.Path(path) + return Path(path) def run_cwl( self, - cwl_path: typing.Optional[StrPath] = None, - io_path: typing.Optional[StrPath] = None, - ) -> typing.Union[CWLObjectType, str, None]: + cwl_path: Optional[StrPath] = None, + io_path: Optional[StrPath] = None, + ) -> Union[CWLObjectType, str, None]: """Run configured plugin in CWL. Run plugin as a CWL command line tool after setting I/O values. @@ -272,31 +278,34 @@ def run_cwl( """ if not self.outDir: - raise ValueError("") + msg = "" + raise ValueError(msg) if not cwl_path: - _p = pathlib.Path.cwd().joinpath(name_cleaner(self.name) + ".cwl") + _p = Path.cwd().joinpath(name_cleaner(self.name) + ".cwl") _cwl = self.save_cwl(_p) else: _cwl = self.save_cwl(cwl_path) if not io_path: - _p = pathlib.Path.cwd().joinpath(name_cleaner(self.name) + ".yml") + _p = Path.cwd().joinpath(name_cleaner(self.name) + ".yml") self.save_cwl_io(_p) # saves io to make it visible to user else: self.save_cwl_io(io_path) # saves io to make it visible to user - outdir_path = self.outDir.parent.relative_to(pathlib.Path.cwd()) + outdir_path = self.outDir.parent.relative_to(Path.cwd()) r_c = RuntimeContext({"outdir": str(outdir_path)}) fac = Factory(runtime_context=r_c) cwl = fac.make(str(_cwl)) return cwl(**self._cwl_io) # object's io dict is used instead of .yml file - def __lt__(self, other): + def __lt__(self, other: "BasePlugin") -> bool: return self.version < other.version - def __gt__(self, other): + def __gt__(self, other: "BasePlugin") -> bool: return other.version < self.version def __repr__(self) -> str: - return f"{self.__class__.__name__}(name='{self.name}', version={self.version.version})" + return ( + f"{self.__class__.__name__}(name='{self.name}', version={self.version!s})" + ) diff --git a/src/polus/plugins/_plugins/classes/plugin_classes.py b/src/polus/plugins/_plugins/classes/plugin_classes_v1.py similarity index 88% rename from src/polus/plugins/_plugins/classes/plugin_classes.py rename to src/polus/plugins/_plugins/classes/plugin_classes_v1.py index 9c4d54c05..90e67ddcc 100644 --- a/src/polus/plugins/_plugins/classes/plugin_classes.py +++ b/src/polus/plugins/_plugins/classes/plugin_classes_v1.py @@ -1,5 +1,5 @@ """Classes for Plugin objects containing methods to configure, run, and save.""" -# pylint: disable=W1203, enable=W1201 +# pylint: disable=W1203, W0212, enable=W1201 import json import logging import shutil @@ -10,14 +10,14 @@ from typing import Optional from typing import Union -from polus.plugins._plugins.classes.plugin_methods import _PluginMethods -from polus.plugins._plugins.io import DuplicateVersionFoundError -from polus.plugins._plugins.io import Version -from polus.plugins._plugins.io import _in_old_to_new -from polus.plugins._plugins.io import _ui_old_to_new -from polus.plugins._plugins.manifests.manifest_utils import InvalidManifest -from polus.plugins._plugins.manifests.manifest_utils import _load_manifest -from polus.plugins._plugins.manifests.manifest_utils import validate_manifest +from polus.plugins._plugins.classes.plugin_base import BasePlugin +from polus.plugins._plugins.io.io_v1 import DuplicateVersionFoundError +from polus.plugins._plugins.io.io_v1 import Version +from polus.plugins._plugins.io.io_v1 import _in_old_to_new +from polus.plugins._plugins.io.io_v1 import _ui_old_to_new +from polus.plugins._plugins.manifests import InvalidManifestError +from polus.plugins._plugins.manifests import _load_manifest +from polus.plugins._plugins.manifests import validate_manifest from polus.plugins._plugins.models import ComputeSchema from polus.plugins._plugins.models import PluginUIInput from polus.plugins._plugins.models import PluginUIOutput @@ -54,7 +54,7 @@ def refresh() -> None: try: plugin = validate_manifest(file) - except InvalidManifest: + except InvalidManifestError: logger.warning(f"Validation error in {file!s}") except BaseException as exc: # pylint: disable=W0718 # noqa: BLE001 logger.warning(f"Unexpected error {exc} with {file!s}") @@ -85,7 +85,21 @@ def list_plugins() -> list: return output -class Plugin(WIPPPluginManifest, _PluginMethods): +def _get_config(plugin: Union["Plugin", "ComputePlugin"], class_: str) -> dict: + model_ = plugin.dict() + # iterate over I/O to convert to dict + for io_name, io in model_["_io_keys"].items(): + # overwrite val if enum + if io["type"] == "enum": + val_ = io["value"].name # mapDirectory.raw + model_["_io_keys"][io_name]["value"] = val_.split(".")[-1] # raw + for inp in model_["inputs"]: + inp["value"] = None + model_["class"] = class_ + return model_ + + +class Plugin(WIPPPluginManifest, BasePlugin): """WIPP Plugin Class. Contains methods to configure, run, and save plugins. @@ -168,26 +182,20 @@ def save_manifest( def __setattr__(self, name: str, value: Any) -> None: # noqa: ANN401 """Set I/O parameters as attributes.""" - _PluginMethods.__setattr__(self, name, value) - - @property - def _config_file(self) -> dict: - config_ = self._config - config_["class"] = "WIPP" - return config_ + BasePlugin.__setattr__(self, name, value) def save_config(self, path: Union[str, Path]) -> None: """Save manifest with configured I/O parameters to specified path.""" with Path(path).open("w", encoding="utf-8") as file: - json.dump(self._config_file, file, indent=4, default=str) + json.dump(_get_config(self, "WIPP"), file, indent=4, default=str) logger.debug(f"Saved config to {path}") def __repr__(self) -> str: """Print plugin name and version.""" - return _PluginMethods.__repr__(self) + return BasePlugin.__repr__(self) -class ComputePlugin(ComputeSchema, _PluginMethods): +class ComputePlugin(ComputeSchema, BasePlugin): """Compute Plugin Class. Contains methods to configure, run, and save plugins. @@ -281,20 +289,14 @@ def versions(self) -> list: # cannot be in PluginMethods because PLUGINS lives """Return list of local versions of a Plugin.""" return list(PLUGINS[name_cleaner(self.name)]) - @property - def _config_file(self) -> dict: - config_ = self._config - config_["class"] = "Compute" - return config_ - def __setattr__(self, name: str, value: Any) -> None: # noqa: ANN401 """Set I/O parameters as attributes.""" - _PluginMethods.__setattr__(self, name, value) + BasePlugin.__setattr__(self, name, value) def save_config(self, path: Union[str, Path]) -> None: """Save configured manifest with I/O parameters to specified path.""" with Path(path).open("w", encoding="utf-8") as file: - json.dump(self._config_file, file, indent=4) + json.dump(_get_config(self, "Compute"), file, indent=4, default=str) logger.debug(f"Saved config to {path}") def save_manifest(self, path: Union[str, Path]) -> None: @@ -305,7 +307,7 @@ def save_manifest(self, path: Union[str, Path]) -> None: def __repr__(self) -> str: """Print plugin name and version.""" - return _PluginMethods.__repr__(self) + return BasePlugin.__repr__(self) def _load_plugin( @@ -383,15 +385,15 @@ def get_plugin( return _load_plugin(PLUGINS[name][Version(**{"version": version})]) -def load_config(config: Union[dict, Path]) -> Union[Plugin, ComputePlugin]: +def load_config(config: Union[dict, Path, str]) -> Union[Plugin, ComputePlugin]: """Load configured plugin from config file/dict.""" - if isinstance(config, Path): - with config.open("r", encoding="utf-8") as file: + if isinstance(config, (Path, str)): + with Path(config).open("r", encoding="utf-8") as file: manifest_ = json.load(file) elif isinstance(config, dict): manifest_ = config else: - msg = "config must be a dict or a path" + msg = "config must be a dict, str, or a path" raise TypeError(msg) io_keys_ = manifest_["_io_keys"] class_ = manifest_["class"] diff --git a/src/polus/plugins/_plugins/classes/plugin_classes_v2.py b/src/polus/plugins/_plugins/classes/plugin_classes_v2.py new file mode 100644 index 000000000..5a2aadae2 --- /dev/null +++ b/src/polus/plugins/_plugins/classes/plugin_classes_v2.py @@ -0,0 +1,429 @@ +"""Classes for Plugin objects containing methods to configure, run, and save.""" +# pylint: disable=W1203, W0212, enable=W1201 +import json +import logging +import shutil +import uuid +from copy import deepcopy +from pathlib import Path +from typing import Any +from typing import Optional +from typing import Union + +from polus.plugins._plugins.classes.plugin_base import BasePlugin +from polus.plugins._plugins.io.io_v2 import DuplicateVersionFoundError +from polus.plugins._plugins.io.io_v2 import Version +from polus.plugins._plugins.io.io_v2 import _in_old_to_new +from polus.plugins._plugins.io.io_v2 import _ui_old_to_new +from polus.plugins._plugins.manifests import InvalidManifestError +from polus.plugins._plugins.manifests import _load_manifest +from polus.plugins._plugins.manifests import validate_manifest +from polus.plugins._plugins.models import ComputeSchema +from polus.plugins._plugins.models import PluginUIInput +from polus.plugins._plugins.models import PluginUIOutput +from polus.plugins._plugins.models import WIPPPluginManifest +from polus.plugins._plugins.utils import cast_version +from polus.plugins._plugins.utils import name_cleaner +from pydantic import ConfigDict + +logger = logging.getLogger("polus.plugins") +PLUGINS: dict[str, dict] = {} +# PLUGINS = {"BasicFlatfieldCorrectionPlugin": +# {Version('0.1.4'): Path(<...>), Version('0.1.5'): Path(<...>)}. +# "VectorToLabel": {Version(...)}} + +""" +Paths and Fields +""" +# Location to store any discovered plugin manifests +_PLUGIN_DIR = Path(__file__).parent.parent.joinpath("manifests") + + +def refresh() -> None: + """Refresh the plugin list.""" + organizations = [ + x for x in _PLUGIN_DIR.iterdir() if x.name != "__pycache__" and x.is_dir() + ] # ignore __pycache__ + + PLUGINS.clear() + + for org in organizations: + for file in org.iterdir(): + if file.suffix == ".py": + continue + + try: + plugin = validate_manifest(file) + except InvalidManifestError: + logger.warning(f"Validation error in {file!s}") + except BaseException as exc: # pylint: disable=W0718 + logger.warning(f"Unexpected error {exc} with {file!s}") + raise exc + + else: + key = name_cleaner(plugin.name) + # Add version and path to VERSIONS + if key not in PLUGINS: + PLUGINS[key] = {} + if ( + plugin.version in PLUGINS[key] + and file != PLUGINS[key][plugin.version] + ): + msg = ( + "Found duplicate version of plugin" + f"{plugin.name} in {_PLUGIN_DIR}" + ) + raise DuplicateVersionFoundError( + msg, + ) + PLUGINS[key][plugin.version] = file + + +def list_plugins() -> list: + """List all local plugins.""" + output = list(PLUGINS.keys()) + output.sort() + return output + + +def _get_config(plugin: Union["Plugin", "ComputePlugin"], class_: str) -> dict: + model_ = json.loads(plugin.model_dump_json()) + model_["_io_keys"] = deepcopy(plugin._io_keys) # type: ignore + # iterate over I/O to convert to dict + for io_name, io in model_["_io_keys"].items(): + model_["_io_keys"][io_name] = json.loads(io.model_dump_json()) + # overwrite val if enum + if io.type.value == "enum": + model_["_io_keys"][io_name]["value"] = io.value.name # str + for inp in model_["inputs"]: + inp["value"] = None + model_["class"] = class_ + return model_ + + +class Plugin(WIPPPluginManifest, BasePlugin): + """WIPP Plugin Class. + + Contains methods to configure, run, and save plugins. + + Attributes: + versions: A list of local available versions for this plugin. + + Methods: + save_manifest(path): save plugin manifest to specified path + """ + + id: uuid.UUID # noqa: A003 + model_config = ConfigDict(extra="allow", frozen=True) + + def __init__(self, _uuid: bool = True, **data: dict) -> None: + """Init a plugin object from manifest.""" + if _uuid: + data["id"] = uuid.uuid4() # type: ignore + else: + data["id"] = uuid.UUID(str(data["id"])) # type: ignore + + super().__init__(**data) + + self._io_keys = {i.name: i for i in self.inputs} + self._io_keys.update({o.name: o for o in self.outputs}) + + if not self.author: + warn_msg = ( + f"The plugin ({self.name}) is missing the author field. " + "This field is not required but should be filled in." + ) + logger.warning(warn_msg) + + @property + def versions(self) -> list: # cannot be in PluginMethods because PLUGINS lives here + """Return list of local versions of a Plugin.""" + return list(PLUGINS[name_cleaner(self.name)]) + + def to_compute( + self, + hardware_requirements: Optional[dict] = None, + ) -> type[ComputeSchema]: + """Convert WIPP Plugin object to Compute Plugin object.""" + data = deepcopy(self.manifest) + return ComputePlugin( + hardware_requirements=hardware_requirements, + _from_old=True, + **data, + ) + + def save_manifest( + self, + path: Union[str, Path], + hardware_requirements: Optional[dict] = None, + compute: bool = False, + ) -> None: + """Save plugin manifest to specified path.""" + if compute: + with Path(path).open("w", encoding="utf-8") as file: + self.to_compute( + hardware_requirements=hardware_requirements, + ).save_manifest(path) + else: + with Path(path).open("w", encoding="utf-8") as file: + dict_ = self.manifest + json.dump( + dict_, + file, + indent=4, + ) + + logger.debug(f"Saved manifest to {path}") + + def __setattr__(self, name: str, value: Any) -> None: # noqa: ANN401 + """Set I/O parameters as attributes.""" + BasePlugin.__setattr__(self, name, value) + + def save_config(self, path: Union[str, Path]) -> None: + """Save manifest with configured I/O parameters to specified path.""" + with Path(path).open("w", encoding="utf-8") as file: + json.dump(_get_config(self, "WIPP"), file, indent=4, default=str) + logger.debug(f"Saved config to {path}") + + def __repr__(self) -> str: + """Print plugin name and version.""" + return BasePlugin.__repr__(self) + + +class ComputePlugin(ComputeSchema, BasePlugin): + """Compute Plugin Class. + + Contains methods to configure, run, and save plugins. + + Attributes: + versions: A list of local available versions for this plugin. + + Methods: + save_manifest(path): save plugin manifest to specified path + """ + + model_config = ConfigDict(extra="allow", frozen=True) + + def __init__( + self, + hardware_requirements: Optional[dict] = None, + _from_old: bool = False, + _uuid: bool = True, + **data: dict, + ) -> None: + """Init a plugin object from manifest.""" + if _uuid: + data["id"] = uuid.uuid4() # type: ignore + else: + data["id"] = uuid.UUID(str(data["id"])) # type: ignore + + if _from_old: + + def _convert_input(dict_: dict) -> dict: + dict_["type"] = _in_old_to_new(dict_["type"]) + return dict_ + + def _convert_output(dict_: dict) -> dict: + dict_["type"] = "path" + return dict_ + + def _ui_in(dict_: dict) -> PluginUIInput: # assuming old all ui input + # assuming format inputs. ___ + inp = dict_["key"].split(".")[-1] # e.g inpDir + try: + type_ = [x["type"] for x in data["inputs"] if x["name"] == inp][ + 0 + ] # get type from i/o + except IndexError: + type_ = "string" # default to string + except BaseException as exc: + raise exc + + dict_["type"] = _ui_old_to_new(type_) + return PluginUIInput(**dict_) + + def _ui_out(dict_: dict) -> PluginUIOutput: + new_dict_ = deepcopy(dict_) + new_dict_["name"] = "outputs." + new_dict_["name"] + new_dict_["type"] = _ui_old_to_new(new_dict_["type"]) + return PluginUIOutput(**new_dict_) + + data["inputs"] = [_convert_input(x) for x in data["inputs"]] # type: ignore + data["outputs"] = [ + _convert_output(x) for x in data["outputs"] + ] # type: ignore + data["pluginHardwareRequirements"] = {} + data["ui"] = [_ui_in(x) for x in data["ui"]] # type: ignore + data["ui"].extend( # type: ignore[attr-defined] + [_ui_out(x) for x in data["outputs"]], + ) + + if hardware_requirements: + for k, v in hardware_requirements.items(): + data["pluginHardwareRequirements"][k] = v + + data["version"] = cast_version(data["version"]) + super().__init__(**data) + self.Config.allow_mutation = True + self._io_keys = {i.name: i for i in self.inputs} + self._io_keys.update({o.name: o for o in self.outputs}) # type: ignore + + if not self.author: + warn_msg = ( + f"The plugin ({self.name}) is missing the author field. " + "This field is not required but should be filled in." + ) + logger.warning(warn_msg) + + @property + def versions(self) -> list: # cannot be in PluginMethods because PLUGINS lives here + """Return list of local versions of a Plugin.""" + return list(PLUGINS[name_cleaner(self.name)]) + + def __setattr__(self, name: str, value: Any) -> None: # noqa: ANN401 + """Set I/O parameters as attributes.""" + BasePlugin.__setattr__(self, name, value) + + def save_config(self, path: Union[str, Path]) -> None: + """Save configured manifest with I/O parameters to specified path.""" + with Path(path).open("w", encoding="utf-8") as file: + json.dump(_get_config(self, "Compute"), file, indent=4, default=str) + logger.debug(f"Saved config to {path}") + + def save_manifest(self, path: Union[str, Path]) -> None: + """Save plugin manifest to specified path.""" + with Path(path).open("w", encoding="utf-8") as file: + json.dump(self.manifest, file, indent=4) + logger.debug(f"Saved manifest to {path}") + + def __repr__(self) -> str: + """Print plugin name and version.""" + return BasePlugin.__repr__(self) + + +def _load_plugin( + manifest: Union[str, dict, Path], +) -> Union[Plugin, ComputePlugin]: + """Parse a manifest and return one of Plugin or ComputePlugin.""" + manifest = _load_manifest(manifest) + if "pluginHardwareRequirements" in manifest: # type: ignore[operator] + # Parse the manifest + plugin = ComputePlugin(**manifest) # type: ignore[arg-type] + else: + # Parse the manifest + plugin = Plugin(**manifest) # type: ignore[arg-type] + return plugin + + +def submit_plugin( + manifest: Union[str, dict, Path], +) -> Union[Plugin, ComputePlugin]: + """Parse a plugin and create a local copy of it. + + This function accepts a plugin manifest as a string, a dictionary (parsed + json), or a pathlib.Path object pointed at a plugin manifest. + + Args: + manifest: + A plugin manifest. It can be a url, a dictionary, + a path to a JSON file or a string that can be parsed as a dictionary + + Returns: + A Plugin object populated with information from the plugin manifest. + """ + plugin = validate_manifest(manifest) + plugin_name = name_cleaner(plugin.name) + + # Get Major/Minor/Patch versions + out_name = ( + plugin_name + + f"_M{plugin.version.major}m{plugin.version.minor}p{plugin.version.patch}.json" + ) + + # Save the manifest if it doesn't already exist in the database + organization = plugin.containerId.split("/")[0] + org_path = _PLUGIN_DIR.joinpath(organization.lower()) + org_path.mkdir(exist_ok=True, parents=True) + if not org_path.joinpath(out_name).exists(): + with org_path.joinpath(out_name).open("w", encoding="utf-8") as file: + manifest_ = json.loads(plugin.model_dump_json()) + json.dump(manifest_, file, indent=4) + + # Refresh plugins list + refresh() + return plugin + + +def get_plugin( + name: str, + version: Optional[str] = None, +) -> Union[Plugin, ComputePlugin]: + """Get a plugin with option to specify version. + + Return a plugin object with the option to specify a version. + The specified version's manifest must exist in manifests folder. + + Args: + name: Name of the plugin. + version: Optional version of the plugin, must follow semver. + + Returns: + Plugin object + """ + if version is None: + return _load_plugin(PLUGINS[name][max(PLUGINS[name])]) + return _load_plugin(PLUGINS[name][Version(version)]) + + +def load_config(config: Union[dict, Path, str]) -> Union[Plugin, ComputePlugin]: + """Load configured plugin from config file/dict.""" + if isinstance(config, (Path, str)): + with Path(config).open("r", encoding="utf-8") as file: + manifest_ = json.load(file) + elif isinstance(config, dict): + manifest_ = config + else: + msg = "config must be a dict, str, or a path" + raise TypeError(msg) + io_keys_ = manifest_["_io_keys"] + class_ = manifest_["class"] + manifest_.pop("class", None) + if class_ == "Compute": + plugin_ = ComputePlugin(_uuid=False, **manifest_) + elif class_ == "WIPP": + plugin_ = Plugin(_uuid=False, **manifest_) + else: + msg = "Invalid value of class" + raise ValueError(msg) + for key, value_ in io_keys_.items(): + val = value_["value"] + if val is not None: # exclude those values not set + setattr(plugin_, key, val) + return plugin_ + + +def remove_plugin(plugin: str, version: Optional[Union[str, list[str]]] = None) -> None: + """Remove plugin from the local database.""" + if version is None: + for plugin_version in PLUGINS[plugin]: + remove_plugin(plugin, plugin_version) + else: + if isinstance(version, list): + for version_ in version: + remove_plugin(plugin, version_) + return + version_ = Version(version) if not isinstance(version, Version) else version + path = PLUGINS[plugin][version_] + path.unlink() + refresh() + + +def remove_all() -> None: + """Remove all plugins from the local database.""" + organizations = [ + x for x in _PLUGIN_DIR.iterdir() if x.name != "__pycache__" and x.is_dir() + ] # ignore __pycache__ + logger.warning("Removing all plugins from local database") + for org in organizations: + shutil.rmtree(org) + refresh() diff --git a/src/polus/plugins/_plugins/io/__init__.py b/src/polus/plugins/_plugins/io/__init__.py new file mode 100644 index 000000000..1e10418b7 --- /dev/null +++ b/src/polus/plugins/_plugins/io/__init__.py @@ -0,0 +1,35 @@ +"""Init IO module.""" + +import pydantic + +PYDANTIC_VERSION = pydantic.__version__ + +if PYDANTIC_VERSION.split(".")[0] == "1": + from polus.plugins._plugins.io.io_v1 import Input + from polus.plugins._plugins.io.io_v1 import IOBase + from polus.plugins._plugins.io.io_v1 import Output + from polus.plugins._plugins.io.io_v1 import Version + from polus.plugins._plugins.io.io_v1 import input_to_cwl + from polus.plugins._plugins.io.io_v1 import io_to_yml + from polus.plugins._plugins.io.io_v1 import output_to_cwl + from polus.plugins._plugins.io.io_v1 import outputs_cwl +elif PYDANTIC_VERSION.split(".")[0] == "2": + from polus.plugins._plugins.io.io_v2 import Input + from polus.plugins._plugins.io.io_v2 import IOBase + from polus.plugins._plugins.io.io_v2 import Output + from polus.plugins._plugins.io.io_v2 import Version + from polus.plugins._plugins.io.io_v2 import input_to_cwl + from polus.plugins._plugins.io.io_v2 import io_to_yml + from polus.plugins._plugins.io.io_v2 import output_to_cwl + from polus.plugins._plugins.io.io_v2 import outputs_cwl + +__all__ = [ + "Input", + "Output", + "IOBase", + "Version", + "io_to_yml", + "outputs_cwl", + "input_to_cwl", + "output_to_cwl", +] diff --git a/src/polus/plugins/_plugins/io.py b/src/polus/plugins/_plugins/io/io_v1.py similarity index 98% rename from src/polus/plugins/_plugins/io.py rename to src/polus/plugins/_plugins/io/io_v1.py index fe6f27c7a..248e1af15 100644 --- a/src/polus/plugins/_plugins/io.py +++ b/src/polus/plugins/_plugins/io/io_v1.py @@ -10,6 +10,7 @@ from functools import singledispatchmethod from typing import Any from typing import Optional +from typing import TypeVar from typing import Union import fsspec @@ -102,6 +103,9 @@ def _ui_old_to_new(old: str) -> str: # map wipp InputType to compute schema's U return "text" +FileSystem = TypeVar("FileSystem", bound=fsspec.spec.AbstractFileSystem) + + class IOBase(BaseModel): # pylint: disable=R0903 """Base Class for I/O arguments.""" @@ -109,7 +113,7 @@ class IOBase(BaseModel): # pylint: disable=R0903 options: Optional[dict] = None value: Optional[Any] = None id_: Optional[Any] = None - _fs: Optional[type[fsspec.spec.AbstractFileSystem]] = PrivateAttr( + _fs: Optional[FileSystem] = PrivateAttr( default=None, ) # type checking is done at plugin level diff --git a/src/polus/plugins/_plugins/io/io_v2.py b/src/polus/plugins/_plugins/io/io_v2.py new file mode 100644 index 000000000..81c42f2bc --- /dev/null +++ b/src/polus/plugins/_plugins/io/io_v2.py @@ -0,0 +1,468 @@ +# type: ignore +# ruff: noqa: S101, A003 +# pylint: disable=no-self-argument +"""Plugins I/O utilities.""" +import enum +import logging +import pathlib +import re +from functools import singledispatch +from functools import singledispatchmethod +from typing import Annotated +from typing import Any +from typing import Optional +from typing import TypeVar +from typing import Union + +import fsspec +from pydantic import BaseModel +from pydantic import Field +from pydantic import PrivateAttr +from pydantic import RootModel +from pydantic import StringConstraints +from pydantic import field_validator + +logger = logging.getLogger("polus.plugins") + +""" +Enums for validating plugin input, output, and ui components. +""" +WIPP_TYPES = { + "collection": pathlib.Path, + "pyramid": pathlib.Path, + "csvCollection": pathlib.Path, + "genericData": pathlib.Path, + "stitchingVector": pathlib.Path, + "notebook": pathlib.Path, + "tensorflowModel": pathlib.Path, + "tensorboardLogs": pathlib.Path, + "pyramidAnnotation": pathlib.Path, + "integer": int, + "number": float, + "string": str, + "boolean": bool, + "array": str, + "enum": enum.Enum, + "path": pathlib.Path, +} + + +class InputTypes(str, enum.Enum): # wipp schema + """Enum of Input Types for WIPP schema.""" + + COLLECTION = "collection" + PYRAMID = "pyramid" + CSVCOLLECTION = "csvCollection" + GENERICDATA = "genericData" + STITCHINGVECTOR = "stitchingVector" + NOTEBOOK = "notebook" + TENSORFLOWMODEL = "tensorflowModel" + TENSORBOARDLOGS = "tensorboardLogs" + PYRAMIDANNOTATION = "pyramidAnnotation" + INTEGER = "integer" + NUMBER = "number" + STRING = "string" + BOOLEAN = "boolean" + ARRAY = "array" + ENUM = "enum" + + +class OutputTypes(str, enum.Enum): # wipp schema + """Enum for Output Types for WIPP schema.""" + + COLLECTION = "collection" + PYRAMID = "pyramid" + CSVCOLLECTION = "csvCollection" + GENERICDATA = "genericData" + STITCHINGVECTOR = "stitchingVector" + NOTEBOOK = "notebook" + TENSORFLOWMODEL = "tensorflowModel" + TENSORBOARDLOGS = "tensorboardLogs" + PYRAMIDANNOTATION = "pyramidAnnotation" + + +def _in_old_to_new(old: str) -> str: # map wipp InputType to compute schema's InputType + """Map an InputType from wipp schema to one of compute schema.""" + d = {"integer": "number", "enum": "string"} + if old in ["string", "array", "number", "boolean"]: + return old + if old in d: + return d[old] # integer or enum + return "path" # everything else + + +def _ui_old_to_new(old: str) -> str: # map wipp InputType to compute schema's UIType + """Map an InputType from wipp schema to a UIType of compute schema.""" + type_dict = { + "string": "text", + "boolean": "checkbox", + "number": "number", + "array": "text", + "integer": "number", + } + if old in type_dict: + return type_dict[old] + return "text" + + +FileSystem = TypeVar("FileSystem", bound=fsspec.spec.AbstractFileSystem) + + +class IOBase(BaseModel): # pylint: disable=R0903 + """Base Class for I/O arguments.""" + + type: Any = None + options: Optional[dict] = None + value: Optional[Any] = None + id_: Optional[Any] = None + _fs: Optional[FileSystem] = PrivateAttr( + default=None, + ) # type checking is done at plugin level + + def _validate(self) -> None: # pylint: disable=R0912 + value = self.value + + if value is None: + if self.required: + msg = f""" + The input value ({self.name}) is required, + but the value was not set.""" + raise TypeError( + msg, + ) + + return + + if self.type == InputTypes.ENUM: + try: + if isinstance(value, str): + value = enum.Enum(self.name, self.options["values"])[value] + elif not isinstance(value, enum.Enum): + raise ValueError + + except KeyError: + logging.error( + f""" + Value ({value}) is not a valid value + for the enum input ({self.name}). + Must be one of {self.options['values']}. + """, + ) + raise + else: + if isinstance(self.type, (InputTypes, OutputTypes)): # wipp + value = WIPP_TYPES[self.type](value) + else: + value = WIPP_TYPES[self.type.value]( + value, + ) # compute, type does not inherit from str + + if isinstance(value, pathlib.Path): + value = value.absolute() + if self._fs: + assert self._fs.exists( + str(value), + ), f"{value} is invalid or does not exist" + assert self._fs.isdir( + str(value), + ), f"{value} is not a valid directory" + else: + assert value.exists(), f"{value} is invalid or does not exist" + assert value.is_dir(), f"{value} is not a valid directory" + + super().__setattr__("value", value) + + def __setattr__(self, name: str, value: Any) -> None: # ruff: noqa: ANN401 + """Set I/O attributes.""" + if name not in ["value", "id", "_fs"]: + # Don't permit any other values to be changed + msg = f"Cannot set property: {name}" + raise TypeError(msg) + + super().__setattr__(name, value) + + if name == "value": + self._validate() + + +class Output(IOBase): # pylint: disable=R0903 + """Required until JSON schema is fixed.""" + + name: Annotated[ + str, + StringConstraints(pattern=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$"), + ] = Field( + ..., + examples=["outputCollection"], + title="Output name", + ) + type: OutputTypes = Field( + ..., + examples=["stitchingVector", "collection"], + title="Output type", + ) + description: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( + ..., + examples=["Output collection"], + title="Output description", + ) + + +class Input(IOBase): # pylint: disable=R0903 + """Required until JSON schema is fixed.""" + + name: Annotated[ + str, + StringConstraints(pattern=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$"), + ] = Field( + ..., + description="Input name as expected by the plugin CLI", + examples=["inputImages", "fileNamePattern", "thresholdValue"], + title="Input name", + ) + type: InputTypes + description: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( + ..., + examples=["Input Images"], + title="Input description", + ) + required: Optional[bool] = Field( + True, + description="Whether an input is required or not", + examples=[True], + title="Required input", + ) + + def __init__(self, **data) -> None: # ruff: noqa: ANN003 + """Initialize input.""" + super().__init__(**data) + + if self.description is None: + logger.warning( + f""" + The input ({self.name}) is missing the description field. + This field is not required but should be filled in. + """, + ) + + +def _check_version_number(value: Union[str, int]) -> bool: + if isinstance(value, int): + value = str(value) + if "-" in value: + value = value.split("-")[0] + if len(value) > 1 and value[0] == "0": + return False + return bool(re.match(r"^\d+$", value)) + + +class Version(RootModel): + """SemVer object.""" + + root: str + + @field_validator("root") + @classmethod + def semantic_version( + cls, + value, + ) -> Any: # ruff: noqa: ANN202, N805, ANN001 + """Pydantic Validator to check semver.""" + version = value.split(".") + + assert ( + len(version) == 3 # ruff: noqa: PLR2004 + ), f""" + Invalid version ({value}). Version must follow + semantic versioning (see semver.org)""" + if "-" in version[-1]: # with hyphen + idn = version[-1].split("-")[-1] + id_reg = re.compile("[0-9A-Za-z-]+") + assert bool( + id_reg.match(idn), + ), f"""Invalid version ({value}). + Version must follow semantic versioning (see semver.org)""" + + assert all( + map(_check_version_number, version), + ), f"""Invalid version ({value}). + Version must follow semantic versioning (see semver.org)""" + return value + + @property + def major(self): + """Return x from x.y.z .""" + return int(self.root.split(".")[0]) + + @property + def minor(self): + """Return y from x.y.z .""" + return int(self.root.split(".")[1]) + + @property + def patch(self): + """Return z from x.y.z .""" + if not self.root.split(".")[2].isdigit(): + msg = "Patch version is not a digit, comparison may not be accurate." + logger.warning(msg) + return self.root.split(".")[2] + return int(self.root.split(".")[2]) + + def __str__(self) -> str: + """Return string representation of Version object.""" + return self.root + + @singledispatchmethod + def __lt__(self, other: Any) -> bool: + """Compare if Version is less than other object.""" + msg = "invalid type for comparison." + raise TypeError(msg) + + @singledispatchmethod + def __gt__(self, other: Any) -> bool: + """Compare if Version is less than other object.""" + msg = "invalid type for comparison." + raise TypeError(msg) + + @singledispatchmethod + def __eq__(self, other: Any) -> bool: + """Compare if two Version objects are equal.""" + msg = "invalid type for comparison." + raise TypeError(msg) + + def __hash__(self) -> int: + """Needed to use Version objects as dict keys.""" + return hash(self.root) + + def __repr__(self) -> str: + """Return string representation of Version object.""" + return self.root + + +@Version.__eq__.register(Version) # pylint: disable=no-member +def _(self, other): + return ( + other.major == self.major + and other.minor == self.minor + and other.patch == self.patch + ) + + +@Version.__eq__.register(str) # pylint: disable=no-member +def _(self, other): + return self == Version(other) + + +@Version.__lt__.register(Version) # pylint: disable=no-member +def _(self, other): + if other.major > self.major: + return True + if other.major == self.major: + if other.minor > self.minor: + return True + if other.minor == self.minor: + if other.patch > self.patch: + return True + return False + return False + return False + + +@Version.__lt__.register(str) # pylint: disable=no-member +def _(self, other): + v = Version(other) + return self < v + + +@Version.__gt__.register(Version) # pylint: disable=no-member +def _(self, other): + return other < self + + +@Version.__gt__.register(str) # pylint: disable=no-member +def _(self, other): + v = Version(other) + return self > v + + +class DuplicateVersionFoundError(Exception): + """Raise when two equal versions found.""" + + +CWL_INPUT_TYPES = { + "path": "Directory", # always Dir? Yes + "string": "string", + "number": "double", + "boolean": "boolean", + "genericData": "Directory", + "collection": "Directory", + "enum": "string", # for compatibility with workflows + "stitchingVector": "Directory", + # not yet implemented: array +} + + +def _type_in(inp: Input): + """Return appropriate value for `type` based on input type.""" + val = inp.type.value + req = "" if inp.required else "?" + + # NOT compatible with CWL workflows, ok in CLT + # if val == "enum": + # if input.required: + + # if val in CWL_INPUT_TYPES: + return CWL_INPUT_TYPES[val] + req if val in CWL_INPUT_TYPES else "string" + req + + +def input_to_cwl(inp: Input): + """Return dict of inputs for cwl.""" + return { + f"{inp.name}": { + "type": _type_in(inp), + "inputBinding": {"prefix": f"--{inp.name}"}, + }, + } + + +def output_to_cwl(out: Output): + """Return dict of output args for cwl for input section.""" + return { + f"{out.name}": { + "type": "Directory", + "inputBinding": {"prefix": f"--{out.name}"}, + }, + } + + +def outputs_cwl(out: Output): + """Return dict of output for `outputs` in cwl.""" + return { + f"{out.name}": { + "type": "Directory", + "outputBinding": {"glob": f"$(inputs.{out.name}.basename)"}, + }, + } + + +# -- I/O as arguments in .yml + + +@singledispatch +def _io_value_to_yml(io) -> Union[str, dict]: + return str(io) + + +@_io_value_to_yml.register +def _(io: pathlib.Path): + return {"class": "Directory", "location": str(io)} + + +@_io_value_to_yml.register +def _(io: enum.Enum): + return io.name + + +def io_to_yml(io): + """Return IO entry for yml file.""" + return _io_value_to_yml(io.value) diff --git a/src/polus/plugins/_plugins/manifests/__init__.py b/src/polus/plugins/_plugins/manifests/__init__.py new file mode 100644 index 000000000..b2642a73f --- /dev/null +++ b/src/polus/plugins/_plugins/manifests/__init__.py @@ -0,0 +1,26 @@ +"""Initialize manifests module.""" + +import pydantic + +PYDANTIC_VERSION = pydantic.__version__ + +if PYDANTIC_VERSION.split(".")[0] == "1": + from polus.plugins._plugins.manifests.manifest_utils_v1 import InvalidManifestError + from polus.plugins._plugins.manifests.manifest_utils_v1 import _error_log + from polus.plugins._plugins.manifests.manifest_utils_v1 import _load_manifest + from polus.plugins._plugins.manifests.manifest_utils_v1 import _scrape_manifests + from polus.plugins._plugins.manifests.manifest_utils_v1 import validate_manifest +elif PYDANTIC_VERSION.split(".")[0] == "2": + from polus.plugins._plugins.manifests.manifest_utils_v2 import InvalidManifestError + from polus.plugins._plugins.manifests.manifest_utils_v2 import _error_log + from polus.plugins._plugins.manifests.manifest_utils_v2 import _load_manifest + from polus.plugins._plugins.manifests.manifest_utils_v2 import _scrape_manifests + from polus.plugins._plugins.manifests.manifest_utils_v2 import validate_manifest + +__all__ = [ + "InvalidManifestError", + "_load_manifest", + "validate_manifest", + "_error_log", + "_scrape_manifests", +] diff --git a/src/polus/plugins/_plugins/manifests/manifest_utils.py b/src/polus/plugins/_plugins/manifests/manifest_utils.py deleted file mode 100644 index ebc9d0556..000000000 --- a/src/polus/plugins/_plugins/manifests/manifest_utils.py +++ /dev/null @@ -1,202 +0,0 @@ -"""Utilities for manifest parsing and validation.""" -import json -import logging -import pathlib -import typing -from urllib.parse import urlparse - -import github -import requests -from pydantic import ValidationError, errors -from tqdm import tqdm - -from polus.plugins._plugins.models import ComputeSchema, WIPPPluginManifest -from polus.plugins._plugins.utils import cast_version - -logger = logging.getLogger("polus.plugins") - -# Fields that must be in a plugin manifest -REQUIRED_FIELDS = [ - "name", - "version", - "description", - "author", - "containerId", - "inputs", - "outputs", - "ui", -] - - -class InvalidManifest(Exception): - """Raised when manifest has validation errors.""" - - -def is_valid_manifest(plugin: dict) -> bool: - """Validate basic attributes of a plugin manifest. - - Args: - plugin: A parsed plugin json file - - Returns: - True if the plugin has the minimal json fields - """ - fields = list(plugin.keys()) - - try: - for field in REQUIRED_FIELDS: - assert field in fields, f"Missing json field, {field}, in plugin manifest." - except AssertionError: - return False - - return True - - -def _load_manifest(m: typing.Union[str, dict, pathlib.Path]) -> dict: - """Convert to dictionary if pathlib.Path or str.""" - if isinstance(m, dict): - return m - elif isinstance(m, pathlib.Path): - assert ( - m.suffix == ".json" - ), "Plugin manifest must be a json file with .json extension." - - with open(m) as fr: - manifest = json.load(fr) - - elif isinstance(m, str): - if urlparse(m).netloc == "": - manifest = json.loads(m) - else: - manifest = requests.get(m).json() - else: - msg = "invalid manifest" - raise ValueError(msg) - return manifest - - -def validate_manifest( - manifest: typing.Union[str, dict, pathlib.Path], -) -> typing.Union[WIPPPluginManifest, ComputeSchema]: - """Validate a plugin manifest against schema.""" - manifest = _load_manifest(manifest) - manifest["version"] = cast_version( - manifest["version"], - ) # cast version to semver object - if "name" in manifest: - name = manifest["name"] - else: - raise InvalidManifest(f"{manifest} has no value for name") - - if "pluginHardwareRequirements" in manifest: - # Parse the manifest - try: - plugin = ComputeSchema(**manifest) - except ValidationError as e: - raise InvalidManifest(f"{name} does not conform to schema") from e - except BaseException as e: - raise e - else: - # Parse the manifest - try: - plugin = WIPPPluginManifest(**manifest) - except ValidationError as e: - raise InvalidManifest( - f"{manifest['name']} does not conform to schema" - ) from e - except BaseException as e: - raise e - return plugin - - -def _scrape_manifests( - repo: typing.Union[str, github.Repository.Repository], # type: ignore - gh: github.Github, - min_depth: int = 1, - max_depth: typing.Optional[int] = None, - return_invalid: bool = False, -) -> typing.Union[list, tuple[list, list]]: - if max_depth is None: - max_depth = min_depth - min_depth = 0 - - assert max_depth >= min_depth, "max_depth is smaller than min_depth" - - if isinstance(repo, str): - repo = gh.get_repo(repo) - - contents = list(repo.get_contents("")) # type: ignore - next_contents = [] - valid_manifests = [] - invalid_manifests = [] - - for d in range(0, max_depth): - for content in tqdm(contents, desc=f"{repo.full_name}: {d}"): - if content.type == "dir": - next_contents.extend(repo.get_contents(content.path)) # type: ignore - elif content.name.endswith(".json"): - if d >= min_depth: - manifest = json.loads(content.decoded_content) - if is_valid_manifest(manifest): - valid_manifests.append(manifest) - else: - invalid_manifests.append(manifest) - - contents = next_contents.copy() - next_contents = [] - - if return_invalid: - return valid_manifests, invalid_manifests - else: - return valid_manifests - - -def _error_log(val_err, manifest, fct): - report = [] - - for err in val_err.args[0]: - if isinstance(err, list): - err = err[0] - - if isinstance(err, AssertionError): - report.append( - "The plugin ({}) failed an assertion check: {}".format( - manifest["name"], - err.args[0], - ), - ) - logger.critical(f"{fct}: {report[-1]}") - elif isinstance(err.exc, errors.MissingError): - report.append( - "The plugin ({}) is missing fields: {}".format( - manifest["name"], - err.loc_tuple(), - ), - ) - logger.critical(f"{fct}: {report[-1]}") - elif errors.ExtraError: - if err.loc_tuple()[0] in ["inputs", "outputs", "ui"]: - report.append( - "The plugin ({}) had unexpected values in the {} ({}): {}".format( - manifest["name"], - err.loc_tuple()[0], - manifest[err.loc_tuple()[0]][err.loc_tuple()[1]]["name"], - err.exc.args[0][0].loc_tuple(), - ), - ) - else: - report.append( - "The plugin ({}) had an error: {}".format( - manifest["name"], - err.exc.args[0][0], - ), - ) - logger.critical(f"{fct}: {report[-1]}") - else: - logger.warning( - "{}: Uncaught manifest Error in ({}): {}".format( - fct, - manifest["name"], - str(val_err).replace("\n", ", ").replace(" ", " "), - ), - ) diff --git a/src/polus/plugins/_plugins/manifests/manifest_utils_v1.py b/src/polus/plugins/_plugins/manifests/manifest_utils_v1.py new file mode 100644 index 000000000..927126f4a --- /dev/null +++ b/src/polus/plugins/_plugins/manifests/manifest_utils_v1.py @@ -0,0 +1,206 @@ +"""Utilities for manifest parsing and validation.""" +import json +import logging +import pathlib +from typing import Optional +from typing import Union + +import github +import requests +import validators +from polus.plugins._plugins.models import ComputeSchema +from polus.plugins._plugins.models import WIPPPluginManifest +from polus.plugins._plugins.utils import cast_version +from pydantic import ValidationError +from pydantic import errors +from tqdm import tqdm + +logger = logging.getLogger("polus.plugins") + +# Fields that must be in a plugin manifest +REQUIRED_FIELDS = [ + "name", + "version", + "description", + "author", + "containerId", + "inputs", + "outputs", + "ui", +] + + +class InvalidManifestError(Exception): + """Raised when manifest has validation errors.""" + + +def is_valid_manifest(plugin: dict) -> bool: + """Validate basic attributes of a plugin manifest. + + Args: + plugin: A parsed plugin json file + + Returns: + True if the plugin has the minimal json fields + """ + fields = list(plugin.keys()) + + for field in REQUIRED_FIELDS: + if field not in fields: + msg = f"Missing json field, {field}, in plugin manifest." + logger.error(msg) + return False + return True + + +def _load_manifest(manifest: Union[str, dict, pathlib.Path]) -> dict: + """Return manifest as dict from str (url or path) or pathlib.Path.""" + if isinstance(manifest, dict): # is dict + return manifest + if isinstance(manifest, pathlib.Path): # is path + if manifest.suffix != ".json": + msg = "plugin manifest must be a json file with .json extension." + raise ValueError(msg) + + with manifest.open("r", encoding="utf-8") as manifest_json: + manifest_ = json.load(manifest_json) + elif isinstance(manifest, str): # is str + if validators.url(manifest): # is url + manifest_ = requests.get(manifest, timeout=10).json() + else: # could (and should) be path + try: + manifest_ = _load_manifest(pathlib.Path(manifest)) + except Exception as exc: # was not a Path? # noqa + msg = "invalid manifest" + raise ValueError(msg) from exc + else: # is not str, dict, or path + msg = f"invalid manifest type {type(manifest)}" + raise ValueError(msg) + return manifest_ + + +def validate_manifest( + manifest: Union[str, dict, pathlib.Path], +) -> Union[WIPPPluginManifest, ComputeSchema]: + """Validate a plugin manifest against schema.""" + manifest = _load_manifest(manifest) + manifest["version"] = cast_version( + manifest["version"], + ) # cast version to semver object + if "name" in manifest: + name = manifest["name"] + else: + msg = f"{manifest} has no value for name" + raise InvalidManifestError(msg) + + if "pluginHardwareRequirements" in manifest: + # Parse the manifest + try: + plugin = ComputeSchema(**manifest) + except ValidationError as e: + msg = f"{name} does not conform to schema" + raise InvalidManifestError(msg) from e + except BaseException as e: + raise e + else: + # Parse the manifest + try: + plugin = WIPPPluginManifest(**manifest) + except ValidationError as e: + msg = f"{manifest['name']} does not conform to schema" + raise InvalidManifestError( + msg, + ) from e + except BaseException as e: + raise e + return plugin + + +def _scrape_manifests( + repo: Union[str, github.Repository.Repository], # type: ignore + gh: github.Github, + min_depth: int = 1, + max_depth: Optional[int] = None, + return_invalid: bool = False, +) -> Union[list, tuple[list, list]]: + if max_depth is None: + max_depth = min_depth + min_depth = 0 + + if not max_depth >= min_depth: + msg = "max_depth is smaller than min_depth" + raise ValueError(msg) + + if isinstance(repo, str): + repo = gh.get_repo(repo) + + contents = list(repo.get_contents("")) # type: ignore + next_contents: list = [] + valid_manifests: list = [] + invalid_manifests: list = [] + + for d in range(0, max_depth): + for content in tqdm(contents, desc=f"{repo.full_name}: {d}"): + if content.type == "dir": + next_contents.extend(repo.get_contents(content.path)) # type: ignore + elif content.name.endswith(".json") and d >= min_depth: + manifest = json.loads(content.decoded_content) + if is_valid_manifest(manifest): + valid_manifests.append(manifest) + else: + invalid_manifests.append(manifest) + + contents = next_contents.copy() + next_contents = [] + + if return_invalid: + return valid_manifests, invalid_manifests + return valid_manifests + + +def _error_log(val_err: ValidationError, manifest: dict, fct: str) -> None: + report = [] + + for error in val_err.args[0]: + if isinstance(error, list): + error = error[0] # noqa + + if isinstance(error, AssertionError): + msg = ( + f"The plugin ({manifest['name']}) " + "failed an assertion check: {err.args[0]}" + ) + report.append(msg) + logger.critical(f"{fct}: {report[-1]}") # pylint: disable=W1203 + elif isinstance(error.exc, errors.MissingError): + msg = ( + f"The plugin ({manifest['name']}) " + "is missing fields: {err.loc_tuple()}" + ) + report.append(msg) + logger.critical(f"{fct}: {report[-1]}") # pylint: disable=W1203 + elif errors.ExtraError: + if error.loc_tuple()[0] in ["inputs", "outputs", "ui"]: + manifest_ = manifest[error.loc_tuple()[0]][error.loc_tuple()[1]]["name"] + msg = ( + f"The plugin ({manifest['name']}) " + "had unexpected values in the " + f"{error.loc_tuple()[0]} " + f"({manifest_}): " + f"{error.exc.args[0][0].loc_tuple()}" + ) + report.append(msg) + else: + msg = ( + f"The plugin ({manifest['name']}) " + "had an error: {err.exc.args[0][0]}" + ) + report.append(msg) + logger.critical(f"{fct}: {report[-1]}") # pylint: disable=W1203 + else: + str_val_err = str(val_err).replace("\n", ", ").replace(" ", " ") + msg = ( + f"{fct}: Uncaught manifest error in ({manifest['name']}): " + f"{str_val_err}" + ) + logger.warning(msg) diff --git a/src/polus/plugins/_plugins/manifests/manifest_utils_v2.py b/src/polus/plugins/_plugins/manifests/manifest_utils_v2.py new file mode 100644 index 000000000..9c37c111b --- /dev/null +++ b/src/polus/plugins/_plugins/manifests/manifest_utils_v2.py @@ -0,0 +1,202 @@ +"""Utilities for manifest parsing and validation.""" +import json +import logging +import pathlib +from typing import Optional +from typing import Union + +import github +import requests +import validators +from polus.plugins._plugins.models import ComputeSchema +from polus.plugins._plugins.models import WIPPPluginManifest +from pydantic import ValidationError +from pydantic import errors +from tqdm import tqdm + +logger = logging.getLogger("polus.plugins") + +# Fields that must be in a plugin manifest +REQUIRED_FIELDS = [ + "name", + "version", + "description", + "author", + "containerId", + "inputs", + "outputs", + "ui", +] + + +class InvalidManifestError(Exception): + """Raised when manifest has validation errors.""" + + +def is_valid_manifest(plugin: dict) -> bool: + """Validate basic attributes of a plugin manifest. + + Args: + plugin: A parsed plugin json file + + Returns: + True if the plugin has the minimal json fields + """ + fields = list(plugin.keys()) + + for field in REQUIRED_FIELDS: + if field not in fields: + msg = f"Missing json field, {field}, in plugin manifest." + logger.error(msg) + return False + return True + + +def _load_manifest(manifest: Union[str, dict, pathlib.Path]) -> dict: + """Return manifest as dict from str (url or path) or pathlib.Path.""" + if isinstance(manifest, dict): # is dict + return manifest + if isinstance(manifest, pathlib.Path): # is path + if manifest.suffix != ".json": + msg = "plugin manifest must be a json file with .json extension." + raise ValueError(msg) + + with manifest.open("r", encoding="utf-8") as manifest_json: + manifest_ = json.load(manifest_json) + elif isinstance(manifest, str): # is str + if validators.url(manifest): # is url + manifest_ = requests.get(manifest, timeout=10).json() + else: # could (and should) be path + try: + manifest_ = _load_manifest(pathlib.Path(manifest)) + except Exception as exc: # was not a Path? # noqa + msg = "invalid manifest" + raise ValueError(msg) from exc + else: # is not str, dict, or path + msg = f"invalid manifest type {type(manifest)}" + raise ValueError(msg) + return manifest_ + + +def validate_manifest( + manifest: Union[str, dict, pathlib.Path], +) -> Union[WIPPPluginManifest, ComputeSchema]: + """Validate a plugin manifest against schema.""" + manifest = _load_manifest(manifest) + if "name" in manifest: + name = manifest["name"] + else: + msg = f"{manifest} has no value for name" + raise InvalidManifestError(msg) + + if "pluginHardwareRequirements" in manifest: + # Parse the manifest + try: + plugin = ComputeSchema(**manifest) + except ValidationError as e: + msg = f"{name} does not conform to schema" + raise InvalidManifestError(msg) from e + except BaseException as e: + raise e + else: + # Parse the manifest + try: + plugin = WIPPPluginManifest(**manifest) + except ValidationError as e: + msg = f"{manifest['name']} does not conform to schema" + raise InvalidManifestError( + msg, + ) from e + except BaseException as e: + raise e + return plugin + + +def _scrape_manifests( + repo: Union[str, github.Repository.Repository], # type: ignore + gh: github.Github, + min_depth: int = 1, + max_depth: Optional[int] = None, + return_invalid: bool = False, +) -> Union[list, tuple[list, list]]: + if max_depth is None: + max_depth = min_depth + min_depth = 0 + + if not max_depth >= min_depth: + msg = "max_depth is smaller than min_depth" + raise ValueError(msg) + + if isinstance(repo, str): + repo = gh.get_repo(repo) + + contents = list(repo.get_contents("")) # type: ignore + next_contents: list = [] + valid_manifests: list = [] + invalid_manifests: list = [] + + for d in range(0, max_depth): + for content in tqdm(contents, desc=f"{repo.full_name}: {d}"): + if content.type == "dir": + next_contents.extend(repo.get_contents(content.path)) # type: ignore + elif content.name.endswith(".json") and d >= min_depth: + manifest = json.loads(content.decoded_content) + if is_valid_manifest(manifest): + valid_manifests.append(manifest) + else: + invalid_manifests.append(manifest) + + contents = next_contents.copy() + next_contents = [] + + if return_invalid: + return valid_manifests, invalid_manifests + return valid_manifests + + +def _error_log(val_err: ValidationError, manifest: dict, fct: str) -> None: + report = [] + + for error in val_err.args[0]: + if isinstance(error, list): + error = error[0] # noqa + + if isinstance(error, AssertionError): + msg = ( + f"The plugin ({manifest['name']}) " + "failed an assertion check: {err.args[0]}" + ) + report.append(msg) + logger.critical(f"{fct}: {report[-1]}") # pylint: disable=W1203 + elif isinstance(error.exc, errors.MissingError): + msg = ( + f"The plugin ({manifest['name']}) " + "is missing fields: {err.loc_tuple()}" + ) + report.append(msg) + logger.critical(f"{fct}: {report[-1]}") # pylint: disable=W1203 + elif errors.ExtraError: + if error.loc_tuple()[0] in ["inputs", "outputs", "ui"]: + manifest_ = manifest[error.loc_tuple()[0]][error.loc_tuple()[1]]["name"] + msg = ( + f"The plugin ({manifest['name']}) " + "had unexpected values in the " + f"{error.loc_tuple()[0]} " + f"({manifest_}): " + f"{error.exc.args[0][0].loc_tuple()}" + ) + report.append(msg) + else: + msg = ( + f"The plugin ({manifest['name']}) " + "had an error: {err.exc.args[0][0]}" + ) + report.append(msg) + logger.critical(f"{fct}: {report[-1]}") # pylint: disable=W1203 + else: + str_val_err = str(val_err).replace("\n", ", ").replace(" ", " ") + msg = ( + f"{fct}: Uncaught manifest error in ({manifest['name']}): " + f"{str_val_err}" + ) + logger.warning(msg) diff --git a/src/polus/plugins/_plugins/models/PolusComputeSchema.py b/src/polus/plugins/_plugins/models/PolusComputeSchema.py deleted file mode 100644 index ea7e1afb8..000000000 --- a/src/polus/plugins/_plugins/models/PolusComputeSchema.py +++ /dev/null @@ -1,136 +0,0 @@ -# generated by datamodel-codegen: -# filename: PolusComputeSchema.json -# timestamp: 2022-09-21T03:41:58+00:00 - -from __future__ import annotations - -from enum import Enum -from typing import Any, List, Optional, Union - -from pydantic import BaseModel, Field, constr - - -class Model(BaseModel): - __root__: Any - - -class PluginInputType(Enum): - path = "path" - string = "string" - number = "number" - array = "array" - boolean = "boolean" - - -class PluginInput(BaseModel): - format: Optional[str] = Field(None, title="Format") - label: Optional[str] = Field(None, title="Label") - name: str = Field(..., title="Name") - required: bool = Field(..., title="Required") - type: PluginInputType - default: Optional[Union[str, float, bool]] = Field(None, title="Default") - - -class PluginOutputType(Enum): - path = "path" - - -class PluginOutput(BaseModel): - format: Optional[str] = Field(None, title="Format") - label: Optional[str] = Field(None, title="Label") - name: str = Field(..., title="Name") - type: PluginOutputType - - -class GpuVendor(Enum): - none = "none" - amd = "amd" - tpu = "tpu" - nvidia = "nvidia" - - -class PluginHardwareRequirements(BaseModel): - coresMax: Optional[Union[str, float]] = Field(None, title="Coresmax") - coresMin: Optional[Union[str, float]] = Field(None, title="Coresmin") - cpuAVX: Optional[bool] = Field(None, title="Cpuavx") - cpuAVX2: Optional[bool] = Field(None, title="Cpuavx2") - cpuMin: Optional[str] = Field(None, title="Cpumin") - gpu: Optional[GpuVendor] = None - gpuCount: Optional[float] = Field(None, title="Gpucount") - gpuDriverVersion: Optional[str] = Field(None, title="Gpudriverversion") - gpuType: Optional[str] = Field(None, title="Gputype") - outDirMax: Optional[Union[str, float]] = Field(None, title="Outdirmax") - outDirMin: Optional[Union[str, float]] = Field(None, title="Outdirmin") - ramMax: Optional[Union[str, float]] = Field(None, title="Rammax") - ramMin: Optional[Union[str, float]] = Field(None, title="Rammin") - tmpDirMax: Optional[Union[str, float]] = Field(None, title="Tmpdirmax") - tmpDirMin: Optional[Union[str, float]] = Field(None, title="Tmpdirmin") - - -class ThenEntry(BaseModel): - action: str = Field(..., title="Action") - input: str = Field(..., title="Input") - value: str = Field(..., title="Value") - - -class ConditionEntry(BaseModel): - expression: str = Field(..., title="Expression") - - -class Validator(BaseModel): - then: Optional[List[ThenEntry]] = Field(None, title="Then") - validator: Optional[List[ConditionEntry]] = Field(None, title="Validator") - - -class PluginUIType(Enum): - checkbox = "checkbox" - color = "color" - date = "date" - email = "email" - number = "number" - password = "password" - radio = "radio" - range = "range" - text = "text" - time = "time" - - -class PluginUIInput(BaseModel): - bind: Optional[str] = Field(None, title="Bind") - condition: Optional[Union[List[Validator], str]] = Field(None, title="Condition") - default: Optional[Union[str, float, bool]] = Field(None, title="Default") - description: Optional[str] = Field(None, title="Description") - fieldset: Optional[List[str]] = Field(None, title="Fieldset") - hidden: Optional[bool] = Field(None, title="Hidden") - key: str = Field(..., title="Key") - title: str = Field(..., title="Title") - type: PluginUIType - - -class PluginUIOutput(BaseModel): - description: str = Field(..., title="Description") - format: Optional[str] = Field(None, title="Format") - name: str = Field(..., title="Name") - type: PluginUIType - website: Optional[str] = Field(None, title="Website") - - -class PluginSchema(BaseModel): - author: Optional[str] = Field(None, title="Author") - baseCommand: Optional[List[str]] = Field(None, title="Basecommand") - citation: Optional[str] = Field(None, title="Citation") - containerId: str = Field(..., title="Containerid") - customInputs: Optional[bool] = Field(None, title="Custominputs") - description: str = Field(..., title="Description") - inputs: List[PluginInput] = Field(..., title="Inputs") - institution: Optional[str] = Field(None, title="Institution") - name: str = Field(..., title="Name") - outputs: List[PluginOutput] = Field(..., title="Outputs") - pluginHardwareRequirements: PluginHardwareRequirements - repository: Optional[str] = Field(None, title="Repository") - title: str = Field(..., title="Title") - ui: List[Union[PluginUIInput, PluginUIOutput]] = Field(..., title="Ui") - version: constr( - regex=r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$" - ) = Field(..., examples=["0.1.0", "0.1.0rc1"], title="Version") - website: Optional[str] = Field(None, title="Website") diff --git a/src/polus/plugins/_plugins/models/__init__.py b/src/polus/plugins/_plugins/models/__init__.py index ce1d984dd..9b9092c75 100644 --- a/src/polus/plugins/_plugins/models/__init__.py +++ b/src/polus/plugins/_plugins/models/__init__.py @@ -1,10 +1,31 @@ """Pydantic Models based on JSON schemas.""" -from polus.plugins._plugins.models.compute import PluginSchema as ComputeSchema -from polus.plugins._plugins.models.PolusComputeSchema import ( - PluginUIInput, - PluginUIOutput, -) -from polus.plugins._plugins.models.wipp import WIPPPluginManifest + +import pydantic + +PYDANTIC_VERSION = pydantic.__version__ + +if PYDANTIC_VERSION.split(".")[0] == "1": + from polus.plugins._plugins.models.pydanticv1.compute import ( + PluginSchema as ComputeSchema, + ) + from polus.plugins._plugins.models.pydanticv1.PolusComputeSchema import ( + PluginUIInput, + ) + from polus.plugins._plugins.models.pydanticv1.PolusComputeSchema import ( + PluginUIOutput, + ) + from polus.plugins._plugins.models.pydanticv1.wipp import WIPPPluginManifest +elif PYDANTIC_VERSION.split(".")[0] == "2": + from polus.plugins._plugins.models.pydanticv2.compute import ( + PluginSchema as ComputeSchema, + ) + from polus.plugins._plugins.models.pydanticv2.PolusComputeSchema import ( + PluginUIInput, + ) + from polus.plugins._plugins.models.pydanticv2.PolusComputeSchema import ( + PluginUIOutput, + ) + from polus.plugins._plugins.models.pydanticv2.wipp import WIPPPluginManifest __all__ = [ "WIPPPluginManifest", diff --git a/src/polus/plugins/_plugins/models/pydanticv1/PolusComputeSchema.py b/src/polus/plugins/_plugins/models/pydanticv1/PolusComputeSchema.py new file mode 100644 index 000000000..a40b5b402 --- /dev/null +++ b/src/polus/plugins/_plugins/models/pydanticv1/PolusComputeSchema.py @@ -0,0 +1,137 @@ +# generated by datamodel-codegen: +# timestamp: 2022-09-21T03:41:58+00:00 + +from __future__ import annotations + +from enum import Enum +from typing import Any + +from pydantic import BaseModel +from pydantic import Field +from pydantic import constr + + +class Model(BaseModel): + __root__: Any + + +class PluginInputType(Enum): + path = "path" + string = "string" + number = "number" + array = "array" + boolean = "boolean" + + +class PluginInput(BaseModel): + format: str | None = Field(None, title="Format") + label: str | None = Field(None, title="Label") + name: str = Field(..., title="Name") + required: bool = Field(..., title="Required") + type: PluginInputType + default: str | float | bool | None = Field(None, title="Default") + + +class PluginOutputType(Enum): + path = "path" + + +class PluginOutput(BaseModel): + format: str | None = Field(None, title="Format") + label: str | None = Field(None, title="Label") + name: str = Field(..., title="Name") + type: PluginOutputType + + +class GpuVendor(Enum): + none = "none" + amd = "amd" + tpu = "tpu" + nvidia = "nvidia" + + +class PluginHardwareRequirements(BaseModel): + coresMax: str | float | None = Field(None, title="Coresmax") + coresMin: str | float | None = Field(None, title="Coresmin") + cpuAVX: bool | None = Field(None, title="Cpuavx") + cpuAVX2: bool | None = Field(None, title="Cpuavx2") + cpuMin: str | None = Field(None, title="Cpumin") + gpu: GpuVendor | None = None + gpuCount: float | None = Field(None, title="Gpucount") + gpuDriverVersion: str | None = Field(None, title="Gpudriverversion") + gpuType: str | None = Field(None, title="Gputype") + outDirMax: str | float | None = Field(None, title="Outdirmax") + outDirMin: str | float | None = Field(None, title="Outdirmin") + ramMax: str | float | None = Field(None, title="Rammax") + ramMin: str | float | None = Field(None, title="Rammin") + tmpDirMax: str | float | None = Field(None, title="Tmpdirmax") + tmpDirMin: str | float | None = Field(None, title="Tmpdirmin") + + +class ThenEntry(BaseModel): + action: str = Field(..., title="Action") + input: str = Field(..., title="Input") + value: str = Field(..., title="Value") + + +class ConditionEntry(BaseModel): + expression: str = Field(..., title="Expression") + + +class Validator(BaseModel): + then: list[ThenEntry] | None = Field(None, title="Then") + validator: list[ConditionEntry] | None = Field(None, title="Validator") + + +class PluginUIType(Enum): + checkbox = "checkbox" + color = "color" + date = "date" + email = "email" + number = "number" + password = "password" + radio = "radio" + range = "range" + text = "text" + time = "time" + + +class PluginUIInput(BaseModel): + bind: str | None = Field(None, title="Bind") + condition: list[Validator] | str | None = Field(None, title="Condition") + default: str | float | bool | None = Field(None, title="Default") + description: str | None = Field(None, title="Description") + fieldset: list[str] | None = Field(None, title="Fieldset") + hidden: bool | None = Field(None, title="Hidden") + key: str = Field(..., title="Key") + title: str = Field(..., title="Title") + type: PluginUIType + + +class PluginUIOutput(BaseModel): + description: str = Field(..., title="Description") + format: str | None = Field(None, title="Format") + name: str = Field(..., title="Name") + type: PluginUIType + website: str | None = Field(None, title="Website") + + +class PluginSchema(BaseModel): + author: str | None = Field(None, title="Author") + baseCommand: list[str] | None = Field(None, title="Basecommand") + citation: str | None = Field(None, title="Citation") + containerId: str = Field(..., title="Containerid") + customInputs: bool | None = Field(None, title="Custominputs") + description: str = Field(..., title="Description") + inputs: list[PluginInput] = Field(..., title="Inputs") + institution: str | None = Field(None, title="Institution") + name: str = Field(..., title="Name") + outputs: list[PluginOutput] = Field(..., title="Outputs") + pluginHardwareRequirements: PluginHardwareRequirements + repository: str | None = Field(None, title="Repository") + title: str = Field(..., title="Title") + ui: list[PluginUIInput | PluginUIOutput] = Field(..., title="Ui") + version: constr( + regex=r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$", + ) = Field(..., examples=["0.1.0", "0.1.0rc1"], title="Version") + website: str | None = Field(None, title="Website") diff --git a/src/polus/plugins/_plugins/models/WIPPPluginSchema.py b/src/polus/plugins/_plugins/models/pydanticv1/WIPPPluginSchema.py similarity index 69% rename from src/polus/plugins/_plugins/models/WIPPPluginSchema.py rename to src/polus/plugins/_plugins/models/pydanticv1/WIPPPluginSchema.py index 8f9e1eb2a..718d3a3fa 100644 --- a/src/polus/plugins/_plugins/models/WIPPPluginSchema.py +++ b/src/polus/plugins/_plugins/models/pydanticv1/WIPPPluginSchema.py @@ -1,13 +1,15 @@ # generated by datamodel-codegen: -# filename: wipp-plugin-manifest-schema.json # timestamp: 2023-01-04T14:54:38+00:00 from __future__ import annotations from enum import Enum -from typing import Any, List, Optional, Union +from typing import Any -from pydantic import AnyUrl, BaseModel, Field, constr +from pydantic import AnyUrl +from pydantic import BaseModel +from pydantic import Field +from pydantic import constr class Type(Enum): @@ -35,12 +37,16 @@ class Input(BaseModel): title="Input name", ) type: Type = Field( - ..., examples=["collection", "string", "number"], title="Input Type" + ..., + examples=["collection", "string", "number"], + title="Input Type", ) description: constr(regex=r"^(.*)$") = Field( - ..., examples=["Input Images"], title="Input description" + ..., + examples=["Input Images"], + title="Input description", ) - required: Optional[bool] = Field( + required: bool | None = Field( True, description="Whether an input is required or not", examples=[True], @@ -61,18 +67,24 @@ class Type1(Enum): class Output(BaseModel): name: constr(regex=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$") = Field( - ..., examples=["outputCollection"], title="Output name" + ..., + examples=["outputCollection"], + title="Output name", ) type: Type1 = Field( - ..., examples=["stitchingVector", "collection"], title="Output type" + ..., + examples=["stitchingVector", "collection"], + title="Output type", ) description: constr(regex=r"^(.*)$") = Field( - ..., examples=["Output collection"], title="Output description" + ..., + examples=["Output collection"], + title="Output description", ) class UiItem(BaseModel): - key: Union[Any, Any] = Field( + key: Any | Any = Field( ..., description="Key of the input which this UI definition applies to, the expected format is 'inputs.inputName'. Special keyword 'fieldsets' can be used to define arrangement of inputs by sections.", examples=["inputs.inputImages", "inputs.fileNamPattern", "fieldsets"], @@ -81,10 +93,12 @@ class UiItem(BaseModel): class CudaRequirements(BaseModel): - deviceMemoryMin: Optional[float] = Field( - 0, examples=[100], title="Minimum device memory" + deviceMemoryMin: float | None = Field( + 0, + examples=[100], + title="Minimum device memory", ) - cudaComputeCapability: Optional[Union[str, List[Any]]] = Field( + cudaComputeCapability: str | list[Any] | None = Field( None, description="Specify either a single minimum value, or an array of valid values", examples=["8.0", ["3.5", "5.0", "6.0", "7.0", "7.5", "8.0"]], @@ -93,26 +107,32 @@ class CudaRequirements(BaseModel): class ResourceRequirements(BaseModel): - ramMin: Optional[float] = Field( - None, examples=[2048], title="Minimum RAM in mebibytes (Mi)" + ramMin: float | None = Field( + None, + examples=[2048], + title="Minimum RAM in mebibytes (Mi)", ) - coresMin: Optional[float] = Field( - None, examples=[1], title="Minimum number of CPU cores" + coresMin: float | None = Field( + None, + examples=[1], + title="Minimum number of CPU cores", ) - cpuAVX: Optional[bool] = Field( + cpuAVX: bool | None = Field( False, examples=[True], title="Advanced Vector Extensions (AVX) CPU capability required", ) - cpuAVX2: Optional[bool] = Field( + cpuAVX2: bool | None = Field( False, examples=[False], title="Advanced Vector Extensions 2 (AVX2) CPU capability required", ) - gpu: Optional[bool] = Field( - False, examples=[True], title="GPU/accelerator required" + gpu: bool | None = Field( + False, + examples=[True], + title="GPU/accelerator required", ) - cudaRequirements: Optional[CudaRequirements] = Field( + cudaRequirements: CudaRequirements | None = Field( {}, examples=[{"deviceMemoryMin": 100, "cudaComputeCapability": "8.0"}], title="GPU Cuda-related requirements", @@ -143,26 +163,30 @@ class WippPluginManifest(BaseModel): examples=["Custom image segmentation plugin"], title="Short description of the plugin", ) - author: Optional[Optional[constr(regex=r"^(.*)$")]] = Field( - "", examples=["FirstName LastName"], title="Author(s)" + author: constr(regex="^(.*)$") | None | None = Field( + "", + examples=["FirstName LastName"], + title="Author(s)", ) - institution: Optional[Optional[constr(regex=r"^(.*)$")]] = Field( + institution: constr(regex="^(.*)$") | None | None = Field( "", examples=["National Institute of Standards and Technology"], title="Institution", ) - repository: Optional[Optional[AnyUrl]] = Field( + repository: AnyUrl | None | None = Field( "", examples=["https://github.com/usnistgov/WIPP"], title="Source code repository", ) - website: Optional[Optional[AnyUrl]] = Field( - "", examples=["http://usnistgov.github.io/WIPP"], title="Website" + website: AnyUrl | None | None = Field( + "", + examples=["http://usnistgov.github.io/WIPP"], + title="Website", ) - citation: Optional[Optional[constr(regex=r"^(.*)$")]] = Field( + citation: constr(regex="^(.*)$") | None | None = Field( "", examples=[ - "Peter Bajcsy, Joe Chalfoun, and Mylene Simon (2018). Web Microanalysis of Big Image Data. Springer-Verlag International" + "Peter Bajcsy, Joe Chalfoun, and Mylene Simon (2018). Web Microanalysis of Big Image Data. Springer-Verlag International", ], title="Citation", ) @@ -172,23 +196,25 @@ class WippPluginManifest(BaseModel): examples=["docker.io/wipp/plugin-example:1.0.0"], title="ContainerId", ) - baseCommand: Optional[List[str]] = Field( + baseCommand: list[str] | None = Field( None, description="Base command to use while running container image", examples=[["python3", "/opt/executable/main.py"]], title="Base command", ) - inputs: List[Input] = Field( + inputs: list[Input] = Field( ..., description="Defines inputs to the plugin", title="List of Inputs", unique_items=True, ) - outputs: List[Output] = Field( - ..., description="Defines the outputs of the plugin", title="List of Outputs" + outputs: list[Output] = Field( + ..., + description="Defines the outputs of the plugin", + title="List of Outputs", ) - ui: List[UiItem] = Field(..., title="Plugin form UI definition") - resourceRequirements: Optional[ResourceRequirements] = Field( + ui: list[UiItem] = Field(..., title="Plugin form UI definition") + resourceRequirements: ResourceRequirements | None = Field( {}, examples=[ { @@ -201,7 +227,7 @@ class WippPluginManifest(BaseModel): "deviceMemoryMin": 100, "cudaComputeCapability": "8.0", }, - } + }, ], title="Plugin Resource Requirements", ) diff --git a/src/polus/plugins/_plugins/models/pydanticv1/__init__.py b/src/polus/plugins/_plugins/models/pydanticv1/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/polus/plugins/_plugins/models/compute.py b/src/polus/plugins/_plugins/models/pydanticv1/compute.py similarity index 57% rename from src/polus/plugins/_plugins/models/compute.py rename to src/polus/plugins/_plugins/models/pydanticv1/compute.py index 8584143c3..2a34a3ae7 100644 --- a/src/polus/plugins/_plugins/models/compute.py +++ b/src/polus/plugins/_plugins/models/pydanticv1/compute.py @@ -4,14 +4,12 @@ functions of PolusComputeSchema.py which is automatically generated by datamodel-codegen from JSON schema. """ -from typing import List -from polus.plugins._plugins.io import IOBase, Version -from polus.plugins._plugins.models.PolusComputeSchema import ( - PluginInput, - PluginOutput, - PluginSchema, -) +from polus.plugins._plugins.io import IOBase +from polus.plugins._plugins.io import Version +from polus.plugins._plugins.models.pydanticv1.PolusComputeSchema import PluginInput +from polus.plugins._plugins.models.pydanticv1.PolusComputeSchema import PluginOutput +from polus.plugins._plugins.models.pydanticv1.PolusComputeSchema import PluginSchema class PluginInput(PluginInput, IOBase): # type: ignore @@ -25,6 +23,6 @@ class PluginOutput(PluginOutput, IOBase): # type: ignore class PluginSchema(PluginSchema): # type: ignore """Extended Compute Plugin Schema with extended IO defs.""" - inputs: List[PluginInput] - outputs: List[PluginOutput] + inputs: list[PluginInput] + outputs: list[PluginOutput] version: Version diff --git a/src/polus/plugins/_plugins/models/pydanticv1/wipp.py b/src/polus/plugins/_plugins/models/pydanticv1/wipp.py new file mode 100644 index 000000000..6557355a2 --- /dev/null +++ b/src/polus/plugins/_plugins/models/pydanticv1/wipp.py @@ -0,0 +1,79 @@ +"""Extending automatically generated wipp model. + +This file modifies and extend certain fields and +functions of WIPPPluginSchema.py which is automatically +generated by datamodel-codegen from JSON schema. +""" +from typing import Literal +from typing import Optional +from typing import Union + +from polus.plugins._plugins.io import Input +from polus.plugins._plugins.io import Output +from polus.plugins._plugins.io import Version +from polus.plugins._plugins.models.pydanticv1.WIPPPluginSchema import ( + ResourceRequirements, +) +from polus.plugins._plugins.models.pydanticv1.WIPPPluginSchema import WippPluginManifest +from pydantic import BaseModel +from pydantic import Field + + +class UI1(BaseModel): + """Base class for UI items.""" + + key: str = Field(constr=r"^inputs.[a-zA-Z0-9][-a-zA-Z0-9]*$") + title: str + description: Optional[str] + condition: Optional[str] + default: Optional[Union[str, float, int, bool]] + hidden: Optional[bool] = Field(default=False) + bind: Optional[str] + + +class FieldSet(BaseModel): + """Base class for FieldSet.""" + + title: str + fields: list[str] = Field(min_items=1, unique_items=True) + + +class UI2(BaseModel): + """UI items class for fieldsets.""" + + key: Literal["fieldsets"] + fieldsets: list[FieldSet] = Field(min_items=1, unique_items=True) + + +class WIPPPluginManifest(WippPluginManifest): + """Extended WIPP Plugin Schema.""" + + inputs: list[Input] = Field( + ..., + description="Defines inputs to the plugin", + title="List of Inputs", + ) + outputs: list[Output] = Field( + ..., + description="Defines the outputs of the plugin", + title="List of Outputs", + ) + ui: list[Union[UI1, UI2]] = Field(..., title="Plugin form UI definition") + version: Version + resourceRequirements: Optional[ResourceRequirements] = Field( # noqa + None, + examples=[ + { + "ramMin": 2048, + "coresMin": 1, + "cpuAVX": True, + "cpuAVX2": False, + "gpu": True, + "cudaRequirements": { + "deviceMemoryMin": 100, + "cudaComputeCapability": "8.0", + }, + }, + ], + title="Plugin Resource Requirements", + ) diff --git a/src/polus/plugins/_plugins/models/pydanticv2/PolusComputeSchema.py b/src/polus/plugins/_plugins/models/pydanticv2/PolusComputeSchema.py new file mode 100644 index 000000000..d87a986fd --- /dev/null +++ b/src/polus/plugins/_plugins/models/pydanticv2/PolusComputeSchema.py @@ -0,0 +1,136 @@ +# generated by datamodel-codegen: edited by Camilo Velez +# timestamp: 2022-09-21T03:41:58+00:00 + +from __future__ import annotations + +from enum import Enum +from typing import Annotated + +from pydantic import BaseModel +from pydantic import Field +from pydantic import StringConstraints + + +class PluginInputType(Enum): + path = "path" + string = "string" + number = "number" + array = "array" + boolean = "boolean" + + +class PluginInput(BaseModel): + format: str | None = Field(None, title="Format") + label: str | None = Field(None, title="Label") + name: str = Field(..., title="Name") + required: bool = Field(..., title="Required") + type: PluginInputType + default: str | float | bool | None = Field(None, title="Default") + + +class PluginOutputType(Enum): + path = "path" + + +class PluginOutput(BaseModel): + format: str | None = Field(None, title="Format") + label: str | None = Field(None, title="Label") + name: str = Field(..., title="Name") + type: PluginOutputType + + +class GpuVendor(Enum): + none = "none" + amd = "amd" + tpu = "tpu" + nvidia = "nvidia" + + +class PluginHardwareRequirements(BaseModel): + coresMax: str | float | None = Field(None, title="Coresmax") + coresMin: str | float | None = Field(None, title="Coresmin") + cpuAVX: bool | None = Field(None, title="Cpuavx") + cpuAVX2: bool | None = Field(None, title="Cpuavx2") + cpuMin: str | None = Field(None, title="Cpumin") + gpu: GpuVendor | None = None + gpuCount: float | None = Field(None, title="Gpucount") + gpuDriverVersion: str | None = Field(None, title="Gpudriverversion") + gpuType: str | None = Field(None, title="Gputype") + outDirMax: str | float | None = Field(None, title="Outdirmax") + outDirMin: str | float | None = Field(None, title="Outdirmin") + ramMax: str | float | None = Field(None, title="Rammax") + ramMin: str | float | None = Field(None, title="Rammin") + tmpDirMax: str | float | None = Field(None, title="Tmpdirmax") + tmpDirMin: str | float | None = Field(None, title="Tmpdirmin") + + +class ThenEntry(BaseModel): + action: str = Field(..., title="Action") + input: str = Field(..., title="Input") + value: str = Field(..., title="Value") + + +class ConditionEntry(BaseModel): + expression: str = Field(..., title="Expression") + + +class Validator(BaseModel): + then: list[ThenEntry] | None = Field(None, title="Then") + validator: list[ConditionEntry] | None = Field(None, title="Validator") + + +class PluginUIType(Enum): + checkbox = "checkbox" + color = "color" + date = "date" + email = "email" + number = "number" + password = "password" + radio = "radio" + range = "range" + text = "text" + time = "time" + + +class PluginUIInput(BaseModel): + bind: str | None = Field(None, title="Bind") + condition: list[Validator] | str | None = Field(None, title="Condition") + default: str | float | bool | None = Field(None, title="Default") + description: str | None = Field(None, title="Description") + fieldset: list[str] | None = Field(None, title="Fieldset") + hidden: bool | None = Field(None, title="Hidden") + key: str = Field(..., title="Key") + title: str = Field(..., title="Title") + type: PluginUIType + + +class PluginUIOutput(BaseModel): + description: str = Field(..., title="Description") + format: str | None = Field(None, title="Format") + name: str = Field(..., title="Name") + type: PluginUIType + website: str | None = Field(None, title="Website") + + +class PluginSchema(BaseModel): + author: str | None = Field(None, title="Author") + baseCommand: list[str] | None = Field(None, title="Basecommand") + citation: str | None = Field(None, title="Citation") + containerId: str = Field(..., title="Containerid") + customInputs: bool | None = Field(None, title="Custominputs") + description: str = Field(..., title="Description") + inputs: list[PluginInput] = Field(..., title="Inputs") + institution: str | None = Field(None, title="Institution") + name: str = Field(..., title="Name") + outputs: list[PluginOutput] = Field(..., title="Outputs") + pluginHardwareRequirements: PluginHardwareRequirements + repository: str | None = Field(None, title="Repository") + title: str = Field(..., title="Title") + ui: list[PluginUIInput | PluginUIOutput] = Field(..., title="Ui") + version: Annotated[ + str, + StringConstraints( + pattern=r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$", + ), + ] = Field(..., examples=["0.1.0", "0.1.0rc1"], title="Version") + website: str | None = Field(None, title="Website") diff --git a/src/polus/plugins/_plugins/models/pydanticv2/WIPPPluginSchema.py b/src/polus/plugins/_plugins/models/pydanticv2/WIPPPluginSchema.py new file mode 100644 index 000000000..099cb32d2 --- /dev/null +++ b/src/polus/plugins/_plugins/models/pydanticv2/WIPPPluginSchema.py @@ -0,0 +1,241 @@ +# generated by datamodel-codegen: edited by Camilo Velez +# timestamp: 2023-01-04T14:54:38+00:00 + +from __future__ import annotations + +from enum import Enum +from typing import Annotated +from typing import Any + +from pydantic import AnyUrl +from pydantic import BaseModel +from pydantic import Field +from pydantic import StringConstraints + + +class Type(Enum): + collection = "collection" + stitchingVector = "stitchingVector" + tensorflowModel = "tensorflowModel" + csvCollection = "csvCollection" + pyramid = "pyramid" + pyramidAnnotation = "pyramidAnnotation" + notebook = "notebook" + genericData = "genericData" + string = "string" + number = "number" + integer = "integer" + enum = "enum" + array = "array" + boolean = "boolean" + + +class Input(BaseModel): + name: Annotated[ + str, + StringConstraints(pattern=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$"), + ] = Field( + ..., + description="Input name as expected by the plugin CLI", + examples=["inputImages", "fileNamePattern", "thresholdValue"], + title="Input name", + ) + type: Type = Field( + ..., + examples=["collection", "string", "number"], + title="Input Type", + ) + description: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( + ..., + examples=["Input Images"], + title="Input description", + ) + required: bool | None = Field( + True, + description="Whether an input is required or not", + examples=[True], + title="Required input", + ) + + +class Type1(Enum): + collection = "collection" + stitchingVector = "stitchingVector" + tensorflowModel = "tensorflowModel" + tensorboardLogs = "tensorboardLogs" + csvCollection = "csvCollection" + pyramid = "pyramid" + pyramidAnnotation = "pyramidAnnotation" + genericData = "genericData" + + +class Output(BaseModel): + name: Annotated[ + str, + StringConstraints(pattern=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$"), + ] = Field(..., examples=["outputCollection"], title="Output name") + type: Type1 = Field( + ..., + examples=["stitchingVector", "collection"], + title="Output type", + ) + description: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( + ..., + examples=["Output collection"], + title="Output description", + ) + + +class UiItem(BaseModel): + key: Any | Any = Field( + ..., + description="Key of the input which this UI definition applies to, the expected format is 'inputs.inputName'. Special keyword 'fieldsets' can be used to define arrangement of inputs by sections.", + examples=["inputs.inputImages", "inputs.fileNamPattern", "fieldsets"], + title="UI key", + ) + + +class CudaRequirements(BaseModel): + deviceMemoryMin: float | None = Field( + 0, + examples=[100], + title="Minimum device memory", + ) + cudaComputeCapability: str | list[Any] | None = Field( + None, + description="Specify either a single minimum value, or an array of valid values", + examples=["8.0", ["3.5", "5.0", "6.0", "7.0", "7.5", "8.0"]], + title="The cudaComputeCapability Schema", + ) + + +class ResourceRequirements(BaseModel): + ramMin: float | None = Field( + None, + examples=[2048], + title="Minimum RAM in mebibytes (Mi)", + ) + coresMin: float | None = Field( + None, + examples=[1], + title="Minimum number of CPU cores", + ) + cpuAVX: bool | None = Field( + False, + examples=[True], + title="Advanced Vector Extensions (AVX) CPU capability required", + ) + cpuAVX2: bool | None = Field( + False, + examples=[False], + title="Advanced Vector Extensions 2 (AVX2) CPU capability required", + ) + gpu: bool | None = Field( + False, + examples=[True], + title="GPU/accelerator required", + ) + cudaRequirements: CudaRequirements | None = Field( + {}, + examples=[{"deviceMemoryMin": 100, "cudaComputeCapability": "8.0"}], + title="GPU Cuda-related requirements", + ) + + +class WippPluginManifest(BaseModel): + name: Annotated[str, StringConstraints(pattern=r"^(.*)$", min_length=1)] = Field( + ..., + description="Name of the plugin (format: org/name)", + examples=["wipp/plugin-example"], + title="Plugin name", + ) + version: Annotated[str, StringConstraints(pattern=r"^(.*)$", min_length=1)] = Field( + ..., + description="Version of the plugin (semantic versioning preferred)", + examples=["1.0.0"], + title="Plugin version", + ) + title: Annotated[str, StringConstraints(pattern=r"^(.*)$", min_length=1)] = Field( + ..., + description="Plugin title to display in WIPP forms", + examples=["WIPP Plugin example"], + title="Plugin title", + ) + description: Annotated[ + str, + StringConstraints(pattern=r"^(.*)$", min_length=1), + ] = Field( + ..., + examples=["Custom image segmentation plugin"], + title="Short description of the plugin", + ) + author: Annotated[str, StringConstraints(pattern="^(.*)$")] | None | None = Field( + "", + examples=["FirstName LastName"], + title="Author(s)", + ) + institution: Annotated[ + str, + StringConstraints(pattern="^(.*)$"), + ] | None | None = Field( + "", + examples=["National Institute of Standards and Technology"], + title="Institution", + ) + repository: AnyUrl | None | None = Field( + "", + examples=["https://github.com/usnistgov/WIPP"], + title="Source code repository", + ) + website: AnyUrl | None | None = Field( + "", + examples=["http://usnistgov.github.io/WIPP"], + title="Website", + ) + citation: Annotated[str, StringConstraints(pattern="^(.*)$")] | None | None = Field( + "", + examples=[ + "Peter Bajcsy, Joe Chalfoun, and Mylene Simon (2018). Web Microanalysis of Big Image Data. Springer-Verlag International", + ], + title="Citation", + ) + containerId: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( + ..., + description="Docker image ID", + examples=["docker.io/wipp/plugin-example:1.0.0"], + title="ContainerId", + ) + baseCommand: list[str] | None = Field( + None, + description="Base command to use while running container image", + examples=[["python3", "/opt/executable/main.py"]], + title="Base command", + ) + inputs: set[Input] = Field( + ..., + description="Defines inputs to the plugin", + title="List of Inputs", + ) + outputs: list[Output] = Field( + ..., + description="Defines the outputs of the plugin", + title="List of Outputs", + ) + ui: list[UiItem] = Field(..., title="Plugin form UI definition") + resourceRequirements: ResourceRequirements | None = Field( + {}, + examples=[ + { + "ramMin": 2048, + "coresMin": 1, + "cpuAVX": True, + "cpuAVX2": False, + "gpu": True, + "cudaRequirements": { + "deviceMemoryMin": 100, + "cudaComputeCapability": "8.0", + }, + }, + ], + title="Plugin Resource Requirements", + ) diff --git a/src/polus/plugins/_plugins/models/pydanticv2/__init__.py b/src/polus/plugins/_plugins/models/pydanticv2/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/polus/plugins/_plugins/models/pydanticv2/compute.py b/src/polus/plugins/_plugins/models/pydanticv2/compute.py new file mode 100644 index 000000000..250574f12 --- /dev/null +++ b/src/polus/plugins/_plugins/models/pydanticv2/compute.py @@ -0,0 +1,28 @@ +"""Extending automatically generated compute model. + +This file modifies and extend certain fields and +functions of PolusComputeSchema.py which is automatically +generated by datamodel-codegen from JSON schema. +""" + +from polus.plugins._plugins.io import IOBase +from polus.plugins._plugins.io import Version +from polus.plugins._plugins.models.pydanticv2.PolusComputeSchema import PluginInput +from polus.plugins._plugins.models.pydanticv2.PolusComputeSchema import PluginOutput +from polus.plugins._plugins.models.pydanticv2.PolusComputeSchema import PluginSchema + + +class PluginInput(PluginInput, IOBase): # type: ignore + """Base Class for Input Args.""" + + +class PluginOutput(PluginOutput, IOBase): # type: ignore + """Base Class for Output Args.""" + + +class PluginSchema(PluginSchema): # type: ignore + """Extended Compute Plugin Schema with extended IO defs.""" + + inputs: list[PluginInput] + outputs: list[PluginOutput] + version: Version diff --git a/src/polus/plugins/_plugins/models/pydanticv2/wipp.py b/src/polus/plugins/_plugins/models/pydanticv2/wipp.py new file mode 100644 index 000000000..41c0dff31 --- /dev/null +++ b/src/polus/plugins/_plugins/models/pydanticv2/wipp.py @@ -0,0 +1,79 @@ +"""Extending automatically generated wipp model. + +This file modifies and extend certain fields and +functions of WIPPPluginSchema.py which is automatically +generated by datamodel-codegen from JSON schema. +""" +from typing import Literal +from typing import Optional +from typing import Union + +from polus.plugins._plugins.io import Input +from polus.plugins._plugins.io import Output +from polus.plugins._plugins.io import Version +from polus.plugins._plugins.models.pydanticv2.WIPPPluginSchema import ( + ResourceRequirements, +) +from polus.plugins._plugins.models.pydanticv2.WIPPPluginSchema import WippPluginManifest +from pydantic import BaseModel +from pydantic import Field + + +class UI1(BaseModel): + """Base class for UI items.""" + + key: str = Field(constr=r"^inputs.[a-zA-Z0-9][-a-zA-Z0-9]*$") + title: str + description: Optional[str] = None + condition: Optional[str] = None + default: Optional[Union[str, float, int, bool]] = None + hidden: Optional[bool] = Field(default=False) + bind: Optional[str] = None + + +class FieldSet(BaseModel): + """Base class for FieldSet.""" + + title: str + fields: set[str] = Field(min_length=1) + + +class UI2(BaseModel): + """UI items class for fieldsets.""" + + key: Literal["fieldsets"] + fieldsets: set[FieldSet] = Field(min_length=1) + + +class WIPPPluginManifest(WippPluginManifest): + """Extended WIPP Plugin Schema.""" + + inputs: list[Input] = Field( + ..., + description="Defines inputs to the plugin", + title="List of Inputs", + ) + outputs: list[Output] = Field( + ..., + description="Defines the outputs of the plugin", + title="List of Outputs", + ) + ui: list[Union[UI1, UI2]] = Field(..., title="Plugin form UI definition") + version: Version + resourceRequirements: Optional[ResourceRequirements] = Field( # noqa + None, + examples=[ + { + "ramMin": 2048, + "coresMin": 1, + "cpuAVX": True, + "cpuAVX2": False, + "gpu": True, + "cudaRequirements": { + "deviceMemoryMin": 100, + "cudaComputeCapability": "8.0", + }, + }, + ], + title="Plugin Resource Requirements", + ) diff --git a/src/polus/plugins/_plugins/models/wipp.py b/src/polus/plugins/_plugins/models/wipp.py deleted file mode 100644 index 619d615fb..000000000 --- a/src/polus/plugins/_plugins/models/wipp.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Extending automatically generated wipp model. - -This file modifies and extend certain fields and -functions of WIPPPluginSchema.py which is automatically -generated by datamodel-codegen from JSON schema. -""" -from typing import List, Literal, Optional, Union - -from pydantic import BaseModel, Field - -from polus.plugins._plugins.io import Input, Output, Version -from polus.plugins._plugins.models.WIPPPluginSchema import WippPluginManifest - - -class ui1(BaseModel): - """Base class for UI items.""" - - key: str = Field(constr=r"^inputs.[a-zA-Z0-9][-a-zA-Z0-9]*$") - title: str - description: Optional[str] - condition: Optional[str] - default: Optional[Union[str, float, int, bool]] - hidden: Optional[bool] = Field(default=False) - bind: Optional[str] - - -class FieldSet(BaseModel): - """Base class for FieldSet.""" - - title: str - fields: List[str] = Field(min_items=1, unique_items=True) - - -class ui2(BaseModel): - """UI items class for fieldsets.""" - - key: Literal["fieldsets"] - fieldsets: List[FieldSet] = Field(min_items=1, unique_items=True) - - -class WIPPPluginManifest(WippPluginManifest): - """Extended WIPP Plugin Schema.""" - - inputs: List[Input] = Field( - ..., description="Defines inputs to the plugin", title="List of Inputs" - ) - outputs: List[Output] = Field( - ..., description="Defines the outputs of the plugin", title="List of Outputs" - ) - ui: List[Union[ui1, ui2]] = Field(..., title="Plugin form UI definition") - version: Version diff --git a/src/polus/plugins/_plugins/update/__init__.py b/src/polus/plugins/_plugins/update/__init__.py new file mode 100644 index 000000000..ff8010c6e --- /dev/null +++ b/src/polus/plugins/_plugins/update/__init__.py @@ -0,0 +1,14 @@ +"""Initialize update module.""" + +import pydantic + +PYDANTIC_VERSION = pydantic.__version__ + +if PYDANTIC_VERSION.split(".")[0] == "1": + from polus.plugins._plugins.update.update_v1 import update_nist_plugins + from polus.plugins._plugins.update.update_v1 import update_polus_plugins +elif PYDANTIC_VERSION.split(".")[0] == "2": + from polus.plugins._plugins.update.update_v2 import update_nist_plugins + from polus.plugins._plugins.update.update_v2 import update_polus_plugins + +__all__ = ["update_polus_plugins", "update_nist_plugins"] diff --git a/src/polus/plugins/_plugins/update.py b/src/polus/plugins/_plugins/update/update_v1.py similarity index 57% rename from src/polus/plugins/_plugins/update.py rename to src/polus/plugins/_plugins/update/update_v1.py index 8651fb92d..9a0dc508f 100644 --- a/src/polus/plugins/_plugins/update.py +++ b/src/polus/plugins/_plugins/update/update_v1.py @@ -1,77 +1,79 @@ +# pylint: disable=W1203, W1201 import json import logging import re import typing -from pydantic import ValidationError -from tqdm import tqdm - -from polus.plugins._plugins.classes import refresh, submit_plugin +from polus.plugins._plugins.classes import refresh +from polus.plugins._plugins.classes import submit_plugin from polus.plugins._plugins.gh import _init_github from polus.plugins._plugins.io import Version -from polus.plugins._plugins.manifests.manifest_utils import ( - _error_log, - _scrape_manifests, -) +from polus.plugins._plugins.manifests import _error_log +from polus.plugins._plugins.manifests import _scrape_manifests +from pydantic import ValidationError +from tqdm import tqdm logger = logging.getLogger("polus.plugins") def update_polus_plugins( - gh_auth: typing.Optional[str] = None, min_depth: int = 2, max_depth: int = 3 -): + gh_auth: typing.Optional[str] = None, + min_depth: int = 2, + max_depth: int = 3, +) -> None: """Scrape PolusAI GitHub repo and create local versions of Plugins.""" logger.info("Updating polus plugins.") # Get all manifests valid, invalid = _scrape_manifests( - "polusai/polus-plugins", _init_github(gh_auth), min_depth, max_depth, True + "polusai/polus-plugins", + _init_github(gh_auth), + min_depth, + max_depth, + True, ) manifests = valid.copy() manifests.extend(invalid) - logger.info("Submitting %s plugins." % len(manifests)) + logger.info(f"Submitting {len(manifests)} plugins.") for manifest in manifests: try: plugin = submit_plugin(manifest) - """ Parsing checks specific to polus-plugins """ + # Parsing checks specific to polus-plugins error_list = [] # Check that plugin version matches container version tag container_name, version = tuple(plugin.containerId.split(":")) version = Version(version=version) organization, container_name = tuple(container_name.split("/")) - try: - assert ( - plugin.version == version - ), f"containerId version ({version}) does not match plugin version ({plugin.version})" - except AssertionError as err: - error_list.append(err) + if plugin.version != version: + msg = ( + f"containerId version ({version}) does not " + f"match plugin version ({plugin.version})" + ) + logger.error(msg) + error_list.append(ValueError(msg)) # Check to see that the plugin is registered to Labshare - try: - assert organization in [ - "polusai", - "labshare", - ], "All polus plugin containers must be under the Labshare organization." - except AssertionError as err: - error_list.append(err) + if organization not in ["polusai", "labshare"]: + msg = ( + "all polus plugin containers must be" + " under the Labshare organization." + ) + logger.error(msg) + error_list.append(ValueError(msg)) # Checks for container name, they are somewhat related to our # Jenkins build - try: - assert container_name.startswith( - "polus" - ), "containerId name must begin with polus-" - except AssertionError as err: - error_list.append(err) + if not container_name.startswith("polus"): + msg = "containerId name must begin with polus-" + logger.error(msg) + error_list.append(ValueError(msg)) - try: - assert container_name.endswith( - "plugin" - ), "containerId name must end with -plugin" - except AssertionError as err: - error_list.append(err) + if not container_name.endswith("plugin"): + msg = "containerId name must end with -plugin" + logger.error(msg) + error_list.append(ValueError(msg)) if len(error_list) > 0: raise ValidationError(error_list, plugin.__class__) @@ -79,16 +81,14 @@ def update_polus_plugins( except ValidationError as val_err: try: _error_log(val_err, manifest, "update_polus_plugins") - except BaseException as e: - # logger.debug(f"There was an error {e} in {plugin.name}") + except BaseException as e: # pylint: disable=W0718 logger.exception(f"In {plugin.name}: {e}") - except BaseException as e: - # logger.debug(f"There was an error {e} in {plugin.name}") + except BaseException as e: # pylint: disable=W0718 logger.exception(f"In {plugin.name}: {e}") refresh() -def update_nist_plugins(gh_auth: typing.Optional[str] = None): +def update_nist_plugins(gh_auth: typing.Optional[str] = None) -> None: """Scrape NIST GitHub repo and create local versions of Plugins.""" # Parse README links gh = _init_github(gh_auth) @@ -96,7 +96,7 @@ def update_nist_plugins(gh_auth: typing.Optional[str] = None): contents = repo.get_contents("plugins") readme = [r for r in contents if r.name == "README.md"][0] pattern = re.compile( - r"\[manifest\]\((https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*))\)" + r"\[manifest\]\((https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*))\)", ) matches = pattern.findall(str(readme.decoded_content)) logger.info("Updating NIST plugins.") @@ -104,7 +104,7 @@ def update_nist_plugins(gh_auth: typing.Optional[str] = None): url_parts = match[0].split("/")[3:] plugin_repo = gh.get_repo("/".join(url_parts[:2])) manifest = json.loads( - plugin_repo.get_contents("/".join(url_parts[4:])).decoded_content + plugin_repo.get_contents("/".join(url_parts[4:])).decoded_content, ) try: diff --git a/src/polus/plugins/_plugins/update/update_v2.py b/src/polus/plugins/_plugins/update/update_v2.py new file mode 100644 index 000000000..67bd0038c --- /dev/null +++ b/src/polus/plugins/_plugins/update/update_v2.py @@ -0,0 +1,115 @@ +# pylint: disable=W1203, W1201 +import json +import logging +import re +import typing + +from polus.plugins._plugins.classes import refresh +from polus.plugins._plugins.classes import submit_plugin +from polus.plugins._plugins.gh import _init_github +from polus.plugins._plugins.io import Version +from polus.plugins._plugins.manifests import _error_log +from polus.plugins._plugins.manifests import _scrape_manifests +from pydantic import ValidationError +from tqdm import tqdm + +logger = logging.getLogger("polus.plugins") + + +def update_polus_plugins( + gh_auth: typing.Optional[str] = None, + min_depth: int = 2, + max_depth: int = 3, +) -> None: + """Scrape PolusAI GitHub repo and create local versions of Plugins.""" + logger.info("Updating polus plugins.") + # Get all manifests + valid, invalid = _scrape_manifests( + "polusai/polus-plugins", + _init_github(gh_auth), + min_depth, + max_depth, + True, + ) + manifests = valid.copy() + manifests.extend(invalid) + logger.info(f"Submitting {len(manifests)} plugins.") + + for manifest in manifests: + try: + plugin = submit_plugin(manifest) + + # Parsing checks specific to polus-plugins + error_list = [] + + # Check that plugin version matches container version tag + container_name, version = tuple(plugin.containerId.split(":")) + version = Version(version) + organization, container_name = tuple(container_name.split("/")) + if plugin.version != version: + msg = ( + f"containerId version ({version}) does not " + f"match plugin version ({plugin.version})" + ) + logger.error(msg) + error_list.append(ValueError(msg)) + + # Check to see that the plugin is registered to Labshare + if organization not in ["polusai", "labshare"]: + msg = ( + "all polus plugin containers must be" + " under the Labshare organization." + ) + logger.error(msg) + error_list.append(ValueError(msg)) + + # Checks for container name, they are somewhat related to our + # Jenkins build + if not container_name.startswith("polus"): + msg = "containerId name must begin with polus-" + logger.error(msg) + error_list.append(ValueError(msg)) + + if not container_name.endswith("plugin"): + msg = "containerId name must end with -plugin" + logger.error(msg) + error_list.append(ValueError(msg)) + + if len(error_list) > 0: + raise ValidationError(error_list, plugin.__class__) + + except ValidationError as val_err: + try: + _error_log(val_err, manifest, "update_polus_plugins") + except BaseException as e: # pylint: disable=W0718 + logger.exception(f"In {plugin.name}: {e}") + except BaseException as e: # pylint: disable=W0718 + logger.exception(f"In {plugin.name}: {e}") + refresh() + + +def update_nist_plugins(gh_auth: typing.Optional[str] = None) -> None: + """Scrape NIST GitHub repo and create local versions of Plugins.""" + # Parse README links + gh = _init_github(gh_auth) + repo = gh.get_repo("usnistgov/WIPP") + contents = repo.get_contents("plugins") + readme = [r for r in contents if r.name == "README.md"][0] + pattern = re.compile( + r"\[manifest\]\((https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*))\)", + ) + matches = pattern.findall(str(readme.decoded_content)) + logger.info("Updating NIST plugins.") + for match in tqdm(matches, desc="NIST Manifests"): + url_parts = match[0].split("/")[3:] + plugin_repo = gh.get_repo("/".join(url_parts[:2])) + manifest = json.loads( + plugin_repo.get_contents("/".join(url_parts[4:])).decoded_content, + ) + + try: + submit_plugin(manifest) + + except ValidationError as val_err: + _error_log(val_err, manifest, "update_nist_plugins") + refresh() diff --git a/tests/test_cwl.py b/tests/test_cwl.py index 94fce2a4f..58ac9fd08 100644 --- a/tests/test_cwl.py +++ b/tests/test_cwl.py @@ -3,12 +3,14 @@ """Tests for CWL utils.""" from pathlib import Path +import pydantic import pytest import yaml import polus.plugins as pp -from polus.plugins._plugins.classes.plugin_methods import MissingInputValues +from polus.plugins._plugins.classes.plugin_base import MissingInputValuesError +PYDANTIC_VERSION = pydantic.__version__.split(".")[0] RSRC_PATH = Path(__file__).parent.joinpath("resources") OMECONVERTER = RSRC_PATH.joinpath("omeconverter030.json") @@ -20,7 +22,7 @@ def submit_plugin(): if "OmeConverter" not in pp.list: pp.submit_plugin(OMECONVERTER) else: - if "0.3.0" not in [x.version for x in pp.OmeConverter.versions]: + if "0.3.0" not in pp.OmeConverter.versions: pp.submit_plugin(OMECONVERTER) @@ -65,7 +67,7 @@ def test_save_read_cwl(plug, cwl_path): def test_save_cwl_io_not_inp(plug, cwl_io_path): """Test save_cwl IO.""" - with pytest.raises(MissingInputValues): + with pytest.raises(MissingInputValuesError): plug.save_cwl_io(cwl_io_path) @@ -73,7 +75,7 @@ def test_save_cwl_io_not_inp2(plug, cwl_io_path): """Test save_cwl IO.""" plug.inpDir = RSRC_PATH.absolute() plug.filePattern = "img_r{rrr}_c{ccc}.tif" - with pytest.raises(MissingInputValues): + with pytest.raises(MissingInputValuesError): plug.save_cwl_io(cwl_io_path) @@ -83,7 +85,7 @@ def test_save_cwl_io_not_yml(plug, cwl_io_path): plug.filePattern = "img_r{rrr}_c{ccc}.tif" plug.fileExtension = ".ome.zarr" plug.outDir = RSRC_PATH.absolute() - with pytest.raises(AssertionError): + with pytest.raises(ValueError): plug.save_cwl_io(cwl_io_path.with_suffix(".txt")) diff --git a/tests/test_io.py b/tests/test_io.py index f1135e504..83e289a6e 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -5,8 +5,8 @@ import pytest from fsspec.implementations.local import LocalFileSystem -from polus.plugins._plugins.classes.plugin_classes import _load_plugin -from polus.plugins._plugins.classes.plugin_methods import IOKeyError +from polus.plugins._plugins.classes import _load_plugin +from polus.plugins._plugins.classes.plugin_base import IOKeyError from polus.plugins._plugins.io import Input, IOBase RSRC_PATH = Path(__file__).parent.joinpath("resources") diff --git a/tests/test_manifests.py b/tests/test_manifests.py index dd4cccb7e..286e93dd6 100644 --- a/tests/test_manifests.py +++ b/tests/test_manifests.py @@ -3,15 +3,26 @@ from collections import OrderedDict from pathlib import Path +import pydantic import pytest -from polus.plugins._plugins.classes import list_plugins -from polus.plugins._plugins.classes.plugin_classes import PLUGINS -from polus.plugins._plugins.manifests.manifest_utils import ( - InvalidManifest, - _load_manifest, - validate_manifest, -) +PYDANTIC_VERSION = pydantic.__version__.split(".")[0] + +from polus.plugins._plugins.classes import PLUGINS, list_plugins + +if PYDANTIC_VERSION == "1": + from polus.plugins._plugins.manifests.manifest_utils_v1 import ( + InvalidManifestError, + _load_manifest, + validate_manifest, + ) +elif PYDANTIC_VERSION == "2": + from polus.plugins._plugins.manifests.manifest_utils_v2 import ( + InvalidManifestError, + _load_manifest, + validate_manifest, + ) + from polus.plugins._plugins.models import ComputeSchema, WIPPPluginManifest RSRC_PATH = Path(__file__).parent.joinpath("resources") @@ -219,12 +230,6 @@ def test_load_manifest(type_): # test path and dict assert _load_manifest(type_) == d_val -def test_load_manifest_str(): - """Test _load_manifest() for str.""" - st_ = """{"a": 2, "b": "Polus"}""" - assert _load_manifest(st_) == {"a": 2, "b": "Polus"} - - bad = [f"b{x}.json" for x in [1, 2, 3]] good = [f"g{x}.json" for x in [1, 2, 3]] @@ -232,7 +237,7 @@ def test_load_manifest_str(): @pytest.mark.parametrize("manifest", bad, ids=bad) def test_bad_manifest(manifest): """Test bad manifests raise InvalidManifest error.""" - with pytest.raises(InvalidManifest): + with pytest.raises(InvalidManifestError): validate_manifest(REPO_PATH.joinpath("tests", "resources", manifest)) diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 302759940..4a0d60ca1 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -6,7 +6,7 @@ import pytest import polus.plugins as pp -from polus.plugins._plugins.classes.plugin_classes import Plugin, _load_plugin +from polus.plugins._plugins.classes import Plugin, _load_plugin RSRC_PATH = Path(__file__).parent.joinpath("resources") OMECONVERTER = RSRC_PATH.joinpath("omeconverter022.json") @@ -101,9 +101,7 @@ def test_attr2(submit_basic131): def test_versions(submit_basic131, submit_basic127): """Test versions.""" - assert sorted( - [x for x in pp.get_plugin("BasicFlatfieldCorrectionPlugin").versions] - ) == [ + assert sorted(pp.get_plugin("BasicFlatfieldCorrectionPlugin").versions) == [ "1.2.7", "1.3.1", ] @@ -141,6 +139,20 @@ def test_remove_all_versions_plugin( assert pp.list == ["OmeConverter"] +def test_submit_str_1(): + """Test submit_plugin with string.""" + pp.remove_all() + pp.submit_plugin(str(OMECONVERTER)) + assert pp.list == ["OmeConverter"] + + +def test_submit_str_2(): + """Test submit_plugin with string.""" + pp.remove_all() + pp.submit_plugin(str(OMECONVERTER.absolute())) + assert pp.list == ["OmeConverter"] + + @pytest.fixture def plug1(): """Configure the class.""" diff --git a/tests/test_version.py b/tests/test_version.py index 2a10f0c63..512bdc420 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -1,10 +1,13 @@ """Test Version object and cast_version utility function.""" +import pydantic import pytest from pydantic import ValidationError from polus.plugins._plugins.io import Version from polus.plugins._plugins.utils import cast_version +PYDANTIC_VERSION = pydantic.__version__.split(".", maxsplit=1)[0] + GOOD_VERSIONS = [ "1.2.3", "1.4.7-rc1", @@ -17,13 +20,19 @@ ] BAD_VERSIONS = ["02.2.3", "002.2.3", "1.2", "1.0", "1.03.2", "23.3.03", "d.2.4"] +PV = PYDANTIC_VERSION +print(PV) + @pytest.mark.parametrize("ver", GOOD_VERSIONS, ids=GOOD_VERSIONS) def test_version(ver): """Test Version pydantic model.""" - Version(version=ver) + if PV == "1": + assert isinstance(Version(version=ver), Version) + assert isinstance(Version(ver), Version) +@pytest.mark.skipif(int(PV) > 1, reason="requires pydantic 1") @pytest.mark.parametrize("ver", GOOD_VERSIONS, ids=GOOD_VERSIONS) def test_cast_version(ver): """Test cast_version utility function.""" @@ -33,8 +42,11 @@ def test_cast_version(ver): @pytest.mark.parametrize("ver", BAD_VERSIONS, ids=BAD_VERSIONS) def test_bad_version1(ver): """Test ValidationError is raised for invalid versions.""" + if PV == "1": + with pytest.raises(ValidationError): + assert isinstance(cast_version(ver), Version) with pytest.raises(ValidationError): - assert isinstance(cast_version(ver), Version) + assert isinstance(Version(ver), Version) MAJOR_VERSION_EQUAL = ["2.4.3", "2.98.28", "2.1.2", "2.0.0", "2.4.0"] @@ -45,74 +57,115 @@ def test_bad_version1(ver): @pytest.mark.parametrize("ver", MAJOR_VERSION_EQUAL, ids=MAJOR_VERSION_EQUAL) def test_major(ver): """Test major version.""" - assert cast_version(ver).major == 2 + if PV == "2": + assert Version(ver).major == 2 + else: + assert cast_version(ver).major == 2 @pytest.mark.parametrize("ver", MINOR_VERSION_EQUAL, ids=MINOR_VERSION_EQUAL) def test_minor(ver): """Test minor version.""" - assert cast_version(ver).minor == 3 + if PV == "2": + assert Version(ver).minor == 3 + else: + assert cast_version(ver).minor == 3 @pytest.mark.parametrize("ver", PATCH_EQUAL, ids=PATCH_EQUAL) def test_patch(ver): """Test patch version.""" - assert cast_version(ver).patch == 7 - + if PV == "2": + assert Version(ver).patch == 7 + else: + assert cast_version(ver).patch == 7 def test_gt1(): """Test greater than operator.""" - assert cast_version("1.2.3") > cast_version("1.2.1") + if PV == "2": + assert Version("1.2.3") > Version("1.2.1") + else: + assert cast_version("1.2.3") > cast_version("1.2.1") def test_gt2(): """Test greater than operator.""" - assert cast_version("5.7.3") > cast_version("5.6.3") + if PV == "2": + assert Version("5.7.3") > Version("5.6.3") + else: + assert cast_version("5.7.3") > cast_version("5.6.3") def test_st1(): """Test less than operator.""" - assert cast_version("5.7.3") < cast_version("5.7.31") + if PV == "2": + assert Version("5.7.3") < Version("5.7.31") + else: + assert cast_version("5.7.3") < cast_version("5.7.31") def test_st2(): """Test less than operator.""" - assert cast_version("1.0.2") < cast_version("2.0.2") + if PV == "2": + assert Version("1.0.2") < Version("2.0.2") + else: + assert cast_version("1.0.2") < cast_version("2.0.2") def test_eq1(): """Test equality operator.""" - assert Version(version="1.3.3") == cast_version("1.3.3") + if PV == "2": + assert Version("1.3.3") == Version("1.3.3") + else: + assert Version(version="1.3.3") == cast_version("1.3.3") def test_eq2(): """Test equality operator.""" - assert Version(version="5.4.3") == cast_version("5.4.3") + if PV == "2": + assert Version("5.4.3") == Version("5.4.3") + else: + assert Version(version="5.4.3") == cast_version("5.4.3") def test_eq3(): """Test equality operator.""" - assert Version(version="1.3.3") != cast_version("1.3.8") + if PV == "2": + assert Version("1.3.3") != Version("1.3.8") + else: + assert Version(version="1.3.3") != cast_version("1.3.8") def test_eq_str1(): """Test equality with str.""" - assert Version(version="1.3.3") == "1.3.3" + if PV == "2": + assert Version("1.3.3") == "1.3.3" + else: + assert Version(version="1.3.3") == "1.3.3" def test_lt_str1(): """Test equality with str.""" - assert Version(version="1.3.3") < "1.5.3" + if PV == "2": + assert Version("1.3.3") < "1.5.3" + else: + assert Version(version="1.3.3") < "1.5.3" def test_gt_str1(): """Test equality with str.""" - assert Version(version="4.5.10") > "4.5.9" + if PV == "2": + assert Version("4.5.10") > "4.5.9" + else: + assert Version(version="4.5.10") > "4.5.9" def test_eq_no_str(): """Test equality with non-string.""" - with pytest.raises(TypeError): - assert Version(version="1.3.3") == 1.3 - + if PV == "2": + with pytest.raises(TypeError): + assert Version("1.3.3") == 1.3 + else: + with pytest.raises(TypeError): + assert Version(version="1.3.3") == 1.3 From b4af6a53a956886b66566121cf4ce8de5ae44673 Mon Sep 17 00:00:00 2001 From: Camilo Velez Date: Thu, 25 Jan 2024 23:42:20 -0500 Subject: [PATCH 2/4] chore: update dependencies pyproject.toml --- pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 67bb0ace1..d7f6d7118 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,7 @@ cwltool = "^3.1.20230513155734" fsspec = "^2023.6.0" pydantic = ">=1.10.0" pygithub = "^1.58.2" -python-on-whales = "^0.57.0" +python-on-whales = "^0.68.0" pyyaml = "^6.0" tqdm = "^4.65.0" validators = "^0.22.0" @@ -35,12 +35,13 @@ mypy = "^1.4.0" nox = "^2022.11.21" poetry = "^1.3.2" pre-commit = "^3.3.3" +pydantic = ">=1.10" pytest = "^7.3.2" pytest-benchmark = "^4.0.0" pytest-cov = "^4.1.0" pytest-sugar = "^0.9.7" pytest-xdist = "^3.3.1" -python-on-whales = "^0.57.0" +python-on-whales = "^0.68.0" pyyaml = "^6.0" ruff = "^0.0.274" tqdm = "^4.64.1" From 2b3d5a44add388fa6318cf07a9d348a15e538ed1 Mon Sep 17 00:00:00 2001 From: Camilo Velez Date: Fri, 26 Jan 2024 23:56:44 -0500 Subject: [PATCH 3/4] chore(api): improved version management --- src/polus/plugins/__init__.py | 9 +++++++-- src/polus/plugins/_plugins/VERSION | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/polus/plugins/__init__.py b/src/polus/plugins/__init__.py index a69b802fa..ac6a7c00e 100644 --- a/src/polus/plugins/__init__.py +++ b/src/polus/plugins/__init__.py @@ -1,6 +1,7 @@ """Initialize polus-plugins module.""" import logging +from pathlib import Path from typing import Union from polus.plugins._plugins.classes import ( @@ -30,7 +31,11 @@ """ logger = logging.getLogger("polus.plugins") -VERSION = "0.1.1" +with Path(__file__).parent.joinpath("_plugins/VERSION").open( + "r", + encoding="utf-8", +) as version_file: + VERSION = version_file.read().strip() refresh() # calls the refresh method when library is imported @@ -41,7 +46,7 @@ def __getattr__(name: str) -> Union[Plugin, ComputePlugin, list]: return list_plugins() if name in list_plugins(): return get_plugin(name) - if name == "__version__": + if name in ["__version__", "VERSION"]: return VERSION msg = f"module '{__name__}' has no attribute '{name}'" raise AttributeError(msg) diff --git a/src/polus/plugins/_plugins/VERSION b/src/polus/plugins/_plugins/VERSION index 6e8bf73aa..17e51c385 100644 --- a/src/polus/plugins/_plugins/VERSION +++ b/src/polus/plugins/_plugins/VERSION @@ -1 +1 @@ -0.1.0 +0.1.1 From cdafa21c2aceed2e66f5ea25b56939291a8a96c3 Mon Sep 17 00:00:00 2001 From: Camilo Velez Date: Wed, 31 Jan 2024 08:50:45 -0500 Subject: [PATCH 4/4] feat: improve Pydantic V2 compat checks --- src/polus/plugins/_plugins/_compat.py | 4 + .../plugins/_plugins/classes/__init__.py | 38 +- ...plugin_classes_v1.py => plugin_classes.py} | 82 +++- .../_plugins/classes/plugin_classes_v2.py | 429 ---------------- src/polus/plugins/_plugins/io/__init__.py | 30 +- .../plugins/_plugins/io/{io_v2.py => _io.py} | 382 ++++++++++----- src/polus/plugins/_plugins/io/io_v1.py | 459 ------------------ .../plugins/_plugins/manifests/__init__.py | 21 +- ...manifest_utils_v1.py => manifest_utils.py} | 12 +- .../_plugins/manifests/manifest_utils_v2.py | 202 -------- src/polus/plugins/_plugins/update/__init__.py | 12 +- .../update/{update_v1.py => _update.py} | 3 +- .../plugins/_plugins/update/update_v2.py | 115 ----- tests/test_manifests.py | 22 +- 14 files changed, 356 insertions(+), 1455 deletions(-) create mode 100644 src/polus/plugins/_plugins/_compat.py rename src/polus/plugins/_plugins/classes/{plugin_classes_v1.py => plugin_classes.py} (86%) delete mode 100644 src/polus/plugins/_plugins/classes/plugin_classes_v2.py rename src/polus/plugins/_plugins/io/{io_v2.py => _io.py} (52%) delete mode 100644 src/polus/plugins/_plugins/io/io_v1.py rename src/polus/plugins/_plugins/manifests/{manifest_utils_v1.py => manifest_utils.py} (95%) delete mode 100644 src/polus/plugins/_plugins/manifests/manifest_utils_v2.py rename src/polus/plugins/_plugins/update/{update_v1.py => _update.py} (96%) delete mode 100644 src/polus/plugins/_plugins/update/update_v2.py diff --git a/src/polus/plugins/_plugins/_compat.py b/src/polus/plugins/_plugins/_compat.py new file mode 100644 index 000000000..190aa0fd3 --- /dev/null +++ b/src/polus/plugins/_plugins/_compat.py @@ -0,0 +1,4 @@ +"""Compat of Pydantic.""" +import pydantic + +PYDANTIC_V2 = pydantic.VERSION.startswith("2.") diff --git a/src/polus/plugins/_plugins/classes/__init__.py b/src/polus/plugins/_plugins/classes/__init__.py index 200b2c821..2d4460519 100644 --- a/src/polus/plugins/_plugins/classes/__init__.py +++ b/src/polus/plugins/_plugins/classes/__init__.py @@ -1,32 +1,16 @@ """Plugin classes and functions.""" -import pydantic -PYDANTIC_VERSION = pydantic.__version__ - -if PYDANTIC_VERSION.split(".")[0] == "1": - from polus.plugins._plugins.classes.plugin_classes_v1 import PLUGINS - from polus.plugins._plugins.classes.plugin_classes_v1 import ComputePlugin - from polus.plugins._plugins.classes.plugin_classes_v1 import Plugin - from polus.plugins._plugins.classes.plugin_classes_v1 import _load_plugin - from polus.plugins._plugins.classes.plugin_classes_v1 import get_plugin - from polus.plugins._plugins.classes.plugin_classes_v1 import list_plugins - from polus.plugins._plugins.classes.plugin_classes_v1 import load_config - from polus.plugins._plugins.classes.plugin_classes_v1 import refresh - from polus.plugins._plugins.classes.plugin_classes_v1 import remove_all - from polus.plugins._plugins.classes.plugin_classes_v1 import remove_plugin - from polus.plugins._plugins.classes.plugin_classes_v1 import submit_plugin -elif PYDANTIC_VERSION.split(".")[0] == "2": - from polus.plugins._plugins.classes.plugin_classes_v2 import PLUGINS - from polus.plugins._plugins.classes.plugin_classes_v2 import ComputePlugin - from polus.plugins._plugins.classes.plugin_classes_v2 import Plugin - from polus.plugins._plugins.classes.plugin_classes_v2 import _load_plugin - from polus.plugins._plugins.classes.plugin_classes_v2 import get_plugin - from polus.plugins._plugins.classes.plugin_classes_v2 import list_plugins - from polus.plugins._plugins.classes.plugin_classes_v2 import load_config - from polus.plugins._plugins.classes.plugin_classes_v2 import refresh - from polus.plugins._plugins.classes.plugin_classes_v2 import remove_all - from polus.plugins._plugins.classes.plugin_classes_v2 import remove_plugin - from polus.plugins._plugins.classes.plugin_classes_v2 import submit_plugin +from polus.plugins._plugins.classes.plugin_classes import PLUGINS +from polus.plugins._plugins.classes.plugin_classes import ComputePlugin +from polus.plugins._plugins.classes.plugin_classes import Plugin +from polus.plugins._plugins.classes.plugin_classes import _load_plugin +from polus.plugins._plugins.classes.plugin_classes import get_plugin +from polus.plugins._plugins.classes.plugin_classes import list_plugins +from polus.plugins._plugins.classes.plugin_classes import load_config +from polus.plugins._plugins.classes.plugin_classes import refresh +from polus.plugins._plugins.classes.plugin_classes import remove_all +from polus.plugins._plugins.classes.plugin_classes import remove_plugin +from polus.plugins._plugins.classes.plugin_classes import submit_plugin __all__ = [ "Plugin", diff --git a/src/polus/plugins/_plugins/classes/plugin_classes_v1.py b/src/polus/plugins/_plugins/classes/plugin_classes.py similarity index 86% rename from src/polus/plugins/_plugins/classes/plugin_classes_v1.py rename to src/polus/plugins/_plugins/classes/plugin_classes.py index 90e67ddcc..ca453c8c9 100644 --- a/src/polus/plugins/_plugins/classes/plugin_classes_v1.py +++ b/src/polus/plugins/_plugins/classes/plugin_classes.py @@ -10,11 +10,12 @@ from typing import Optional from typing import Union +from polus.plugins._plugins._compat import PYDANTIC_V2 from polus.plugins._plugins.classes.plugin_base import BasePlugin -from polus.plugins._plugins.io.io_v1 import DuplicateVersionFoundError -from polus.plugins._plugins.io.io_v1 import Version -from polus.plugins._plugins.io.io_v1 import _in_old_to_new -from polus.plugins._plugins.io.io_v1 import _ui_old_to_new +from polus.plugins._plugins.io._io import DuplicateVersionFoundError +from polus.plugins._plugins.io._io import Version +from polus.plugins._plugins.io._io import _in_old_to_new +from polus.plugins._plugins.io._io import _ui_old_to_new from polus.plugins._plugins.manifests import InvalidManifestError from polus.plugins._plugins.manifests import _load_manifest from polus.plugins._plugins.manifests import validate_manifest @@ -24,7 +25,7 @@ from polus.plugins._plugins.models import WIPPPluginManifest from polus.plugins._plugins.utils import cast_version from polus.plugins._plugins.utils import name_cleaner -from pydantic import Extra +from pydantic import ConfigDict logger = logging.getLogger("polus.plugins") PLUGINS: dict[str, dict] = {} @@ -56,8 +57,9 @@ def refresh() -> None: plugin = validate_manifest(file) except InvalidManifestError: logger.warning(f"Validation error in {file!s}") - except BaseException as exc: # pylint: disable=W0718 # noqa: BLE001 + except BaseException as exc: # pylint: disable=W0718 logger.warning(f"Unexpected error {exc} with {file!s}") + raise exc else: key = name_cleaner(plugin.name) @@ -86,11 +88,20 @@ def list_plugins() -> list: def _get_config(plugin: Union["Plugin", "ComputePlugin"], class_: str) -> dict: - model_ = plugin.dict() + if PYDANTIC_V2: + model_ = json.loads(plugin.model_dump_json()) + model_["_io_keys"] = deepcopy(plugin._io_keys) # type: ignore + else: + # ignore mypy if pydantic < 2.0.0 + model_ = plugin.dict() # type: ignore # iterate over I/O to convert to dict for io_name, io in model_["_io_keys"].items(): - # overwrite val if enum - if io["type"] == "enum": + if PYDANTIC_V2: + model_["_io_keys"][io_name] = json.loads(io.model_dump_json()) + # overwrite val if enum + if io.type.value == "enum": + model_["_io_keys"][io_name]["value"] = io.value.name # str + elif io["type"] == "enum": # pydantic V1 val_ = io["value"].name # mapDirectory.raw model_["_io_keys"][io_name]["value"] = val_.split(".")[-1] # raw for inp in model_["inputs"]: @@ -112,12 +123,15 @@ class Plugin(WIPPPluginManifest, BasePlugin): """ id: uuid.UUID # noqa: A003 + if PYDANTIC_V2: + model_config = ConfigDict(extra="allow", frozen=True) + else: - class Config: - """Config class for Pydantic Model.""" + class Config: # pylint: disable=R0903 + """Config.""" - extra = Extra.allow - allow_mutation = False + extra = "allow" + allow_mutation = False def __init__(self, _uuid: bool = True, **data: dict) -> None: """Init a plugin object from manifest.""" @@ -126,10 +140,14 @@ def __init__(self, _uuid: bool = True, **data: dict) -> None: else: data["id"] = uuid.UUID(str(data["id"])) # type: ignore - data["version"] = cast_version(data["version"]) + if not PYDANTIC_V2: # pydantic V1 + data["version"] = cast_version(data["version"]) + super().__init__(**data) - self.Config.allow_mutation = True + if not PYDANTIC_V2: # pydantic V1 + self.Config.allow_mutation = True + self._io_keys = {i.name: i for i in self.inputs} self._io_keys.update({o.name: o for o in self.outputs}) @@ -207,11 +225,15 @@ class ComputePlugin(ComputeSchema, BasePlugin): save_manifest(path): save plugin manifest to specified path """ - class Config: - """Config class for Pydantic Model.""" + if PYDANTIC_V2: + model_config = ConfigDict(extra="allow", frozen=True) + else: # pydantic V1 + + class Config: # pylint: disable=R0903 + """Config.""" - extra = Extra.allow - allow_mutation = False + extra = "allow" + allow_mutation = False def __init__( self, @@ -355,8 +377,11 @@ def submit_plugin( org_path.mkdir(exist_ok=True, parents=True) if not org_path.joinpath(out_name).exists(): with org_path.joinpath(out_name).open("w", encoding="utf-8") as file: - manifest_ = plugin.dict() - manifest_["version"] = manifest_["version"]["version"] + if not PYDANTIC_V2: # pydantic V1 + manifest_ = plugin.dict() # type: ignore + manifest_["version"] = manifest_["version"]["version"] + else: # PYDANTIC V2 + manifest_ = json.loads(plugin.model_dump_json()) json.dump(manifest_, file, indent=4) # Refresh plugins list @@ -382,7 +407,9 @@ def get_plugin( """ if version is None: return _load_plugin(PLUGINS[name][max(PLUGINS[name])]) - return _load_plugin(PLUGINS[name][Version(**{"version": version})]) + if PYDANTIC_V2: + return _load_plugin(PLUGINS[name][Version(version)]) + return _load_plugin(PLUGINS[name][Version(**{"version": version})]) # Pydantic V1 def load_config(config: Union[dict, Path, str]) -> Union[Plugin, ComputePlugin]: @@ -422,10 +449,13 @@ def remove_plugin(plugin: str, version: Optional[Union[str, list[str]]] = None) for version_ in version: remove_plugin(plugin, version_) return - if not isinstance(version, Version): - version_ = cast_version(version) - else: - version_ = version + if not PYDANTIC_V2: # pydantic V1 + if not isinstance(version, Version): + version_ = cast_version(version) + else: + version_ = version + else: # pydanitc V2 + version_ = Version(version) if not isinstance(version, Version) else version path = PLUGINS[plugin][version_] path.unlink() refresh() diff --git a/src/polus/plugins/_plugins/classes/plugin_classes_v2.py b/src/polus/plugins/_plugins/classes/plugin_classes_v2.py deleted file mode 100644 index 5a2aadae2..000000000 --- a/src/polus/plugins/_plugins/classes/plugin_classes_v2.py +++ /dev/null @@ -1,429 +0,0 @@ -"""Classes for Plugin objects containing methods to configure, run, and save.""" -# pylint: disable=W1203, W0212, enable=W1201 -import json -import logging -import shutil -import uuid -from copy import deepcopy -from pathlib import Path -from typing import Any -from typing import Optional -from typing import Union - -from polus.plugins._plugins.classes.plugin_base import BasePlugin -from polus.plugins._plugins.io.io_v2 import DuplicateVersionFoundError -from polus.plugins._plugins.io.io_v2 import Version -from polus.plugins._plugins.io.io_v2 import _in_old_to_new -from polus.plugins._plugins.io.io_v2 import _ui_old_to_new -from polus.plugins._plugins.manifests import InvalidManifestError -from polus.plugins._plugins.manifests import _load_manifest -from polus.plugins._plugins.manifests import validate_manifest -from polus.plugins._plugins.models import ComputeSchema -from polus.plugins._plugins.models import PluginUIInput -from polus.plugins._plugins.models import PluginUIOutput -from polus.plugins._plugins.models import WIPPPluginManifest -from polus.plugins._plugins.utils import cast_version -from polus.plugins._plugins.utils import name_cleaner -from pydantic import ConfigDict - -logger = logging.getLogger("polus.plugins") -PLUGINS: dict[str, dict] = {} -# PLUGINS = {"BasicFlatfieldCorrectionPlugin": -# {Version('0.1.4'): Path(<...>), Version('0.1.5'): Path(<...>)}. -# "VectorToLabel": {Version(...)}} - -""" -Paths and Fields -""" -# Location to store any discovered plugin manifests -_PLUGIN_DIR = Path(__file__).parent.parent.joinpath("manifests") - - -def refresh() -> None: - """Refresh the plugin list.""" - organizations = [ - x for x in _PLUGIN_DIR.iterdir() if x.name != "__pycache__" and x.is_dir() - ] # ignore __pycache__ - - PLUGINS.clear() - - for org in organizations: - for file in org.iterdir(): - if file.suffix == ".py": - continue - - try: - plugin = validate_manifest(file) - except InvalidManifestError: - logger.warning(f"Validation error in {file!s}") - except BaseException as exc: # pylint: disable=W0718 - logger.warning(f"Unexpected error {exc} with {file!s}") - raise exc - - else: - key = name_cleaner(plugin.name) - # Add version and path to VERSIONS - if key not in PLUGINS: - PLUGINS[key] = {} - if ( - plugin.version in PLUGINS[key] - and file != PLUGINS[key][plugin.version] - ): - msg = ( - "Found duplicate version of plugin" - f"{plugin.name} in {_PLUGIN_DIR}" - ) - raise DuplicateVersionFoundError( - msg, - ) - PLUGINS[key][plugin.version] = file - - -def list_plugins() -> list: - """List all local plugins.""" - output = list(PLUGINS.keys()) - output.sort() - return output - - -def _get_config(plugin: Union["Plugin", "ComputePlugin"], class_: str) -> dict: - model_ = json.loads(plugin.model_dump_json()) - model_["_io_keys"] = deepcopy(plugin._io_keys) # type: ignore - # iterate over I/O to convert to dict - for io_name, io in model_["_io_keys"].items(): - model_["_io_keys"][io_name] = json.loads(io.model_dump_json()) - # overwrite val if enum - if io.type.value == "enum": - model_["_io_keys"][io_name]["value"] = io.value.name # str - for inp in model_["inputs"]: - inp["value"] = None - model_["class"] = class_ - return model_ - - -class Plugin(WIPPPluginManifest, BasePlugin): - """WIPP Plugin Class. - - Contains methods to configure, run, and save plugins. - - Attributes: - versions: A list of local available versions for this plugin. - - Methods: - save_manifest(path): save plugin manifest to specified path - """ - - id: uuid.UUID # noqa: A003 - model_config = ConfigDict(extra="allow", frozen=True) - - def __init__(self, _uuid: bool = True, **data: dict) -> None: - """Init a plugin object from manifest.""" - if _uuid: - data["id"] = uuid.uuid4() # type: ignore - else: - data["id"] = uuid.UUID(str(data["id"])) # type: ignore - - super().__init__(**data) - - self._io_keys = {i.name: i for i in self.inputs} - self._io_keys.update({o.name: o for o in self.outputs}) - - if not self.author: - warn_msg = ( - f"The plugin ({self.name}) is missing the author field. " - "This field is not required but should be filled in." - ) - logger.warning(warn_msg) - - @property - def versions(self) -> list: # cannot be in PluginMethods because PLUGINS lives here - """Return list of local versions of a Plugin.""" - return list(PLUGINS[name_cleaner(self.name)]) - - def to_compute( - self, - hardware_requirements: Optional[dict] = None, - ) -> type[ComputeSchema]: - """Convert WIPP Plugin object to Compute Plugin object.""" - data = deepcopy(self.manifest) - return ComputePlugin( - hardware_requirements=hardware_requirements, - _from_old=True, - **data, - ) - - def save_manifest( - self, - path: Union[str, Path], - hardware_requirements: Optional[dict] = None, - compute: bool = False, - ) -> None: - """Save plugin manifest to specified path.""" - if compute: - with Path(path).open("w", encoding="utf-8") as file: - self.to_compute( - hardware_requirements=hardware_requirements, - ).save_manifest(path) - else: - with Path(path).open("w", encoding="utf-8") as file: - dict_ = self.manifest - json.dump( - dict_, - file, - indent=4, - ) - - logger.debug(f"Saved manifest to {path}") - - def __setattr__(self, name: str, value: Any) -> None: # noqa: ANN401 - """Set I/O parameters as attributes.""" - BasePlugin.__setattr__(self, name, value) - - def save_config(self, path: Union[str, Path]) -> None: - """Save manifest with configured I/O parameters to specified path.""" - with Path(path).open("w", encoding="utf-8") as file: - json.dump(_get_config(self, "WIPP"), file, indent=4, default=str) - logger.debug(f"Saved config to {path}") - - def __repr__(self) -> str: - """Print plugin name and version.""" - return BasePlugin.__repr__(self) - - -class ComputePlugin(ComputeSchema, BasePlugin): - """Compute Plugin Class. - - Contains methods to configure, run, and save plugins. - - Attributes: - versions: A list of local available versions for this plugin. - - Methods: - save_manifest(path): save plugin manifest to specified path - """ - - model_config = ConfigDict(extra="allow", frozen=True) - - def __init__( - self, - hardware_requirements: Optional[dict] = None, - _from_old: bool = False, - _uuid: bool = True, - **data: dict, - ) -> None: - """Init a plugin object from manifest.""" - if _uuid: - data["id"] = uuid.uuid4() # type: ignore - else: - data["id"] = uuid.UUID(str(data["id"])) # type: ignore - - if _from_old: - - def _convert_input(dict_: dict) -> dict: - dict_["type"] = _in_old_to_new(dict_["type"]) - return dict_ - - def _convert_output(dict_: dict) -> dict: - dict_["type"] = "path" - return dict_ - - def _ui_in(dict_: dict) -> PluginUIInput: # assuming old all ui input - # assuming format inputs. ___ - inp = dict_["key"].split(".")[-1] # e.g inpDir - try: - type_ = [x["type"] for x in data["inputs"] if x["name"] == inp][ - 0 - ] # get type from i/o - except IndexError: - type_ = "string" # default to string - except BaseException as exc: - raise exc - - dict_["type"] = _ui_old_to_new(type_) - return PluginUIInput(**dict_) - - def _ui_out(dict_: dict) -> PluginUIOutput: - new_dict_ = deepcopy(dict_) - new_dict_["name"] = "outputs." + new_dict_["name"] - new_dict_["type"] = _ui_old_to_new(new_dict_["type"]) - return PluginUIOutput(**new_dict_) - - data["inputs"] = [_convert_input(x) for x in data["inputs"]] # type: ignore - data["outputs"] = [ - _convert_output(x) for x in data["outputs"] - ] # type: ignore - data["pluginHardwareRequirements"] = {} - data["ui"] = [_ui_in(x) for x in data["ui"]] # type: ignore - data["ui"].extend( # type: ignore[attr-defined] - [_ui_out(x) for x in data["outputs"]], - ) - - if hardware_requirements: - for k, v in hardware_requirements.items(): - data["pluginHardwareRequirements"][k] = v - - data["version"] = cast_version(data["version"]) - super().__init__(**data) - self.Config.allow_mutation = True - self._io_keys = {i.name: i for i in self.inputs} - self._io_keys.update({o.name: o for o in self.outputs}) # type: ignore - - if not self.author: - warn_msg = ( - f"The plugin ({self.name}) is missing the author field. " - "This field is not required but should be filled in." - ) - logger.warning(warn_msg) - - @property - def versions(self) -> list: # cannot be in PluginMethods because PLUGINS lives here - """Return list of local versions of a Plugin.""" - return list(PLUGINS[name_cleaner(self.name)]) - - def __setattr__(self, name: str, value: Any) -> None: # noqa: ANN401 - """Set I/O parameters as attributes.""" - BasePlugin.__setattr__(self, name, value) - - def save_config(self, path: Union[str, Path]) -> None: - """Save configured manifest with I/O parameters to specified path.""" - with Path(path).open("w", encoding="utf-8") as file: - json.dump(_get_config(self, "Compute"), file, indent=4, default=str) - logger.debug(f"Saved config to {path}") - - def save_manifest(self, path: Union[str, Path]) -> None: - """Save plugin manifest to specified path.""" - with Path(path).open("w", encoding="utf-8") as file: - json.dump(self.manifest, file, indent=4) - logger.debug(f"Saved manifest to {path}") - - def __repr__(self) -> str: - """Print plugin name and version.""" - return BasePlugin.__repr__(self) - - -def _load_plugin( - manifest: Union[str, dict, Path], -) -> Union[Plugin, ComputePlugin]: - """Parse a manifest and return one of Plugin or ComputePlugin.""" - manifest = _load_manifest(manifest) - if "pluginHardwareRequirements" in manifest: # type: ignore[operator] - # Parse the manifest - plugin = ComputePlugin(**manifest) # type: ignore[arg-type] - else: - # Parse the manifest - plugin = Plugin(**manifest) # type: ignore[arg-type] - return plugin - - -def submit_plugin( - manifest: Union[str, dict, Path], -) -> Union[Plugin, ComputePlugin]: - """Parse a plugin and create a local copy of it. - - This function accepts a plugin manifest as a string, a dictionary (parsed - json), or a pathlib.Path object pointed at a plugin manifest. - - Args: - manifest: - A plugin manifest. It can be a url, a dictionary, - a path to a JSON file or a string that can be parsed as a dictionary - - Returns: - A Plugin object populated with information from the plugin manifest. - """ - plugin = validate_manifest(manifest) - plugin_name = name_cleaner(plugin.name) - - # Get Major/Minor/Patch versions - out_name = ( - plugin_name - + f"_M{plugin.version.major}m{plugin.version.minor}p{plugin.version.patch}.json" - ) - - # Save the manifest if it doesn't already exist in the database - organization = plugin.containerId.split("/")[0] - org_path = _PLUGIN_DIR.joinpath(organization.lower()) - org_path.mkdir(exist_ok=True, parents=True) - if not org_path.joinpath(out_name).exists(): - with org_path.joinpath(out_name).open("w", encoding="utf-8") as file: - manifest_ = json.loads(plugin.model_dump_json()) - json.dump(manifest_, file, indent=4) - - # Refresh plugins list - refresh() - return plugin - - -def get_plugin( - name: str, - version: Optional[str] = None, -) -> Union[Plugin, ComputePlugin]: - """Get a plugin with option to specify version. - - Return a plugin object with the option to specify a version. - The specified version's manifest must exist in manifests folder. - - Args: - name: Name of the plugin. - version: Optional version of the plugin, must follow semver. - - Returns: - Plugin object - """ - if version is None: - return _load_plugin(PLUGINS[name][max(PLUGINS[name])]) - return _load_plugin(PLUGINS[name][Version(version)]) - - -def load_config(config: Union[dict, Path, str]) -> Union[Plugin, ComputePlugin]: - """Load configured plugin from config file/dict.""" - if isinstance(config, (Path, str)): - with Path(config).open("r", encoding="utf-8") as file: - manifest_ = json.load(file) - elif isinstance(config, dict): - manifest_ = config - else: - msg = "config must be a dict, str, or a path" - raise TypeError(msg) - io_keys_ = manifest_["_io_keys"] - class_ = manifest_["class"] - manifest_.pop("class", None) - if class_ == "Compute": - plugin_ = ComputePlugin(_uuid=False, **manifest_) - elif class_ == "WIPP": - plugin_ = Plugin(_uuid=False, **manifest_) - else: - msg = "Invalid value of class" - raise ValueError(msg) - for key, value_ in io_keys_.items(): - val = value_["value"] - if val is not None: # exclude those values not set - setattr(plugin_, key, val) - return plugin_ - - -def remove_plugin(plugin: str, version: Optional[Union[str, list[str]]] = None) -> None: - """Remove plugin from the local database.""" - if version is None: - for plugin_version in PLUGINS[plugin]: - remove_plugin(plugin, plugin_version) - else: - if isinstance(version, list): - for version_ in version: - remove_plugin(plugin, version_) - return - version_ = Version(version) if not isinstance(version, Version) else version - path = PLUGINS[plugin][version_] - path.unlink() - refresh() - - -def remove_all() -> None: - """Remove all plugins from the local database.""" - organizations = [ - x for x in _PLUGIN_DIR.iterdir() if x.name != "__pycache__" and x.is_dir() - ] # ignore __pycache__ - logger.warning("Removing all plugins from local database") - for org in organizations: - shutil.rmtree(org) - refresh() diff --git a/src/polus/plugins/_plugins/io/__init__.py b/src/polus/plugins/_plugins/io/__init__.py index 1e10418b7..4687f4624 100644 --- a/src/polus/plugins/_plugins/io/__init__.py +++ b/src/polus/plugins/_plugins/io/__init__.py @@ -1,27 +1,13 @@ """Init IO module.""" -import pydantic - -PYDANTIC_VERSION = pydantic.__version__ - -if PYDANTIC_VERSION.split(".")[0] == "1": - from polus.plugins._plugins.io.io_v1 import Input - from polus.plugins._plugins.io.io_v1 import IOBase - from polus.plugins._plugins.io.io_v1 import Output - from polus.plugins._plugins.io.io_v1 import Version - from polus.plugins._plugins.io.io_v1 import input_to_cwl - from polus.plugins._plugins.io.io_v1 import io_to_yml - from polus.plugins._plugins.io.io_v1 import output_to_cwl - from polus.plugins._plugins.io.io_v1 import outputs_cwl -elif PYDANTIC_VERSION.split(".")[0] == "2": - from polus.plugins._plugins.io.io_v2 import Input - from polus.plugins._plugins.io.io_v2 import IOBase - from polus.plugins._plugins.io.io_v2 import Output - from polus.plugins._plugins.io.io_v2 import Version - from polus.plugins._plugins.io.io_v2 import input_to_cwl - from polus.plugins._plugins.io.io_v2 import io_to_yml - from polus.plugins._plugins.io.io_v2 import output_to_cwl - from polus.plugins._plugins.io.io_v2 import outputs_cwl +from polus.plugins._plugins.io._io import Input +from polus.plugins._plugins.io._io import IOBase +from polus.plugins._plugins.io._io import Output +from polus.plugins._plugins.io._io import Version +from polus.plugins._plugins.io._io import input_to_cwl +from polus.plugins._plugins.io._io import io_to_yml +from polus.plugins._plugins.io._io import output_to_cwl +from polus.plugins._plugins.io._io import outputs_cwl __all__ = [ "Input", diff --git a/src/polus/plugins/_plugins/io/io_v2.py b/src/polus/plugins/_plugins/io/_io.py similarity index 52% rename from src/polus/plugins/_plugins/io/io_v2.py rename to src/polus/plugins/_plugins/io/_io.py index 81c42f2bc..6aff15a5f 100644 --- a/src/polus/plugins/_plugins/io/io_v2.py +++ b/src/polus/plugins/_plugins/io/_io.py @@ -1,6 +1,6 @@ # type: ignore # ruff: noqa: S101, A003 -# pylint: disable=no-self-argument +# pylint: disable=no-self-argument, C0412 """Plugins I/O utilities.""" import enum import logging @@ -8,19 +8,26 @@ import re from functools import singledispatch from functools import singledispatchmethod -from typing import Annotated from typing import Any from typing import Optional from typing import TypeVar from typing import Union import fsspec +from polus.plugins._plugins._compat import PYDANTIC_V2 from pydantic import BaseModel from pydantic import Field from pydantic import PrivateAttr -from pydantic import RootModel -from pydantic import StringConstraints -from pydantic import field_validator + +if PYDANTIC_V2: + from typing import Annotated + + from pydantic import RootModel + from pydantic import StringConstraints + from pydantic import field_validator +else: + from pydantic import constr + from pydantic import validator logger = logging.getLogger("polus.plugins") @@ -188,44 +195,69 @@ def __setattr__(self, name: str, value: Any) -> None: # ruff: noqa: ANN401 class Output(IOBase): # pylint: disable=R0903 """Required until JSON schema is fixed.""" - name: Annotated[ - str, - StringConstraints(pattern=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$"), - ] = Field( - ..., - examples=["outputCollection"], - title="Output name", - ) + if PYDANTIC_V2: + name: Annotated[ + str, + StringConstraints(pattern=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$"), + ] = Field( + ..., + examples=["outputCollection"], + title="Output name", + ) + description: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( + ..., + examples=["Output collection"], + title="Output description", + ) + else: + name: constr(regex=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$") = Field( + ..., + examples=["outputCollection"], + title="Output name", + ) + description: constr(regex=r"^(.*)$") = Field( + ..., + examples=["Output collection"], + title="Output description", + ) type: OutputTypes = Field( ..., examples=["stitchingVector", "collection"], title="Output type", ) - description: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( - ..., - examples=["Output collection"], - title="Output description", - ) class Input(IOBase): # pylint: disable=R0903 """Required until JSON schema is fixed.""" - name: Annotated[ - str, - StringConstraints(pattern=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$"), - ] = Field( - ..., - description="Input name as expected by the plugin CLI", - examples=["inputImages", "fileNamePattern", "thresholdValue"], - title="Input name", - ) + if PYDANTIC_V2: + name: Annotated[ + str, + StringConstraints(pattern=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$"), + ] = Field( + ..., + description="Input name as expected by the plugin CLI", + examples=["inputImages", "fileNamePattern", "thresholdValue"], + title="Input name", + ) + description: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( + ..., + examples=["Input Images"], + title="Input description", + ) + else: + name: constr(regex=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$") = Field( + ..., + description="Input name as expected by the plugin CLI", + examples=["inputImages", "fileNamePattern", "thresholdValue"], + title="Input name", + ) + description: constr(regex=r"^(.*)$") = Field( + ..., + examples=["Input Images"], + title="Input description", + ) type: InputTypes - description: Annotated[str, StringConstraints(pattern=r"^(.*)$")] = Field( - ..., - examples=["Input Images"], - title="Input description", - ) required: Optional[bool] = Field( True, description="Whether an input is required or not", @@ -256,87 +288,200 @@ def _check_version_number(value: Union[str, int]) -> bool: return bool(re.match(r"^\d+$", value)) -class Version(RootModel): - """SemVer object.""" - - root: str - - @field_validator("root") - @classmethod - def semantic_version( - cls, - value, - ) -> Any: # ruff: noqa: ANN202, N805, ANN001 - """Pydantic Validator to check semver.""" - version = value.split(".") - - assert ( - len(version) == 3 # ruff: noqa: PLR2004 - ), f""" - Invalid version ({value}). Version must follow - semantic versioning (see semver.org)""" - if "-" in version[-1]: # with hyphen - idn = version[-1].split("-")[-1] - id_reg = re.compile("[0-9A-Za-z-]+") - assert bool( - id_reg.match(idn), +if PYDANTIC_V2: + + class Version(RootModel): + """SemVer object.""" + + root: str + + @field_validator("root") + @classmethod + def semantic_version( + cls, + value, + ) -> Any: # ruff: noqa: ANN202, N805, ANN001 + """Pydantic Validator to check semver.""" + version = value.split(".") + + assert ( + len(version) == 3 # ruff: noqa: PLR2004 + ), f""" + Invalid version ({value}). Version must follow + semantic versioning (see semver.org)""" + if "-" in version[-1]: # with hyphen + idn = version[-1].split("-")[-1] + id_reg = re.compile("[0-9A-Za-z-]+") + assert bool( + id_reg.match(idn), + ), f"""Invalid version ({value}). + Version must follow semantic versioning (see semver.org)""" + + assert all( + map(_check_version_number, version), ), f"""Invalid version ({value}). Version must follow semantic versioning (see semver.org)""" + return value + + @property + def major(self): + """Return x from x.y.z .""" + return int(self.root.split(".")[0]) + + @property + def minor(self): + """Return y from x.y.z .""" + return int(self.root.split(".")[1]) + + @property + def patch(self): + """Return z from x.y.z .""" + if not self.root.split(".")[2].isdigit(): + msg = "Patch version is not a digit, comparison may not be accurate." + logger.warning(msg) + return self.root.split(".")[2] + return int(self.root.split(".")[2]) + + def __str__(self) -> str: + """Return string representation of Version object.""" + return self.root + + @singledispatchmethod + def __lt__(self, other: Any) -> bool: + """Compare if Version is less than other object.""" + msg = "invalid type for comparison." + raise TypeError(msg) - assert all( - map(_check_version_number, version), - ), f"""Invalid version ({value}). - Version must follow semantic versioning (see semver.org)""" - return value - - @property - def major(self): - """Return x from x.y.z .""" - return int(self.root.split(".")[0]) - - @property - def minor(self): - """Return y from x.y.z .""" - return int(self.root.split(".")[1]) - - @property - def patch(self): - """Return z from x.y.z .""" - if not self.root.split(".")[2].isdigit(): - msg = "Patch version is not a digit, comparison may not be accurate." - logger.warning(msg) - return self.root.split(".")[2] - return int(self.root.split(".")[2]) - - def __str__(self) -> str: - """Return string representation of Version object.""" - return self.root - - @singledispatchmethod - def __lt__(self, other: Any) -> bool: - """Compare if Version is less than other object.""" - msg = "invalid type for comparison." - raise TypeError(msg) - - @singledispatchmethod - def __gt__(self, other: Any) -> bool: - """Compare if Version is less than other object.""" - msg = "invalid type for comparison." - raise TypeError(msg) - - @singledispatchmethod - def __eq__(self, other: Any) -> bool: - """Compare if two Version objects are equal.""" - msg = "invalid type for comparison." - raise TypeError(msg) - - def __hash__(self) -> int: - """Needed to use Version objects as dict keys.""" - return hash(self.root) - - def __repr__(self) -> str: - """Return string representation of Version object.""" - return self.root + @singledispatchmethod + def __gt__(self, other: Any) -> bool: + """Compare if Version is less than other object.""" + msg = "invalid type for comparison." + raise TypeError(msg) + + @singledispatchmethod + def __eq__(self, other: Any) -> bool: + """Compare if two Version objects are equal.""" + msg = "invalid type for comparison." + raise TypeError(msg) + + def __hash__(self) -> int: + """Needed to use Version objects as dict keys.""" + return hash(self.root) + + def __repr__(self) -> str: + """Return string representation of Version object.""" + return self.root + + @Version.__eq__.register(str) # pylint: disable=no-member + def _(self, other): + return self == Version(other) + + @Version.__lt__.register(str) # pylint: disable=no-member + def _(self, other): + v = Version(other) + return self < v + + @Version.__gt__.register(str) # pylint: disable=no-member + def _(self, other): + v = Version(other) + return self > v + +else: # PYDANTIC_V1 + + class Version(BaseModel): + """SemVer object.""" + + version: str + + def __init__(self, version: str) -> None: + """Initialize Version object.""" + super().__init__(version=version) + + @validator("version") + def semantic_version( + cls, + value, + ): # ruff: noqa: ANN202, N805, ANN001 + """Pydantic Validator to check semver.""" + version = value.split(".") + + assert ( + len(version) == 3 # ruff: noqa: PLR2004 + ), f""" + Invalid version ({value}). Version must follow + semantic versioning (see semver.org)""" + if "-" in version[-1]: # with hyphen + idn = version[-1].split("-")[-1] + id_reg = re.compile("[0-9A-Za-z-]+") + assert bool( + id_reg.match(idn), + ), f"""Invalid version ({value}). + Version must follow semantic versioning (see semver.org)""" + + assert all( + map(_check_version_number, version), + ), f"""Invalid version ({value}). + Version must follow semantic versioning (see semver.org)""" + return value + + @property + def major(self): + """Return x from x.y.z .""" + return int(self.version.split(".")[0]) + + @property + def minor(self): + """Return y from x.y.z .""" + return int(self.version.split(".")[1]) + + @property + def patch(self): + """Return z from x.y.z .""" + if not self.version.split(".")[2].isdigit(): + msg = "Patch version is not a digit, comparison may not be accurate." + logger.warning(msg) + return self.version.split(".")[2] + return int(self.version.split(".")[2]) + + def __str__(self) -> str: + """Return string representation of Version object.""" + return self.version + + @singledispatchmethod + def __lt__(self, other: Any) -> bool: + """Compare if Version is less than other object.""" + msg = "invalid type for comparison." + raise TypeError(msg) + + @singledispatchmethod + def __gt__(self, other: Any) -> bool: + """Compare if Version is less than other object.""" + msg = "invalid type for comparison." + raise TypeError(msg) + + @singledispatchmethod + def __eq__(self, other: Any) -> bool: + """Compare if two Version objects are equal.""" + msg = "invalid type for comparison." + raise TypeError(msg) + + def __hash__(self) -> int: + """Needed to use Version objects as dict keys.""" + return hash(self.version) + + @Version.__eq__.register(str) # pylint: disable=no-member + def _(self, other): + return self == Version(**{"version": other}) + + @Version.__lt__.register(str) # pylint: disable=no-member + def _(self, other): + v = Version(**{"version": other}) + return self < v + + @Version.__gt__.register(str) # pylint: disable=no-member + def _(self, other): + v = Version(**{"version": other}) + return self > v @Version.__eq__.register(Version) # pylint: disable=no-member @@ -348,11 +493,6 @@ def _(self, other): ) -@Version.__eq__.register(str) # pylint: disable=no-member -def _(self, other): - return self == Version(other) - - @Version.__lt__.register(Version) # pylint: disable=no-member def _(self, other): if other.major > self.major: @@ -368,23 +508,11 @@ def _(self, other): return False -@Version.__lt__.register(str) # pylint: disable=no-member -def _(self, other): - v = Version(other) - return self < v - - @Version.__gt__.register(Version) # pylint: disable=no-member def _(self, other): return other < self -@Version.__gt__.register(str) # pylint: disable=no-member -def _(self, other): - v = Version(other) - return self > v - - class DuplicateVersionFoundError(Exception): """Raise when two equal versions found.""" diff --git a/src/polus/plugins/_plugins/io/io_v1.py b/src/polus/plugins/_plugins/io/io_v1.py deleted file mode 100644 index 248e1af15..000000000 --- a/src/polus/plugins/_plugins/io/io_v1.py +++ /dev/null @@ -1,459 +0,0 @@ -# type: ignore -# ruff: noqa: S101, A003 -# pylint: disable=no-self-argument -"""Plugins I/O utilities.""" -import enum -import logging -import pathlib -import re -from functools import singledispatch -from functools import singledispatchmethod -from typing import Any -from typing import Optional -from typing import TypeVar -from typing import Union - -import fsspec -from pydantic import BaseModel -from pydantic import Field -from pydantic import PrivateAttr -from pydantic import constr -from pydantic import validator - -logger = logging.getLogger("polus.plugins") - -""" -Enums for validating plugin input, output, and ui components. -""" -WIPP_TYPES = { - "collection": pathlib.Path, - "pyramid": pathlib.Path, - "csvCollection": pathlib.Path, - "genericData": pathlib.Path, - "stitchingVector": pathlib.Path, - "notebook": pathlib.Path, - "tensorflowModel": pathlib.Path, - "tensorboardLogs": pathlib.Path, - "pyramidAnnotation": pathlib.Path, - "integer": int, - "number": float, - "string": str, - "boolean": bool, - "array": str, - "enum": enum.Enum, - "path": pathlib.Path, -} - - -class InputTypes(str, enum.Enum): # wipp schema - """Enum of Input Types for WIPP schema.""" - - COLLECTION = "collection" - PYRAMID = "pyramid" - CSVCOLLECTION = "csvCollection" - GENERICDATA = "genericData" - STITCHINGVECTOR = "stitchingVector" - NOTEBOOK = "notebook" - TENSORFLOWMODEL = "tensorflowModel" - TENSORBOARDLOGS = "tensorboardLogs" - PYRAMIDANNOTATION = "pyramidAnnotation" - INTEGER = "integer" - NUMBER = "number" - STRING = "string" - BOOLEAN = "boolean" - ARRAY = "array" - ENUM = "enum" - - -class OutputTypes(str, enum.Enum): # wipp schema - """Enum for Output Types for WIPP schema.""" - - COLLECTION = "collection" - PYRAMID = "pyramid" - CSVCOLLECTION = "csvCollection" - GENERICDATA = "genericData" - STITCHINGVECTOR = "stitchingVector" - NOTEBOOK = "notebook" - TENSORFLOWMODEL = "tensorflowModel" - TENSORBOARDLOGS = "tensorboardLogs" - PYRAMIDANNOTATION = "pyramidAnnotation" - - -def _in_old_to_new(old: str) -> str: # map wipp InputType to compute schema's InputType - """Map an InputType from wipp schema to one of compute schema.""" - d = {"integer": "number", "enum": "string"} - if old in ["string", "array", "number", "boolean"]: - return old - if old in d: - return d[old] # integer or enum - return "path" # everything else - - -def _ui_old_to_new(old: str) -> str: # map wipp InputType to compute schema's UIType - """Map an InputType from wipp schema to a UIType of compute schema.""" - type_dict = { - "string": "text", - "boolean": "checkbox", - "number": "number", - "array": "text", - "integer": "number", - } - if old in type_dict: - return type_dict[old] - return "text" - - -FileSystem = TypeVar("FileSystem", bound=fsspec.spec.AbstractFileSystem) - - -class IOBase(BaseModel): # pylint: disable=R0903 - """Base Class for I/O arguments.""" - - type: Any - options: Optional[dict] = None - value: Optional[Any] = None - id_: Optional[Any] = None - _fs: Optional[FileSystem] = PrivateAttr( - default=None, - ) # type checking is done at plugin level - - def _validate(self) -> None: # pylint: disable=R0912 - value = self.value - - if value is None: - if self.required: - msg = f""" - The input value ({self.name}) is required, - but the value was not set.""" - raise TypeError( - msg, - ) - - return - - if self.type == InputTypes.ENUM: - try: - if isinstance(value, str): - value = enum.Enum(self.name, self.options["values"])[value] - elif not isinstance(value, enum.Enum): - raise ValueError - - except KeyError: - logging.error( - f""" - Value ({value}) is not a valid value - for the enum input ({self.name}). - Must be one of {self.options['values']}. - """, - ) - raise - else: - if isinstance(self.type, (InputTypes, OutputTypes)): # wipp - value = WIPP_TYPES[self.type](value) - else: - value = WIPP_TYPES[self.type.value]( - value, - ) # compute, type does not inherit from str - - if isinstance(value, pathlib.Path): - value = value.absolute() - if self._fs: - assert self._fs.exists( - str(value), - ), f"{value} is invalid or does not exist" - assert self._fs.isdir( - str(value), - ), f"{value} is not a valid directory" - else: - assert value.exists(), f"{value} is invalid or does not exist" - assert value.is_dir(), f"{value} is not a valid directory" - - super().__setattr__("value", value) - - def __setattr__(self, name: str, value: Any) -> None: # ruff: noqa: ANN401 - """Set I/O attributes.""" - if name not in ["value", "id", "_fs"]: - # Don't permit any other values to be changed - msg = f"Cannot set property: {name}" - raise TypeError(msg) - - super().__setattr__(name, value) - - if name == "value": - self._validate() - - -class Output(IOBase): # pylint: disable=R0903 - """Required until JSON schema is fixed.""" - - name: constr(regex=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$") = Field( - ..., - examples=["outputCollection"], - title="Output name", - ) - type: OutputTypes = Field( - ..., - examples=["stitchingVector", "collection"], - title="Output type", - ) - description: constr(regex=r"^(.*)$") = Field( - ..., - examples=["Output collection"], - title="Output description", - ) - - -class Input(IOBase): # pylint: disable=R0903 - """Required until JSON schema is fixed.""" - - name: constr(regex=r"^[a-zA-Z0-9][-a-zA-Z0-9]*$") = Field( - ..., - description="Input name as expected by the plugin CLI", - examples=["inputImages", "fileNamePattern", "thresholdValue"], - title="Input name", - ) - type: InputTypes - description: constr(regex=r"^(.*)$") = Field( - ..., - examples=["Input Images"], - title="Input description", - ) - required: Optional[bool] = Field( - True, - description="Whether an input is required or not", - examples=[True], - title="Required input", - ) - - def __init__(self, **data) -> None: # ruff: noqa: ANN003 - """Initialize input.""" - super().__init__(**data) - - if self.description is None: - logger.warning( - f""" - The input ({self.name}) is missing the description field. - This field is not required but should be filled in. - """, - ) - - -def _check_version_number(value: Union[str, int]) -> bool: - if isinstance(value, int): - value = str(value) - if "-" in value: - value = value.split("-")[0] - if len(value) > 1 and value[0] == "0": - return False - return bool(re.match(r"^\d+$", value)) - - -class Version(BaseModel): - """SemVer object.""" - - version: str - - def __init__(self, version: str) -> None: - """Initialize Version object.""" - super().__init__(version=version) - - @validator("version") - def semantic_version( - cls, - value, - ): # ruff: noqa: ANN202, N805, ANN001 - """Pydantic Validator to check semver.""" - version = value.split(".") - - assert ( - len(version) == 3 # ruff: noqa: PLR2004 - ), f""" - Invalid version ({value}). Version must follow - semantic versioning (see semver.org)""" - if "-" in version[-1]: # with hyphen - idn = version[-1].split("-")[-1] - id_reg = re.compile("[0-9A-Za-z-]+") - assert bool( - id_reg.match(idn), - ), f"""Invalid version ({value}). - Version must follow semantic versioning (see semver.org)""" - - assert all( - map(_check_version_number, version), - ), f"""Invalid version ({value}). - Version must follow semantic versioning (see semver.org)""" - return value - - @property - def major(self): - """Return x from x.y.z .""" - return int(self.version.split(".")[0]) - - @property - def minor(self): - """Return y from x.y.z .""" - return int(self.version.split(".")[1]) - - @property - def patch(self): - """Return z from x.y.z .""" - if not self.version.split(".")[2].isdigit(): - msg = "Patch version is not a digit, comparison may not be accurate." - logger.warning(msg) - return self.version.split(".")[2] - return int(self.version.split(".")[2]) - - def __str__(self) -> str: - """Return string representation of Version object.""" - return self.version - - @singledispatchmethod - def __lt__(self, other: Any) -> bool: - """Compare if Version is less than other object.""" - msg = "invalid type for comparison." - raise TypeError(msg) - - @singledispatchmethod - def __gt__(self, other: Any) -> bool: - """Compare if Version is less than other object.""" - msg = "invalid type for comparison." - raise TypeError(msg) - - @singledispatchmethod - def __eq__(self, other: Any) -> bool: - """Compare if two Version objects are equal.""" - msg = "invalid type for comparison." - raise TypeError(msg) - - def __hash__(self) -> int: - """Needed to use Version objects as dict keys.""" - return hash(self.version) - - -@Version.__eq__.register(Version) # pylint: disable=no-member -def _(self, other): - return ( - other.major == self.major - and other.minor == self.minor - and other.patch == self.patch - ) - - -@Version.__eq__.register(str) # pylint: disable=no-member -def _(self, other): - return self == Version(**{"version": other}) - - -@Version.__lt__.register(Version) # pylint: disable=no-member -def _(self, other): - if other.major > self.major: - return True - if other.major == self.major: - if other.minor > self.minor: - return True - if other.minor == self.minor: - if other.patch > self.patch: - return True - return False - return False - return False - - -@Version.__lt__.register(str) # pylint: disable=no-member -def _(self, other): - v = Version(**{"version": other}) - return self < v - - -@Version.__gt__.register(Version) # pylint: disable=no-member -def _(self, other): - return other < self - - -@Version.__gt__.register(str) # pylint: disable=no-member -def _(self, other): - v = Version(**{"version": other}) - return self > v - - -class DuplicateVersionFoundError(Exception): - """Raise when two equal versions found.""" - - -CWL_INPUT_TYPES = { - "path": "Directory", # always Dir? Yes - "string": "string", - "number": "double", - "boolean": "boolean", - "genericData": "Directory", - "collection": "Directory", - "enum": "string", # for compatibility with workflows - "stitchingVector": "Directory", - # not yet implemented: array -} - - -def _type_in(inp: Input): - """Return appropriate value for `type` based on input type.""" - val = inp.type.value - req = "" if inp.required else "?" - - # NOT compatible with CWL workflows, ok in CLT - # if val == "enum": - # if input.required: - - # if val in CWL_INPUT_TYPES: - return CWL_INPUT_TYPES[val] + req if val in CWL_INPUT_TYPES else "string" + req - - -def input_to_cwl(inp: Input): - """Return dict of inputs for cwl.""" - return { - f"{inp.name}": { - "type": _type_in(inp), - "inputBinding": {"prefix": f"--{inp.name}"}, - }, - } - - -def output_to_cwl(out: Output): - """Return dict of output args for cwl for input section.""" - return { - f"{out.name}": { - "type": "Directory", - "inputBinding": {"prefix": f"--{out.name}"}, - }, - } - - -def outputs_cwl(out: Output): - """Return dict of output for `outputs` in cwl.""" - return { - f"{out.name}": { - "type": "Directory", - "outputBinding": {"glob": f"$(inputs.{out.name}.basename)"}, - }, - } - - -# -- I/O as arguments in .yml - - -@singledispatch -def _io_value_to_yml(io) -> Union[str, dict]: - return str(io) - - -@_io_value_to_yml.register -def _(io: pathlib.Path): - return {"class": "Directory", "location": str(io)} - - -@_io_value_to_yml.register -def _(io: enum.Enum): - return io.name - - -def io_to_yml(io): - """Return IO entry for yml file.""" - return _io_value_to_yml(io.value) diff --git a/src/polus/plugins/_plugins/manifests/__init__.py b/src/polus/plugins/_plugins/manifests/__init__.py index b2642a73f..9f1456e8a 100644 --- a/src/polus/plugins/_plugins/manifests/__init__.py +++ b/src/polus/plugins/_plugins/manifests/__init__.py @@ -1,21 +1,10 @@ """Initialize manifests module.""" -import pydantic - -PYDANTIC_VERSION = pydantic.__version__ - -if PYDANTIC_VERSION.split(".")[0] == "1": - from polus.plugins._plugins.manifests.manifest_utils_v1 import InvalidManifestError - from polus.plugins._plugins.manifests.manifest_utils_v1 import _error_log - from polus.plugins._plugins.manifests.manifest_utils_v1 import _load_manifest - from polus.plugins._plugins.manifests.manifest_utils_v1 import _scrape_manifests - from polus.plugins._plugins.manifests.manifest_utils_v1 import validate_manifest -elif PYDANTIC_VERSION.split(".")[0] == "2": - from polus.plugins._plugins.manifests.manifest_utils_v2 import InvalidManifestError - from polus.plugins._plugins.manifests.manifest_utils_v2 import _error_log - from polus.plugins._plugins.manifests.manifest_utils_v2 import _load_manifest - from polus.plugins._plugins.manifests.manifest_utils_v2 import _scrape_manifests - from polus.plugins._plugins.manifests.manifest_utils_v2 import validate_manifest +from polus.plugins._plugins.manifests.manifest_utils import InvalidManifestError +from polus.plugins._plugins.manifests.manifest_utils import _error_log +from polus.plugins._plugins.manifests.manifest_utils import _load_manifest +from polus.plugins._plugins.manifests.manifest_utils import _scrape_manifests +from polus.plugins._plugins.manifests.manifest_utils import validate_manifest __all__ = [ "InvalidManifestError", diff --git a/src/polus/plugins/_plugins/manifests/manifest_utils_v1.py b/src/polus/plugins/_plugins/manifests/manifest_utils.py similarity index 95% rename from src/polus/plugins/_plugins/manifests/manifest_utils_v1.py rename to src/polus/plugins/_plugins/manifests/manifest_utils.py index 927126f4a..f2e8dbcf4 100644 --- a/src/polus/plugins/_plugins/manifests/manifest_utils_v1.py +++ b/src/polus/plugins/_plugins/manifests/manifest_utils.py @@ -8,13 +8,16 @@ import github import requests import validators +from polus.plugins._plugins._compat import PYDANTIC_V2 from polus.plugins._plugins.models import ComputeSchema from polus.plugins._plugins.models import WIPPPluginManifest -from polus.plugins._plugins.utils import cast_version from pydantic import ValidationError from pydantic import errors from tqdm import tqdm +if not PYDANTIC_V2: + from polus.plugins._plugins.utils import cast_version + logger = logging.getLogger("polus.plugins") # Fields that must be in a plugin manifest @@ -84,9 +87,10 @@ def validate_manifest( ) -> Union[WIPPPluginManifest, ComputeSchema]: """Validate a plugin manifest against schema.""" manifest = _load_manifest(manifest) - manifest["version"] = cast_version( - manifest["version"], - ) # cast version to semver object + if not PYDANTIC_V2: # Pydantic V1 + manifest["version"] = cast_version( + manifest["version"], + ) # cast version to semver object if "name" in manifest: name = manifest["name"] else: diff --git a/src/polus/plugins/_plugins/manifests/manifest_utils_v2.py b/src/polus/plugins/_plugins/manifests/manifest_utils_v2.py deleted file mode 100644 index 9c37c111b..000000000 --- a/src/polus/plugins/_plugins/manifests/manifest_utils_v2.py +++ /dev/null @@ -1,202 +0,0 @@ -"""Utilities for manifest parsing and validation.""" -import json -import logging -import pathlib -from typing import Optional -from typing import Union - -import github -import requests -import validators -from polus.plugins._plugins.models import ComputeSchema -from polus.plugins._plugins.models import WIPPPluginManifest -from pydantic import ValidationError -from pydantic import errors -from tqdm import tqdm - -logger = logging.getLogger("polus.plugins") - -# Fields that must be in a plugin manifest -REQUIRED_FIELDS = [ - "name", - "version", - "description", - "author", - "containerId", - "inputs", - "outputs", - "ui", -] - - -class InvalidManifestError(Exception): - """Raised when manifest has validation errors.""" - - -def is_valid_manifest(plugin: dict) -> bool: - """Validate basic attributes of a plugin manifest. - - Args: - plugin: A parsed plugin json file - - Returns: - True if the plugin has the minimal json fields - """ - fields = list(plugin.keys()) - - for field in REQUIRED_FIELDS: - if field not in fields: - msg = f"Missing json field, {field}, in plugin manifest." - logger.error(msg) - return False - return True - - -def _load_manifest(manifest: Union[str, dict, pathlib.Path]) -> dict: - """Return manifest as dict from str (url or path) or pathlib.Path.""" - if isinstance(manifest, dict): # is dict - return manifest - if isinstance(manifest, pathlib.Path): # is path - if manifest.suffix != ".json": - msg = "plugin manifest must be a json file with .json extension." - raise ValueError(msg) - - with manifest.open("r", encoding="utf-8") as manifest_json: - manifest_ = json.load(manifest_json) - elif isinstance(manifest, str): # is str - if validators.url(manifest): # is url - manifest_ = requests.get(manifest, timeout=10).json() - else: # could (and should) be path - try: - manifest_ = _load_manifest(pathlib.Path(manifest)) - except Exception as exc: # was not a Path? # noqa - msg = "invalid manifest" - raise ValueError(msg) from exc - else: # is not str, dict, or path - msg = f"invalid manifest type {type(manifest)}" - raise ValueError(msg) - return manifest_ - - -def validate_manifest( - manifest: Union[str, dict, pathlib.Path], -) -> Union[WIPPPluginManifest, ComputeSchema]: - """Validate a plugin manifest against schema.""" - manifest = _load_manifest(manifest) - if "name" in manifest: - name = manifest["name"] - else: - msg = f"{manifest} has no value for name" - raise InvalidManifestError(msg) - - if "pluginHardwareRequirements" in manifest: - # Parse the manifest - try: - plugin = ComputeSchema(**manifest) - except ValidationError as e: - msg = f"{name} does not conform to schema" - raise InvalidManifestError(msg) from e - except BaseException as e: - raise e - else: - # Parse the manifest - try: - plugin = WIPPPluginManifest(**manifest) - except ValidationError as e: - msg = f"{manifest['name']} does not conform to schema" - raise InvalidManifestError( - msg, - ) from e - except BaseException as e: - raise e - return plugin - - -def _scrape_manifests( - repo: Union[str, github.Repository.Repository], # type: ignore - gh: github.Github, - min_depth: int = 1, - max_depth: Optional[int] = None, - return_invalid: bool = False, -) -> Union[list, tuple[list, list]]: - if max_depth is None: - max_depth = min_depth - min_depth = 0 - - if not max_depth >= min_depth: - msg = "max_depth is smaller than min_depth" - raise ValueError(msg) - - if isinstance(repo, str): - repo = gh.get_repo(repo) - - contents = list(repo.get_contents("")) # type: ignore - next_contents: list = [] - valid_manifests: list = [] - invalid_manifests: list = [] - - for d in range(0, max_depth): - for content in tqdm(contents, desc=f"{repo.full_name}: {d}"): - if content.type == "dir": - next_contents.extend(repo.get_contents(content.path)) # type: ignore - elif content.name.endswith(".json") and d >= min_depth: - manifest = json.loads(content.decoded_content) - if is_valid_manifest(manifest): - valid_manifests.append(manifest) - else: - invalid_manifests.append(manifest) - - contents = next_contents.copy() - next_contents = [] - - if return_invalid: - return valid_manifests, invalid_manifests - return valid_manifests - - -def _error_log(val_err: ValidationError, manifest: dict, fct: str) -> None: - report = [] - - for error in val_err.args[0]: - if isinstance(error, list): - error = error[0] # noqa - - if isinstance(error, AssertionError): - msg = ( - f"The plugin ({manifest['name']}) " - "failed an assertion check: {err.args[0]}" - ) - report.append(msg) - logger.critical(f"{fct}: {report[-1]}") # pylint: disable=W1203 - elif isinstance(error.exc, errors.MissingError): - msg = ( - f"The plugin ({manifest['name']}) " - "is missing fields: {err.loc_tuple()}" - ) - report.append(msg) - logger.critical(f"{fct}: {report[-1]}") # pylint: disable=W1203 - elif errors.ExtraError: - if error.loc_tuple()[0] in ["inputs", "outputs", "ui"]: - manifest_ = manifest[error.loc_tuple()[0]][error.loc_tuple()[1]]["name"] - msg = ( - f"The plugin ({manifest['name']}) " - "had unexpected values in the " - f"{error.loc_tuple()[0]} " - f"({manifest_}): " - f"{error.exc.args[0][0].loc_tuple()}" - ) - report.append(msg) - else: - msg = ( - f"The plugin ({manifest['name']}) " - "had an error: {err.exc.args[0][0]}" - ) - report.append(msg) - logger.critical(f"{fct}: {report[-1]}") # pylint: disable=W1203 - else: - str_val_err = str(val_err).replace("\n", ", ").replace(" ", " ") - msg = ( - f"{fct}: Uncaught manifest error in ({manifest['name']}): " - f"{str_val_err}" - ) - logger.warning(msg) diff --git a/src/polus/plugins/_plugins/update/__init__.py b/src/polus/plugins/_plugins/update/__init__.py index ff8010c6e..936f7b803 100644 --- a/src/polus/plugins/_plugins/update/__init__.py +++ b/src/polus/plugins/_plugins/update/__init__.py @@ -1,14 +1,6 @@ """Initialize update module.""" -import pydantic - -PYDANTIC_VERSION = pydantic.__version__ - -if PYDANTIC_VERSION.split(".")[0] == "1": - from polus.plugins._plugins.update.update_v1 import update_nist_plugins - from polus.plugins._plugins.update.update_v1 import update_polus_plugins -elif PYDANTIC_VERSION.split(".")[0] == "2": - from polus.plugins._plugins.update.update_v2 import update_nist_plugins - from polus.plugins._plugins.update.update_v2 import update_polus_plugins +from polus.plugins._plugins.update._update import update_nist_plugins +from polus.plugins._plugins.update._update import update_polus_plugins __all__ = ["update_polus_plugins", "update_nist_plugins"] diff --git a/src/polus/plugins/_plugins/update/update_v1.py b/src/polus/plugins/_plugins/update/_update.py similarity index 96% rename from src/polus/plugins/_plugins/update/update_v1.py rename to src/polus/plugins/_plugins/update/_update.py index 9a0dc508f..cc184d493 100644 --- a/src/polus/plugins/_plugins/update/update_v1.py +++ b/src/polus/plugins/_plugins/update/_update.py @@ -4,6 +4,7 @@ import re import typing +from polus.plugins._plugins._compat import PYDANTIC_V2 from polus.plugins._plugins.classes import refresh from polus.plugins._plugins.classes import submit_plugin from polus.plugins._plugins.gh import _init_github @@ -44,7 +45,7 @@ def update_polus_plugins( # Check that plugin version matches container version tag container_name, version = tuple(plugin.containerId.split(":")) - version = Version(version=version) + version = Version(version) if PYDANTIC_V2 else Version(version=version) organization, container_name = tuple(container_name.split("/")) if plugin.version != version: msg = ( diff --git a/src/polus/plugins/_plugins/update/update_v2.py b/src/polus/plugins/_plugins/update/update_v2.py deleted file mode 100644 index 67bd0038c..000000000 --- a/src/polus/plugins/_plugins/update/update_v2.py +++ /dev/null @@ -1,115 +0,0 @@ -# pylint: disable=W1203, W1201 -import json -import logging -import re -import typing - -from polus.plugins._plugins.classes import refresh -from polus.plugins._plugins.classes import submit_plugin -from polus.plugins._plugins.gh import _init_github -from polus.plugins._plugins.io import Version -from polus.plugins._plugins.manifests import _error_log -from polus.plugins._plugins.manifests import _scrape_manifests -from pydantic import ValidationError -from tqdm import tqdm - -logger = logging.getLogger("polus.plugins") - - -def update_polus_plugins( - gh_auth: typing.Optional[str] = None, - min_depth: int = 2, - max_depth: int = 3, -) -> None: - """Scrape PolusAI GitHub repo and create local versions of Plugins.""" - logger.info("Updating polus plugins.") - # Get all manifests - valid, invalid = _scrape_manifests( - "polusai/polus-plugins", - _init_github(gh_auth), - min_depth, - max_depth, - True, - ) - manifests = valid.copy() - manifests.extend(invalid) - logger.info(f"Submitting {len(manifests)} plugins.") - - for manifest in manifests: - try: - plugin = submit_plugin(manifest) - - # Parsing checks specific to polus-plugins - error_list = [] - - # Check that plugin version matches container version tag - container_name, version = tuple(plugin.containerId.split(":")) - version = Version(version) - organization, container_name = tuple(container_name.split("/")) - if plugin.version != version: - msg = ( - f"containerId version ({version}) does not " - f"match plugin version ({plugin.version})" - ) - logger.error(msg) - error_list.append(ValueError(msg)) - - # Check to see that the plugin is registered to Labshare - if organization not in ["polusai", "labshare"]: - msg = ( - "all polus plugin containers must be" - " under the Labshare organization." - ) - logger.error(msg) - error_list.append(ValueError(msg)) - - # Checks for container name, they are somewhat related to our - # Jenkins build - if not container_name.startswith("polus"): - msg = "containerId name must begin with polus-" - logger.error(msg) - error_list.append(ValueError(msg)) - - if not container_name.endswith("plugin"): - msg = "containerId name must end with -plugin" - logger.error(msg) - error_list.append(ValueError(msg)) - - if len(error_list) > 0: - raise ValidationError(error_list, plugin.__class__) - - except ValidationError as val_err: - try: - _error_log(val_err, manifest, "update_polus_plugins") - except BaseException as e: # pylint: disable=W0718 - logger.exception(f"In {plugin.name}: {e}") - except BaseException as e: # pylint: disable=W0718 - logger.exception(f"In {plugin.name}: {e}") - refresh() - - -def update_nist_plugins(gh_auth: typing.Optional[str] = None) -> None: - """Scrape NIST GitHub repo and create local versions of Plugins.""" - # Parse README links - gh = _init_github(gh_auth) - repo = gh.get_repo("usnistgov/WIPP") - contents = repo.get_contents("plugins") - readme = [r for r in contents if r.name == "README.md"][0] - pattern = re.compile( - r"\[manifest\]\((https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*))\)", - ) - matches = pattern.findall(str(readme.decoded_content)) - logger.info("Updating NIST plugins.") - for match in tqdm(matches, desc="NIST Manifests"): - url_parts = match[0].split("/")[3:] - plugin_repo = gh.get_repo("/".join(url_parts[:2])) - manifest = json.loads( - plugin_repo.get_contents("/".join(url_parts[4:])).decoded_content, - ) - - try: - submit_plugin(manifest) - - except ValidationError as val_err: - _error_log(val_err, manifest, "update_nist_plugins") - refresh() diff --git a/tests/test_manifests.py b/tests/test_manifests.py index 286e93dd6..d24a5d91f 100644 --- a/tests/test_manifests.py +++ b/tests/test_manifests.py @@ -3,26 +3,14 @@ from collections import OrderedDict from pathlib import Path -import pydantic import pytest -PYDANTIC_VERSION = pydantic.__version__.split(".")[0] - from polus.plugins._plugins.classes import PLUGINS, list_plugins - -if PYDANTIC_VERSION == "1": - from polus.plugins._plugins.manifests.manifest_utils_v1 import ( - InvalidManifestError, - _load_manifest, - validate_manifest, - ) -elif PYDANTIC_VERSION == "2": - from polus.plugins._plugins.manifests.manifest_utils_v2 import ( - InvalidManifestError, - _load_manifest, - validate_manifest, - ) - +from polus.plugins._plugins.manifests import ( + InvalidManifestError, + _load_manifest, + validate_manifest, +) from polus.plugins._plugins.models import ComputeSchema, WIPPPluginManifest RSRC_PATH = Path(__file__).parent.joinpath("resources")