From 76bb9b509ac5ba8f5aabd3877d9c285da0247c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Wed, 23 Oct 2024 12:50:22 +0200 Subject: [PATCH] Revert "Add new command `nf-core rocrate` to create a Research Object (RO) crate for a pipeline" --- .github/actions/create-lint-wf/action.yml | 6 - .../create-test-lint-wf-template.yml | 8 - nf_core/__main__.py | 44 +- nf_core/commands_pipelines.py | 28 -- nf_core/components/info.py | 4 +- nf_core/modules/modules_json.py | 4 +- nf_core/pipelines/lint/files_exist.py | 14 +- nf_core/pipelines/rocrate.py | 400 ------------------ nf_core/pipelines/schema.py | 4 +- nf_core/utils.py | 109 +---- requirements.txt | 2 - tests/test_rocrate.py | 83 ---- 12 files changed, 27 insertions(+), 679 deletions(-) delete mode 100644 nf_core/pipelines/rocrate.py delete mode 100644 tests/test_rocrate.py diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 3ef076051..6ad6b7b15 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -65,12 +65,6 @@ runs: run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \; working-directory: create-lint-wf - # Add empty ro-crate file - - name: add empty ro-crate file - shell: bash - run: touch nf-core-testpipeline/ro-crate-metadata.json - working-directory: create-lint-wf - # Run nf-core pipelines linting - name: nf-core pipelines lint shell: bash diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 5871919ca..627b73fb6 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -112,9 +112,6 @@ jobs: run: | cd create-test-lint-wf nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml template_skip_${{ matrix.TEMPLATE }}.yml - # fake ro-crate - touch my-prefix-testpipeline/ro-crate-metadata.json - git commit -am "add ro-crate" - name: run the pipeline run: | @@ -150,11 +147,6 @@ jobs: run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \; working-directory: create-test-lint-wf - # Add empty ro-crate file - - name: add empty ro-crate file - run: touch my-prefix-testpipeline/ro-crate-metadata.json - working-directory: create-test-lint-wf - # Run nf-core linting - name: nf-core pipelines lint run: nf-core --log-file log.txt --hide-progress pipelines lint --dir my-prefix-testpipeline --fail-warned diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 4af5a99c5..356b16f7f 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -4,7 +4,6 @@ import logging import os import sys -from pathlib import Path import rich import rich.console @@ -36,7 +35,6 @@ pipelines_launch, pipelines_lint, pipelines_list, - pipelines_rocrate, pipelines_schema_build, pipelines_schema_docs, pipelines_schema_lint, @@ -87,7 +85,7 @@ }, { "name": "For developers", - "commands": ["create", "lint", "bump-version", "sync", "schema", "rocrate", "create-logo"], + "commands": ["create", "lint", "bump-version", "sync", "schema", "create-logo"], }, ], "nf-core modules": [ @@ -571,44 +569,6 @@ def command_pipelines_list(ctx, keywords, sort, json, show_archived): pipelines_list(ctx, keywords, sort, json, show_archived) -# nf-core pipelines rocrate -@pipelines.command("rocrate") -@click.argument( - "pipeline_dir", - type=click.Path(exists=True), - default=Path.cwd(), - required=True, - metavar="", -) -@click.option( - "-j", - "--json_path", - default=Path.cwd(), - type=str, - help="Path to save RO Crate metadata json file to", -) -@click.option("-z", "--zip_path", type=str, help="Path to save RO Crate zip file to") -@click.option( - "-pv", - "--pipeline_version", - type=str, - help="Version of pipeline to use for RO Crate", - default="", -) -@click.pass_context -def rocrate( - ctx, - pipeline_dir: str, - json_path: str, - zip_path: str, - pipeline_version: str, -): - """ - Make an Research Object Crate - """ - pipelines_rocrate(ctx, pipeline_dir, json_path, zip_path, pipeline_version) - - # nf-core pipelines sync @pipelines.command("sync") @click.pass_context @@ -1732,7 +1692,7 @@ def command_schema_validate(pipeline, params): @click.option( "--url", type=str, - default="https://oldsite.nf-co.re/pipeline_schema_builder", + default="https://nf-co.re/pipeline_schema_builder", help="Customise the builder URL (for development work)", ) def command_schema_build(directory, no_prompts, web_only, url): diff --git a/nf_core/commands_pipelines.py b/nf_core/commands_pipelines.py index 9699dc53a..1186935e5 100644 --- a/nf_core/commands_pipelines.py +++ b/nf_core/commands_pipelines.py @@ -2,7 +2,6 @@ import os import sys from pathlib import Path -from typing import Optional, Union import rich @@ -278,33 +277,6 @@ def pipelines_list(ctx, keywords, sort, json, show_archived): stdout.print(list_workflows(keywords, sort, json, show_archived)) -# nf-core pipelines rocrate -def pipelines_rocrate( - ctx, - pipeline_dir: Union[str, Path], - json_path: Optional[Union[str, Path]], - zip_path: Optional[Union[str, Path]], - pipeline_version: str, -) -> None: - from nf_core.pipelines.rocrate import ROCrate - - if json_path is None and zip_path is None: - log.error("Either `--json_path` or `--zip_path` must be specified.") - sys.exit(1) - else: - pipeline_dir = Path(pipeline_dir) - if json_path is not None: - json_path = Path(json_path) - if zip_path is not None: - zip_path = Path(zip_path) - try: - rocrate_obj = ROCrate(pipeline_dir, pipeline_version) - rocrate_obj.create_rocrate(pipeline_dir, json_path=json_path, zip_path=zip_path) - except (UserWarning, LookupError, FileNotFoundError) as e: - log.error(e) - sys.exit(1) - - # nf-core pipelines sync def pipelines_sync(ctx, directory, from_branch, pull_request, github_repository, username, template_yaml, force_pr): """ diff --git a/nf_core/components/info.py b/nf_core/components/info.py index 31769785a..f3e5bf617 100644 --- a/nf_core/components/info.py +++ b/nf_core/components/info.py @@ -211,9 +211,9 @@ def get_local_yaml(self) -> Optional[Dict]: return yaml.safe_load(fh) log.debug(f"{self.component_type[:-1].title()} '{self.component}' meta.yml not found locally") - return {} + return None - def get_remote_yaml(self) -> Optional[Dict]: + def get_remote_yaml(self) -> Optional[dict]: """Attempt to get the meta.yml file from a remote repo. Returns: diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index 0dbd87f77..05c64b6de 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -1119,10 +1119,8 @@ def dump(self, run_prettier: bool = False) -> None: """ Sort the modules.json, and write it to file """ - # Sort the modules.json - if self.modules_json is None: - self.load() if self.modules_json is not None: + # Sort the modules.json self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) if run_prettier: dump_json_with_prettier(self.modules_json_path, self.modules_json) diff --git a/nf_core/pipelines/lint/files_exist.py b/nf_core/pipelines/lint/files_exist.py index 19c249826..9dd307d8b 100644 --- a/nf_core/pipelines/lint/files_exist.py +++ b/nf_core/pipelines/lint/files_exist.py @@ -66,7 +66,6 @@ def files_exist(self) -> Dict[str, List[str]]: conf/igenomes.config .github/workflows/awstest.yml .github/workflows/awsfulltest.yml - ro-crate-metadata.json Files that *must not* be present, due to being renamed or removed in the template: @@ -172,7 +171,6 @@ def files_exist(self) -> Dict[str, List[str]]: [Path(".github", "workflows", "awstest.yml")], [Path(".github", "workflows", "awsfulltest.yml")], [Path("modules.json")], - [Path("ro-crate-metadata.json")], ] # List of strings. Fails / warns if any of the strings exist. @@ -200,12 +198,6 @@ def files_exist(self) -> Dict[str, List[str]]: ] files_warn_ifexists = [Path(".travis.yml")] - files_hint = [ - [ - ["ro-crate-metadata.json"], - ". Run `nf-core rocrate` to generate this file. Read more about RO-Crates in the [nf-core/tools docs](https://nf-co.re/tools#create-a-ro-crate-metadata-file).", - ], - ] # Remove files that should be ignored according to the linting config ignore_files = self.lint_config.get("files_exist", []) if self.lint_config is not None else [] @@ -233,11 +225,7 @@ def pf(file_path: Union[str, Path]) -> Path: if any([pf(f).is_file() for f in files]): passed.append(f"File found: {self._wrap_quotes(files)}") else: - hint = "" - for file_hint in files_hint: - if file_hint[0] == files: - hint = str(file_hint[1]) - warned.append(f"File not found: {self._wrap_quotes(files)}{hint}") + warned.append(f"File not found: {self._wrap_quotes(files)}") # Files that cause an error if they exist for file in files_fail_ifexists: diff --git a/nf_core/pipelines/rocrate.py b/nf_core/pipelines/rocrate.py deleted file mode 100644 index de00189a2..000000000 --- a/nf_core/pipelines/rocrate.py +++ /dev/null @@ -1,400 +0,0 @@ -#!/usr/bin/env python -"""Code to deal with pipeline RO (Research Object) Crates""" - -import logging -import os -import re -import sys -from datetime import datetime -from pathlib import Path -from typing import Dict, List, Optional, Set, Union - -import requests -import rocrate.rocrate -from git import GitCommandError, InvalidGitRepositoryError -from repo2rocrate.nextflow import NextflowCrateBuilder -from rich.progress import BarColumn, Progress -from rocrate.model.person import Person -from rocrate.rocrate import ROCrate as BaseROCrate - -from nf_core.pipelines.schema import PipelineSchema -from nf_core.utils import Pipeline - -log = logging.getLogger(__name__) - - -class CustomNextflowCrateBuilder(NextflowCrateBuilder): - DATA_ENTITIES = NextflowCrateBuilder.DATA_ENTITIES + [ - ("docs/usage.md", "File", "Usage documentation"), - ("docs/output.md", "File", "Output documentation"), - ("suborkflows/local", "Dataset", "Pipeline-specific suborkflows"), - ("suborkflows/nf-core", "Dataset", "nf-core suborkflows"), - (".nf-core.yml", "File", "nf-core configuration file, configuring template features and linting rules"), - (".pre-commit-config.yaml", "File", "Configuration file for pre-commit hooks"), - (".prettierignore", "File", "Ignore file for prettier"), - (".prettierrc", "File", "Configuration file for prettier"), - ] - - -def custom_make_crate( - root: Path, - workflow: Optional[Path] = None, - repo_url: Optional[str] = None, - wf_name: Optional[str] = None, - wf_version: Optional[str] = None, - lang_version: Optional[str] = None, - ci_workflow: Optional[str] = "ci.yml", - diagram: Optional[Path] = None, -) -> BaseROCrate: - builder = CustomNextflowCrateBuilder(root, repo_url=repo_url) - - return builder.build( - workflow, - wf_name=wf_name, - wf_version=wf_version, - lang_version=lang_version, - license=None, - ci_workflow=ci_workflow, - diagram=diagram, - ) - - -class ROCrate: - """ - Class to generate an RO Crate for a pipeline - - """ - - def __init__(self, pipeline_dir: Path, version="") -> None: - """ - Initialise the ROCrate object - - Args: - pipeline_dir (Path): Path to the pipeline directory - version (str): Version of the pipeline to checkout - """ - from nf_core.utils import is_pipeline_directory, setup_requests_cachedir - - is_pipeline_directory(pipeline_dir) - self.pipeline_dir = pipeline_dir - self.version: str = version - self.crate: rocrate.rocrate.ROCrate - self.pipeline_obj = Pipeline(self.pipeline_dir) - self.pipeline_obj._load() - self.pipeline_obj.schema_obj = PipelineSchema() - # Assume we're in a pipeline dir root if schema path not set - self.pipeline_obj.schema_obj.get_schema_path(self.pipeline_dir) - self.pipeline_obj.schema_obj.load_schema() - - setup_requests_cachedir() - - def create_rocrate( - self, outdir: Path, json_path: Union[None, Path] = None, zip_path: Union[None, Path] = None - ) -> None: - """ - Create an RO Crate for a pipeline - - Args: - outdir (Path): Path to the output directory - json_path (Path): Path to the metadata file - zip_path (Path): Path to the zip file - - """ - # Set input paths - try: - self.set_crate_paths(outdir) - except OSError as e: - log.error(e) - sys.exit(1) - - # Change to the pipeline directory, because the RO Crate doesn't handle relative paths well - - # Check that the checkout pipeline version is the same as the requested version - if self.version != "": - if self.version != self.pipeline_obj.nf_config.get("manifest.version"): - # using git checkout to get the requested version - log.info(f"Checking out pipeline version {self.version}") - if self.pipeline_obj.repo is None: - log.error(f"Pipeline repository not found in {self.pipeline_dir}") - sys.exit(1) - try: - self.pipeline_obj.repo.git.checkout(self.version) - self.pipeline_obj = Pipeline(self.pipeline_dir) - self.pipeline_obj._load() - except InvalidGitRepositoryError: - log.error(f"Could not find a git repository in {self.pipeline_dir}") - sys.exit(1) - except GitCommandError: - log.error(f"Could not checkout version {self.version}") - sys.exit(1) - self.version = self.pipeline_obj.nf_config.get("manifest.version", "") - self.make_workflow_rocrate() - - # Save just the JSON metadata file - if json_path is not None: - if json_path.name != "ro-crate-metadata.json": - json_path = json_path / "ro-crate-metadata.json" - - log.info(f"Saving metadata file to '{json_path}'") - self.crate.metadata.write(json_path) - - # Save the whole crate zip file - if zip_path is not None: - if zip_path.name != "ro-crate.crate.zip": - zip_path = zip_path / "ro-crate.crate.zip" - log.info(f"Saving zip file '{zip_path}") - self.crate.write_zip(zip_path) - - def make_workflow_rocrate(self) -> None: - """ - Create an RO Crate for a pipeline - """ - if self.pipeline_obj is None: - raise ValueError("Pipeline object not loaded") - - diagram: Optional[Path] = None - # find files (metro|tube)_?(map)?.png in the pipeline directory or docs/ using pathlib - pattern = re.compile(r".*?(metro|tube|subway)_(map).*?\.png", re.IGNORECASE) - for file in self.pipeline_dir.rglob("*.png"): - if pattern.match(file.name): - log.debug(f"Found diagram: {file}") - diagram = file.relative_to(self.pipeline_dir) - break - - # Create the RO Crate object - - self.crate = custom_make_crate( - self.pipeline_dir, - self.pipeline_dir / "main.nf", - self.pipeline_obj.nf_config.get("manifest.homePage", ""), - self.pipeline_obj.nf_config.get("manifest.name", ""), - self.pipeline_obj.nf_config.get("manifest.version", ""), - self.pipeline_obj.nf_config.get("manifest.nextflowVersion", ""), - diagram=diagram, - ) - - # add readme as description - readme = Path("README.md") - - try: - self.crate.description = readme.read_text() - except FileNotFoundError: - log.error(f"Could not find README.md in {self.pipeline_dir}") - # get license from LICENSE file - license_file = Path("LICENSE") - try: - license = license_file.read_text() - if license.startswith("MIT"): - self.crate.license = "MIT" - else: - # prompt for license - log.info("Could not determine license from LICENSE file") - self.crate.license = input("Please enter the license for this pipeline: ") - except FileNotFoundError: - log.error(f"Could not find LICENSE file in {self.pipeline_dir}") - - self.crate.add_jsonld( - {"@id": "https://nf-co.re/", "@type": "Organization", "name": "nf-core", "url": "https://nf-co.re/"} - ) - - # Set metadata for main entity file - self.set_main_entity("main.nf") - - def set_main_entity(self, main_entity_filename: str): - """ - Set the main.nf as the main entity of the crate and add necessary metadata - """ - if self.crate.mainEntity is None: - raise ValueError("Main entity not set") - - self.crate.mainEntity.append_to( - "dct:conformsTo", "https://bioschemas.org/profiles/ComputationalWorkflow/1.0-RELEASE/", compact=True - ) - # add dateCreated and dateModified, based on the current data - self.crate.mainEntity.append_to("dateCreated", self.crate.root_dataset.get("dateCreated", ""), compact=True) - self.crate.mainEntity.append_to( - "dateModified", str(datetime.now().strftime("%Y-%m-%dT%H:%M:%SZ")), compact=True - ) - self.crate.mainEntity.append_to("sdPublisher", {"@id": "https://nf-co.re/"}, compact=True) - if self.version.endswith("dev"): - url = "dev" - else: - url = self.version - self.crate.mainEntity.append_to( - "url", f"https://nf-co.re/{self.crate.name.replace('nf-core/','')}/{url}/", compact=True - ) - self.crate.mainEntity.append_to("version", self.version, compact=True) - if self.pipeline_obj.schema_obj is not None: - log.debug("input value") - - schema_input = self.pipeline_obj.schema_obj.schema["definitions"]["input_output_options"]["properties"][ - "input" - ] - input_value: Dict[str, Union[str, List[str], bool]] = { - "@id": "#input", - "@type": ["FormalParameter"], - "default": schema_input.get("default", ""), - "encodingFormat": schema_input.get("mimetype", ""), - "valueRequired": "input" - in self.pipeline_obj.schema_obj.schema["definitions"]["input_output_options"]["required"], - "dct:conformsTo": "https://bioschemas.org/types/FormalParameter/1.0-RELEASE", - } - self.crate.add_jsonld(input_value) - self.crate.mainEntity.append_to( - "input", - {"@id": "#input"}, - ) - - # get keywords from nf-core website - remote_workflows = requests.get("https://nf-co.re/pipelines.json").json()["remote_workflows"] - # go through all remote workflows and find the one that matches the pipeline name - topics = ["nf-core", "nextflow"] - for remote_wf in remote_workflows: - assert self.pipeline_obj.pipeline_name is not None # mypy - if remote_wf["name"] == self.pipeline_obj.pipeline_name.replace("nf-core/", ""): - topics = topics + remote_wf["topics"] - break - - log.debug(f"Adding topics: {topics}") - self.crate.mainEntity.append_to("keywords", topics) - - self.add_main_authors(self.crate.mainEntity) - - self.crate.mainEntity = self.crate.mainEntity - - self.crate.mainEntity.append_to("license", self.crate.license) - self.crate.mainEntity.append_to("name", self.crate.name) - - if "dev" in self.version: - self.crate.creativeWorkStatus = "InProgress" - else: - self.crate.creativeWorkStatus = "Stable" - if self.pipeline_obj.repo is None: - log.error(f"Pipeline repository not found in {self.pipeline_dir}") - else: - tags = self.pipeline_obj.repo.tags - if tags: - # get the tag for this version - for tag in tags: - if tag.commit.hexsha == self.pipeline_obj.repo.head.commit.hexsha: - self.crate.mainEntity.append_to( - "dateCreated", - tag.commit.committed_datetime.strftime("%Y-%m-%dT%H:%M:%SZ"), - compact=True, - ) - - def add_main_authors(self, wf_file: rocrate.model.entity.Entity) -> None: - """ - Add workflow authors to the crate - """ - # add author entity to crate - - try: - authors = self.pipeline_obj.nf_config["manifest.author"].split(",") - # remove spaces - authors = [a.strip() for a in authors] - # add manifest authors as maintainer to crate - - except KeyError: - log.error("No author field found in manifest of nextflow.config") - return - # look at git contributors for author names - try: - git_contributors: Set[str] = set() - assert self.pipeline_obj.repo is not None # mypy - commits_touching_path = list(self.pipeline_obj.repo.iter_commits(paths="main.nf")) - - for commit in commits_touching_path: - if commit.author.name is not None: - git_contributors.add(commit.author.name) - # exclude bots - contributors = {c for c in git_contributors if not c.endswith("bot") and c != "Travis CI User"} - - log.debug(f"Found {len(contributors)} git authors") - - progress_bar = Progress( - "[bold blue]{task.description}", - BarColumn(bar_width=None), - "[magenta]{task.completed} of {task.total}[reset] ยป [bold yellow]{task.fields[test_name]}", - transient=True, - disable=os.environ.get("HIDE_PROGRESS", None) is not None, - ) - with progress_bar: - bump_progress = progress_bar.add_task( - "Searching for author names on GitHub", total=len(contributors), test_name="" - ) - - for git_author in contributors: - progress_bar.update(bump_progress, advance=1, test_name=git_author) - git_author = ( - requests.get(f"https://api.github.com/users/{git_author}").json().get("name", git_author) - ) - if git_author is None: - log.debug(f"Could not find name for {git_author}") - continue - - except AttributeError: - log.debug("Could not find git contributors") - - # remove usernames (just keep names with spaces) - named_contributors = {c for c in contributors if " " in c} - - for author in named_contributors: - log.debug(f"Adding author: {author}") - assert self.pipeline_obj.repo is not None # mypy - # get email from git log - email = self.pipeline_obj.repo.git.log(f"--author={author}", "--pretty=format:%ae", "-1") - orcid = get_orcid(author) - author_entitity = self.crate.add( - Person( - self.crate, orcid if orcid is not None else "#" + email, properties={"name": author, "email": email} - ) - ) - wf_file.append_to("creator", author_entitity) - if author in authors: - wf_file.append_to("maintainer", author_entitity) - - def set_crate_paths(self, path: Path) -> None: - """Given a pipeline name, directory, or path, set wf_crate_filename""" - - if path.is_dir(): - self.pipeline_dir = path - # wf_crate_filename = path / "ro-crate-metadata.json" - elif path.is_file(): - self.pipeline_dir = path.parent - # wf_crate_filename = path - - # Check that the schema file exists - if self.pipeline_dir is None: - raise OSError(f"Could not find pipeline '{path}'") - - -def get_orcid(name: str) -> Optional[str]: - """ - Get the ORCID for a given name - - Args: - name (str): Name of the author - - Returns: - str: ORCID URI or None - """ - base_url = "https://pub.orcid.org/v3.0/search/" - headers = { - "Accept": "application/json", - } - params = {"q": f'family-name:"{name.split()[-1]}" AND given-names:"{name.split()[0]}"'} - response = requests.get(base_url, params=params, headers=headers) - - if response.status_code == 200: - json_response = response.json() - if json_response.get("num-found") == 1: - orcid_uri = json_response.get("result")[0].get("orcid-identifier", {}).get("uri") - log.info(f"Using found ORCID for {name}. Please double-check: {orcid_uri}") - return orcid_uri - else: - log.debug(f"No exact ORCID found for {name}. See {response.url}") - return None - else: - log.info(f"API request to ORCID unsuccessful. Status code: {response.status_code}") - return None diff --git a/nf_core/pipelines/schema.py b/nf_core/pipelines/schema.py index 9bec75c9c..2c1eb397f 100644 --- a/nf_core/pipelines/schema.py +++ b/nf_core/pipelines/schema.py @@ -43,7 +43,7 @@ def __init__(self): self.schema_from_scratch = False self.no_prompts = False self.web_only = False - self.web_schema_build_url = "https://oldsite.nf-co.re/pipeline_schema_builder" + self.web_schema_build_url = "https://nf-co.re/pipeline_schema_builder" self.web_schema_build_web_url = None self.web_schema_build_api_url = None self.validation_plugin = None @@ -956,7 +956,6 @@ def launch_web_builder(self): """ Send pipeline schema to web builder and wait for response """ - content = { "post_content": "json_schema", "api": "true", @@ -965,7 +964,6 @@ def launch_web_builder(self): "schema": json.dumps(self.schema), } web_response = nf_core.utils.poll_nfcore_web_api(self.web_schema_build_url, content) - try: if "api_url" not in web_response: raise AssertionError('"api_url" not in web_response') diff --git a/nf_core/utils.py b/nf_core/utils.py index 9aa0bd589..16125aed3 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -5,7 +5,6 @@ import concurrent.futures import datetime import errno -import fnmatch import hashlib import io import json @@ -20,7 +19,7 @@ import time from contextlib import contextmanager from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, List, Literal, Optional, Tuple, Union +from typing import Any, Callable, Dict, Generator, List, Literal, Optional, Tuple, Union import git import prompt_toolkit.styles @@ -37,9 +36,6 @@ import nf_core -if TYPE_CHECKING: - from nf_core.pipelines.schema import PipelineSchema - log = logging.getLogger(__name__) # ASCII nf-core logo @@ -56,29 +52,14 @@ [ ("qmark", "fg:ansiblue bold"), # token in front of the question ("question", "bold"), # question text - ( - "answer", - "fg:ansigreen nobold bg:", - ), # submitted answer text behind the question - ( - "pointer", - "fg:ansiyellow bold", - ), # pointer used in select and checkbox prompts - ( - "highlighted", - "fg:ansiblue bold", - ), # pointed-at choice in select and checkbox prompts - ( - "selected", - "fg:ansiyellow noreverse bold", - ), # style for a selected item of a checkbox + ("answer", "fg:ansigreen nobold bg:"), # submitted answer text behind the question + ("pointer", "fg:ansiyellow bold"), # pointer used in select and checkbox prompts + ("highlighted", "fg:ansiblue bold"), # pointed-at choice in select and checkbox prompts + ("selected", "fg:ansiyellow noreverse bold"), # style for a selected item of a checkbox ("separator", "fg:ansiblack"), # separator in lists ("instruction", ""), # user instructions for select, rawselect, checkbox ("text", ""), # plain text - ( - "disabled", - "fg:gray italic", - ), # disabled choices for select and checkbox prompts + ("disabled", "fg:gray italic"), # disabled choices for select and checkbox prompts ("choice-default", "fg:ansiblack"), ("choice-default-changed", "fg:ansiyellow"), ("choice-required", "fg:ansired"), @@ -98,11 +79,7 @@ def fetch_remote_version(source_url): return remote_version -def check_if_outdated( - current_version=None, - remote_version=None, - source_url="https://nf-co.re/tools_version", -): +def check_if_outdated(current_version=None, remote_version=None, source_url="https://nf-co.re/tools_version"): """ Check if the current version of nf-core is outdated """ @@ -169,12 +146,11 @@ def __init__(self, wf_path: Path) -> None: self.wf_path = Path(wf_path) self.pipeline_name: Optional[str] = None self.pipeline_prefix: Optional[str] = None - self.schema_obj: Optional[PipelineSchema] = None - self.repo: Optional[git.Repo] = None + self.schema_obj: Optional[Dict] = None try: - self.repo = git.Repo(self.wf_path) - self.git_sha = self.repo.head.object.hexsha + repo = git.Repo(self.wf_path) + self.git_sha = repo.head.object.hexsha except Exception as e: log.debug(f"Could not find git hash for pipeline: {self.wf_path}. {e}") @@ -278,7 +254,7 @@ def fetch_wf_config(wf_path: Path, cache_config: bool = True) -> dict: """ log.debug(f"Got '{wf_path}' as path") - wf_path = Path(wf_path) + config = {} cache_fn = None cache_basedir = None @@ -465,7 +441,6 @@ def poll_nfcore_web_api(api_url: str, post_data: Optional[Dict] = None) -> Dict: if post_data is None: response = requests.get(api_url, headers={"Cache-Control": "no-cache"}) else: - log.debug(f"requesting {api_url} with {post_data}") response = requests.post(url=api_url, data=post_data) except requests.exceptions.Timeout: raise AssertionError(f"URL timed out: {api_url}") @@ -551,8 +526,7 @@ def __call__(self, r): with open(gh_cli_config_fn) as fh: gh_cli_config = yaml.safe_load(fh) self.auth = requests.auth.HTTPBasicAuth( - gh_cli_config["github.com"]["user"], - gh_cli_config["github.com"]["oauth_token"], + gh_cli_config["github.com"]["user"], gh_cli_config["github.com"]["oauth_token"] ) self.auth_mode = f"gh CLI config: {gh_cli_config['github.com']['user']}" except Exception: @@ -820,18 +794,12 @@ def get_tag_date(tag_date): # Obtain version and build match = re.search(r"(?::)+([A-Za-z\d\-_.]+)", img["image_name"]) if match is not None: - all_docker[match.group(1)] = { - "date": get_tag_date(img["updated"]), - "image": img, - } + all_docker[match.group(1)] = {"date": get_tag_date(img["updated"]), "image": img} elif img["image_type"] == "Singularity": # Obtain version and build match = re.search(r"(?::)+([A-Za-z\d\-_.]+)", img["image_name"]) if match is not None: - all_singularity[match.group(1)] = { - "date": get_tag_date(img["updated"]), - "image": img, - } + all_singularity[match.group(1)] = {"date": get_tag_date(img["updated"]), "image": img} # Obtain common builds from Docker and Singularity images common_keys = list(all_docker.keys() & all_singularity.keys()) current_date = None @@ -961,19 +929,13 @@ def prompt_pipeline_release_branch( # Releases if len(wf_releases) > 0: for tag in map(lambda release: release.get("tag_name"), wf_releases): - tag_display = [ - ("fg:ansiblue", f"{tag} "), - ("class:choice-default", "[release]"), - ] + tag_display = [("fg:ansiblue", f"{tag} "), ("class:choice-default", "[release]")] choices.append(questionary.Choice(title=tag_display, value=tag)) tag_set.append(str(tag)) # Branches for branch in wf_branches.keys(): - branch_display = [ - ("fg:ansiyellow", f"{branch} "), - ("class:choice-default", "[branch]"), - ] + branch_display = [("fg:ansiyellow", f"{branch} "), ("class:choice-default", "[branch]")] choices.append(questionary.Choice(title=branch_display, value=branch)) tag_set.append(branch) @@ -1004,8 +966,7 @@ def validate(self, value): return True else: raise questionary.ValidationError( - message="Invalid remote cache index file", - cursor_position=len(value.text), + message="Invalid remote cache index file", cursor_position=len(value.text) ) else: return True @@ -1035,13 +996,7 @@ def get_repo_releases_branches(pipeline, wfs): pipeline = wf.full_name # Store releases and stop loop - wf_releases = list( - sorted( - wf.releases, - key=lambda k: k.get("published_at_timestamp", 0), - reverse=True, - ) - ) + wf_releases = list(sorted(wf.releases, key=lambda k: k.get("published_at_timestamp", 0), reverse=True)) break # Arbitrary GitHub repo @@ -1061,13 +1016,7 @@ def get_repo_releases_branches(pipeline, wfs): raise AssertionError(f"Not able to find pipeline '{pipeline}'") except AttributeError: # Success! We have a list, which doesn't work with .get() which is looking for a dict key - wf_releases = list( - sorted( - rel_r.json(), - key=lambda k: k.get("published_at_timestamp", 0), - reverse=True, - ) - ) + wf_releases = list(sorted(rel_r.json(), key=lambda k: k.get("published_at_timestamp", 0), reverse=True)) # Get release tag commit hashes if len(wf_releases) > 0: @@ -1275,7 +1224,7 @@ def get_first_available_path(directory: Union[Path, str], paths: List[str]) -> U return None -def sort_dictionary(d: Dict) -> Dict: +def sort_dictionary(d): """Sorts a nested dictionary recursively""" result = {} for k, v in sorted(d.items()): @@ -1416,21 +1365,3 @@ def set_wd(path: Path) -> Generator[None, None, None]: yield finally: os.chdir(start_wd) - - -def get_wf_files(wf_path: Path): - """Return a list of all files in a directory (ignores .gitigore files)""" - - wf_files = [] - - with open(Path(wf_path, ".gitignore")) as f: - lines = f.read().splitlines() - ignore = [line for line in lines if line and not line.startswith("#")] - - for path in Path(wf_path).rglob("*"): - if any(fnmatch.fnmatch(str(path), pattern) for pattern in ignore): - continue - if path.is_file(): - wf_files.append(str(path)) - - return wf_files diff --git a/requirements.txt b/requirements.txt index 51259938a..b7f1c39ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,8 +18,6 @@ requests requests_cache rich-click==1.8.* rich>=13.3.1 -rocrate -repo2rocrate tabulate textual==0.71.0 trogon diff --git a/tests/test_rocrate.py b/tests/test_rocrate.py deleted file mode 100644 index 6defd5d5e..000000000 --- a/tests/test_rocrate.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Test the nf-core pipelines rocrate command""" - -import shutil -import tempfile -import unittest -from pathlib import Path - -import rocrate.rocrate -from git import Repo - -import nf_core.pipelines.create -import nf_core.pipelines.create.create -import nf_core.pipelines.rocrate -import nf_core.utils - - -class TestROCrate(unittest.TestCase): - """Class for lint tests""" - - def setUp(self): - """Function that runs at start of tests for common resources - - Use nf_core.create() to make a pipeline that we can use for testing - """ - - self.tmp_dir = Path(tempfile.mkdtemp()) - self.test_pipeline_dir = Path(self.tmp_dir, "nf-core-testpipeline") - self.create_obj = nf_core.pipelines.create.create.PipelineCreate( - name="testpipeline", - description="This is a test pipeline", - author="Test McTestFace", - outdir=str(self.test_pipeline_dir), - version="1.0.0", - no_git=False, - force=True, - ) - self.create_obj.init_pipeline() - - # add fake metro map - Path(self.test_pipeline_dir, "docs", "images", "nf-core-testpipeline_metro_map.png").touch() - # commit the changes - repo = Repo(self.test_pipeline_dir) - repo.git.add(A=True) - repo.index.commit("Initial commit") - - def tearDown(self): - """Clean up temporary files and folders""" - - if self.tmp_dir.exists(): - shutil.rmtree(self.tmp_dir) - - def test_rocrate_creation(self): - """Run the nf-core rocrate command""" - - # Run the command - self.rocrate_obj = nf_core.pipelines.rocrate.ROCrate(self.test_pipeline_dir) - self.rocrate_obj.create_rocrate(self.test_pipeline_dir, metadata_path=Path(self.test_pipeline_dir)) - - # Check that the crate was created - self.assertTrue(Path(self.test_pipeline_dir, "ro-crate-metadata.json").exists()) - - # Check that the entries in the crate are correct - crate = rocrate.rocrate.ROCrate(self.test_pipeline_dir) - entities = crate.get_entities() - - # Check if the correct entities are set: - for entity in entities: - entity_json = entity.as_jsonld() - if entity_json["@id"] == "./": - self.assertEqual(entity_json.get("name"), "nf-core/testpipeline") - self.assertEqual(entity_json["mainEntity"], {"@id": "main.nf"}) - elif entity_json["@id"] == "#main.nf": - self.assertEqual(entity_json["programmingLanguage"], [{"@id": "#nextflow"}]) - self.assertEqual(entity_json["image"], [{"@id": "nf-core-testpipeline_metro_map.png"}]) - # assert there is a metro map - # elif entity_json["@id"] == "nf-core-testpipeline_metro_map.png": # FIXME waiting for https://github.com/ResearchObject/ro-crate-py/issues/174 - # self.assertEqual(entity_json["@type"], ["File", "ImageObject"]) - # assert that author is set as a person - elif "name" in entity_json and entity_json["name"] == "Test McTestFace": - self.assertEqual(entity_json["@type"], "Person") - # check that it is set as author of the main entity - if crate.mainEntity is not None: - self.assertEqual(crate.mainEntity["author"][0].id, entity_json["@id"])