Merge branch 'dev' into pipeline-test-conda-syngularity

.github/actions/create-lint-wf/action.yml

@@ -53,6 +53,12 @@ runs:
       run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
       working-directory: create-lint-wf
 
+    # Uncomment includeConfig statement
+    - name: uncomment include config
+      shell: bash
+      run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \;
+      working-directory: create-lint-wf
+
     # Replace zenodo.XXXXXX to pass readme linting
     - name: replace zenodo.XXXXXX
       shell: bash

.github/workflows/create-lint-wf.yml

@@ -38,7 +38,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.10.0"
+          - "24.04.2"
           - "latest-everything"
     steps:
       - name: go to subdirectory and change nextflow workdir
@@ -78,6 +78,11 @@ jobs:
         run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
         working-directory: create-lint-wf
 
+      # Uncomment includeConfig statement
+      - name: uncomment include config
+        run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \;
+        working-directory: create-lint-wf
+
       # Run the other nf-core commands
       - name: nf-core pipelines list
         run: nf-core --log-file log.txt pipelines list

.github/workflows/create-test-lint-wf-template.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Create Matrix
         id: create_matrix
         run: |
-          echo "matrix=$(yq 'keys | filter(. != "github") | filter(. != "is_nfcore") | filter(. != "test_config") | tojson(0)' nf_core/pipelines/create/templatefeatures.yml)" >> $GITHUB_OUTPUT
+          echo "matrix=$(yq 'keys | filter(. != "github") | filter(. != "is_nfcore") | filter(. != "test_config") | tojson(0)' nf_core/pipelines/create/template_features.yml)" >> $GITHUB_OUTPUT
 
   RunTestWorkflow:
     runs-on: ${{ matrix.runner }}
@@ -137,6 +137,11 @@ jobs:
         run: find my-prefix-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
         working-directory: create-test-lint-wf
 
+      # Uncomment includeConfig statement
+      - name: uncomment include config
+        run: find my-prefix-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \;
+        working-directory: create-test-lint-wf
+
       # Replace zenodo.XXXXXX to pass readme linting
       - name: replace zenodo.XXXXXX
         run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;

.github/workflows/create-test-wf.yml

@@ -39,7 +39,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.10.0"
+          - "24.04.2"
           - "latest-everything"
     steps:
       - name: go to working directory
@@ -75,7 +75,7 @@ jobs:
           pwd
           # echo content of current directory
           ls -la
-          nextflow run nf-core-testpipeline -profile test,self_hosted_runner --outdir ./results
+          nextflow run nf-core-testpipeline -profile self_hosted_runner,test --outdir ./results
 
       - name: Upload log file artifact
         if: ${{ always() }}

.pre-commit-config.yaml

@@ -13,13 +13,13 @@ repos:
           - prettier@3.3.3
 
   - repo: https://github.com/editorconfig-checker/editorconfig-checker.python
-    rev: "2.7.3"
+    rev: "3.0.3"
     hooks:
       - id: editorconfig-checker
         alias: ec
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.11.1"
+    rev: "v1.11.2"
     hooks:
       - id: mypy
         additional_dependencies:

CHANGELOG.md

@@ -37,6 +37,12 @@
 - add option to exclude test configs from pipeline template ([#3133](https://github.com/nf-core/tools/pull/3133))
 - add option to exclude tower.yml from pipeline template ([#3134](https://github.com/nf-core/tools/pull/3134))
 - test pipeline with conda and singularity on PRs to master ([#3149](https://github.com/nf-core/tools/pull/3149))
+- run nf-core lint `--release` on PRs to master ([#3148](https://github.com/nf-core/tools/pull/3148))
+- Add tests to ensure all files are part of a template customisation group and all groups are tested ([#3099](https://github.com/nf-core/tools/pull/3099))
+- Remove if/else block to include `igenomes.config` ([#3168](https://github.com/nf-core/tools/pull/3168))
+- Replaces the old custom `check_max()` function with the Nextflow native `resourceLimits` directive ([#3037](https://github.com/nf-core/tools/pull/3037))
+- Fixed release announcement hashtags for Mastodon ([#3099](https://github.com/nf-core/tools/pull/3176))
+- Remove try/catch blocks from `nextflow.config` ([#3167](https://github.com/nf-core/tools/pull/3167))
 
 ### Linting
 
@@ -50,6 +56,7 @@
 - Linting will now check the use of the right validation plugin include statements in the workflow scripts ([#3116])(https://github.com/nf-core/tools/pull/3116)
 - Full linting for correct use of nf-schema and nf-validation ([#3116](https://github.com/nf-core/tools/pull/3116))
 - Handle cases where the directory path contains the name of the component ([#3147](https://github.com/nf-core/tools/pull/3147))
+- Don't test conda `environment.yml` `name` attribute (which should no longer be there) ([#3161](https://github.com/nf-core/tools/pull/3161))
 
 ### Pipeline create command
 
@@ -63,6 +70,7 @@
 - The `modules_nfcore` tag in the `main.nf.test` file of modules/subworkflows now displays the organization name in custom modules repositories ([#3005](https://github.com/nf-core/tools/pull/3005))
 - Add `--migrate_pytest` option to `nf-core <modules|subworkflows> test` command ([#3085](https://github.com/nf-core/tools/pull/3085))
 - Components: allow spaces at the beginning of include statements ([#3115](https://github.com/nf-core/tools/pull/3115))
+- Add option `--fix` to update the `meta.yml` file of subworkflows ([#3077](https://github.com/nf-core/tools/pull/3077))
 
 ### General
 
@@ -83,6 +91,7 @@
 - Add bot action to update textual snapshots and write bot documentation ([#3102](https://github.com/nf-core/tools/pull/3102))
 - Update gitpod setup ([#3136](https://github.com/nf-core/tools/pull/3136))
 - fix syncing a pipeline from current directory ([#3143](https://github.com/nf-core/tools/pull/3143))
+- Patch gitpod conda setup to not use defaults channel ([#3159](https://github.com/nf-core/tools/pull/3159))
 
 ## Version updates
 

MANIFEST.in

@@ -10,4 +10,4 @@ include nf_core/assets/logo/nf-core-repo-logo-base-darkbg.png
 include nf_core/assets/logo/placeholder_logo.svg
 include nf_core/assets/logo/MavenPro-Bold.ttf
 include nf_core/pipelines/create/create.tcss
-include nf_core/pipelines/create/templatefeatures.yml
+include nf_core/pipelines/create/template_features.yml

docs/api/_src/pipeline_lint_tests/included_configs.md

@@ -0,0 +1,5 @@
+# included_configs
+
+    ```{eval-rst}
+    .. automethod:: nf_core.pipelines.lint.PipelineLint.included_configs
+    ```

docs/api/_src/pipeline_lint_tests/index.md

@@ -7,6 +7,7 @@
     - [base_config](./base_config/)
     - [files_exist](./files_exist/)
     - [files_unchanged](./files_unchanged/)
+    - [included_configs](./included_configs/)
     - [merge_markers](./merge_markers/)
     - [modules_config](./modules_config/)
     - [modules_json](./modules_json/)
@@ -16,6 +17,7 @@
     - [nfcore_yml](./nfcore_yml/)
     - [pipeline_name_conventions](./pipeline_name_conventions/)
     - [pipeline_todos](./pipeline_todos/)
+    - [plugin_includes](./plugin_includes/)
     - [readme](./readme/)
     - [schema_description](./schema_description/)
     - [schema_lint](./schema_lint/)

nf_core/__main__.py

@@ -1231,11 +1231,14 @@ def command_modules_test(ctx, tool, directory, no_prompts, update, once, profile
     is_flag=True,
     help="Fix the module version if a newer version is available",
 )
-def command_modules_lint(ctx, tool, directory, registry, key, all, fail_warned, local, passed, sort_by, fix_version):
+@click.option("--fix", is_flag=True, help="Fix all linting tests if possible.")
+def command_modules_lint(
+    ctx, tool, directory, registry, key, all, fail_warned, local, passed, sort_by, fix_version, fix
+):
     """
     Lint one or more modules in a directory.
     """
-    modules_lint(ctx, tool, directory, registry, key, all, fail_warned, local, passed, sort_by, fix_version)
+    modules_lint(ctx, tool, directory, registry, key, all, fail_warned, local, passed, sort_by, fix_version, fix)
 
 
 # nf-core modules info
@@ -1476,11 +1479,14 @@ def command_subworkflows_list_local(ctx, keywords, json, directory):  # pylint:
     help="Sort lint output by subworkflow or test name.",
     show_default=True,
 )
-def command_subworkflows_lint(ctx, subworkflow, directory, registry, key, all, fail_warned, local, passed, sort_by):
+@click.option("--fix", is_flag=True, help="Fix all linting tests if possible.")
+def command_subworkflows_lint(
+    ctx, subworkflow, directory, registry, key, all, fail_warned, local, passed, sort_by, fix
+):
     """
     Lint one or more subworkflows in a directory.
     """
-    subworkflows_lint(ctx, subworkflow, directory, registry, key, all, fail_warned, local, passed, sort_by)
+    subworkflows_lint(ctx, subworkflow, directory, registry, key, all, fail_warned, local, passed, sort_by, fix)
 
 
 # nf-core subworkflows info

nf_core/commands_modules.py

@@ -261,7 +261,7 @@ def modules_test(ctx, tool, directory, no_prompts, update, once, profile, migrat
         sys.exit(1)
 
 
-def modules_lint(ctx, tool, directory, registry, key, all, fail_warned, local, passed, sort_by, fix_version):
+def modules_lint(ctx, tool, directory, registry, key, all, fail_warned, local, passed, sort_by, fix_version, fix):
     """
     Lint one or more modules in a directory.
 
@@ -278,6 +278,7 @@ def modules_lint(ctx, tool, directory, registry, key, all, fail_warned, local, p
         module_lint = ModuleLint(
             directory,
             fail_warned=fail_warned,
+            fix=fix,
             registry=ctx.params["registry"],
             remote_url=ctx.obj["modules_repo_url"],
             branch=ctx.obj["modules_repo_branch"],

nf_core/commands_subworkflows.py

@@ -104,7 +104,7 @@ def subworkflows_list_local(ctx, keywords, json, directory):  # pylint: disable=
         sys.exit(1)
 
 
-def subworkflows_lint(ctx, subworkflow, directory, registry, key, all, fail_warned, local, passed, sort_by):
+def subworkflows_lint(ctx, subworkflow, directory, registry, key, all, fail_warned, local, passed, sort_by, fix):
     """
     Lint one or more subworkflows in a directory.
 
@@ -121,6 +121,7 @@ def subworkflows_lint(ctx, subworkflow, directory, registry, key, all, fail_warn
         subworkflow_lint = SubworkflowLint(
             directory,
             fail_warned=fail_warned,
+            fix=fix,
             registry=ctx.params["registry"],
             remote_url=ctx.obj["modules_repo_url"],
             branch=ctx.obj["modules_repo_branch"],

nf_core/components/components_utils.py

@@ -4,6 +4,7 @@
 from typing import TYPE_CHECKING, List, Optional, Tuple, Union
 
 import questionary
+import requests
 import rich.prompt
 
 if TYPE_CHECKING:
@@ -162,3 +163,29 @@ def get_components_to_install(subworkflow_dir: Union[str, Path]) -> Tuple[List[s
                 elif link.startswith("../"):
                     subworkflows.append(name.lower())
     return modules, subworkflows
+
+
+def get_biotools_id(tool_name) -> str:
+    """
+    Try to find a bio.tools ID for 'tool'
+    """
+    url = f"https://bio.tools/api/t/?q={tool_name}&format=json"
+    try:
+        # Send a GET request to the API
+        response = requests.get(url)
+        response.raise_for_status()  # Raise an error for bad status codes
+        # Parse the JSON response
+        data = response.json()
+
+        # Iterate through the tools in the response to find the tool name
+        for tool in data["list"]:
+            if tool["name"].lower() == tool_name:
+                return tool["biotoolsCURIE"]
+
+        # If the tool name was not found in the response
+        log.warning(f"Could not find a bio.tools ID for '{tool_name}'")
+        return ""
+
+    except requests.exceptions.RequestException as e:
+        log.warning(f"Could not find a bio.tools ID for '{tool_name}': {e}")
+        return ""

nf_core/components/create.py

@@ -21,6 +21,7 @@
 import nf_core
 import nf_core.utils
 from nf_core.components.components_command import ComponentCommand
+from nf_core.components.components_utils import get_biotools_id
 from nf_core.pipelines.lint_utils import run_prettier_on_file
 
 log = logging.getLogger(__name__)
@@ -61,6 +62,7 @@ def __init__(
         self.file_paths: Dict[str, Path] = {}
         self.not_empty_template = not empty_template
         self.migrate_pytest = migrate_pytest
+        self.tool_identifier = ""
 
     def create(self) -> bool:
         """
@@ -149,6 +151,8 @@ def create(self) -> bool:
             if self.component_type == "modules":
                 # Try to find a bioconda package for 'component'
                 self._get_bioconda_tool()
+                # Try to find a biotools entry for 'component'
+                self.tool_identifier = get_biotools_id(self.component)
 
             # Prompt for GitHub username
             self._get_username()

nf_core/components/info.py

@@ -229,6 +229,25 @@ def get_remote_yaml(self) -> Optional[dict]:
         self.remote_location = self.modules_repo.remote_url
         return yaml.safe_load(file_contents)
 
+    def generate_params_table(self, type) -> Table:
+        "Generate a rich table for inputs and outputs"
+        table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0)
+        table.add_column(f":inbox_tray: {type}")
+        table.add_column("Description")
+        if self.component_type == "modules":
+            table.add_column("Pattern", justify="right", style="green")
+        elif self.component_type == "subworkflows":
+            table.add_column("Structure", justify="right", style="green")
+        return table
+
+    def get_channel_structure(self, structure: dict) -> str:
+        "Get the structure of a channel"
+        structure_str = ""
+        for key, info in structure.items():
+            pattern = f" - {info['pattern']}" if info.get("pattern") else ""
+            structure_str += f"{key} ({info['type']}{pattern})"
+        return structure_str
+
     def generate_component_info_help(self):
         """Take the parsed meta.yml and generate rich help.
 
@@ -277,33 +296,48 @@ def generate_component_info_help(self):
 
         # Inputs
         if self.meta.get("input"):
-            inputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0)
-            inputs_table.add_column(":inbox_tray: Inputs")
-            inputs_table.add_column("Description")
-            inputs_table.add_column("Pattern", justify="right", style="green")
-            for input in self.meta["input"]:
-                for key, info in input.items():
-                    inputs_table.add_row(
-                        f"[orange1 on black] {key} [/][dim i] ({info['type']})",
-                        Markdown(info["description"] if info["description"] else ""),
-                        info.get("pattern", ""),
-                    )
+            inputs_table = self.generate_params_table("Inputs")
+            for i, input in enumerate(self.meta["input"]):
+                inputs_table.add_row(f"[italic]input[{i}][/]", "", "")
+                if self.component_type == "modules":
+                    for element in input:
+                        for key, info in element.items():
+                            inputs_table.add_row(
+                                f"[orange1 on black] {key} [/][dim i] ({info['type']})",
+                                Markdown(info["description"] if info["description"] else ""),
+                                info.get("pattern", ""),
+                            )
+                elif self.component_type == "subworkflows":
+                    for key, info in input.items():
+                        inputs_table.add_row(
+                            f"[orange1 on black] {key} [/][dim i]",
+                            Markdown(info["description"] if info["description"] else ""),
+                            self.get_channel_structure(info["structure"]) if info.get("structure") else "",
+                        )
 
             renderables.append(inputs_table)
 
         # Outputs
         if self.meta.get("output"):
-            outputs_table = Table(expand=True, show_lines=True, box=box.MINIMAL_HEAVY_HEAD, padding=0)
-            outputs_table.add_column(":outbox_tray: Outputs")
-            outputs_table.add_column("Description")
-            outputs_table.add_column("Pattern", justify="right", style="green")
+            outputs_table = self.generate_params_table("Outputs")
             for output in self.meta["output"]:
-                for key, info in output.items():
-                    outputs_table.add_row(
-                        f"[orange1 on black] {key} [/][dim i] ({info['type']})",
-                        Markdown(info["description"] if info["description"] else ""),
-                        info.get("pattern", ""),
-                    )
+                if self.component_type == "modules":
+                    for ch_name, elements in output.items():
+                        outputs_table.add_row(f"{ch_name}", "", "")
+                        for element in elements:
+                            for key, info in element.items():
+                                outputs_table.add_row(
+                                    f"[orange1 on black] {key} [/][dim i] ({info['type']})",
+                                    Markdown(info["description"] if info["description"] else ""),
+                                    info.get("pattern", ""),
+                                )
+                elif self.component_type == "subworkflows":
+                    for key, info in output.items():
+                        outputs_table.add_row(
+                            f"[orange1 on black] {key} [/][dim i]",
+                            Markdown(info["description"] if info["description"] else ""),
+                            self.get_channel_structure(info["structure"]) if info.get("structure") else "",
+                        )
 
             renderables.append(outputs_table)
 

nf_core/components/lint/__init__.py

@@ -57,6 +57,7 @@ def __init__(
         component_type: str,
         directory: Union[str, Path],
         fail_warned: bool = False,
+        fix: bool = False,
         remote_url: Optional[str] = None,
         branch: Optional[str] = None,
         no_pull: bool = False,
@@ -73,6 +74,7 @@ def __init__(
         )
 
         self.fail_warned = fail_warned
+        self.fix = fix
         self.passed: List[LintResult] = []
         self.warned: List[LintResult] = []
         self.failed: List[LintResult] = []