Merge pull request #3197 from nf-core/dev

Dev -> Master for v3.0.0

.editorconfig

@@ -20,3 +20,11 @@ indent_style = unset
 
 [**/Makefile]
 indent_style = unset
+
+[tests/pipelines/__snapshots__/*]
+charset = unset
+end_of_line = unset
+insert_final_newline = unset
+trim_trailing_whitespace = unset
+indent_style = unset
+indent_size = unset

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -24,7 +24,7 @@ body:
       description: Steps to reproduce the behaviour. Please paste the command and output from your terminal.
       render: console
       placeholder: |
-        $ nf-core lint ...
+        $ nf-core pipelines lint ...
 
         Some output where something broke
 

.github/RELEASE_CHECKLIST.md

@@ -9,15 +9,15 @@
 7. Create a PR from `dev` to `master`
 8. Make sure all CI tests are passing again (additional tests are run on PRs to `master`)
 9. Request review (2 approvals required)
-10. Run `rich-codex` to regenerate docs screengrabs (actions `workflow_dispatch` button)
-11. Merge the PR into `master`
-12. Wait for CI tests on the commit to passed
-13. (Optional but a good idea) Run a manual sync on `nf-core/testpipeline` and check that CI is passing on the resulting PR.
-14. Create a new release copying the `CHANGELOG` for that release into the description section.
+10. Merge the PR into `master`
+11. Wait for CI tests on the commit to passed
+12. (Optional but a good idea) Run a manual sync on `nf-core/testpipeline` and check that CI is passing on the resulting PR.
+13. Create a new release copying the `CHANGELOG` for that release into the description section.
 
 ## After release
 
 1. Check the automated template synchronisation has been triggered properly. This should automatically open PRs directly to individual pipeline repos with the appropriate changes to update the pipeline template.
 2. Check that the automatic `PyPi` deployment has worked: [pypi.org/project/nf-core](https://pypi.org/project/nf-core/)
 3. Check `BioConda` has an automated PR to bump the version, and merge. eg. [bioconda/bioconda-recipes #20065](https://github.com/bioconda/bioconda-recipes/pull/20065)
 4. Create a tools PR to `dev` to bump back to the next development version in `CHANGELOG.md` and `setup.py` and change the gitpod container to `nfcore/gitpod:dev`.
+5. Run `rich-codex` on the [tools/website repo](https://github.com/nf-core/website/actions/workflows/rich-codex.yml) to regenerate docs screengrabs (actions `workflow_dispatch` button)

.github/actions/create-lint-wf/action.yml

@@ -27,12 +27,12 @@ runs:
       run: |
         mkdir -p create-lint-wf && cd create-lint-wf
         export NXF_WORK=$(pwd)
-        nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain
+        nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface"
 
     # Try syncing it before we change anything
-    - name: nf-core sync
+    - name: nf-core pipelines sync
       shell: bash
-      run: nf-core --log-file log.txt sync --dir nf-core-testpipeline/
+      run: nf-core --log-file log.txt pipelines sync --dir nf-core-testpipeline/
       working-directory: create-lint-wf
 
     # Run code style linting
@@ -53,26 +53,32 @@ runs:
       run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
       working-directory: create-lint-wf
 
+    # Uncomment includeConfig statement
+    - name: uncomment include config
+      shell: bash
+      run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \;
+      working-directory: create-lint-wf
+
     # Replace zenodo.XXXXXX to pass readme linting
     - name: replace zenodo.XXXXXX
       shell: bash
       run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
       working-directory: create-lint-wf
 
-    # Run nf-core linting
-    - name: nf-core lint
+    # Run nf-core pipelines linting
+    - name: nf-core pipelines lint
       shell: bash
-      run: nf-core --verbose --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned
+      run: nf-core --verbose --log-file log.txt --hide-progress pipelines lint --dir nf-core-testpipeline --fail-ignored --fail-warned
       working-directory: create-lint-wf
 
-    - name: nf-core bump-version to release
+    - name: nf-core pipelines bump-version to release
       shell: bash
-      run: nf-core --log-file log.txt bump-version --dir nf-core-testpipeline/ 1.1
+      run: nf-core --log-file log.txt pipelines bump-version --dir nf-core-testpipeline/ 1.1
       working-directory: create-lint-wf
 
-    - name: nf-core lint in release mode
+    - name: nf-core pipelines lint in release mode
       shell: bash
-      run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release
+      run: nf-core --log-file log.txt --hide-progress pipelines lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release
       working-directory: create-lint-wf
 
     - name: Upload log file artifact

.github/workflows/changelog.py

@@ -1,3 +1,4 @@
+#! /usr/bin/env python3
 """
 Taken from https://github.com/MultiQC/MultiQC/blob/main/.github/workflows/changelog.py and updated for nf-core
 
@@ -18,7 +19,7 @@
 import re
 import sys
 from pathlib import Path
-from typing import List
+from typing import List, Tuple
 
 REPO_URL = "https://github.com/nf-core/tools"
 
@@ -32,7 +33,7 @@
 assert pr_number, pr_number
 
 # Trim the PR number added when GitHub squashes commits, e.g. "Template: Updated (#2026)"
-pr_title = pr_title.removesuffix(f" (#{pr_number})")
+pr_title = pr_title.removesuffix(f" (#{pr_number})")  # type: ignore
 
 changelog_path = workspace_path / "CHANGELOG.md"
 
@@ -50,7 +51,7 @@
     sys.exit(0)
 
 
-def _determine_change_type(pr_title) -> tuple[str, str]:
+def _determine_change_type(pr_title) -> Tuple[str, str]:
     """
     Determine the type of the PR: Template, Download, Linting, Modules, Subworkflows, or General
     Returns a tuple of the section name and the module info.
@@ -85,7 +86,7 @@ def _determine_change_type(pr_title) -> tuple[str, str]:
 pr_link = f"([#{pr_number}]({REPO_URL}/pull/{pr_number}))"
 
 # Handle manual changelog entries through comments.
-if comment := comment.removeprefix("@nf-core-bot changelog").strip():
+if comment := comment.removeprefix("@nf-core-bot changelog").strip():  # type: ignore
     print(f"Adding manual changelog entry: {comment}")
     pr_title = comment
 new_lines = [

.github/workflows/create-lint-wf.yml

@@ -38,7 +38,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "24.04.2"
           - "latest-everything"
     steps:
       - name: go to subdirectory and change nextflow workdir
@@ -78,13 +78,18 @@ jobs:
         run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
         working-directory: create-lint-wf
 
+      # Uncomment includeConfig statement
+      - name: uncomment include config
+        run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \;
+        working-directory: create-lint-wf
+
       # Run the other nf-core commands
-      - name: nf-core list
-        run: nf-core --log-file log.txt list
+      - name: nf-core pipelines list
+        run: nf-core --log-file log.txt pipelines list
         working-directory: create-lint-wf
 
-      - name: nf-core schema
-        run: nf-core --log-file log.txt schema build --dir nf-core-testpipeline/ --no-prompts
+      - name: nf-core pipelines schema
+        run: nf-core --log-file log.txt pipelines schema build --dir nf-core-testpipeline/ --no-prompts
         working-directory: create-lint-wf
 
       - name: Cleanup work directory

.github/workflows/create-test-lint-wf-template.yml

@@ -31,27 +31,43 @@ env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
+  prepare-matrix:
+    name: Retrieve all template features
+    runs-on: ubuntu-latest
+    outputs:
+      all_features: ${{ steps.create_matrix.outputs.matrix }}
+    steps:
+      - name: 🏗 Set up yq
+        uses: frenck/action-setup-yq@v1
+      - name: checkout
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+      - name: Create Matrix
+        id: create_matrix
+        run: |
+          echo "matrix=$(yq 'keys | filter(. != "github") | filter(. != "is_nfcore") | filter(. != "test_config") | tojson(0)' nf_core/pipelines/create/template_features.yml)" >> $GITHUB_OUTPUT
+
   RunTestWorkflow:
     runs-on: ${{ matrix.runner }}
+    needs: prepare-matrix
     env:
       NXF_ANSI_LOG: false
     strategy:
       matrix:
-        TEMPLATE:
-          - "template_skip_github_badges.yml"
-          - "template_skip_igenomes.yml"
-          - "template_skip_ci.yml"
+        TEMPLATE: ${{ fromJson(needs.prepare-matrix.outputs.all_features) }}
         runner:
           # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default
           - ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }}
         profile: ["self_hosted_runner"]
         include:
-          - TEMPLATE: "template_skip_all.yml"
+          - TEMPLATE: all
             runner: ubuntu-latest
             profile: "docker"
-          - TEMPLATE: "template_skip_nf_core_configs.yml"
+          - TEMPLATE: nf_core_configs
             runner: ubuntu-latest
             profile: "docker"
+        exclude:
+          - TEMPLATE: nf_core_configs
+            profile: "self_hosted_runner"
       fail-fast: false
 
     steps:
@@ -80,33 +96,22 @@ jobs:
           version: latest-everything
 
       # Create template files
-      - name: Create template skip all (except github)
+      - name: Create template skip ${{ matrix.TEMPLATE }}
         run: |
           mkdir create-test-lint-wf
           export NXF_WORK=$(pwd)
-          printf "prefix: my-prefix\nskip: ['ci', 'github_badges', 'igenomes', 'nf_core_configs']" > create-test-lint-wf/template_skip_all.yml
-
-      - name: Create template skip github_badges
-        run: |
-          printf "prefix: my-prefix\nskip: github_badges" > create-test-lint-wf/template_skip_github_badges.yml
-
-      - name: Create template skip igenomes
-        run: |
-          printf "prefix: my-prefix\nskip: igenomes" > create-test-lint-wf/template_skip_igenomes.yml
-
-      - name: Create template skip ci
-        run: |
-          printf "prefix: my-prefix\nskip: ci" > create-test-lint-wf/template_skip_ci.yml
-
-      - name: Create template skip nf_core_configs
-        run: |
-          printf "prefix: my-prefix\nskip: nf_core_configs" > create-test-lint-wf/template_skip_nf_core_configs.yml
+          if [ ${{ matrix.TEMPLATE }} == "all" ]
+          then
+            printf "org: my-prefix\nskip_features: ${{ needs.prepare-matrix.outputs.all_features }}" > create-test-lint-wf/template_skip_all.yml
+          else
+            printf "org: my-prefix\nskip_features: [${{ matrix.TEMPLATE }}]" > create-test-lint-wf/template_skip_${{ matrix.TEMPLATE }}.yml
+          fi
 
       # Create a pipeline from the template
       - name: create a pipeline from the template ${{ matrix.TEMPLATE }}
         run: |
           cd create-test-lint-wf
-          nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml ${{ matrix.TEMPLATE }}
+          nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml template_skip_${{ matrix.TEMPLATE }}.yml
 
       - name: run the pipeline
         run: |
@@ -119,8 +124,8 @@ jobs:
           rm -rf create-test-lint-wf/results
 
       # Try syncing it before we change anything
-      - name: nf-core sync
-        run: nf-core --log-file log.txt sync --dir create-test-lint-wf/my-prefix-testpipeline/
+      - name: nf-core pipelines sync
+        run: nf-core --log-file log.txt pipelines sync --dir create-test-lint-wf/my-prefix-testpipeline/
 
       # Run code style linting
       - name: Run pre-commit
@@ -132,24 +137,29 @@ jobs:
         run: find my-prefix-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
         working-directory: create-test-lint-wf
 
+      # Uncomment includeConfig statement
+      - name: uncomment include config
+        run: find my-prefix-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \;
+        working-directory: create-test-lint-wf
+
       # Replace zenodo.XXXXXX to pass readme linting
       - name: replace zenodo.XXXXXX
         run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
         working-directory: create-test-lint-wf
 
       # Run nf-core linting
-      - name: nf-core lint
-        run: nf-core --log-file log.txt --hide-progress lint --dir my-prefix-testpipeline --fail-warned
+      - name: nf-core pipelines lint
+        run: nf-core --log-file log.txt --hide-progress pipelines lint --dir my-prefix-testpipeline --fail-warned
         working-directory: create-test-lint-wf
 
       # Run bump-version
-      - name: nf-core bump-version
-        run: nf-core --log-file log.txt bump-version --dir my-prefix-testpipeline/ 1.1
+      - name: nf-core pipelines bump-version
+        run: nf-core --log-file log.txt pipelines bump-version --dir my-prefix-testpipeline/ 1.1
         working-directory: create-test-lint-wf
 
       # Run nf-core linting in release mode
-      - name: nf-core lint in release mode
-        run: nf-core --log-file log.txt --hide-progress lint --dir my-prefix-testpipeline --fail-warned --release
+      - name: nf-core pipelines lint in release mode
+        run: nf-core --log-file log.txt --hide-progress pipelines lint --dir my-prefix-testpipeline --fail-warned --release
         working-directory: create-test-lint-wf
 
       - name: Tar files

.github/workflows/create-test-wf.yml

@@ -39,7 +39,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "24.04.2"
           - "latest-everything"
     steps:
       - name: go to working directory
@@ -70,8 +70,8 @@ jobs:
         run: |
           mkdir create-test-wf && cd create-test-wf
           export NXF_WORK=$(pwd)
-          nf-core --log-file log.txt create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --plain
-          nextflow run nf-core-testpipeline -profile test,self_hosted_runner --outdir ./results
+          nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface"
+          nextflow run nf-core-testpipeline -profile self_hosted_runner,test --outdir ./results
 
       - name: Upload log file artifact
         if: ${{ always() }}

.github/workflows/pytest.yml

@@ -67,7 +67,7 @@ jobs:
       - name: List tests
         id: list_tests
         run: |
-          echo "tests=$(find tests/test_* | tac | sed 's/tests\///g' | jq -R -s -c '{test: (split("\n")[:-1])}')" >> $GITHUB_OUTPUT
+          echo "tests=$(find tests -type f -name "test_*.py" | tac | sed 's/tests\///g'  | jq -R -s -c '{test: (split("\n")[:-1])}')" >> $GITHUB_OUTPUT
     outputs:
       tests: ${{ steps.list_tests.outputs.tests }}
 
@@ -142,10 +142,22 @@ jobs:
             exit 1
           fi
 
+      - name: Store snapshot report
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4
+        if: always()
+        with:
+          name: Snapshot Report ${{ matrix.test }}
+          path: ./snapshot_report.html
+
+      - name: remove slashes from test name
+        run: |
+          test=$(echo ${{ matrix.test }} | sed 's/\//__/g')
+          echo "test=${test}" >> $GITHUB_ENV
+
       - name: Upload coverage
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4
         with:
-          name: coverage_${{ matrix.test }}
+          name: coverage_${{ env.test }}
           path: .coverage
 
   coverage: