Merge branch 'dev' into release-checklist

.editorconfig

@@ -21,7 +21,7 @@ indent_style = unset
 [**/Makefile]
 indent_style = unset
 
-[tests/__snapshots__/*]
+[tests/pipelines/__snapshots__/*]
 charset = unset
 end_of_line = unset
 insert_final_newline = unset

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -24,7 +24,7 @@ body:
       description: Steps to reproduce the behaviour. Please paste the command and output from your terminal.
       render: console
       placeholder: |
-        $ nf-core lint ...
+        $ nf-core pipelines lint ...
 
         Some output where something broke
 

.github/actions/create-lint-wf/action.yml

@@ -30,9 +30,9 @@ runs:
         nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface"
 
     # Try syncing it before we change anything
-    - name: nf-core sync
+    - name: nf-core pipelines sync
       shell: bash
-      run: nf-core --log-file log.txt sync --dir nf-core-testpipeline/
+      run: nf-core --log-file log.txt pipelines sync --dir nf-core-testpipeline/
       working-directory: create-lint-wf
 
     # Run code style linting
@@ -53,26 +53,32 @@ runs:
       run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
       working-directory: create-lint-wf
 
+    # Uncomment includeConfig statement
+    - name: uncomment include config
+      shell: bash
+      run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \;
+      working-directory: create-lint-wf
+
     # Replace zenodo.XXXXXX to pass readme linting
     - name: replace zenodo.XXXXXX
       shell: bash
       run: find nf-core-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
       working-directory: create-lint-wf
 
-    # Run nf-core linting
-    - name: nf-core lint
+    # Run nf-core pipelines linting
+    - name: nf-core pipelines lint
       shell: bash
-      run: nf-core --verbose --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned
+      run: nf-core --verbose --log-file log.txt --hide-progress pipelines lint --dir nf-core-testpipeline --fail-ignored --fail-warned
       working-directory: create-lint-wf
 
-    - name: nf-core bump-version to release
+    - name: nf-core pipelines bump-version to release
       shell: bash
-      run: nf-core --log-file log.txt bump-version --dir nf-core-testpipeline/ 1.1
+      run: nf-core --log-file log.txt pipelines bump-version --dir nf-core-testpipeline/ 1.1
       working-directory: create-lint-wf
 
-    - name: nf-core lint in release mode
+    - name: nf-core pipelines lint in release mode
       shell: bash
-      run: nf-core --log-file log.txt --hide-progress lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release
+      run: nf-core --log-file log.txt --hide-progress pipelines lint --dir nf-core-testpipeline --fail-ignored --fail-warned --release
       working-directory: create-lint-wf
 
     - name: Upload log file artifact

.github/workflows/changelog.py

@@ -1,3 +1,4 @@
+#! /usr/bin/env python3
 """
 Taken from https://github.com/MultiQC/MultiQC/blob/main/.github/workflows/changelog.py and updated for nf-core
 
@@ -18,7 +19,7 @@
 import re
 import sys
 from pathlib import Path
-from typing import List
+from typing import List, Tuple
 
 REPO_URL = "https://github.com/nf-core/tools"
 
@@ -32,7 +33,7 @@
 assert pr_number, pr_number
 
 # Trim the PR number added when GitHub squashes commits, e.g. "Template: Updated (#2026)"
-pr_title = pr_title.removesuffix(f" (#{pr_number})")
+pr_title = pr_title.removesuffix(f" (#{pr_number})")  # type: ignore
 
 changelog_path = workspace_path / "CHANGELOG.md"
 
@@ -50,7 +51,7 @@
     sys.exit(0)
 
 
-def _determine_change_type(pr_title) -> tuple[str, str]:
+def _determine_change_type(pr_title) -> Tuple[str, str]:
     """
     Determine the type of the PR: Template, Download, Linting, Modules, Subworkflows, or General
     Returns a tuple of the section name and the module info.
@@ -85,7 +86,7 @@ def _determine_change_type(pr_title) -> tuple[str, str]:
 pr_link = f"([#{pr_number}]({REPO_URL}/pull/{pr_number}))"
 
 # Handle manual changelog entries through comments.
-if comment := comment.removeprefix("@nf-core-bot changelog").strip():
+if comment := comment.removeprefix("@nf-core-bot changelog").strip():  # type: ignore
     print(f"Adding manual changelog entry: {comment}")
     pr_title = comment
 new_lines = [

.github/workflows/create-lint-wf.yml

@@ -38,7 +38,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "24.04.2"
           - "latest-everything"
     steps:
       - name: go to subdirectory and change nextflow workdir
@@ -78,13 +78,18 @@ jobs:
         run: find nf-core-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
         working-directory: create-lint-wf
 
+      # Uncomment includeConfig statement
+      - name: uncomment include config
+        run: find nf-core-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \;
+        working-directory: create-lint-wf
+
       # Run the other nf-core commands
-      - name: nf-core list
-        run: nf-core --log-file log.txt list
+      - name: nf-core pipelines list
+        run: nf-core --log-file log.txt pipelines list
         working-directory: create-lint-wf
 
-      - name: nf-core schema
-        run: nf-core --log-file log.txt schema build --dir nf-core-testpipeline/ --no-prompts
+      - name: nf-core pipelines schema
+        run: nf-core --log-file log.txt pipelines schema build --dir nf-core-testpipeline/ --no-prompts
         working-directory: create-lint-wf
 
       - name: Cleanup work directory

.github/workflows/create-test-lint-wf-template.yml

@@ -31,27 +31,43 @@ env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
+  prepare-matrix:
+    name: Retrieve all template features
+    runs-on: ubuntu-latest
+    outputs:
+      all_features: ${{ steps.create_matrix.outputs.matrix }}
+    steps:
+      - name: 🏗 Set up yq
+        uses: frenck/action-setup-yq@v1
+      - name: checkout
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+      - name: Create Matrix
+        id: create_matrix
+        run: |
+          echo "matrix=$(yq 'keys | filter(. != "github") | filter(. != "is_nfcore") | filter(. != "test_config") | tojson(0)' nf_core/pipelines/create/template_features.yml)" >> $GITHUB_OUTPUT
+
   RunTestWorkflow:
     runs-on: ${{ matrix.runner }}
+    needs: prepare-matrix
     env:
       NXF_ANSI_LOG: false
     strategy:
       matrix:
-        TEMPLATE:
-          - "template_skip_github_badges.yml"
-          - "template_skip_igenomes.yml"
-          - "template_skip_ci.yml"
+        TEMPLATE: ${{ fromJson(needs.prepare-matrix.outputs.all_features) }}
         runner:
           # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default
           - ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }}
         profile: ["self_hosted_runner"]
         include:
-          - TEMPLATE: "template_skip_all.yml"
+          - TEMPLATE: all
             runner: ubuntu-latest
             profile: "docker"
-          - TEMPLATE: "template_skip_nf_core_configs.yml"
+          - TEMPLATE: nf_core_configs
             runner: ubuntu-latest
             profile: "docker"
+        exclude:
+          - TEMPLATE: nf_core_configs
+            profile: "self_hosted_runner"
       fail-fast: false
 
     steps:
@@ -80,33 +96,22 @@ jobs:
           version: latest-everything
 
       # Create template files
-      - name: Create template skip all (except github)
+      - name: Create template skip ${{ matrix.TEMPLATE }}
         run: |
           mkdir create-test-lint-wf
           export NXF_WORK=$(pwd)
-          printf "org: my-prefix\nskip: ['ci', 'github_badges', 'igenomes', 'nf_core_configs']" > create-test-lint-wf/template_skip_all.yml
-
-      - name: Create template skip github_badges
-        run: |
-          printf "org: my-prefix\nskip: github_badges" > create-test-lint-wf/template_skip_github_badges.yml
-
-      - name: Create template skip igenomes
-        run: |
-          printf "org: my-prefix\nskip: igenomes" > create-test-lint-wf/template_skip_igenomes.yml
-
-      - name: Create template skip ci
-        run: |
-          printf "org: my-prefix\nskip: ci" > create-test-lint-wf/template_skip_ci.yml
-
-      - name: Create template skip nf_core_configs
-        run: |
-          printf "org: my-prefix\nskip: nf_core_configs" > create-test-lint-wf/template_skip_nf_core_configs.yml
+          if [ ${{ matrix.TEMPLATE }} == "all" ]
+          then
+            printf "org: my-prefix\nskip_features: ${{ needs.prepare-matrix.outputs.all_features }}" > create-test-lint-wf/template_skip_all.yml
+          else
+            printf "org: my-prefix\nskip_features: [${{ matrix.TEMPLATE }}]" > create-test-lint-wf/template_skip_${{ matrix.TEMPLATE }}.yml
+          fi
 
       # Create a pipeline from the template
       - name: create a pipeline from the template ${{ matrix.TEMPLATE }}
         run: |
           cd create-test-lint-wf
-          nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml ${{ matrix.TEMPLATE }}
+          nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml template_skip_${{ matrix.TEMPLATE }}.yml
 
       - name: run the pipeline
         run: |
@@ -119,8 +124,8 @@ jobs:
           rm -rf create-test-lint-wf/results
 
       # Try syncing it before we change anything
-      - name: nf-core sync
-        run: nf-core --log-file log.txt sync --dir create-test-lint-wf/my-prefix-testpipeline/
+      - name: nf-core pipelines sync
+        run: nf-core --log-file log.txt pipelines sync --dir create-test-lint-wf/my-prefix-testpipeline/
 
       # Run code style linting
       - name: Run pre-commit
@@ -132,24 +137,29 @@ jobs:
         run: find my-prefix-testpipeline -type f -exec sed -i '/TODO nf-core:/d' {} \;
         working-directory: create-test-lint-wf
 
+      # Uncomment includeConfig statement
+      - name: uncomment include config
+        run: find my-prefix-testpipeline -type f -exec sed -i 's/\/\/ includeConfig/includeConfig/' {} \;
+        working-directory: create-test-lint-wf
+
       # Replace zenodo.XXXXXX to pass readme linting
       - name: replace zenodo.XXXXXX
         run: find my-prefix-testpipeline -type f -exec sed -i 's/zenodo.XXXXXX/zenodo.123456/g' {} \;
         working-directory: create-test-lint-wf
 
       # Run nf-core linting
-      - name: nf-core lint
-        run: nf-core --log-file log.txt --hide-progress lint --dir my-prefix-testpipeline --fail-warned
+      - name: nf-core pipelines lint
+        run: nf-core --log-file log.txt --hide-progress pipelines lint --dir my-prefix-testpipeline --fail-warned
         working-directory: create-test-lint-wf
 
       # Run bump-version
-      - name: nf-core bump-version
-        run: nf-core --log-file log.txt bump-version --dir my-prefix-testpipeline/ 1.1
+      - name: nf-core pipelines bump-version
+        run: nf-core --log-file log.txt pipelines bump-version --dir my-prefix-testpipeline/ 1.1
         working-directory: create-test-lint-wf
 
       # Run nf-core linting in release mode
-      - name: nf-core lint in release mode
-        run: nf-core --log-file log.txt --hide-progress lint --dir my-prefix-testpipeline --fail-warned --release
+      - name: nf-core pipelines lint in release mode
+        run: nf-core --log-file log.txt --hide-progress pipelines lint --dir my-prefix-testpipeline --fail-warned --release
         working-directory: create-test-lint-wf
 
       - name: Tar files

.github/workflows/create-test-wf.yml

@@ -39,7 +39,7 @@ jobs:
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "24.04.2"
           - "latest-everything"
     steps:
       - name: go to working directory
@@ -75,7 +75,7 @@ jobs:
           pwd
           # echo content of current directory
           ls -la
-          nextflow run nf-core-testpipeline -profile test,self_hosted_runner --outdir ./results
+          nextflow run nf-core-testpipeline -profile self_hosted_runner,test --outdir ./results
 
       - name: Upload log file artifact
         if: ${{ always() }}

.github/workflows/pytest.yml

@@ -67,7 +67,7 @@ jobs:
       - name: List tests
         id: list_tests
         run: |
-          echo "tests=$(find tests/test_* | tac | sed 's/tests\///g' | jq -R -s -c '{test: (split("\n")[:-1])}')" >> $GITHUB_OUTPUT
+          echo "tests=$(find tests -type f -name "test_*.py" | tac | sed 's/tests\///g'  | jq -R -s -c '{test: (split("\n")[:-1])}')" >> $GITHUB_OUTPUT
     outputs:
       tests: ${{ steps.list_tests.outputs.tests }}
 
@@ -149,10 +149,15 @@ jobs:
           name: Snapshot Report ${{ matrix.test }}
           path: ./snapshot_report.html
 
+      - name: remove slashes from test name
+        run: |
+          test=$(echo ${{ matrix.test }} | sed 's/\//__/g')
+          echo "test=${test}" >> $GITHUB_ENV
+
       - name: Upload coverage
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4
         with:
-          name: coverage_${{ matrix.test }}
+          name: coverage_${{ env.test }}
           path: .coverage
 
   coverage:

.github/workflows/sync.yml

@@ -19,6 +19,10 @@ on:
         description: "Force a PR to be created"
         type: boolean
         default: false
+      pipeline:
+        description: "Pipeline to sync"
+        type: string
+        default: "all"
 
 # Cancel if a newer run is started
 concurrency:
@@ -35,6 +39,14 @@ jobs:
         run: |
           if [ "${{ github.event.inputs.testpipeline }}" == "true" ]; then
             echo '{"pipeline":["testpipeline"]}' > pipeline_names.json
+          elif [ "${{ github.event.inputs.pipeline }}" != "all" ]; then
+            curl -O https://nf-co.re/pipeline_names.json
+            # check if the pipeline exists
+            if ! grep -q "\"${{ github.event.inputs.pipeline }}\"" pipeline_names.json; then
+              echo "Pipeline ${{ github.event.inputs.pipeline }} does not exist"
+              exit 1
+            fi
+            echo '{"pipeline":["${{ github.event.inputs.pipeline }}"]}' > pipeline_names.json
           else
             curl -O https://nf-co.re/pipeline_names.json
           fi
@@ -82,7 +94,7 @@ jobs:
         run: |
           git config --global user.email "core@nf-co.re"
           git config --global user.name "nf-core-bot"
-          nf-core --log-file sync_log_${{ matrix.pipeline }}.txt sync -d nf-core/${{ matrix.pipeline }} \
+          nf-core --log-file sync_log_${{ matrix.pipeline }}.txt pipelines sync -d nf-core/${{ matrix.pipeline }} \
             --from-branch dev \
             --pull-request \
             --username nf-core-bot \