Merge branch 'dev' into module-conda-env-remove-defaults

nf-core · Aug 23, 2024 · e9e52a5 · e9e52a5
2 parents b2336ad + 7785b4c
commit e9e52a5
Show file tree

Hide file tree

Showing 189 changed files with 7,712 additions and 7,215 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -21,7 +21,7 @@ indent_style = unset
 [**/Makefile]
 indent_style = unset
 
-[tests/__snapshots__/*]
+[tests/pipelines/__snapshots__/*]
 charset = unset
 end_of_line = unset
 insert_final_newline = unset

diff --git a/.github/workflows/changelog.py b/.github/workflows/changelog.py
@@ -1,3 +1,4 @@
+#! /usr/bin/env python3
 """
 Taken from https://github.com/MultiQC/MultiQC/blob/main/.github/workflows/changelog.py and updated for nf-core
 
@@ -18,7 +19,7 @@
 import re
 import sys
 from pathlib import Path
-from typing import List
+from typing import List, Tuple
 
 REPO_URL = "https://github.com/nf-core/tools"
 
@@ -32,7 +33,7 @@
 assert pr_number, pr_number
 
 # Trim the PR number added when GitHub squashes commits, e.g. "Template: Updated (#2026)"
-pr_title = pr_title.removesuffix(f" (#{pr_number})")
+pr_title = pr_title.removesuffix(f" (#{pr_number})")  # type: ignore
 
 changelog_path = workspace_path / "CHANGELOG.md"
 
@@ -50,7 +51,7 @@
     sys.exit(0)
 
 
-def _determine_change_type(pr_title) -> tuple[str, str]:
+def _determine_change_type(pr_title) -> Tuple[str, str]:
     """
     Determine the type of the PR: Template, Download, Linting, Modules, Subworkflows, or General
     Returns a tuple of the section name and the module info.
@@ -85,7 +86,7 @@ def _determine_change_type(pr_title) -> tuple[str, str]:
 pr_link = f"([#{pr_number}]({REPO_URL}/pull/{pr_number}))"
 
 # Handle manual changelog entries through comments.
-if comment := comment.removeprefix("@nf-core-bot changelog").strip():
+if comment := comment.removeprefix("@nf-core-bot changelog").strip():  # type: ignore
     print(f"Adding manual changelog entry: {comment}")
     pr_title = comment
 new_lines = [

diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml
@@ -31,27 +31,45 @@ env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
+  prepare-matrix:
+    name: Retrieve all template features
+    runs-on: ubuntu-latest
+    outputs:
+      all_features: ${{ steps.create_matrix.outputs.matrix }}
+    steps:
+      - name: 🏗 Set up yq
+        uses: frenck/action-setup-yq@v1
+      - name: checkout
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+      - name: Create Matrix
+        id: create_matrix
+        run: |
+          echo "matrix=$(yq 'keys | tojson(0)' nf_core/pipelines/create/templatefeatures.yml)" >> $GITHUB_OUTPUT
+
   RunTestWorkflow:
     runs-on: ${{ matrix.runner }}
+    needs: prepare-matrix
     env:
       NXF_ANSI_LOG: false
     strategy:
       matrix:
-        TEMPLATE:
-          - "template_skip_github_badges.yml"
-          - "template_skip_igenomes.yml"
-          - "template_skip_ci.yml"
+        TEMPLATE: ${{ fromJson(needs.prepare-matrix.outputs.all_features) }}
         runner:
           # use the runner given by the input if it is dispatched manually, run on github if it is a rerun or on self-hosted by default
           - ${{ github.event.inputs.runners || github.run_number > 1 && 'ubuntu-latest' || 'self-hosted' }}
         profile: ["self_hosted_runner"]
         include:
-          - TEMPLATE: "template_skip_all.yml"
+          - TEMPLATE: all
             runner: ubuntu-latest
             profile: "docker"
-          - TEMPLATE: "template_skip_nf_core_configs.yml"
+          - TEMPLATE: nf_core_configs
             runner: ubuntu-latest
             profile: "docker"
+        exclude:
+          - TEMPLATE: github
+          - TEMPLATE: is_nfcore
+          - TEMPLATE: nf_core_configs
+            profile: "self_hosted_runner"
       fail-fast: false
 
     steps:
@@ -80,33 +98,22 @@ jobs:
           version: latest-everything
 
       # Create template files
-      - name: Create template skip all (except github)
+      - name: Create template skip ${{ matrix.TEMPLATE }}
         run: |
           mkdir create-test-lint-wf
           export NXF_WORK=$(pwd)
-          printf "org: my-prefix\nskip: ['ci', 'github_badges', 'igenomes', 'nf_core_configs']" > create-test-lint-wf/template_skip_all.yml
-
-      - name: Create template skip github_badges
-        run: |
-          printf "org: my-prefix\nskip: github_badges" > create-test-lint-wf/template_skip_github_badges.yml
-
-      - name: Create template skip igenomes
-        run: |
-          printf "org: my-prefix\nskip: igenomes" > create-test-lint-wf/template_skip_igenomes.yml
-
-      - name: Create template skip ci
-        run: |
-          printf "org: my-prefix\nskip: ci" > create-test-lint-wf/template_skip_ci.yml
-
-      - name: Create template skip nf_core_configs
-        run: |
-          printf "org: my-prefix\nskip: nf_core_configs" > create-test-lint-wf/template_skip_nf_core_configs.yml
+          if [ ${{ matrix.TEMPLATE }} == "all" ]
+          then
+            printf "org: my-prefix\nskip_features: ${{ needs.prepare-matrix.outputs.all_features }}" > create-test-lint-wf/template_skip_all.yml
+          else
+            printf "org: my-prefix\nskip_features: [${{ matrix.TEMPLATE }}]" > create-test-lint-wf/template_skip_${{ matrix.TEMPLATE }}.yml
+          fi
 
       # Create a pipeline from the template
       - name: create a pipeline from the template ${{ matrix.TEMPLATE }}
         run: |
           cd create-test-lint-wf
-          nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml ${{ matrix.TEMPLATE }}
+          nf-core --log-file log.txt pipelines create -n testpipeline -d "This pipeline is for testing" -a "Testing McTestface" --template-yaml template_skip_${{ matrix.TEMPLATE }}.yml
 
       - name: run the pipeline
         run: |

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -67,7 +67,7 @@ jobs:
       - name: List tests
         id: list_tests
         run: |
-          echo "tests=$(find tests/test_* | tac | sed 's/tests\///g' | jq -R -s -c '{test: (split("\n")[:-1])}')" >> $GITHUB_OUTPUT
+          echo "tests=$(find tests -type f -name "test_*.py" | tac | sed 's/tests\///g'  | jq -R -s -c '{test: (split("\n")[:-1])}')" >> $GITHUB_OUTPUT
     outputs:
       tests: ${{ steps.list_tests.outputs.tests }}
 
@@ -149,10 +149,15 @@ jobs:
           name: Snapshot Report ${{ matrix.test }}
           path: ./snapshot_report.html
 
+      - name: remove slashes from test name
+        run: |
+          test=$(echo ${{ matrix.test }} | sed 's/\//__/g')
+          echo "test=${test}" >> $GITHUB_ENV
+
       - name: Upload coverage
         uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4
         with:
-          name: coverage_${{ matrix.test }}
+          name: coverage_${{ env.test }}
           path: .coverage
 
   coverage:

diff --git a/.github/workflows/update-textual-snapshots.yml b/.github/workflows/update-textual-snapshots.yml
@@ -0,0 +1,93 @@
+name: Update Textual snapshots from a comment
+on:
+  issue_comment:
+    types: [created]
+
+jobs:
+  update-snapshots:
+    # Only run if comment is on a PR with the main repo, and if it contains the magic keywords
+    if: >
+      contains(github.event.comment.html_url, '/pull/') &&
+      contains(github.event.comment.body, '@nf-core-bot update snapshots') &&
+      github.repository == 'nf-core/tools'
+    runs-on: ubuntu-latest
+    steps:
+      # Use the @nf-core-bot token to check out so we can push later
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
+        with:
+          token: ${{ secrets.nf_core_bot_auth_token }}
+
+      # indication that the command is running
+      - name: React on comment
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: eyes
+
+      # Action runs on the issue comment, so we don't get the PR by default
+      # Use the gh cli to check out the PR
+      - name: Checkout Pull Request
+        run: gh pr checkout ${{ github.event.issue.number }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
+
+      # Install dependencies and run pytest
+      - name: Set up Python
+        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5
+        with:
+          python-version: "3.12"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip -r requirements-dev.txt
+          pip install -e .
+
+      - name: Run pytest to update snapshots
+        id: pytest
+        run: |
+          python3 -m pytest tests/test_create_app.py --snapshot-update --color=yes  --durations=0
+        continue-on-error: true
+
+      # indication that the run has finished
+      - name: react if finished succesfully
+        if: steps.pytest.outcome == 'success'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: "+1"
+
+      - name: Commit & push changes
+        id: commit-and-push
+        if: steps.pytest.outcome == 'failure'
+        run: |
+          git config user.email "core@nf-co.re"
+          git config user.name "nf-core-bot"
+          git config push.default upstream
+          git add .
+          git status
+          git commit -m "[automated] Update Textual snapshots"
+          git push
+
+      - name: react if snapshots were updated
+        id: react-if-updated
+        if: steps.commit-and-push.outcome == 'success'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: hooray
+
+      - name: react if snapshots were not updated
+        if: steps.commit-and-push.outcome == 'failure'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          reactions: confused
+
+      - name: react if snapshots were not updated
+        if: steps.commit-and-push.outcome  == 'failure'
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4
+        with:
+          issue-number: ${{ github.event.issue.number }}
+          body: |
+            @${{ github.actor }} I tried to update the snapshots, but it didn't work. Please update them manually.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.1
+    rev: v0.6.0
     hooks:
       - id: ruff # linter
         args: [--fix, --exit-non-zero-on-fix] # sort imports and fix
@@ -10,7 +10,7 @@ repos:
     hooks:
       - id: prettier
         additional_dependencies:
-          - prettier@3.2.5
+          - prettier@3.3.3
 
   - repo: https://github.com/editorconfig-checker/editorconfig-checker.python
     rev: "2.7.3"
@@ -19,7 +19,7 @@ repos:
         alias: ec
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.10.1"
+    rev: "v1.11.1"
     hooks:
       - id: mypy
         additional_dependencies:
@@ -28,3 +28,4 @@ repos:
           - types-jsonschema
           - types-Markdown
           - types-setuptools
+          - pydantic
diff --git a/.prettierignore b/.prettierignore
@@ -5,6 +5,10 @@ docs/api/_build
 testing
 nf_core/module-template/meta.yml
 nf_core/pipeline-template/nextflow_schema.json
+nf_core/pipeline-template/modules.json
+nf_core/pipeline-template/tower.yml
+nf_core/pipeline-template/.github/ISSUE_TEMPLATE/bug_report.yml
+tests/data/pipeline_create_template_skip.yml
 # don't run on things handled by ruff
 *.py
 *.pyc

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,12 +9,24 @@
 - Remove deprecated syntax ([#3046](https://github.com/nf-core/tools/pull/3046))
 - Use filename in code block for `params.yml` ([#3055](https://github.com/nf-core/tools/pull/3055))
 - Remove release announcement for non nf-core pipelines ([#3072](https://github.com/nf-core/tools/pull/3072))
+- add option to exclude code linters for custom pipeline template ([#3084](https://github.com/nf-core/tools/pull/3084))
+- add option to exclude citations for custom pipeline template ([#3101](https://github.com/nf-core/tools/pull/3101))
+- add option to exclude gitpod for custom pipeline template ([#3100](https://github.com/nf-core/tools/pull/3100))
+- add option to exclude codespaces from pipeline template ([#3105](https://github.com/nf-core/tools/pull/3105))
+- add option to exclude multiqc from pipeline template ([#3103](https://github.com/nf-core/tools/pull/3103))
+- add option to exclude changelog from custom pipeline template ([#3104](https://github.com/nf-core/tools/pull/3104))
+- handle template features with a yaml file ([#3108](https://github.com/nf-core/tools/pull/3108))
+- add templatefeatures.yml to python package ([#3112](https://github.com/nf-core/tools/pull/3112))
+- add option to exclude license from pipeline template ([#3125](https://github.com/nf-core/tools/pull/3125))
+- add option to exclude email from pipeline template ([#3126](https://github.com/nf-core/tools/pull/3126))
 
 ### Linting
 
 - Fix linting fail on nfcore_external_java_deps if nf_schema is used ([#2976](https://github.com/nf-core/tools/pull/2976))
 - Conda module linting: Include package name in log file ([#3014](https://github.com/nf-core/tools/pull/3014))
-- Remove defaults from conda environment.yml file. ([#3029](https://github.com/nf-core/tools/pull/3029))
+- Remove defaults from conda `environment.yml` file. ([#3029](https://github.com/nf-core/tools/pull/3029))
+- Restructure pipeline tests and move pipeline linting into subfolder ([#3070](https://github.com/nf-core/tools/pull/3070))
+- Fix module linting warning for process_high_memory ([#3086](https://github.com/nf-core/tools/issues/3086))
 
 ### Download
 
@@ -54,6 +66,14 @@
 - handle new jsonschema error type ([#3061](https://github.com/nf-core/tools/pull/3061))
 - Update python:3.12-slim Docker digest to f11725a ([#3071](https://github.com/nf-core/tools/pull/3071))
 - Fix number of arguments for pipelines_create within the command_create function ([#3074](https://github.com/nf-core/tools/pull/3074))
+- Update python:3.12-slim Docker digest to 740d94a ([#3079](https://github.com/nf-core/tools/pull/3079))
+- Add `--migrate_pytest` option to `nf-core <modules|subworkflows> test` command ([#3085](https://github.com/nf-core/tools/pull/3085))
+- Update pre-commit hook pre-commit/mirrors-mypy to v1.11.1 ([#3091](https://github.com/nf-core/tools/pull/3091))
+- Pipelines: allow numbers in custom pipeline name ([#3094](https://github.com/nf-core/tools/pull/3094))
+- Add bot action to update textual snapshots and write bot documentation ([#3102](https://github.com/nf-core/tools/pull/3102))
+- Components: allow spaces at the betinning of include statements ([#3115](https://github.com/nf-core/tools/pull/3115))
+- Update pre-commit hook astral-sh/ruff-pre-commit to v0.6.0 ([#3122](https://github.com/nf-core/tools/pull/3122))
+- Update python:3.12-slim Docker digest to 59c7332 ([#3124](https://github.com/nf-core/tools/pull/3124))
 
 ## [v2.14.1 - Tantalum Toad - Patch](https://github.com/nf-core/tools/releases/tag/2.14.1) - [2024-05-09]
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -125,3 +125,40 @@ To get started:
 Devcontainer specs:
 
 - [DevContainer config](.devcontainer/devcontainer.json)
+
+## nf-core-bot
+
+nf-core has a bot which you can use to perform certain actions on a PR.
+
+- Fix linting:
+
+If the linting tests is failing on a PR to nf-core/tools, you can post a comment with the magic words:
+
+```
+@nf-core-bot fix linting
+```
+
+The bot will try to fix the linting, push to your branch, and react to the comment when it starts running (👀) and if the fix was successful (👍🏻) or not (😕).
+
+- Update the `CHANGELOG.md`:
+
+The nf-core-bot runs automatically on every PR updating the `CHANGELOG.md` if it was not updated. It will add the new change using the title of your PR.
+If the action didn't run automatically, or you want to provide a different title, you can post a comment with:
+
+```
+@nf-core-bot changelog
+```
+
+Optionally followed by the description that you want to add to the changelog.
+
+- Update Textual snapshots:
+
+If the Textual snapshots (run by `tests/test_crate_app.py`) fail, an HTML report is generated and uploaded as an artifact.
+If you are sure that these changes are correct, you can automatically update the snapshots form the PR by posting a comment with the magic words:
+
+```
+@nf-core-bot update snapshots
+```
+
+> [!WARNING]
+> Please always check the HTML report to make sure that the changes are expected.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.12-slim@sha256:f11725aba18c19664a408902103365eaf8013823ffc56270f921d1dc78a198cb
+FROM python:3.12-slim@sha256:59c7332a4a24373861c4a5f0eec2c92b87e3efeb8ddef011744ef9a751b1d11c
 LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \
     description="Docker image containing requirements for nf-core/tools"
 

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -10,3 +10,4 @@ include nf_core/assets/logo/nf-core-repo-logo-base-darkbg.png
 include nf_core/assets/logo/placeholder_logo.svg
 include nf_core/assets/logo/MavenPro-Bold.ttf
 include nf_core/pipelines/create/create.tcss
+include nf_core/pipelines/create/templatefeatures.yml