diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 8fdd2bd7e1..5043b37acc 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -37,5 +37,5 @@ body: * Hardware _(eg. HPC, Desktop, Cloud)_ * Executor _(eg. slurm, local, awsbatch)_ * OS _(eg. CentOS Linux, macOS, Linux Mint)_ - * Version of nf-core/tools _(eg. 1.1, 1.5, 1.8.2)_ - * Python version _(eg. 3.10, 3.11)_ + * Version of nf-core/tools _(eg. 1.10, 1.12.1, 1.13)_ + * Python version _(eg. 3.11, 3.12)_ diff --git a/.github/actions/create-lint-wf/action.yml b/.github/actions/create-lint-wf/action.yml index 1741b934f2..0bc5e432e7 100644 --- a/.github/actions/create-lint-wf/action.yml +++ b/.github/actions/create-lint-wf/action.yml @@ -77,7 +77,7 @@ runs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-lint-wf/log.txt diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 63bf8fa5ed..cebcc854bc 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -19,7 +19,7 @@ jobs: ) steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.NF_CORE_BOT_AUTH_TOKEN }} @@ -36,9 +36,9 @@ jobs: fi gh pr checkout $PR_NUMBER - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: "3.11" + python-version: "3.12" - name: Install packages run: | @@ -64,10 +64,10 @@ jobs: git diff --exit-code ${GITHUB_WORKSPACE}/CHANGELOG.md || echo "changed=YES" >> $GITHUB_ENV echo "File changed: ${{ env.changed }}" - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install pre-commit diff --git a/.github/workflows/create-lint-wf.yml b/.github/workflows/create-lint-wf.yml index 7a72e18d22..03b9aa2411 100644 --- a/.github/workflows/create-lint-wf.yml +++ b/.github/workflows/create-lint-wf.yml @@ -48,14 +48,14 @@ jobs: export NXF_WORK=$(pwd) # Get the repo code - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository # Set up nf-core/tools - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" cache: pip - name: Install python dependencies diff --git a/.github/workflows/create-test-lint-wf-template.yml b/.github/workflows/create-test-lint-wf-template.yml index 0d65c40b07..0de7287a57 100644 --- a/.github/workflows/create-test-lint-wf-template.yml +++ b/.github/workflows/create-test-lint-wf-template.yml @@ -61,13 +61,13 @@ jobs: cd create-lint-wf-template export NXF_WORK=$(pwd) - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | @@ -158,7 +158,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: nf-core-log-file-${{ matrix.TEMPLATE }} path: create-test-lint-wf/artifact_files.tar diff --git a/.github/workflows/create-test-wf.yml b/.github/workflows/create-test-wf.yml index 9800ac1910..87cdf2e7bb 100644 --- a/.github/workflows/create-test-wf.yml +++ b/.github/workflows/create-test-wf.yml @@ -48,13 +48,13 @@ jobs: cd create-test-wf export NXF_WORK=$(pwd) - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | @@ -75,7 +75,7 @@ jobs: - name: Upload log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: nf-core-log-file-${{ matrix.NXF_VER }} path: create-test-wf/log.txt diff --git a/.github/workflows/deploy-pypi.yml b/.github/workflows/deploy-pypi.yml index e53d2f2f5a..1202891e4d 100644 --- a/.github/workflows/deploy-pypi.yml +++ b/.github/workflows/deploy-pypi.yml @@ -13,13 +13,13 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 95a03c70fe..4334871c4c 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} @@ -32,9 +32,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} # Install and run pre-commit - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install pre-commit run: pip install pre-commit diff --git a/.github/workflows/lint-code.yml b/.github/workflows/lint-code.yml index 8ed52a0582..3bddd42d49 100644 --- a/.github/workflows/lint-code.yml +++ b/.github/workflows/lint-code.yml @@ -18,12 +18,12 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install pre-commit diff --git a/.github/workflows/push_dockerhub_dev.yml b/.github/workflows/push_dockerhub_dev.yml index af11a79990..c613e13a2d 100644 --- a/.github/workflows/push_dockerhub_dev.yml +++ b/.github/workflows/push_dockerhub_dev.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Build nfcore/tools:dev docker image run: docker build --no-cache . -t nfcore/tools:dev diff --git a/.github/workflows/push_dockerhub_release.yml b/.github/workflows/push_dockerhub_release.yml index 0b3f381d86..5a076f6d3b 100644 --- a/.github/workflows/push_dockerhub_release.yml +++ b/.github/workflows/push_dockerhub_release.yml @@ -23,7 +23,7 @@ jobs: fail-fast: false steps: - name: Check out code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Build nfcore/tools:latest docker image run: docker build --no-cache . -t nfcore/tools:latest diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2aed1752ff..70b9cfd0a8 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -40,7 +40,7 @@ jobs: runs-on: "ubuntu-latest" strategy: matrix: - python-version: ["3.8", "3.11"] + python-version: ["3.8", "3.12"] runner: ["ubuntu-latest"] include: - python-version: "3.8" @@ -61,7 +61,7 @@ jobs: name: Get test file matrix runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: List tests @@ -87,11 +87,11 @@ jobs: cd pytest export NXF_WORK=$(pwd) - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out source-code repository - name: Set up Python ${{ needs.setup.outputs.python-version }} - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: ${{ needs.setup.outputs.python-version }} cache: "pip" @@ -143,7 +143,7 @@ jobs: fi - name: Upload coverage - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: coverage_${{ matrix.test }} path: .coverage @@ -158,13 +158,13 @@ jobs: mkdir -p pytest cd pytest - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 env: AGENT_TOOLSDIRECTORY: /opt/actions-runner/_work/tools/tools/ with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install dependencies @@ -177,14 +177,14 @@ jobs: mv .github/.coveragerc . - name: Download all artifacts - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4 + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4 - name: Run coverage run: | coverage combine --keep coverage*/.coverage* coverage report coverage xml - - uses: codecov/codecov-action@54bcd8715eee62d40e33596ef5e8f0f48dbbccab # v4 + - uses: codecov/codecov-action@5ecb98a3c6b747ed38dc09f787459979aebb39be # v4 with: files: coverage.xml env: diff --git a/.github/workflows/rich-codex.yml b/.github/workflows/rich-codex.yml index 13fd71a883..cd12b139d3 100644 --- a/.github/workflows/rich-codex.yml +++ b/.github/workflows/rich-codex.yml @@ -6,9 +6,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out the repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: 3.x cache: pip diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 19e895f452..15802cdef2 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -44,10 +44,10 @@ jobs: matrix: ${{fromJson(needs.get-pipelines.outputs.matrix)}} fail-fast: false steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out nf-core/tools - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 name: Check out nf-core/${{ matrix.pipeline }} with: repository: nf-core/${{ matrix.pipeline }} @@ -56,10 +56,10 @@ jobs: path: nf-core/${{ matrix.pipeline }} fetch-depth: "0" - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install python dependencies run: | @@ -86,7 +86,7 @@ jobs: - name: Upload sync log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: sync_log_${{ matrix.pipeline }} path: sync_log_${{ matrix.pipeline }}.txt diff --git a/.github/workflows/update_components_template.yml b/.github/workflows/update_components_template.yml index 0fa3adaf20..5ba513735e 100644 --- a/.github/workflows/update_components_template.yml +++ b/.github/workflows/update_components_template.yml @@ -11,10 +11,10 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Set up Python - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.x" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8a68848237..887cbe027c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.3 + rev: v0.4.3 hooks: - id: ruff # linter args: [--fix, --exit-non-zero-on-fix] # sort imports and fix @@ -19,7 +19,7 @@ repos: alias: ec - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.9.0" + rev: "v1.10.0" hooks: - id: mypy additional_dependencies: diff --git a/CHANGELOG.md b/CHANGELOG.md index ab0d37627a..88ce77a3b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,17 +10,28 @@ - Add fallback to `download_pipeline.yml` in case the pipeline does not support stub runs ([#2846](https://github.com/nf-core/tools/pull/2846)) - Set topic variable correctly in the mastodon announcement ([#2848](https://github.com/nf-core/tools/pull/2848)) - Add a cleanup action to `download_pipeline.yml` to fix failures caused by inadequate storage space on the runner ([#2849](https://github.com/nf-core/tools/pull/2849)) +- Update python to 3.12 ([#2805](https://github.com/nf-core/tools/pull/2805)) +- Remove `pyproject.toml` from template root +- Shorten lines in pipeline template ([#2908](https://github.com/nf-core/tools/pull/2908)) +- Permanently activated pipeline-specific institutional configs support for all pipelines without need for manual intervention ([#2936](https://github.com/nf-core/tools/pull/2936)) +- Template config: `conda.channels`, not `channels` ([#2950](https://github.com/nf-core/tools/pull/2950)) +- Handles multiple DOIs + doi.org resolver from manifest.doi ([#2946](https://github.com/nf-core/tools/pull/2946)) +- Update included components ([#2949](https://github.com/nf-core/tools/pull/2949)) ### Linting - Only match assignments of params in `main.nf` and not references like `params.aligner == ` ([#2833](https://github.com/nf-core/tools/pull/2833)) +- Include test for presence of versions in snapshot ([#2888](https://github.com/nf-core/tools/pull/2888)) ### Download - Replace `--tower` with `--platform`. The former will remain for backwards compatability for now but will be removed in a future release. +- Better error message when GITHUB_TOKEN exists but is wrong/outdated ### Components +- Handle more complete list of possible git URL forms (ssh:// and ftp:// prefixes specifically) ([#2945](https://github.com/nf-core/tools/pull/2945)) + ### General - Update CI to use nf-core/setup-nextflow v2 @@ -36,6 +47,30 @@ - Update python:3.11-slim Docker digest to a2eb07f ([#2847](https://github.com/nf-core/tools/pull/2847)) - Strip out mention of "Nextflow Tower" and replace with "Seqera Platform" wherever possible - Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.3 ([#2850](https://github.com/nf-core/tools/pull/2850)) +- Fix issue with config resolution that was causing nested configs to behave unexpectedly ([#2862](https://github.com/nf-core/tools/pull/2862)) +- Fix schema docs console output truncating ([#2880](https://github.com/nf-core/tools/pull/2880)) +- fix: ensure path object converted to string before stripping quotes ([#2878](https://github.com/nf-core/tools/pull/2878)) +- Fix incorrect assertions for called_with on mocks ([#2891](https://github.com/nf-core/tools/pull/2891)) +- Make cli-provided module/subworkflow names case insensitive ([#2869](https://github.com/nf-core/tools/pull/2869)) +- Update gitpod/workspace-base Docker digest to 168d78b ([#2899](https://github.com/nf-core/tools/pull/2899)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.4 ([#2894](https://github.com/nf-core/tools/pull/2894)) +- Update GitHub Actions ([#2902](https://github.com/nf-core/tools/pull/2902)) +- Get immediate parent path name for schema creation ([#2886](https://github.com/nf-core/tools/pull/2886)) +- Update pre-commit hook astral-sh/ruff-pre-commit to v0.3.5 ([#2903](https://github.com/nf-core/tools/pull/2903)) +- Remove old references to CUSTOMDUMPSOFTWAREVERSIONS and add linting checks ([#2897](https://github.com/nf-core/tools/pull/2897)) +- Update codecov/codecov-action digest to 7afa10e ([#2909](https://github.com/nf-core/tools/pull/2909)) +- Update codecov/codecov-action digest to 8450866 ([#2913](https://github.com/nf-core/tools/pull/2913)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.6 ([#2915](https://github.com/nf-core/tools/pull/2915)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.3.7 ([#2917](https://github.com/nf-core/tools/pull/2917)) +- Update gitpod/workspace-base Docker digest to 0af257e ([#2919](https://github.com/nf-core/tools/pull/2919)) +- Update GitHub Actions ([#2927](https://github.com/nf-core/tools/pull/2927)) +- Update gitpod/workspace-base Docker digest to 5aeb24f ([#2929](https://github.com/nf-core/tools/pull/2929)) +- Update pre-commit hook pre-commit/mirrors-mypy to v1.10.0 ([#2933](https://github.com/nf-core/tools/pull/2933)) +- Update GitHub Actions ([#2939](https://github.com/nf-core/tools/pull/2939)) +- Update codecov/codecov-action digest to 5ecb98a ([#2948](https://github.com/nf-core/tools/pull/2948)) +- Update gitpod/workspace-base Docker digest to 124f2b8 ([#2943](https://github.com/nf-core/tools/pull/2943)) +- fix(collectfile): sort true for methods_description_mqc.yaml ([#2947](https://github.com/nf-core/tools/pull/2947)) +- chore(deps): update pre-commit hook astral-sh/ruff-pre-commit to v0.4.3 ([#2951](https://github.com/nf-core/tools/pull/2951)) ## [v2.13.1 - Tin Puppy Patch](https://github.com/nf-core/tools/releases/tag/2.13) - [2024-02-29] @@ -46,6 +81,7 @@ - Fix topic extraction step for hashtags in toots ([#2810](https://github.com/nf-core/tools/pull/2810)) - Update modules and subworkflows in the template ([#2811](https://github.com/nf-core/tools/pull/2811)) - Unpin setup-nextflow and action-tower-launch ([#2806](https://github.com/nf-core/tools/pull/2806)) +- Add nf-core-version to `.nf-core.yml` ([#2874](https://github.com/nf-core/tools/pull/2874)) ### Download diff --git a/Dockerfile b/Dockerfile index 60768499a1..ae3a4e1a3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM python:3.11-slim@sha256:a2eb07f336e4f194358382611b4fea136c632b40baa6314cb27a366deeaf0144 -LABEL authors="phil.ewels@scilifelab.se,erik.danielsson@scilifelab.se" \ - description="Docker image containing requirements for the nfcore tools" +FROM python:3.12-slim@sha256:2be8daddbb82756f7d1f2c7ece706aadcb284bf6ab6d769ea695cc3ed6016743 +LABEL authors="phil.ewels@seqera.io,erik.danielsson@scilifelab.se" \ + description="Docker image containing requirements for nf-core/tools" # Do not pick up python packages from $HOME ENV PYTHONNUSERSITE=1 diff --git a/README.md b/README.md index 53e9f77c97..a5e6799861 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ conda install nf-core Alternatively, you can create a new environment with both nf-core/tools and nextflow: ```bash -conda create --name nf-core python=3.11 nf-core nextflow +conda create --name nf-core python=3.12 nf-core nextflow conda activate nf-core ``` diff --git a/docs/api/_src/pipeline_lint_tests/base_config.md b/docs/api/_src/pipeline_lint_tests/base_config.md new file mode 100644 index 0000000000..4a56ef9789 --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/base_config.md @@ -0,0 +1,5 @@ +# base_config + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.base_config +``` diff --git a/docs/api/_src/pipeline_lint_tests/modules_config.md b/docs/api/_src/pipeline_lint_tests/modules_config.md new file mode 100644 index 0000000000..2a4f51c5a4 --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/modules_config.md @@ -0,0 +1,5 @@ +# modules_config + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.modules_config +``` diff --git a/docs/api/_src/pipeline_lint_tests/nfcore_yml.md b/docs/api/_src/pipeline_lint_tests/nfcore_yml.md new file mode 100644 index 0000000000..f7e797a29c --- /dev/null +++ b/docs/api/_src/pipeline_lint_tests/nfcore_yml.md @@ -0,0 +1,5 @@ +# nfcore_yml + +```{eval-rst} +.. automethod:: nf_core.lint.PipelineLint.nfcore_yml +``` diff --git a/nf_core/__main__.py b/nf_core/__main__.py index 20cba43e4e..807bc776bb 100644 --- a/nf_core/__main__.py +++ b/nf_core/__main__.py @@ -103,6 +103,14 @@ def selective_traceback_hook(exctype, value, traceback): sys.excepthook = selective_traceback_hook +# Define callback function to normalize the case of click arguments, +# which is used to make the module/subworkflow names, provided by the +# user on the cli, case insensitive. +def normalize_case(ctx, param, component_name): + if component_name is not None: + return component_name.casefold() + + def run_nf_core(): # print nf-core header if environment variable is not set if os.environ.get("_NF_CORE_COMPLETE") is None: @@ -786,7 +794,7 @@ def modules_list_local(ctx, keywords, json, dir): # pylint: disable=redefined-b # nf-core modules install @modules.command("install") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -838,7 +846,7 @@ def modules_install(ctx, tool, dir, prompt, force, sha): # nf-core modules update @modules.command("update") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -930,7 +938,7 @@ def modules_update( # nf-core modules patch @modules.command() @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -967,7 +975,7 @@ def patch(ctx, tool, dir, remove): # nf-core modules remove @modules.command("remove") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1121,7 +1129,7 @@ def create_module( # nf-core modules test @modules.command("test") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1180,7 +1188,7 @@ def test_module(ctx, tool, dir, no_prompts, update, once, profile): # nf-core modules lint @modules.command("lint") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1267,7 +1275,7 @@ def modules_lint(ctx, tool, dir, registry, key, all, fail_warned, local, passed, # nf-core modules info @modules.command("info") @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1306,7 +1314,7 @@ def modules_info(ctx, tool, dir): # nf-core modules bump-versions @modules.command() @click.pass_context -@click.argument("tool", type=str, required=False, metavar=" or ") +@click.argument("tool", type=str, callback=normalize_case, required=False, metavar=" or ") @click.option( "-d", "--dir", @@ -1392,7 +1400,7 @@ def create_subworkflow(ctx, subworkflow, dir, author, force, migrate_pytest): # nf-core subworkflows test @subworkflows.command("test") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1519,7 +1527,7 @@ def subworkflows_list_local(ctx, keywords, json, dir): # pylint: disable=redefi # nf-core subworkflows lint @subworkflows.command("lint") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1600,7 +1608,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo # nf-core subworkflows info @subworkflows.command("info") @click.pass_context -@click.argument("tool", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1608,7 +1616,7 @@ def subworkflows_lint(ctx, subworkflow, dir, registry, key, all, fail_warned, lo default=".", help=r"Pipeline directory. [dim]\[default: Current working directory][/]", ) -def subworkflows_info(ctx, tool, dir): +def subworkflows_info(ctx, subworkflow, dir): """ Show developer usage information about a given subworkflow. @@ -1625,7 +1633,7 @@ def subworkflows_info(ctx, tool, dir): try: subworkflow_info = SubworkflowInfo( dir, - tool, + subworkflow, ctx.obj["modules_repo_url"], ctx.obj["modules_repo_branch"], ctx.obj["modules_repo_no_pull"], @@ -1639,7 +1647,7 @@ def subworkflows_info(ctx, tool, dir): # nf-core subworkflows install @subworkflows.command("install") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1697,7 +1705,7 @@ def subworkflows_install(ctx, subworkflow, dir, prompt, force, sha): # nf-core subworkflows remove @subworkflows.command("remove") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", @@ -1727,7 +1735,7 @@ def subworkflows_remove(ctx, dir, subworkflow): # nf-core subworkflows update @subworkflows.command("update") @click.pass_context -@click.argument("subworkflow", type=str, required=False, metavar="subworkflow name") +@click.argument("subworkflow", type=str, callback=normalize_case, required=False, metavar="subworkflow name") @click.option( "-d", "--dir", diff --git a/nf_core/components/components_command.py b/nf_core/components/components_command.py index 8332429835..4df67639e2 100644 --- a/nf_core/components/components_command.py +++ b/nf_core/components/components_command.py @@ -245,7 +245,7 @@ def check_patch_paths(self, patch_path: Path, module_name: str) -> None: # Update path in modules.json if the file is in the correct format modules_json = ModulesJson(self.dir) modules_json.load() - if modules_json.has_git_url_and_modules(): + if modules_json.has_git_url_and_modules() and modules_json.modules_json is not None: modules_json.modules_json["repos"][self.modules_repo.remote_url]["modules"][ self.modules_repo.repo_path ][module_name]["patch"] = str(patch_path.relative_to(Path(self.dir).resolve())) diff --git a/nf_core/components/create.py b/nf_core/components/create.py index d2169e3a72..6c9c01b496 100644 --- a/nf_core/components/create.py +++ b/nf_core/components/create.py @@ -394,7 +394,7 @@ def _get_component_dirs(self): if self.component_type == "modules": file_paths["environment.yml"] = component_dir / "environment.yml" file_paths["tests/tags.yml"] = component_dir / "tests" / "tags.yml" - file_paths["tests/main.nf.test"] = component_dir / "tests" / "main.nf.test" + file_paths["tests/main.nf.test.j2"] = component_dir / "tests" / "main.nf.test" return file_paths diff --git a/nf_core/download.py b/nf_core/download.py index 9d4decc424..8ffecc0f55 100644 --- a/nf_core/download.py +++ b/nf_core/download.py @@ -925,7 +925,7 @@ def rectify_raw_container_matches(self, raw_findings): if bool(container_definition) and bool(container_definition.group(1)): pattern = re.escape(container_definition.group(1)) # extract the quoted string(s) following the variable assignment - container_names = re.findall(r"%s\s*=\s*[\"\']([^\"\']+)[\"\']" % pattern, search_space) + container_names = re.findall(rf"{pattern}\s*=\s*[\"\']([^\"\']+)[\"\']", search_space) if bool(container_names): if isinstance(container_names, str): diff --git a/nf_core/gitpod/gitpod.Dockerfile b/nf_core/gitpod/gitpod.Dockerfile index 3136d02e97..6afca0e479 100644 --- a/nf_core/gitpod/gitpod.Dockerfile +++ b/nf_core/gitpod/gitpod.Dockerfile @@ -1,7 +1,7 @@ # Test build locally before making a PR # docker build -t gitpod:test -f nf_core/gitpod/gitpod.Dockerfile . -FROM gitpod/workspace-base@sha256:1e133e5691add6c19443672594b9f3d7d9c3372ead4c86a4490c2701dbfa32e3 +FROM gitpod/workspace-base@sha256:124f2b8cbefe9b4abbb6a14538da8846770dde20b93f038d9551b6230aec1d1c USER root diff --git a/nf_core/lint/__init__.py b/nf_core/lint/__init__.py index be9ac183a6..9292a07fd8 100644 --- a/nf_core/lint/__init__.py +++ b/nf_core/lint/__init__.py @@ -9,7 +9,7 @@ import logging import os from pathlib import Path -from typing import List, Union +from typing import List, Tuple, Union import git import rich @@ -25,6 +25,7 @@ import nf_core.subworkflows.lint import nf_core.utils from nf_core import __version__ +from nf_core.components.lint import ComponentLint from nf_core.lint_utils import console from nf_core.utils import plural_s as _s from nf_core.utils import strip_ansi_codes @@ -32,148 +33,6 @@ log = logging.getLogger(__name__) -def run_linting( - pipeline_dir, - release_mode=False, - fix=(), - key=(), - show_passed=False, - fail_ignored=False, - fail_warned=False, - sort_by="test", - md_fn=None, - json_fn=None, - hide_progress=False, -): - """Runs all nf-core linting checks on a given Nextflow pipeline project - in either `release` mode or `normal` mode (default). Returns an object - of type :class:`PipelineLint` after finished. - - Args: - pipeline_dir (str): The path to the Nextflow pipeline root directory - release_mode (bool): Set this to `True`, if the linting should be run in the `release` mode. - See :class:`PipelineLint` for more information. - - Returns: - An object of type :class:`PipelineLint` that contains all the linting results. - An object of type :class:`ComponentLint` that contains all the linting results for the modules. - An object of type :class:`ComponentLint` that contains all the linting results for the subworkflows. - """ - - # Verify that the requested tests exist - if key: - all_tests = set(PipelineLint._get_all_lint_tests(release_mode)).union( - set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True)) - ) - bad_keys = [k for k in key if k not in all_tests] - if len(bad_keys) > 0: - raise AssertionError( - "Test name{} not recognised: '{}'".format( - _s(bad_keys), - "', '".join(bad_keys), - ) - ) - log.info("Only running tests: '{}'".format("', '".join(key))) - - # Check if we were given any keys, and if they match any pipeline tests - if key: - pipeline_keys = list(set(key).intersection(set(PipelineLint._get_all_lint_tests(release_mode)))) - else: - # If no key is supplied, run all tests - pipeline_keys = None - - # Create the lint object - lint_obj = PipelineLint(pipeline_dir, release_mode, fix, pipeline_keys, fail_ignored, fail_warned, hide_progress) - - # Load the various pipeline configs - lint_obj._load_lint_config() - lint_obj._load_pipeline_config() - lint_obj._list_files() - - # Create the modules lint object - module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) - # Create the subworkflows lint object - try: - subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) - except LookupError: - subworkflow_lint_obj = None - - # Verify that the pipeline is correctly configured and has a modules.json file - module_lint_obj.has_valid_directory() - module_lint_obj.has_modules_file() - - # Run only the tests we want - if key: - # Select only the module lint tests - module_lint_tests = list( - set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True))) - ) - # Select only the subworkflow lint tests - subworkflow_lint_tests = list( - set(key).intersection( - set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True)) - ) - ) - else: - # If no key is supplied, run the default modules tests - module_lint_tests = ("module_changes", "module_version") - subworkflow_lint_tests = ("subworkflow_changes", "subworkflow_version") - module_lint_obj.filter_tests_by_key(module_lint_tests) - if subworkflow_lint_obj is not None: - subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) - - # Set up files for component linting test - module_lint_obj.set_up_pipeline_files() - if subworkflow_lint_obj is not None: - subworkflow_lint_obj.set_up_pipeline_files() - - # Run the pipeline linting tests - try: - lint_obj._lint_pipeline() - except AssertionError as e: - log.critical(f"Critical error: {e}") - log.info("Stopping tests...") - return lint_obj, module_lint_obj - - # Run the module lint tests - if len(module_lint_obj.all_local_components) > 0: - module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True) - if len(module_lint_obj.all_remote_components) > 0: - module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False) - # Run the subworkflows lint tests - if subworkflow_lint_obj is not None: - if len(subworkflow_lint_obj.all_local_components) > 0: - subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) - if len(subworkflow_lint_obj.all_remote_components) > 0: - subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) - - # Print the results - lint_obj._print_results(show_passed) - module_lint_obj._print_results(show_passed, sort_by=sort_by) - if subworkflow_lint_obj is not None: - subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) - nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj) - nf_core.lint_utils.print_fixes(lint_obj) - - # Save results to Markdown file - if md_fn is not None: - log.info(f"Writing lint results to {md_fn}") - markdown = lint_obj._get_results_md() - with open(md_fn, "w") as fh: - fh.write(markdown) - - # Save results to JSON file - if json_fn is not None: - lint_obj._save_json_results(json_fn) - - # Reminder about --release mode flag if we had failures - if len(lint_obj.failed) > 0: - if release_mode: - log.info("Reminder: Lint tests were run in --release mode.") - - return lint_obj, module_lint_obj, subworkflow_lint_obj - - class PipelineLint(nf_core.utils.Pipeline): """Object to hold linting information and results. @@ -199,6 +58,7 @@ class PipelineLint(nf_core.utils.Pipeline): from .actions_schema_validation import ( # type: ignore[misc] actions_schema_validation, ) + from .configs import base_config, modules_config # type: ignore[misc] from .files_exist import files_exist # type: ignore[misc] from .files_unchanged import files_unchanged # type: ignore[misc] from .merge_markers import merge_markers # type: ignore[misc] @@ -206,6 +66,7 @@ class PipelineLint(nf_core.utils.Pipeline): from .modules_structure import modules_structure # type: ignore[misc] from .multiqc_config import multiqc_config # type: ignore[misc] from .nextflow_config import nextflow_config # type: ignore[misc] + from .nfcore_yml import nfcore_yml # type: ignore[misc] from .pipeline_name_conventions import ( # type: ignore[misc] pipeline_name_conventions, ) @@ -264,6 +125,9 @@ def _get_all_lint_tests(release_mode): "modules_json", "multiqc_config", "modules_structure", + "base_config", + "modules_config", + "nfcore_yml", ] + (["version_consistency"] if release_mode else []) def _load(self): @@ -409,7 +273,7 @@ def format_result(test_results): # Table of passed tests if len(self.passed) > 0 and show_passed: console.print( - rich.panel.Panel( + Panel( format_result(self.passed), title=rf"[bold][✔] {len(self.passed)} Pipeline Test{_s(self.passed)} Passed", title_align="left", @@ -421,7 +285,7 @@ def format_result(test_results): # Table of fixed tests if len(self.fixed) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.fixed), title=rf"[bold][?] {len(self.fixed)} Pipeline Test{_s(self.fixed)} Fixed", title_align="left", @@ -433,7 +297,7 @@ def format_result(test_results): # Table of ignored tests if len(self.ignored) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.ignored), title=rf"[bold][?] {len(self.ignored)} Pipeline Test{_s(self.ignored)} Ignored", title_align="left", @@ -445,7 +309,7 @@ def format_result(test_results): # Table of warning tests if len(self.warned) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.warned), title=rf"[bold][!] {len(self.warned)} Pipeline Test Warning{_s(self.warned)}", title_align="left", @@ -457,7 +321,7 @@ def format_result(test_results): # Table of failing tests if len(self.failed) > 0: console.print( - rich.panel.Panel( + Panel( format_result(self.failed), title=rf"[bold][✗] {len(self.failed)} Pipeline Test{_s(self.failed)} Failed", title_align="left", @@ -638,3 +502,144 @@ def _wrap_quotes(self, files: Union[List[str], List[Path], Path]) -> str: files = [files] bfiles = [f"`{str(f)}`" for f in files] return " or ".join(bfiles) + + +def run_linting( + pipeline_dir, + release_mode: bool = False, + fix=(), + key=(), + show_passed: bool = False, + fail_ignored: bool = False, + fail_warned: bool = False, + sort_by: str = "test", + md_fn=None, + json_fn=None, + hide_progress: bool = False, +) -> Tuple[PipelineLint, ComponentLint, Union[ComponentLint, None]]: + """Runs all nf-core linting checks on a given Nextflow pipeline project + in either `release` mode or `normal` mode (default). Returns an object + of type :class:`PipelineLint` after finished. + + Args: + pipeline_dir (str): The path to the Nextflow pipeline root directory + release_mode (bool): Set this to `True`, if the linting should be run in the `release` mode. + See :class:`PipelineLint` for more information. + + Returns: + An object of type :class:`PipelineLint` that contains all the linting results. + An object of type :class:`ComponentLint` that contains all the linting results for the modules. + An object of type :class:`ComponentLint` that contains all the linting results for the subworkflows. + """ + + # Verify that the requested tests exist + if key: + all_tests = set(PipelineLint._get_all_lint_tests(release_mode)).union( + set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True)) + ) + bad_keys = [k for k in key if k not in all_tests] + if len(bad_keys) > 0: + raise AssertionError( + "Test name{} not recognised: '{}'".format( + _s(bad_keys), + "', '".join(bad_keys), + ) + ) + log.info("Only running tests: '{}'".format("', '".join(key))) + + # Check if we were given any keys, and if they match any pipeline tests + if key: + pipeline_keys = list(set(key).intersection(set(PipelineLint._get_all_lint_tests(release_mode)))) + else: + # If no key is supplied, run all tests + pipeline_keys = None + + # Create the lint object + lint_obj = PipelineLint(pipeline_dir, release_mode, fix, pipeline_keys, fail_ignored, fail_warned, hide_progress) + + # Load the various pipeline configs + lint_obj._load_lint_config() + lint_obj._load_pipeline_config() + lint_obj._list_files() + + # Create the modules lint object + module_lint_obj = nf_core.modules.lint.ModuleLint(pipeline_dir, hide_progress=hide_progress) + # Create the subworkflows lint object + try: + subworkflow_lint_obj = nf_core.subworkflows.lint.SubworkflowLint(pipeline_dir, hide_progress=hide_progress) + except LookupError: + subworkflow_lint_obj = None + + # Verify that the pipeline is correctly configured and has a modules.json file + module_lint_obj.has_valid_directory() + module_lint_obj.has_modules_file() + # Run only the tests we want + if key: + # Select only the module lint tests + module_lint_tests = list( + set(key).intersection(set(nf_core.modules.lint.ModuleLint.get_all_module_lint_tests(is_pipeline=True))) + ) + # Select only the subworkflow lint tests + subworkflow_lint_tests = list( + set(key).intersection( + set(nf_core.subworkflows.lint.SubworkflowLint.get_all_subworkflow_lint_tests(is_pipeline=True)) + ) + ) + else: + # If no key is supplied, run the default modules tests + module_lint_tests = list(("module_changes", "module_version")) + subworkflow_lint_tests = list(("subworkflow_changes", "subworkflow_version")) + module_lint_obj.filter_tests_by_key(module_lint_tests) + if subworkflow_lint_obj is not None: + subworkflow_lint_obj.filter_tests_by_key(subworkflow_lint_tests) + + # Set up files for component linting test + module_lint_obj.set_up_pipeline_files() + if subworkflow_lint_obj is not None: + subworkflow_lint_obj.set_up_pipeline_files() + + # Run the pipeline linting tests + try: + lint_obj._lint_pipeline() + except AssertionError as e: + log.critical(f"Critical error: {e}") + log.info("Stopping tests...") + return lint_obj, module_lint_obj, subworkflow_lint_obj + + # Run the module lint tests + if len(module_lint_obj.all_local_components) > 0: + module_lint_obj.lint_modules(module_lint_obj.all_local_components, local=True) + if len(module_lint_obj.all_remote_components) > 0: + module_lint_obj.lint_modules(module_lint_obj.all_remote_components, local=False) + # Run the subworkflows lint tests + if subworkflow_lint_obj is not None: + if len(subworkflow_lint_obj.all_local_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_local_components, local=True) + if len(subworkflow_lint_obj.all_remote_components) > 0: + subworkflow_lint_obj.lint_subworkflows(subworkflow_lint_obj.all_remote_components, local=False) + + # Print the results + lint_obj._print_results(show_passed) + module_lint_obj._print_results(show_passed, sort_by=sort_by) + if subworkflow_lint_obj is not None: + subworkflow_lint_obj._print_results(show_passed, sort_by=sort_by) + nf_core.lint_utils.print_joint_summary(lint_obj, module_lint_obj, subworkflow_lint_obj) + nf_core.lint_utils.print_fixes(lint_obj) + + # Save results to Markdown file + if md_fn is not None: + log.info(f"Writing lint results to {md_fn}") + markdown = lint_obj._get_results_md() + with open(md_fn, "w") as fh: + fh.write(markdown) + + # Save results to JSON file + if json_fn is not None: + lint_obj._save_json_results(json_fn) + + # Reminder about --release mode flag if we had failures + if len(lint_obj.failed) > 0: + if release_mode: + log.info("Reminder: Lint tests were run in --release mode.") + + return lint_obj, module_lint_obj, subworkflow_lint_obj diff --git a/nf_core/lint/configs.py b/nf_core/lint/configs.py new file mode 100644 index 0000000000..2741529198 --- /dev/null +++ b/nf_core/lint/configs.py @@ -0,0 +1,101 @@ +import logging +import re +from pathlib import Path +from typing import Dict, List + +from nf_core.lint_utils import ignore_file + +log = logging.getLogger(__name__) + + +class LintConfig: + def __init__(self, wf_path: str, lint_config: Dict[str, List[str]]): + self.wf_path = wf_path + self.lint_config = lint_config + + def lint_file(self, lint_name: str, file_path: Path) -> Dict[str, List[str]]: + """Lint a file and add the result to the passed or failed list.""" + + passed: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + ignore_configs: List[str] = [] + + fn = Path(self.wf_path, file_path) + + passed, failed, ignored, ignore_configs = ignore_file(lint_name, file_path, Path(self.wf_path)) + + error_message = f"`{file_path}` not found" + # check for partial match in failed or ignored + if not any(f.startswith(error_message) for f in (failed + ignored)): + try: + with open(fn) as fh: + config = fh.read() + except Exception as e: + return {"failed": [f"Could not parse file: {fn}, {e}"]} + + # find sections with a withName: prefix + sections = re.findall(r"withName:\s*['\"]?(\w+)['\"]?", config) + log.debug(f"found sections: {sections}") + + # find all .nf files in the workflow directory + nf_files = list(Path(self.wf_path).rglob("*.nf")) + log.debug(f"found nf_files: {[str(f) for f in nf_files]}") + + # check if withName sections are present in config, but not in workflow files + for section in sections: + if section not in ignore_configs or section.lower() not in ignore_configs: + if not any(section in nf_file.read_text() for nf_file in nf_files): + failed.append( + f"`{file_path}` contains `withName:{section}`, but the corresponding process is not present in any of the Nextflow scripts." + ) + else: + passed.append(f"`{section}` found in `{file_path}` and Nextflow scripts.") + else: + ignored.append(f"``{section}` is ignored.") + + return {"passed": passed, "failed": failed, "ignored": ignored} + + +def modules_config(self) -> Dict[str, List[str]]: + """Make sure the conf/modules.config file follows the nf-core template, especially removed sections. + + .. note:: You can choose to ignore this lint tests by editing the file called + ``.nf-core.yml`` in the root of your pipeline and setting the test to false: + + .. code-block:: yaml + + lint: + modules_config: False + + To disable this test only for specific modules, you can specify a list of module names. + + .. code-block:: yaml + + lint: + modules_config: + - fastqc + + """ + + result = LintConfig(self.wf_path, self.lint_config).lint_file("modules_config", Path("conf", "modules.config")) + + return result + + +def base_config(self) -> Dict[str, List[str]]: + """Make sure the conf/base.config file follows the nf-core template, especially removed sections. + + .. note:: You can choose to ignore this lint tests by editing the file called + ``.nf-core.yml`` in the root of your pipeline and setting the test to false: + + .. code-block:: yaml + + lint: + base_config: False + + """ + + result = LintConfig(self.wf_path, self.lint_config).lint_file("base_config", Path("conf", "base.config")) + + return result diff --git a/nf_core/lint/files_exist.py b/nf_core/lint/files_exist.py index 5d62a23bf8..d801caf704 100644 --- a/nf_core/lint/files_exist.py +++ b/nf_core/lint/files_exist.py @@ -66,31 +66,30 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: conf/igenomes.config .github/workflows/awstest.yml .github/workflows/awsfulltest.yml - pyproject.toml Files that *must not* be present, due to being renamed or removed in the template: .. code-block:: bash - Singularity - parameters.settings.json - pipeline_template.yml # saving information in .nf-core.yml - .nf-core.yaml # NB: Should be yml, not yaml - bin/markdown_to_html.r - conf/aws.config - .github/workflows/push_dockerhub.yml .github/ISSUE_TEMPLATE/bug_report.md .github/ISSUE_TEMPLATE/feature_request.md - docs/images/nf-core-PIPELINE_logo.png + .github/workflows/push_dockerhub.yml .markdownlint.yml + .nf-core.yaml # NB: Should be yml, not yaml .yamllint.yml + bin/markdown_to_html.r + conf/aws.config + docs/images/nf-core-PIPELINE_logo.png lib/Checks.groovy lib/Completion.groovy - lib/Workflow.groovy - lib/WorkflowPIPELINE.groovy lib/NfcoreTemplate.groovy lib/Utils.groovy + lib/Workflow.groovy lib/WorkflowMain.groovy + lib/WorkflowPIPELINE.groovy + parameters.settings.json + pipeline_template.yml # saving information in .nf-core.yml + Singularity Files that *should not* be present: @@ -176,30 +175,29 @@ def files_exist(self) -> Dict[str, Union[List[str], bool]]: [Path(".github", "workflows", "awstest.yml")], [Path(".github", "workflows", "awsfulltest.yml")], [Path("modules.json")], - [Path("pyproject.toml")], ] # List of strings. Fails / warns if any of the strings exist. files_fail_ifexists = [ - Path("Singularity"), - Path("parameters.settings.json"), - Path("pipeline_template.yml"), # saving information in .nf-core.yml - Path(".nf-core.yaml"), # yml not yaml - Path("bin", "markdown_to_html.r"), - Path("conf", "aws.config"), - Path(".github", "workflows", "push_dockerhub.yml"), Path(".github", "ISSUE_TEMPLATE", "bug_report.md"), Path(".github", "ISSUE_TEMPLATE", "feature_request.md"), - Path("docs", "images", f"nf-core-{short_name}_logo.png"), + Path(".github", "workflows", "push_dockerhub.yml"), Path(".markdownlint.yml"), + Path(".nf-core.yaml"), # yml not yaml Path(".yamllint.yml"), + Path("bin", "markdown_to_html.r"), + Path("conf", "aws.config"), + Path("docs", "images", f"nf-core-{short_name}_logo.png"), Path("lib", "Checks.groovy"), Path("lib", "Completion.groovy"), - Path("lib", "Workflow.groovy"), + Path("lib", "NfcoreTemplate.groovy"), Path("lib", "Utils.groovy"), + Path("lib", "Workflow.groovy"), Path("lib", "WorkflowMain.groovy"), - Path("lib", "NfcoreTemplate.groovy"), Path("lib", f"Workflow{short_name[0].upper()}{short_name[1:]}.groovy"), + Path("parameters.settings.json"), + Path("pipeline_template.yml"), # saving information in .nf-core.yml + Path("Singularity"), ] files_warn_ifexists = [Path(".travis.yml")] files_fail_ifinconfig: List[Tuple[Path, Dict[str, str]]] = [ diff --git a/nf_core/lint/files_unchanged.py b/nf_core/lint/files_unchanged.py index 3a3a0cb74a..1cd1f7fdb1 100644 --- a/nf_core/lint/files_unchanged.py +++ b/nf_core/lint/files_unchanged.py @@ -47,7 +47,6 @@ def files_unchanged(self) -> Dict[str, Union[List[str], bool]]: .gitignore .prettierignore - pyproject.toml .. tip:: You can configure the ``nf-core lint`` tests to ignore any of these checks by setting @@ -106,7 +105,7 @@ def files_unchanged(self) -> Dict[str, Union[List[str], bool]]: [Path("docs", "README.md")], ] files_partial = [ - [Path(".gitignore"), Path(".prettierignore"), Path("pyproject.toml")], + [Path(".gitignore"), Path(".prettierignore")], ] # Only show error messages from pipeline creation diff --git a/nf_core/lint/multiqc_config.py b/nf_core/lint/multiqc_config.py index 9b9c80c44e..8b4fa2120f 100644 --- a/nf_core/lint/multiqc_config.py +++ b/nf_core/lint/multiqc_config.py @@ -1,8 +1,10 @@ -import os +from pathlib import Path from typing import Dict, List import yaml +from nf_core.lint_utils import ignore_file + def multiqc_config(self) -> Dict[str, List[str]]: """Make sure basic multiQC plugins are installed and plots are exported @@ -21,100 +23,108 @@ def multiqc_config(self) -> Dict[str, List[str]]: order: -1001 export_plots: true - """ - - passed: List[str] = [] - failed: List[str] = [] - - # Remove field that should be ignored according to the linting config - ignore_configs = self.lint_config.get("multiqc_config", []) + .. note:: You can choose to ignore this lint tests by editing the file called + ``.nf-core.yml`` in the root of your pipeline and setting the test to false: - fn = os.path.join(self.wf_path, "assets", "multiqc_config.yml") + .. code-block:: yaml - # Return a failed status if we can't find the file - if not os.path.isfile(fn): - return {"ignored": ["'assets/multiqc_config.yml' not found"]} + lint: + multiqc_config: False - try: - with open(fn) as fh: - mqc_yml = yaml.safe_load(fh) - except Exception as e: - return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} - - # check if requried sections are present - required_sections = ["report_section_order", "export_plots", "report_comment"] - for section in required_sections: - if section not in mqc_yml and section not in ignore_configs: - failed.append(f"'assets/multiqc_config.yml' does not contain `{section}`") - return {"passed": passed, "failed": failed} - else: - passed.append(f"'assets/multiqc_config.yml' contains `{section}`") - - try: - orders = {} - summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary" - min_plugins = ["software_versions", summary_plugin_name] - for plugin in min_plugins: - if plugin not in mqc_yml["report_section_order"]: - raise AssertionError(f"Section {plugin} missing in report_section_order") - if "order" not in mqc_yml["report_section_order"][plugin]: - raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0") - plugin_order = mqc_yml["report_section_order"][plugin]["order"] - if plugin_order >= 0: - raise AssertionError(f"Section {plugin} 'order' must be < 0") - - for plugin in mqc_yml["report_section_order"]: - if "order" in mqc_yml["report_section_order"][plugin]: - orders[plugin] = mqc_yml["report_section_order"][plugin]["order"] - - if orders[summary_plugin_name] != min(orders.values()): - raise AssertionError(f"Section {summary_plugin_name} should have the lowest order") - orders.pop(summary_plugin_name) - if orders["software_versions"] != min(orders.values()): - raise AssertionError("Section software_versions should have the second lowest order") - except (AssertionError, KeyError, TypeError) as e: - failed.append(f"'assets/multiqc_config.yml' does not meet requirements: {e}") - else: - passed.append("'assets/multiqc_config.yml' follows the ordering scheme of the minimally required plugins.") - - if "report_comment" not in ignore_configs: - # Check that the minimum plugins exist and are coming first in the summary - version = self.nf_config.get("manifest.version", "").strip(" '\"") - if "dev" in version: - version = "dev" - report_comments = ( - f'This report has been generated by the nf-core/{self.pipeline_name}' - f" analysis pipeline. For information about how to interpret these results, please see the " - f'documentation.' - ) + """ + passed: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + + fn = Path(self.wf_path, "assets", "multiqc_config.yml") + file_path = fn.relative_to(self.wf_path) + passed, failed, ignored, ignore_configs = ignore_file("multiqc_config", file_path, self.wf_path) + + # skip other tests if the file is not found + error_message = f"`{file_path}` not found" + # check for partial match in failed or ignored + if not any(f.startswith(error_message) for f in (failed + ignored)): + try: + with open(fn) as fh: + mqc_yml = yaml.safe_load(fh) + except Exception as e: + return {"failed": [f"Could not parse yaml file: {fn}, {e}"]} + + # check if required sections are present + required_sections = ["report_section_order", "export_plots", "report_comment"] + for section in required_sections: + if section not in mqc_yml and section not in ignore_configs: + failed.append(f"`assets/multiqc_config.yml` does not contain `{section}`") + return {"passed": passed, "failed": failed} + else: + passed.append(f"`assets/multiqc_config.yml` contains `{section}`") + + try: + orders = {} + summary_plugin_name = f"{self.pipeline_prefix}-{self.pipeline_name}-summary" + min_plugins = ["software_versions", summary_plugin_name] + for plugin in min_plugins: + if plugin not in mqc_yml["report_section_order"]: + raise AssertionError(f"Section {plugin} missing in report_section_order") + if "order" not in mqc_yml["report_section_order"][plugin]: + raise AssertionError(f"Section {plugin} 'order' missing. Must be < 0") + plugin_order = mqc_yml["report_section_order"][plugin]["order"] + if plugin_order >= 0: + raise AssertionError(f"Section {plugin} 'order' must be < 0") + + for plugin in mqc_yml["report_section_order"]: + if "order" in mqc_yml["report_section_order"][plugin]: + orders[plugin] = mqc_yml["report_section_order"][plugin]["order"] + + if orders[summary_plugin_name] != min(orders.values()): + raise AssertionError(f"Section {summary_plugin_name} should have the lowest order") + orders.pop(summary_plugin_name) + if orders["software_versions"] != min(orders.values()): + raise AssertionError("Section software_versions should have the second lowest order") + except (AssertionError, KeyError, TypeError) as e: + failed.append(f"`assets/multiqc_config.yml` does not meet requirements: {e}") else: - report_comments = ( - f'This report has been generated by the nf-core/{self.pipeline_name}' - f" analysis pipeline. For information about how to interpret these results, please see the " - f'documentation.' - ) - - if mqc_yml["report_comment"].strip() != report_comments: - # find where the report_comment is wrong and give it as a hint - hint = report_comments - failed.append( - f"'assets/multiqc_config.yml' does not contain a matching 'report_comment'. \n" - f"The expected comment is: \n" - f"```{hint}``` \n" - f"The current comment is: \n" - f"```{ mqc_yml['report_comment'].strip()}```" - ) + passed.append("`assets/multiqc_config.yml` follows the ordering scheme of the minimally required plugins.") + + if "report_comment" not in ignore_configs: + # Check that the minimum plugins exist and are coming first in the summary + version = self.nf_config.get("manifest.version", "").strip(" '\"") + if "dev" in version: + version = "dev" + report_comments = ( + f'This report has been generated by the nf-core/{self.pipeline_name}' + f" analysis pipeline. For information about how to interpret these results, please see the " + f'documentation.' + ) + + else: + report_comments = ( + f'This report has been generated by the nf-core/{self.pipeline_name}' + f" analysis pipeline. For information about how to interpret these results, please see the " + f'documentation.' + ) + + if mqc_yml["report_comment"].strip() != report_comments: + # find where the report_comment is wrong and give it as a hint + hint = report_comments + failed.append( + f"`assets/multiqc_config.yml` does not contain a matching 'report_comment'. \n" + f"The expected comment is: \n" + f"```{hint}``` \n" + f"The current comment is: \n" + f"```{ mqc_yml['report_comment'].strip()}```" + ) + else: + passed.append("`assets/multiqc_config.yml` contains a matching 'report_comment'.") + + # Check that export_plots is activated + try: + if not mqc_yml["export_plots"]: + raise AssertionError() + except (AssertionError, KeyError, TypeError): + failed.append("`assets/multiqc_config.yml` does not contain 'export_plots: true'.") else: - passed.append("'assets/multiqc_config.yml' contains a matching 'report_comment'.") - - # Check that export_plots is activated - try: - if not mqc_yml["export_plots"]: - raise AssertionError() - except (AssertionError, KeyError, TypeError): - failed.append("'assets/multiqc_config.yml' does not contain 'export_plots: true'.") - else: - passed.append("'assets/multiqc_config.yml' contains 'export_plots: true'.") - - return {"passed": passed, "failed": failed} + passed.append("`assets/multiqc_config.yml` contains 'export_plots: true'.") + + return {"passed": passed, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint/nfcore_yml.py b/nf_core/lint/nfcore_yml.py new file mode 100644 index 0000000000..f23b2f1a84 --- /dev/null +++ b/nf_core/lint/nfcore_yml.py @@ -0,0 +1,75 @@ +import re +from pathlib import Path +from typing import Dict, List + +from nf_core import __version__ + +REPOSITORY_TYPES = ["pipeline", "modules"] + + +def nfcore_yml(self) -> Dict[str, List[str]]: + """Repository ``.nf-core.yml`` tests + + The ``.nf-core.yml`` contains metadata for nf-core tools to correctly apply its features. + + * repository type: + + * Check that the repository type is set. + + * nf core version: + + * Check if the nf-core version is set to the latest version. + + """ + passed: List[str] = [] + warned: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + + # Remove field that should be ignored according to the linting config + ignore_configs = self.lint_config.get(".nf-core", []) + + try: + with open(Path(self.wf_path, ".nf-core.yml")) as fh: + content = fh.read() + except FileNotFoundError: + with open(Path(self.wf_path, ".nf-core.yaml")) as fh: + content = fh.read() + + if "repository_type" not in ignore_configs: + # Check that the repository type is set in the .nf-core.yml + repo_type_re = r"repository_type: (.+)" + match = re.search(repo_type_re, content) + if match: + repo_type = match.group(1) + if repo_type not in REPOSITORY_TYPES: + failed.append( + f"Repository type in `.nf-core.yml` is not valid. " + f"Should be one of `[{', '.join(REPOSITORY_TYPES)}]` but was `{repo_type}`" + ) + else: + passed.append(f"Repository type in `.nf-core.yml` is valid: `{repo_type}`") + else: + warned.append("Repository type not set in `.nf-core.yml`") + else: + ignored.append("`.nf-core.yml` variable ignored 'repository_type'") + + if "nf_core_version" not in ignore_configs: + # Check that the nf-core version is set in the .nf-core.yml + nf_core_version_re = r"nf_core_version: (.+)" + match = re.search(nf_core_version_re, content) + if match: + nf_core_version = match.group(1).strip('"') + if nf_core_version != __version__ and "dev" not in nf_core_version: + warned.append( + f"nf-core version in `.nf-core.yml` is not set to the latest version. " + f"Should be `{__version__}` but was `{nf_core_version}`" + ) + else: + passed.append(f"nf-core version in `.nf-core.yml` is set to the latest version: `{nf_core_version}`") + else: + warned.append("nf-core version not set in `.nf-core.yml`") + else: + ignored.append("`.nf-core.yml` variable ignored 'nf_core_version'") + + return {"passed": passed, "warned": warned, "failed": failed, "ignored": ignored} diff --git a/nf_core/lint_utils.py b/nf_core/lint_utils.py index 6eca6522d4..167600bfc0 100644 --- a/nf_core/lint_utils.py +++ b/nf_core/lint_utils.py @@ -2,6 +2,7 @@ import logging import subprocess from pathlib import Path +from typing import List import rich from rich.console import Console @@ -101,3 +102,30 @@ def dump_json_with_prettier(file_name, file_content): with open(file_name, "w") as fh: json.dump(file_content, fh, indent=4) run_prettier_on_file(file_name) + + +def ignore_file(lint_name: str, file_path: Path, dir_path: Path) -> List[List[str]]: + """Ignore a file and add the result to the ignored list. Return the passed, failed, ignored and ignore_configs lists.""" + + passed: List[str] = [] + failed: List[str] = [] + ignored: List[str] = [] + _, lint_conf = nf_core.utils.load_tools_config(dir_path) + lint_conf = lint_conf.get("lint", {}) + ignore_entry: List[str] | bool = lint_conf.get(lint_name, []) + full_path = dir_path / file_path + # Return a failed status if we can't find the file + if not full_path.is_file(): + if isinstance(ignore_entry, bool) and not ignore_entry: + ignored.append(f"`{file_path}` not found, but it is ignored.") + ignore_entry = [] + else: + failed.append(f"`{file_path}` not found.") + else: + passed.append(f"`{file_path}` found and not ignored.") + + # we handled the only case where ignore_entry should be a bool, convert it to a list, to make downstream code easier + if isinstance(ignore_entry, bool): + ignore_entry = [] + + return [passed, failed, ignored, ignore_entry] diff --git a/nf_core/module-template/tests/main.nf.test b/nf_core/module-template/tests/main.nf.test.j2 similarity index 100% rename from nf_core/module-template/tests/main.nf.test rename to nf_core/module-template/tests/main.nf.test.j2 diff --git a/nf_core/modules/lint/module_tests.py b/nf_core/modules/lint/module_tests.py index b1a611d70a..b2b6c2221f 100644 --- a/nf_core/modules/lint/module_tests.py +++ b/nf_core/modules/lint/module_tests.py @@ -107,6 +107,22 @@ def module_tests(_, module: NFCoreComponent): snap_file, ) ) + if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): + module.passed.append( + ( + "test_snap_versions", + "versions found in snapshot file", + snap_file, + ) + ) + else: + module.failed.append( + ( + "test_snap_versions", + "versions not found in snapshot file", + snap_file, + ) + ) except json.decoder.JSONDecodeError as e: module.failed.append( ( diff --git a/nf_core/modules/modules_json.py b/nf_core/modules/modules_json.py index f68c27b2d8..7d78268e92 100644 --- a/nf_core/modules/modules_json.py +++ b/nf_core/modules/modules_json.py @@ -6,7 +6,6 @@ import shutil import tempfile from pathlib import Path -from typing import Union import git import questionary @@ -32,7 +31,7 @@ class ModulesJson: An object for handling a 'modules.json' file in a pipeline """ - def __init__(self, pipeline_dir): + def __init__(self, pipeline_dir: str): """ Initialise the object. @@ -43,7 +42,7 @@ def __init__(self, pipeline_dir): self.modules_dir = Path(self.dir, "modules") self.subworkflows_dir = Path(self.dir, "subworkflows") self.modules_json_path = Path(self.dir, "modules.json") - self.modules_json: Union(dict, None) = None + self.modules_json = None self.pipeline_modules = None self.pipeline_subworkflows = None self.pipeline_components = None @@ -1051,17 +1050,18 @@ def get_component_branch(self, component_type, component, repo_url, install_dir) ) return branch - def dump(self, run_prettier: bool = False): + def dump(self, run_prettier: bool = False) -> None: """ Sort the modules.json, and write it to file """ - # Sort the modules.json - self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) - if run_prettier: - dump_json_with_prettier(self.modules_json_path, self.modules_json) - else: - with open(self.modules_json_path, "w") as fh: - json.dump(self.modules_json, fh, indent=4) + if self.modules_json is not None: + # Sort the modules.json + self.modules_json["repos"] = nf_core.utils.sort_dictionary(self.modules_json["repos"]) + if run_prettier: + dump_json_with_prettier(self.modules_json_path, self.modules_json) + else: + with open(self.modules_json_path, "w") as fh: + json.dump(self.modules_json, fh, indent=4) def resolve_missing_installation(self, missing_installation, component_type): missing_but_in_mod_json = [ diff --git a/nf_core/modules/modules_utils.py b/nf_core/modules/modules_utils.py index ca8993483b..6796de41ec 100644 --- a/nf_core/modules/modules_utils.py +++ b/nf_core/modules/modules_utils.py @@ -20,20 +20,19 @@ def repo_full_name_from_remote(remote_url: str) -> str: Extracts the path from the remote URL See https://mirrors.edge.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS for the possible URL patterns """ - # Check whether we have a https or ssh url - if remote_url.startswith("https"): - path = urlparse(remote_url).path - # Remove the intial '/' - path = path[1:] - # Remove extension - path = os.path.splitext(path)[0] + + if remote_url.startswith(("https://", "http://", "ftps://", "ftp://", "ssh://")): + # Parse URL and remove the initial '/' + path = urlparse(remote_url).path.lstrip("/") + elif "git@" in remote_url: + # Extract the part after 'git@' and parse it + path = urlparse(remote_url.split("git@")[-1]).path else: - # Remove the initial `git@`` - split_path: list = remote_url.split("@") - path = split_path[-1] if len(split_path) > 1 else split_path[0] - path = urlparse(path).path - # Remove extension - path = os.path.splitext(path)[0] + path = urlparse(remote_url).path + + # Remove the file extension from the path + path, _ = os.path.splitext(path) + return path diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 3880b2c4dc..6b2547765d 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -28,7 +28,7 @@ jobs: - "latest-everything" steps: - name: Check out pipeline code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v2 diff --git a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml index 44d06c0cd5..ebea16c5cb 100644 --- a/nf_core/pipeline-template/.github/workflows/download_pipeline.yml +++ b/nf_core/pipeline-template/.github/workflows/download_pipeline.yml @@ -35,9 +35,9 @@ jobs: - name: Disk space cleanup uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: "3.11" + python-version: "3.12" architecture: "x64" - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 with: diff --git a/nf_core/pipeline-template/.github/workflows/fix-linting.yml b/nf_core/pipeline-template/.github/workflows/fix-linting.yml index 28e6605b96..18e6f9e158 100644 --- a/nf_core/pipeline-template/.github/workflows/fix-linting.yml +++ b/nf_core/pipeline-template/.github/workflows/fix-linting.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} @@ -32,9 +32,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} # Install and run pre-commit - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" - name: Install pre-commit run: pip install pre-commit diff --git a/nf_core/pipeline-template/.github/workflows/linting.yml b/nf_core/pipeline-template/.github/workflows/linting.yml index 612467ff6e..d1ecae12b7 100644 --- a/nf_core/pipeline-template/.github/workflows/linting.yml +++ b/nf_core/pipeline-template/.github/workflows/linting.yml @@ -14,12 +14,12 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - name: Set up Python 3.12 + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: 3.11 + python-version: "3.12" cache: "pip" - name: Install pre-commit @@ -32,14 +32,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v2 - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: - python-version: "3.11" + python-version: "3.12" architecture: "x64" - name: Install dependencies @@ -60,7 +60,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4 with: name: linting-logs path: | diff --git a/nf_core/pipeline-template/.github/workflows/linting_comment.yml b/nf_core/pipeline-template/.github/workflows/linting_comment.yml index 67ef7b534d..ea408fd6f8 100644 --- a/nf_core/pipeline-template/.github/workflows/linting_comment.yml +++ b/nf_core/pipeline-template/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@a430ac5786b39ad5869da25a98130624d2ce340c # v3 + uses: dawidd6/action-download-artifact@09f2f74827fd3a8607589e5ad7f9398816f540fe # v3 with: workflow: linting.yml workflow_conclusion: completed diff --git a/nf_core/pipeline-template/.github/workflows/release-announcements.yml b/nf_core/pipeline-template/.github/workflows/release-announcements.yml index 2c57060257..8fee061fdd 100644 --- a/nf_core/pipeline-template/.github/workflows/release-announcements.yml +++ b/nf_core/pipeline-template/.github/workflows/release-announcements.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5 with: python-version: "3.10" - name: Install dependencies diff --git a/nf_core/pipeline-template/.nf-core.yml b/nf_core/pipeline-template/.nf-core.yml index 3805dc81c1..e8140bfb12 100644 --- a/nf_core/pipeline-template/.nf-core.yml +++ b/nf_core/pipeline-template/.nf-core.yml @@ -1 +1,2 @@ repository_type: pipeline +nf_core_version: "{{ nf_core_version }}" diff --git a/nf_core/pipeline-template/conf/base.config b/nf_core/pipeline-template/conf/base.config index f73c5afaa4..9c62bf0634 100644 --- a/nf_core/pipeline-template/conf/base.config +++ b/nf_core/pipeline-template/conf/base.config @@ -59,7 +59,4 @@ process { errorStrategy = 'retry' maxRetries = 2 } - withName:CUSTOM_DUMPSOFTWAREVERSIONS { - cache = false - } } diff --git a/nf_core/pipeline-template/conf/modules.config b/nf_core/pipeline-template/conf/modules.config index e3ea8fa6c4..d203d2b6e6 100644 --- a/nf_core/pipeline-template/conf/modules.config +++ b/nf_core/pipeline-template/conf/modules.config @@ -22,14 +22,6 @@ process { ext.args = '--quiet' } - withName: CUSTOM_DUMPSOFTWAREVERSIONS { - publishDir = [ - path: { "${params.outdir}/pipeline_info" }, - mode: params.publish_dir_mode, - pattern: '*_versions.yml' - ] - } - withName: 'MULTIQC' { ext.args = { params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' } publishDir = [ diff --git a/nf_core/pipeline-template/docs/usage.md b/nf_core/pipeline-template/docs/usage.md index c908d3d38c..d46dfca04c 100644 --- a/nf_core/pipeline-template/docs/usage.md +++ b/nf_core/pipeline-template/docs/usage.md @@ -163,6 +163,8 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to be used with [Charliecloud](https://hpc.github.io/charliecloud/) - `apptainer` - A generic configuration profile to be used with [Apptainer](https://apptainer.org/) +- `wave` + - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above (requires Nextflow ` 24.03.0-edge` or later). - `conda` - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. diff --git a/nf_core/pipeline-template/modules.json b/nf_core/pipeline-template/modules.json index 38aa622063..9137c59675 100644 --- a/nf_core/pipeline-template/modules.json +++ b/nf_core/pipeline-template/modules.json @@ -7,7 +7,7 @@ "nf-core": { "fastqc": { "branch": "master", - "git_sha": "f4ae1d942bd50c5c0b9bd2de1393ce38315ba57c", + "git_sha": "285a50500f9e02578d90b3ce6382ea3c30216acd", "installed_by": ["modules"] }, "multiqc": { @@ -26,7 +26,7 @@ }, "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "git_sha": "92de218a329bfc9a9033116eb5f65fd270e72ba3", "installed_by": ["subworkflows"] }, "utils_nfvalidation_plugin": { diff --git a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf index 9e19a74c56..d79f1c862d 100644 --- a/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf +++ b/nf_core/pipeline-template/modules/nf-core/fastqc/main.nf @@ -25,6 +25,11 @@ process FASTQC { def old_new_pairs = reads instanceof Path || reads.size() == 1 ? [[ reads, "${prefix}.${reads.extension}" ]] : reads.withIndex().collect { entry, index -> [ entry, "${prefix}_${index + 1}.${entry.extension}" ] } def rename_to = old_new_pairs*.join(' ').join(' ') def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ') + + def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') + // FastQC memory value allowed range (100 - 10000) + def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb) + """ printf "%s %s\\n" $rename_to | while read old_name new_name; do [ -f "\${new_name}" ] || ln -s \$old_name \$new_name @@ -33,6 +38,7 @@ process FASTQC { fastqc \\ $args \\ --threads $task.cpus \\ + --memory $fastqc_memory \\ $renamed_files cat <<-END_VERSIONS > versions.yml diff --git a/nf_core/pipeline-template/nextflow.config b/nf_core/pipeline-template/nextflow.config index 2f9b349916..936e5e80d7 100644 --- a/nf_core/pipeline-template/nextflow.config +++ b/nf_core/pipeline-template/nextflow.config @@ -76,105 +76,111 @@ try { } // Load {{ name }} custom profiles from different institutions. -// Warning: Uncomment only if a pipeline-specific institutional config already exists on nf-core/configs! -// try { -// includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" -// } catch (Exception e) { -// System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") -// } +try { + includeConfig "${params.custom_config_base}/pipeline/{{ short_name }}.config" +} catch (Exception e) { + System.err.println("WARNING: Could not load nf-core/config/{{ short_name }} profiles: ${params.custom_config_base}/pipeline/{{ short_name }}.config") +} {% endif -%} profiles { debug { - dumpHashes = true - process.beforeScript = 'echo $HOSTNAME' - cleanup = false + dumpHashes = true + process.beforeScript = 'echo $HOSTNAME' + cleanup = false nextflow.enable.configProcessNamesValidation = true } conda { - conda.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - channels = ['conda-forge', 'bioconda', 'defaults'] - apptainer.enabled = false + conda.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + conda.channels = ['conda-forge', 'bioconda', 'defaults'] + apptainer.enabled = false } mamba { - conda.enabled = true - conda.useMamba = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + conda.enabled = true + conda.useMamba = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } docker { - docker.enabled = true - conda.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false - docker.runOptions = '-u $(id -u):$(id -g)' + docker.enabled = true + conda.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false + docker.runOptions = '-u $(id -u):$(id -g)' } arm { - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } singularity { - singularity.enabled = true - singularity.autoMounts = true - conda.enabled = false - docker.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + singularity.enabled = true + singularity.autoMounts = true + conda.enabled = false + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } podman { - podman.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - shifter.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + podman.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } shifter { - shifter.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - charliecloud.enabled = false - apptainer.enabled = false + shifter.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + apptainer.enabled = false } charliecloud { - charliecloud.enabled = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - apptainer.enabled = false + charliecloud.enabled = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + apptainer.enabled = false } apptainer { - apptainer.enabled = true - apptainer.autoMounts = true - conda.enabled = false - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false + apptainer.enabled = true + apptainer.autoMounts = true + conda.enabled = false + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + wave { + apptainer.ociAutoPull = true + singularity.ociAutoPull = true + wave.enabled = true + wave.freeze = true + wave.strategy = 'conda,container' } gitpod { - executor.name = 'local' - executor.cpus = 4 - executor.memory = 8.GB + executor.name = 'local' + executor.cpus = 4 + executor.memory = 8.GB } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } diff --git a/nf_core/pipeline-template/pyproject.toml b/nf_core/pipeline-template/pyproject.toml deleted file mode 100644 index 56110621e7..0000000000 --- a/nf_core/pipeline-template/pyproject.toml +++ /dev/null @@ -1,15 +0,0 @@ -# Config file for Python. Mostly used to configure linting of bin/*.py with Ruff. -# Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. -[tool.ruff] -line-length = 120 -target-version = "py38" -cache-dir = "~/.cache/ruff" - -[tool.ruff.lint] -select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] - -[tool.ruff.lint.isort] -known-first-party = ["nf_core"] - -[tool.ruff.lint.per-file-ignores] -"__init__.py" = ["E402", "F401"] diff --git a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf index e02a113755..a4bfb9f8be 100644 --- a/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/local/utils_nfcore_pipeline_pipeline/main.nf @@ -243,8 +243,16 @@ def methodsDescriptionText(mqc_methods_yaml) { meta["manifest_map"] = workflow.manifest.toMap() // Pipeline DOI - meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" - meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + if (meta.manifest_map.doi) { + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list + def temp_doi_ref = "" + String[] manifest_doi = meta.manifest_map.doi.tokenize(",") + for (String doi_ref: manifest_doi) temp_doi_ref += "(doi: ${doi_ref.replace("https://doi.org/", "").replace(" ", "")}), " + meta["doi_text"] = temp_doi_ref.substring(0, temp_doi_ref.length() - 2) + } else meta["doi_text"] = "" + meta["nodoi_text"] = meta.manifest_map.doi ? "" : "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " // Tool references meta["tool_citations"] = "" diff --git a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf index a8b55d6fe1..14558c3927 100644 --- a/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf +++ b/nf_core/pipeline-template/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -65,9 +65,15 @@ def checkProfileProvided(nextflow_cli_args) { // Citation string for pipeline // def workflowCitation() { + def temp_doi_ref = "" + String[] manifest_doi = workflow.manifest.doi.tokenize(",") + // Using a loop to handle multiple DOIs + // Removing `https://doi.org/` to handle pipelines using DOIs vs DOI resolvers + // Removing ` ` since the manifest.doi is a string and not a proper list + for (String doi_ref: manifest_doi) temp_doi_ref += " https://doi.org/${doi_ref.replace('https://doi.org/', '').replace(' ', '')}\n" return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + "* The pipeline\n" + - " ${workflow.manifest.doi}\n\n" + + temp_doi_ref + "\n" + "* The nf-core framework\n" + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + "* Software dependencies\n" + diff --git a/nf_core/pipeline-template/workflows/pipeline.nf b/nf_core/pipeline-template/workflows/pipeline.nf index 68adbaa328..de0f21fe38 100644 --- a/nf_core/pipeline-template/workflows/pipeline.nf +++ b/nf_core/pipeline-template/workflows/pipeline.nf @@ -40,22 +40,44 @@ workflow {{ short_name|upper }} { // Collate and save software versions // softwareVersionsToYAML(ch_versions) - .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_pipeline_software_mqc_versions.yml', sort: true, newLine: true) - .set { ch_collated_versions } + .collectFile( + storeDir: "${params.outdir}/pipeline_info", + name: 'nf_core_pipeline_software_mqc_versions.yml', + sort: true, + newLine: true + ).set { ch_collated_versions } // // MODULE: MultiQC // - ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) - ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config, checkIfExists: true) : Channel.empty() - ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() - summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") - ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) - ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) - ch_methods_description = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) - ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) - ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) - ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: false)) + ch_multiqc_config = Channel.fromPath( + "$projectDir/assets/multiqc_config.yml", checkIfExists: true) + ch_multiqc_custom_config = params.multiqc_config ? + Channel.fromPath(params.multiqc_config, checkIfExists: true) : + Channel.empty() + ch_multiqc_logo = params.multiqc_logo ? + Channel.fromPath(params.multiqc_logo, checkIfExists: true) : + Channel.empty() + + summary_params = paramsSummaryMap( + workflow, parameters_schema: "nextflow_schema.json") + ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) + + ch_multiqc_custom_methods_description = params.multiqc_methods_description ? + file(params.multiqc_methods_description, checkIfExists: true) : + file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) + ch_methods_description = Channel.value( + methodsDescriptionText(ch_multiqc_custom_methods_description)) + + ch_multiqc_files = ch_multiqc_files.mix( + ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + ch_multiqc_files = ch_multiqc_files.mix( + ch_methods_description.collectFile( + name: 'methods_description_mqc.yaml', + sort: true + ) + ) MULTIQC ( ch_multiqc_files.collect(), diff --git a/nf_core/schema.py b/nf_core/schema.py index 373f8bbaa1..4f5acfa0af 100644 --- a/nf_core/schema.py +++ b/nf_core/schema.py @@ -6,6 +6,7 @@ import tempfile import webbrowser from pathlib import Path +from typing import Union import jinja2 import jsonschema @@ -46,11 +47,14 @@ def __init__(self): self.web_schema_build_web_url = None self.web_schema_build_api_url = None - def get_schema_path(self, path, local_only=False, revision=None): + def get_schema_path( + self, path: Union[str, Path], local_only: bool = False, revision: Union[str, None] = None + ) -> None: """Given a pipeline name, directory, or path, set self.schema_filename""" path = Path(path) # Supplied path exists - assume a local pipeline directory or schema if path.exists(): + log.debug(f"Path exists: {path}. Assuming local pipeline directory or schema") if revision is not None: log.warning(f"Local workflow supplied, ignoring revision '{revision}'") if path.is_dir(): @@ -63,14 +67,16 @@ def get_schema_path(self, path, local_only=False, revision=None): # Path does not exist - assume a name of a remote workflow elif not local_only: self.pipeline_dir = nf_core.list.get_local_wf(path, revision=revision) - self.schema_filename = Path(self.pipeline_dir, "nextflow_schema.json") - + self.schema_filename = Path(self.pipeline_dir or "", "nextflow_schema.json") + # check if the schema file exists + if not self.schema_filename.exists(): + self.schema_filename = None # Only looking for local paths, overwrite with None to be safe else: self.schema_filename = None # Check that the schema file exists - if self.schema_filename is None or not Path(self.schema_filename).exists(): + if self.schema_filename is None or not Path(self.schema_filename).exists() and local_only: error = f"Could not find pipeline schema for '{path}': {self.schema_filename}" log.error(error) raise AssertionError(error) @@ -103,7 +109,7 @@ def load_lint_schema(self): def load_schema(self): """Load a pipeline schema from a file""" - if self.schema_filename is None: + if self.schema_filename is None or not Path(self.schema_filename).exists(): raise AssertionError("Pipeline schema filename could not be found.") with open(self.schema_filename) as fh: @@ -494,7 +500,7 @@ def print_documentation( if not output_fn: console = rich.console.Console() - console.print("\n", Syntax(prettified_docs, format), "\n") + console.print("\n", Syntax(prettified_docs, format, word_wrap=True), "\n") else: if Path(output_fn).exists() and not force: log.error(f"File '{output_fn}' exists! Please delete first, or use '--force'") @@ -581,10 +587,12 @@ def make_skeleton_schema(self): loader=jinja2.PackageLoader("nf_core", "pipeline-template"), keep_trailing_newline=True ) schema_template = env.get_template("nextflow_schema.json") + template_vars = { - "name": self.pipeline_manifest.get("name", Path(self.schema_filename).parent).strip("'"), + "name": self.pipeline_manifest.get("name", Path(self.schema_filename).parent.name).strip("'"), "description": self.pipeline_manifest.get("description", "").strip("'"), } + self.schema = json.loads(schema_template.render(template_vars)) self.get_schema_defaults() @@ -800,7 +808,7 @@ def add_schema_found_configs(self): p_def := self.build_schema_param(p_val).get("default") ): if self.no_prompts or Confirm.ask( - f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{s_def}' | config: '{p_def}'). " + f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{type(s_def)}: {s_def}' | config: '{type(p_def)}: {p_def}'). " "[blue]Update pipeline schema?" ): s_key_def = s_key + ("default",) diff --git a/nf_core/subworkflow-template/tests/main.nf.test b/nf_core/subworkflow-template/tests/main.nf.test.j2 similarity index 100% rename from nf_core/subworkflow-template/tests/main.nf.test rename to nf_core/subworkflow-template/tests/main.nf.test.j2 diff --git a/nf_core/subworkflows/lint/subworkflow_tests.py b/nf_core/subworkflows/lint/subworkflow_tests.py index 796a56d018..cfae2d553c 100644 --- a/nf_core/subworkflows/lint/subworkflow_tests.py +++ b/nf_core/subworkflows/lint/subworkflow_tests.py @@ -20,7 +20,7 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): It verifies that the test directory exists and contains a ``main.nf.test`` a ``main.nf.test.snap`` and ``tags.yml``. - Additionally, hecks that all included components in test ``main.nf`` are specified in ``test.yml`` + Additionally, checks that all included components in test ``main.nf`` are specified in ``test.yml`` """ repo_dir = subworkflow.component_dir.parts[ @@ -114,6 +114,22 @@ def subworkflow_tests(_, subworkflow: NFCoreComponent): snap_file, ) ) + if "versions" in str(snap_content[test_name]) or "versions" in str(snap_content.keys()): + subworkflow.passed.append( + ( + "test_snap_versions", + "versions found in snapshot file", + snap_file, + ) + ) + else: + subworkflow.warned.append( + ( + "test_snap_versions", + "versions not found in snapshot file", + snap_file, + ) + ) except json.decoder.JSONDecodeError as e: subworkflow.failed.append( ( diff --git a/nf_core/utils.py b/nf_core/utils.py index 84b653ffe2..6af2f25165 100644 --- a/nf_core/utils.py +++ b/nf_core/utils.py @@ -213,10 +213,14 @@ def is_pipeline_directory(wf_path): for fn in ["main.nf", "nextflow.config"]: path = os.path.join(wf_path, fn) if not os.path.isfile(path): - raise UserWarning(f"'{wf_path}' is not a pipeline - '{fn}' is missing") + if wf_path == ".": + warning = f"Current directory is not a pipeline - '{fn}' is missing." + else: + warning = f"'{wf_path}' is not a pipeline - '{fn}' is missing." + raise UserWarning(warning) -def fetch_wf_config(wf_path, cache_config=True): +def fetch_wf_config(wf_path: str, cache_config: bool = True) -> dict: """Uses Nextflow to retrieve the the configuration variables from a Nextflow workflow. @@ -236,13 +240,13 @@ def fetch_wf_config(wf_path, cache_config=True): cache_path = None # Nextflow home directory - use env var if set, or default to ~/.nextflow - nxf_home = os.environ.get("NXF_HOME", os.path.join(os.getenv("HOME"), ".nextflow")) + nxf_home = Path(os.environ.get("NXF_HOME", Path(os.getenv("HOME") or "", ".nextflow"))) # Build a cache directory if we can - if os.path.isdir(nxf_home): - cache_basedir = os.path.join(nxf_home, "nf-core") - if not os.path.isdir(cache_basedir): - os.mkdir(cache_basedir) + if nxf_home.is_dir(): + cache_basedir = Path(nxf_home, "nf-core") + if not cache_basedir.is_dir(): + cache_basedir.mkdir(parents=True, exist_ok=True) # If we're given a workflow object with a commit, see if we have a cached copy cache_fn = None @@ -250,7 +254,7 @@ def fetch_wf_config(wf_path, cache_config=True): concat_hash = "" for fn in ["nextflow.config", "main.nf"]: try: - with open(os.path.join(wf_path, fn), "rb") as fh: + with open(Path(wf_path, fn), "rb") as fh: concat_hash += hashlib.sha256(fh.read()).hexdigest() except FileNotFoundError: pass @@ -260,8 +264,8 @@ def fetch_wf_config(wf_path, cache_config=True): cache_fn = f"wf-config-cache-{bighash[:25]}.json" if cache_basedir and cache_fn: - cache_path = os.path.join(cache_basedir, cache_fn) - if os.path.isfile(cache_path) and cache_config is True: + cache_path = Path(cache_basedir, cache_fn) + if cache_path.is_file() and cache_config is True: log.debug(f"Found a config cache, loading: {cache_path}") with open(cache_path) as fh: try: @@ -286,10 +290,11 @@ def fetch_wf_config(wf_path, cache_config=True): # Scrape main.nf for additional parameter declarations # Values in this file are likely to be complex, so don't both trying to capture them. Just get the param name. try: - main_nf = os.path.join(wf_path, "main.nf") - with open(main_nf) as fh: + main_nf = Path(wf_path, "main.nf") + with open(main_nf, "rb") as fh: for line in fh: - match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=(?!=)", line) + line_str = line.decode("utf-8") + match = re.match(r"^\s*(params\.[a-zA-Z0-9_]+)\s*=(?!=)", line_str) if match: config[match.group(1)] = "null" except FileNotFoundError as e: @@ -454,6 +459,7 @@ def __init__(self): # pylint: disable=super-init-not-called self.auth_mode = None self.return_ok = [200, 201] self.return_retry = [403] + self.return_unauthorised = [401] self.has_init = False def lazy_init(self): @@ -546,6 +552,8 @@ def safe_get(self, url): raise e else: return r + elif request.status_code in self.return_unauthorised: + raise RuntimeError("GitHub API PR failed, probably due to an expired GITHUB_TOKEN.") return request @@ -1009,7 +1017,7 @@ def get_repo_releases_branches(pipeline, wfs): DEPRECATED_CONFIG_PATHS = [".nf-core-lint.yml", ".nf-core-lint.yaml"] -def load_tools_config(directory: Union[str, Path] = "."): +def load_tools_config(directory: Union[str, Path] = ".") -> Tuple[Path, dict]: """ Parse the nf-core.yml configuration file @@ -1037,7 +1045,6 @@ def load_tools_config(directory: Union[str, Path] = "."): with open(config_fn) as fh: tools_config = yaml.safe_load(fh) - # If the file is empty tools_config = tools_config or {} @@ -1047,12 +1054,13 @@ def load_tools_config(directory: Union[str, Path] = "."): def determine_base_dir(directory="."): base_dir = start_dir = Path(directory).absolute() - while base_dir != base_dir.parent: + # Only iterate up the tree if the start dir doesn't have a config + while not get_first_available_path(base_dir, CONFIG_PATHS) and base_dir != base_dir.parent: base_dir = base_dir.parent config_fn = get_first_available_path(base_dir, CONFIG_PATHS) if config_fn: - return directory if base_dir == start_dir else base_dir - return directory + break + return directory if base_dir == start_dir else base_dir def get_first_available_path(directory, paths): diff --git a/tests/components/generate_snapshot.py b/tests/components/generate_snapshot.py index 50024a8ebb..3176569ec8 100644 --- a/tests/components/generate_snapshot.py +++ b/tests/components/generate_snapshot.py @@ -120,9 +120,12 @@ def test_test_not_found(self): remote_url=GITLAB_URL, branch=GITLAB_NFTEST_BRANCH, ) + test_file = Path("modules", "nf-core-test", "fastp", "tests", "main.nf.test") + test_file.rename(test_file.parent / "main.nf.test.bak") with pytest.raises(UserWarning) as e: snap_generator.run() assert "Test file 'main.nf.test' not found" in str(e.value) + Path(test_file.parent / "main.nf.test.bak").rename(test_file) def test_unstable_snapshot(self): diff --git a/tests/lint/configs.py b/tests/lint/configs.py new file mode 100644 index 0000000000..b50a1393aa --- /dev/null +++ b/tests/lint/configs.py @@ -0,0 +1,89 @@ +from pathlib import Path + +import yaml + +import nf_core.create +import nf_core.lint + + +def test_withname_in_modules_config(self): + """Tests finding withName in modules.config passes linting.""" + + new_pipeline = self._make_pipeline_copy() + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.modules_config() + assert len(result["failed"]) == 0 + assert any( + ["`FASTQC` found in `conf/modules.config` and Nextflow scripts." in passed for passed in result["passed"]] + ) + + +def test_superfluous_withname_in_modules_config_fails(self): + """Tests finding withName in modules.config fails linting.""" + new_pipeline = self._make_pipeline_copy() + # Add withName to modules.config + modules_config = Path(new_pipeline) / "conf" / "modules.config" + with open(modules_config, "a") as f: + f.write("\nwithName: 'BPIPE' {\n cache = false \n}") + lint_obj = nf_core.lint.PipelineLint(new_pipeline, hide_progress=False) + lint_obj._load() + result = lint_obj.modules_config() + assert len(result["failed"]) == 1 + assert result["failed"][0].startswith("`conf/modules.config` contains `withName:BPIPE`") + + +def test_ignore_modules_config(self): + """Tests ignoring the modules.config passes linting.""" + new_pipeline = self._make_pipeline_copy() + # ignore modules.config in linting + with open(Path(new_pipeline) / ".nf-core.yml") as f: + content = yaml.safe_load(f) + old_content = content.copy() + content["lint"] = {"modules_config": False} + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(content, f) + Path(new_pipeline, "conf", "modules.config").unlink() + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.modules_config() + assert len(result["ignored"]) == 1 + assert result["ignored"][0].startswith("`conf/modules.config` not found, but it is ignored.") + # cleanup + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(old_content, f) + + +def test_superfluous_withname_in_base_config_fails(self): + """Tests finding withName in base.config fails linting.""" + new_pipeline = self._make_pipeline_copy() + # Add withName to base.config + base_config = Path(new_pipeline) / "conf" / "base.config" + with open(base_config, "a") as f: + f.write("\nwithName:CUSTOM_DUMPSOFTWAREVERSIONS {\n cache = false \n}") + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.base_config() + assert len(result["failed"]) == 1 + assert result["failed"][0].startswith("`conf/base.config` contains `withName:CUSTOM_DUMPSOFTWAREVERSIONS`") + + +def test_ignore_base_config(self): + """Tests ignoring the base.config passes linting.""" + new_pipeline = self._make_pipeline_copy() + # ignore base.config in linting + with open(Path(new_pipeline) / ".nf-core.yml") as f: + content = yaml.safe_load(f) + old_content = content.copy() + content["lint"] = {"base_config": False} + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(content, f) + Path(new_pipeline, "conf", "base.config").unlink() + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.base_config() + assert len(result["ignored"]) == 1 + assert result["ignored"][0].startswith("`conf/base.config` not found, but it is ignored.") + # cleanup + with open(Path(new_pipeline) / ".nf-core.yml", "w") as f: + yaml.dump(old_content, f) diff --git a/tests/lint/multiqc_config.py b/tests/lint/multiqc_config.py index 721560ce81..463d5e7654 100644 --- a/tests/lint/multiqc_config.py +++ b/tests/lint/multiqc_config.py @@ -5,14 +5,37 @@ import nf_core.lint -def test_multiqc_config_exists_ignore(self): +def test_multiqc_config_exists(self): """Test that linting fails if the multiqc_config.yml file is missing""" # Delete the file new_pipeline = self._make_pipeline_copy() Path(Path(new_pipeline, "assets", "multiqc_config.yml")).unlink() lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() result = lint_obj.multiqc_config() - assert result["ignored"] == ["'assets/multiqc_config.yml' not found"] + assert result["failed"] == ["`assets/multiqc_config.yml` not found."] + + +def test_multiqc_config_ignore(self): + """Test that linting succeeds if the multiqc_config.yml file is missing but ignored""" + # Delete the file + new_pipeline = self._make_pipeline_copy() + Path(Path(new_pipeline, "assets", "multiqc_config.yml")).unlink() + with open(Path(new_pipeline, ".nf-core.yml")) as f: + content = yaml.safe_load(f) + old_content = content.copy() + content["lint"] = {"multiqc_config": False} + with open(Path(new_pipeline, ".nf-core.yml"), "w") as f: + yaml.dump(content, f) + + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + result = lint_obj.multiqc_config() + assert result["ignored"] == ["`assets/multiqc_config.yml` not found, but it is ignored."] + + # cleanup + with open(Path(new_pipeline, ".nf-core.yml"), "w") as f: + yaml.dump(old_content, f) def test_multiqc_config_missing_report_section_order(self): @@ -20,7 +43,7 @@ def test_multiqc_config_missing_report_section_order(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() mqc_yml.pop("report_section_order") with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) @@ -30,7 +53,7 @@ def test_multiqc_config_missing_report_section_order(self): # Reset the file with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) - assert result["failed"] == ["'assets/multiqc_config.yml' does not contain `report_section_order`"] + assert result["failed"] == ["`assets/multiqc_config.yml` does not contain `report_section_order`"] def test_multiqc_incorrect_export_plots(self): @@ -38,7 +61,7 @@ def test_multiqc_incorrect_export_plots(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() mqc_yml["export_plots"] = False with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) @@ -48,7 +71,7 @@ def test_multiqc_incorrect_export_plots(self): # Reset the file with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) - assert result["failed"] == ["'assets/multiqc_config.yml' does not contain 'export_plots: true'."] + assert result["failed"] == ["`assets/multiqc_config.yml` does not contain 'export_plots: true'."] def test_multiqc_config_report_comment_fail(self): @@ -56,7 +79,7 @@ def test_multiqc_config_report_comment_fail(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() mqc_yml["report_comment"] = "This is a test" with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) @@ -67,7 +90,7 @@ def test_multiqc_config_report_comment_fail(self): with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) assert len(result["failed"]) == 1 - assert result["failed"][0].startswith("'assets/multiqc_config.yml' does not contain a matching 'report_comment'.") + assert result["failed"][0].startswith("`assets/multiqc_config.yml` does not contain a matching 'report_comment'.") def test_multiqc_config_report_comment_release_fail(self): @@ -75,7 +98,7 @@ def test_multiqc_config_report_comment_release_fail(self): new_pipeline = self._make_pipeline_copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml")) as fh: mqc_yml = yaml.safe_load(fh) - mqc_yml_tmp = mqc_yml + mqc_yml_tmp = mqc_yml.copy() with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml, fh) lint_obj = nf_core.lint.PipelineLint(new_pipeline) @@ -87,7 +110,7 @@ def test_multiqc_config_report_comment_release_fail(self): with open(Path(new_pipeline, "assets", "multiqc_config.yml"), "w") as fh: yaml.safe_dump(mqc_yml_tmp, fh) assert len(result["failed"]) == 1 - assert result["failed"][0].startswith("'assets/multiqc_config.yml' does not contain a matching 'report_comment'.") + assert result["failed"][0].startswith("`assets/multiqc_config.yml` does not contain a matching 'report_comment'.") def test_multiqc_config_report_comment_release_succeed(self): @@ -103,4 +126,4 @@ def test_multiqc_config_report_comment_release_succeed(self): # lint again lint_obj._load() result = lint_obj.multiqc_config() - assert "'assets/multiqc_config.yml' contains a matching 'report_comment'." in result["passed"] + assert "`assets/multiqc_config.yml` contains a matching 'report_comment'." in result["passed"] diff --git a/tests/lint/nfcore_yml.py b/tests/lint/nfcore_yml.py new file mode 100644 index 0000000000..474ccd48fc --- /dev/null +++ b/tests/lint/nfcore_yml.py @@ -0,0 +1,53 @@ +import re +from pathlib import Path + +import nf_core.create +import nf_core.lint + + +def test_nfcore_yml_pass(self): + """Lint test: nfcore_yml - PASS""" + self.lint_obj._load() + results = self.lint_obj.nfcore_yml() + + assert "Repository type in `.nf-core.yml` is valid" in str(results["passed"]) + assert "nf-core version in `.nf-core.yml` is set to the latest version" in str(results["passed"]) + assert len(results.get("warned", [])) == 0 + assert len(results.get("failed", [])) == 0 + assert len(results.get("ignored", [])) == 0 + + +def test_nfcore_yml_fail_repo_type(self): + """Lint test: nfcore_yml - FAIL - repository type not set""" + new_pipeline = self._make_pipeline_copy() + nf_core_yml = Path(new_pipeline) / ".nf-core.yml" + with open(nf_core_yml) as fh: + content = fh.read() + new_content = content.replace("repository_type: pipeline", "repository_type: foo") + with open(nf_core_yml, "w") as fh: + fh.write(new_content) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + results = lint_obj.nfcore_yml() + assert "Repository type in `.nf-core.yml` is not valid." in str(results["failed"]) + assert len(results.get("warned", [])) == 0 + assert len(results.get("passed", [])) >= 0 + assert len(results.get("ignored", [])) == 0 + + +def test_nfcore_yml_fail_nfcore_version(self): + """Lint test: nfcore_yml - FAIL - nf-core version not set""" + new_pipeline = self._make_pipeline_copy() + nf_core_yml = Path(new_pipeline) / ".nf-core.yml" + with open(nf_core_yml) as fh: + content = fh.read() + new_content = re.sub(r"nf_core_version:.+", "nf_core_version: foo", content) + with open(nf_core_yml, "w") as fh: + fh.write(new_content) + lint_obj = nf_core.lint.PipelineLint(new_pipeline) + lint_obj._load() + results = lint_obj.nfcore_yml() + assert "nf-core version in `.nf-core.yml` is not set to the latest version." in str(results["warned"]) + assert len(results.get("failed", [])) == 0 + assert len(results.get("passed", [])) >= 0 + assert len(results.get("ignored", [])) == 0 diff --git a/tests/modules/lint.py b/tests/modules/lint.py index 9bd280ddd8..595509de4e 100644 --- a/tests/modules/lint.py +++ b/tests/modules/lint.py @@ -1,3 +1,4 @@ +import json from pathlib import Path import pytest @@ -563,30 +564,29 @@ def test_modules_missing_required_tag(self): def test_modules_missing_tags_yml(self): """Test linting a module with a missing tags.yml file""" - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml").rename( - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml.bak") - ) + tags_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + tags_path.rename(tags_path.parent / "tags.yml.bak") module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=False, module="bpipe/test") - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml.bak").rename( - Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") - ) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 assert len(module_lint.warned) >= 0 assert module_lint.failed[0].lint_test == "test_tags_yml_exists" + # cleanup + Path(tags_path.parent / "tags.yml.bak").rename(tags_path.parent / "tags.yml") def test_modules_incorrect_tags_yml_key(self): """Test linting a module with an incorrect key in tags.yml file""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml")) as fh: + tags_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + with open(tags_path) as fh: content = fh.read() new_content = content.replace("bpipe/test:", "bpipe_test:") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(new_content) module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=True, module="bpipe/test") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(content) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 @@ -596,14 +596,15 @@ def test_modules_incorrect_tags_yml_key(self): def test_modules_incorrect_tags_yml_values(self): """Test linting a module with an incorrect path in tags.yml file""" - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml")) as fh: + tags_path = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml") + with open(tags_path) as fh: content = fh.read() new_content = content.replace("modules/nf-core/bpipe/test/**", "foo") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(new_content) module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) module_lint.lint(print_results=False, module="bpipe/test") - with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "tags.yml"), "w") as fh: + with open(tags_path, "w") as fh: fh.write(content) assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" assert len(module_lint.passed) >= 0 @@ -642,3 +643,68 @@ def test_nftest_failing_linting(self): assert module_lint.failed[2].lint_test == "test_main_tags" assert "kallisto/index" in module_lint.failed[2].message assert module_lint.failed[3].lint_test == "test_tags_yml" + + +def test_modules_absent_version(self): + """Test linting a nf-test module if the versions is absent in the snapshot file `""" + with open(Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap")) as fh: + content = fh.read() + new_content = content.replace("versions", "foo") + with open( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap"), "w" + ) as fh: + fh.write(new_content) + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + with open( + Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap"), "w" + ) as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_snap_versions" + + +def test_modules_empty_file_in_snapshot(self): + """Test linting a nf-test module with an empty file sha sum in the test snapshot, which should make it fail (if it is not a stub)""" + snap_file = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my test"]["content"][0]["0"] = "test:md5,d41d8cd98f00b204e9800998ecf8427e" + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_snap_md5sum" + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_modules_empty_file_in_stub_snapshot(self): + """Test linting a nf-test module with an empty file sha sum in the stub test snapshot, which should make it not fail""" + snap_file = Path(self.nfcore_modules, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my_test_stub"] = {"content": [{"0": "test:md5,d41d8cd98f00b204e9800998ecf8427e", "versions": {}}]} + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + module_lint = nf_core.modules.ModuleLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, module="bpipe/test") + assert len(module_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) > 0 + assert len(module_lint.warned) >= 0 + assert any(x.lint_test == "test_snap_md5sum" for x in module_lint.passed) + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) diff --git a/tests/subworkflows/lint.py b/tests/subworkflows/lint.py index b89b7b78ce..73d2452b37 100644 --- a/tests/subworkflows/lint.py +++ b/tests/subworkflows/lint.py @@ -1,3 +1,5 @@ +import json +import shutil from pathlib import Path import pytest @@ -181,3 +183,160 @@ def test_subworkflows_lint_capitalization_fail(self): # cleanup self.subworkflow_remove.remove("bam_stats_samtools", force=True) + + +def test_subworkflows_absent_version(self): + """Test linting a nf-test module if the versions is absent in the snapshot file `""" + snap_file = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") + with open(snap_file) as fh: + content = fh.read() + new_content = content.replace("versions", "foo") + with open(snap_file, "w") as fh: + fh.write(new_content) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0 + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0, f"Linting warned with {[x.__dict__ for x in subworkflow_lint.warned]}" + assert any([x.lint_test == "test_snap_versions" for x in subworkflow_lint.warned]) + + # cleanup + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_subworkflows_missing_test_dir(self): + """Test linting a nf-test subworkflow if the tests directory is missing""" + test_dir = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests") + test_dir_copy = shutil.copytree(test_dir, test_dir.parent / "tests_copy") + shutil.rmtree(test_dir) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0 + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0, f"Linting warned with {[x.__dict__ for x in subworkflow_lint.warned]}" + assert any([x.lint_test == "test_dir_versions" for x in subworkflow_lint.warned]) + + # cleanup + shutil.copytree(test_dir_copy, test_dir) + + +def test_subworkflows_missing_main_nf(self): + """Test linting a nf-test subworkflow if the main.nf file is missing""" + main_nf = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "main.nf") + main_nf_copy = shutil.copy(main_nf, main_nf.parent / "main_nf_copy") + main_nf.unlink() + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert subworkflow_lint.failed[0].lint_test == "main_nf_exists" + + # cleanup + shutil.copy(main_nf_copy, main_nf) + + +def test_subworkflows_empty_file_in_snapshot(self): + """Test linting a nf-test subworkflow with an empty file sha sum in the test snapshot, which should make it fail (if it is not a stub)""" + snap_file = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my test"]["content"][0]["0"] = "test:md5,d41d8cd98f00b204e9800998ecf8427e" + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert subworkflow_lint.failed[0].lint_test == "test_snap_md5sum" + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_subworkflows_empty_file_in_stub_snapshot(self): + """Test linting a nf-test subworkflow with an empty file sha sum in the stub test snapshot, which should make it not fail""" + snap_file = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") + snap = json.load(snap_file.open()) + content = snap_file.read_text() + snap["my_test_stub"] = {"content": [{"0": "test:md5,d41d8cd98f00b204e9800998ecf8427e", "versions": {}}]} + + with open(snap_file, "w") as fh: + json.dump(snap, fh) + + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + assert len(subworkflow_lint.failed) == 0, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) > 0 + assert len(subworkflow_lint.warned) >= 0 + assert any(x.lint_test == "test_snap_md5sum" for x in subworkflow_lint.passed) + + # reset the file + with open(snap_file, "w") as fh: + fh.write(content) + + +def test_subworkflows_missing_tags_yml(self): + """Test linting a subworkflow with a missing tags.yml file""" + tags_path = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + tags_path.rename(tags_path.parent / "tags.yml.bak") + subworkflow_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + subworkflow_lint.lint(print_results=False, subworkflow="test_subworkflow") + + assert len(subworkflow_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in subworkflow_lint.failed]}" + assert len(subworkflow_lint.passed) >= 0 + assert len(subworkflow_lint.warned) >= 0 + assert subworkflow_lint.failed[0].lint_test == "test_tags_yml_exists" + + # cleanup + Path(tags_path.parent / "tags.yml.bak").rename(tags_path.parent / "tags.yml") + + +def test_subworkflows_incorrect_tags_yml_key(self): + """Test linting a subworkflow with an incorrect key in tags.yml file""" + tags_path = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + with open(tags_path) as fh: + content = fh.read() + new_content = content.replace("test_subworkflow:", "subworkflow:") + with open(tags_path, "w") as fh: + fh.write(new_content) + module_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + module_lint.lint(print_results=True, subworkflow="test_subworkflow") + with open(tags_path, "w") as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_tags_yml" + # cleanup + with open(tags_path, "w") as fh: + fh.write(content) + + +def test_subworkflows_incorrect_tags_yml_values(self): + """Test linting a subworkflow with an incorrect path in tags.yml file""" + tags_path = Path(self.nfcore_modules, "subworkflows", "nf-core", "test_subworkflow", "tests", "tags.yml") + with open(tags_path) as fh: + content = fh.read() + new_content = content.replace("subworkflows/nf-core/test_subworkflow/**", "foo") + with open(tags_path, "w") as fh: + fh.write(new_content) + module_lint = nf_core.subworkflows.SubworkflowLint(dir=self.nfcore_modules) + module_lint.lint(print_results=False, subworkflow="test_subworkflow") + with open(tags_path, "w") as fh: + fh.write(content) + assert len(module_lint.failed) == 1, f"Linting failed with {[x.__dict__ for x in module_lint.failed]}" + assert len(module_lint.passed) >= 0 + assert len(module_lint.warned) >= 0 + assert module_lint.failed[0].lint_test == "test_tags_yml" + # cleanup + with open(tags_path, "w") as fh: + fh.write(content) diff --git a/tests/test_cli.py b/tests/test_cli.py index 54e420f5e4..913a4aac1d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -31,13 +31,13 @@ def test_header_outdated(mock_check_outdated, mock_nf_core_cli, capsys): class TestCli(unittest.TestCase): - """Class for testing the commandline interface""" + """Class for testing the command line interface""" def setUp(self): self.runner = CliRunner() def assemble_params(self, params): - """Assemble a dictionnary of parameters into a list of arguments for the cli + """Assemble a dictionary of parameters into a list of arguments for the cli Note: if the value of a parameter is None, it will be considered a flag. @@ -363,18 +363,21 @@ def test_lint_log_user_warning(self, mock_lint, mock_is_pipeline): def test_schema_lint(self, mock_get_schema_path): """Test nf-core schema lint defaults to nextflow_schema.json""" cmd = ["schema", "lint"] - result = self.invoke_cli(cmd) - assert mock_get_schema_path.called_with("nextflow_schema.json") - assert "nextflow_schema.json" in result.output + with self.runner.isolated_filesystem(): + with open("nextflow_schema.json", "w") as f: + f.write("{}") + self.invoke_cli(cmd) + mock_get_schema_path.assert_called_with("nextflow_schema.json") @mock.patch("nf_core.schema.PipelineSchema.get_schema_path") def test_schema_lint_filename(self, mock_get_schema_path): """Test nf-core schema lint accepts a filename""" cmd = ["schema", "lint", "some_other_filename"] - result = self.invoke_cli(cmd) - assert mock_get_schema_path.called_with("some_other_filename") - assert "some_other_filename" in result.output - assert "nextflow_schema.json" not in result.output + with self.runner.isolated_filesystem(): + with open("some_other_filename", "w") as f: + f.write("{}") + self.invoke_cli(cmd) + mock_get_schema_path.assert_called_with("some_other_filename") @mock.patch("nf_core.create_logo.create_logo") def test_create_logo(self, mock_create_logo): diff --git a/tests/test_lint.py b/tests/test_lint.py index d10cef37e4..b72a6bfdfa 100644 --- a/tests/test_lint.py +++ b/tests/test_lint.py @@ -198,6 +198,13 @@ def test_sphinx_md_files(self): test_actions_schema_validation_missing_jobs, test_actions_schema_validation_missing_on, ) + from .lint.configs import ( # type: ignore[misc] + test_ignore_base_config, + test_ignore_modules_config, + test_superfluous_withname_in_base_config_fails, + test_superfluous_withname_in_modules_config_fails, + test_withname_in_modules_config, + ) from .lint.files_exist import ( # type: ignore[misc] test_files_exist_depreciated_file, test_files_exist_fail_conditional, @@ -213,7 +220,8 @@ def test_sphinx_md_files(self): from .lint.merge_markers import test_merge_markers_found # type: ignore[misc] from .lint.modules_json import test_modules_json_pass # type: ignore[misc] from .lint.multiqc_config import ( # type: ignore[misc] - test_multiqc_config_exists_ignore, + test_multiqc_config_exists, + test_multiqc_config_ignore, test_multiqc_config_missing_report_section_order, test_multiqc_config_report_comment_fail, test_multiqc_config_report_comment_release_fail, @@ -233,6 +241,11 @@ def test_sphinx_md_files(self): test_nextflow_config_example_pass, test_nextflow_config_missing_test_profile_failed, ) + from .lint.nfcore_yml import ( # type: ignore[misc] + test_nfcore_yml_fail_nfcore_version, + test_nfcore_yml_fail_repo_type, + test_nfcore_yml_pass, + ) from .lint.template_strings import ( # type: ignore[misc] test_template_strings, test_template_strings_ignore_file, diff --git a/tests/test_modules.py b/tests/test_modules.py index 944a09f670..d3d99abadd 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,5 +1,6 @@ """Tests covering the modules commands""" +import json import os import shutil import unittest @@ -53,9 +54,22 @@ def create_modules_repo_dummy(tmp_dir): yaml.dump(meta_yml, fh) # Add dummy content to main.nf.test.snap test_snap_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "tests", "main.nf.test.snap") - test_snap_path.touch() + with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {}\n}') + json.dump( + { + "my test": { + "content": [ + { + "0": [], + "versions": {}, + } + ] + } + }, + fh, + indent=4, + ) # remove "TODO" statements from main.nf main_nf_path = Path(root_dir, "modules", "nf-core", "bpipe", "test", "main.nf") @@ -179,6 +193,9 @@ def test_modulesrepo_class(self): test_modules_install_trimgalore_twice, ) from .modules.lint import ( # type: ignore[misc] + test_modules_absent_version, + test_modules_empty_file_in_snapshot, + test_modules_empty_file_in_stub_snapshot, test_modules_environment_yml_file_doesnt_exists, test_modules_environment_yml_file_name_mismatch, test_modules_environment_yml_file_not_array, diff --git a/tests/test_schema.py b/tests/test_schema.py index 29f4921985..e0921908d4 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -5,6 +5,7 @@ import shutil import tempfile import unittest +from pathlib import Path from unittest import mock import pytest @@ -216,6 +217,15 @@ def test_make_skeleton_schema(self): self.schema_obj.pipeline_manifest["description"] = "Test pipeline" self.schema_obj.make_skeleton_schema() self.schema_obj.validate_schema(self.schema_obj.schema) + assert self.schema_obj.schema["title"] == "nf-core/test pipeline parameters" + + def test_make_skeleton_schema_absent_name(self): + """Test making a new schema skeleton""" + self.schema_obj.schema_filename = self.template_schema + self.schema_obj.pipeline_manifest["description"] = "Test pipeline" + self.schema_obj.make_skeleton_schema() + self.schema_obj.validate_schema(self.schema_obj.schema) + assert self.schema_obj.schema["title"] == "wf pipeline parameters" def test_get_wf_params(self): """Test getting the workflow parameters from a pipeline""" @@ -314,9 +324,9 @@ def test_build_schema_from_scratch(self, tmp_dir): Pretty much a copy of test_launch.py test_make_pipeline_schema """ - test_pipeline_dir = os.path.join(tmp_dir, "wf") + test_pipeline_dir = Path(tmp_dir, "wf") shutil.copytree(self.template_dir, test_pipeline_dir) - os.remove(os.path.join(test_pipeline_dir, "nextflow_schema.json")) + Path(test_pipeline_dir, "nextflow_schema.json").unlink() self.schema_obj.build_schema(test_pipeline_dir, True, False, None) diff --git a/tests/test_subworkflows.py b/tests/test_subworkflows.py index 6163faa7a9..0a9224002a 100644 --- a/tests/test_subworkflows.py +++ b/tests/test_subworkflows.py @@ -1,5 +1,6 @@ """Tests covering the subworkflows commands""" +import json import os import shutil import unittest @@ -23,7 +24,6 @@ def create_modules_repo_dummy(tmp_dir): root_dir = Path(tmp_dir, "modules") Path(root_dir, "modules").mkdir(parents=True, exist_ok=True) - Path(root_dir, "subworkflows").mkdir(parents=True, exist_ok=True) Path(root_dir, "subworkflows", "nf-core").mkdir(parents=True, exist_ok=True) Path(root_dir, "tests", "config").mkdir(parents=True, exist_ok=True) with open(Path(root_dir, ".nf-core.yml"), "w") as fh: @@ -34,9 +34,22 @@ def create_modules_repo_dummy(tmp_dir): # Add dummy content to main.nf.test.snap test_snap_path = Path(root_dir, "subworkflows", "nf-core", "test_subworkflow", "tests", "main.nf.test.snap") - test_snap_path.touch() + test_snap_path.parent.mkdir(parents=True, exist_ok=True) with open(test_snap_path, "w") as fh: - fh.write('{\n "my test": {}\n}') + json.dump( + { + "my test": { + "content": [ + { + "0": [], + "versions": {}, + } + ] + } + }, + fh, + indent=4, + ) return root_dir @@ -119,6 +132,11 @@ def tearDown(self): test_subworkflows_install_tracking_added_super_subworkflow, ) from .subworkflows.lint import ( # type: ignore[misc] + test_subworkflows_absent_version, + test_subworkflows_empty_file_in_snapshot, + test_subworkflows_empty_file_in_stub_snapshot, + test_subworkflows_incorrect_tags_yml_key, + test_subworkflows_incorrect_tags_yml_values, test_subworkflows_lint, test_subworkflows_lint_capitalization_fail, test_subworkflows_lint_empty, @@ -131,6 +149,7 @@ def tearDown(self): test_subworkflows_lint_snapshot_file, test_subworkflows_lint_snapshot_file_missing_fail, test_subworkflows_lint_snapshot_file_not_needed, + test_subworkflows_missing_tags_yml, ) from .subworkflows.list import ( # type: ignore[misc] test_subworkflows_install_and_list_subworkflows, diff --git a/tests/test_sync.py b/tests/test_sync.py index 6f0e502e8b..b94968cd4c 100644 --- a/tests/test_sync.py +++ b/tests/test_sync.py @@ -26,7 +26,12 @@ def setUp(self): self.pipeline_dir = os.path.join(self.tmp_dir, "testpipeline") default_branch = "master" self.create_obj = nf_core.create.PipelineCreate( - "testing", "test pipeline", "tester", outdir=self.pipeline_dir, plain=True, default_branch=default_branch + "testing", + "test pipeline", + "tester", + outdir=self.pipeline_dir, + plain=True, + default_branch=default_branch, ) self.create_obj.init_pipeline() self.remote_path = os.path.join(self.tmp_dir, "remote_repo") @@ -374,7 +379,7 @@ def test_close_open_pr(self, mock_patch, mock_post): } assert psync.close_open_pr(pr) - assert mock_patch.called_once_with("url_to_update_pr") + mock_patch.assert_called_once_with(url="url_to_update_pr", data='{"state": "closed"}') @mock.patch("nf_core.utils.gh_api.post", side_effect=mocked_requests_post) @mock.patch("nf_core.utils.gh_api.patch", side_effect=mocked_requests_patch) @@ -397,7 +402,7 @@ def test_close_open_pr_fail(self, mock_patch, mock_post): } assert not psync.close_open_pr(pr) - assert mock_patch.called_once_with("bad_url_to_update_pr") + mock_patch.assert_called_once_with(url="bad_url_to_update_pr", data='{"state": "closed"}') def test_reset_target_dir(self): """Try resetting target pipeline directory"""