From 972de9db993ad0a5af83851b991d4f4bc8690c32 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 10 Oct 2024 14:17:53 +0200 Subject: [PATCH 01/15] Parallelize pipeline GHA tests over docker/conda/singularity --- .../.github/workflows/ci.yml | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index eefb59496..f332eee5a 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -27,6 +27,10 @@ jobs: NXF_VER: - "24.04.2" - "latest-everything" + engine: + - "conda" + - "docker" + - "singularity" steps: - name: Check out pipeline code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 @@ -39,21 +43,10 @@ jobs: - name: Disk space cleanup uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - name: Run pipeline with test data (docker) + - name: "Run pipeline with test data {% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.engine }}{% endraw %}" # TODO nf-core: You can customise CI pipeline run tests as required # For example: adding multiple test runs with different parameters # Remember that you can parallelise this by using strategy.matrix run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,docker --outdir ./results - - - name: Run pipeline with test data (singularity) - # TODO nf-core: You can customise CI pipeline run tests as required - run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,singularity --outdir ./results - if: "{% raw %}${{ github.base_ref == 'master' }}{% endraw %}" - - - name: Run pipeline with test data (conda) - # TODO nf-core: You can customise CI pipeline run tests as required - run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,conda --outdir ./results - if: "{% raw %}${{ github.base_ref == 'master' }}{% endraw %}" + nextflow run ${GITHUB_WORKSPACE} -profile test,{% raw %}${{ matrix.engine }}{% endraw %} --outdir ./results + if: "!{% raw %}${{ github.base_ref == 'master' }}{% endraw %} || {% raw %}${{ matrix.engine == 'docker' }}{% endraw %}" From 8ab88cbc8189ae1f3bce0f1117adc8a04d677640 Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 10 Oct 2024 12:23:22 +0000 Subject: [PATCH 02/15] [automated] Fix code linting --- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index f332eee5a..cddac0d45 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: engine: - "conda" - "docker" - - "singularity" + - "singularity" steps: - name: Check out pipeline code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 From ba6b661e6866b427a0531b5a193491bd7ee0ed7a Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 10 Oct 2024 16:22:05 +0200 Subject: [PATCH 03/15] fix GHA --- .../.github/workflows/ci.yml | 61 ++++++++++++++++--- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index cddac0d45..7fc18a245 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -11,26 +11,44 @@ on: env: NXF_ANSI_LOG: false + NXF_SINGULARITY_CACHEDIR: {% raw %}${{ github.workspace }}{% endraw %}/.singularity + NXF_SINGULARITY_LIBRARYDIR: {% raw %}${{ github.workspace }}{% endraw %}/.singularity concurrency: - group: "{% raw %}${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}{% endraw %}" + group: "{% raw %}${{ github.workflow }}{% endraw %}-{% raw %}${{ github.event.pull_request.number || github.ref }}{% endraw %}" cancel-in-progress: true jobs: test: - name: Run pipeline with test data + name: "Run pipeline with test data ({% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.test }}{% endraw %} | {% raw %}${{ matrix.profile }}{% endraw %})" # Only run on push if this is the nf-core dev branch (merged PRs) - if: "{% raw %}${{{% endraw %} github.event_name != 'push' || (github.event_name == 'push' && github.repository == '{{ name }}') {% raw %}}}{% endraw %}" + if: "{% raw %}${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/bamtofastq') }}{% endraw %}" runs-on: ubuntu-latest strategy: matrix: NXF_VER: - "24.04.2" - "latest-everything" - engine: + profile: - "conda" - "docker" - "singularity" + test: + - "test" + - "test_cram" + - "test_chr" + - "test_no_bai" + - "test_no_crai" + - "test_no_stats" + - "test_no_qc" + - "test_collate_fast" + isMaster: + - {% raw %}${{ github.base_ref == 'master' }}{% endraw %} + exclude: + - isMaster: false + profile: "conda" + - isMaster: false + profile: "singularity" steps: - name: Check out pipeline code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 @@ -40,13 +58,36 @@ jobs: with: version: "{% raw %}${{ matrix.NXF_VER }}{% endraw %}" + - name: Setup apptainer + if: matrix.profile == 'singularity' + uses: eWaterCycle/setup-apptainer@main + + - name: Set up Singularity + if: matrix.profile == 'singularity' + run: | + mkdir -p $NXF_SINGULARITY_CACHEDIR + mkdir -p $NXF_SINGULARITY_LIBRARYDIR + + - name: Set up miniconda + if: matrix.profile == 'conda' + uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 + with: + miniconda-version: "latest" + auto-update-conda: true + channels: conda-forge,bioconda + + - name: Conda setup + if: matrix.profile == 'conda' + run: | + conda clean -a + conda install -n base conda-libmamba-solver + conda config --set solver libmamba + echo $(realpath $CONDA)/condabin >> $GITHUB_PATH + echo $(realpath python) >> $GITHUB_PATH + - name: Disk space cleanup uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - name: "Run pipeline with test data {% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.engine }}{% endraw %}" - # TODO nf-core: You can customise CI pipeline run tests as required - # For example: adding multiple test runs with different parameters - # Remember that you can parallelise this by using strategy.matrix + - name: " Run pipeline with test data {% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.test }}{% endraw %} | {% raw %}${{ matrix.profile }}{% endraw %}" run: | - nextflow run ${GITHUB_WORKSPACE} -profile test,{% raw %}${{ matrix.engine }}{% endraw %} --outdir ./results - if: "!{% raw %}${{ github.base_ref == 'master' }}{% endraw %} || {% raw %}${{ matrix.engine == 'docker' }}{% endraw %}" + nextflow run ${GITHUB_WORKSPACE} -profile {% raw %}${{ matrix.test }}{% endraw %},{% raw %}${{ matrix.profile }}{% endraw %} --outdir ./results From 35d7e083d74596487171f1ef7af3a24607b715bb Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 10 Oct 2024 16:24:15 +0200 Subject: [PATCH 04/15] fix template strings --- nf_core/pipeline-template/.github/workflows/ci.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 7fc18a245..35dcdfb47 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: test: name: "Run pipeline with test data ({% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.test }}{% endraw %} | {% raw %}${{ matrix.profile }}{% endraw %})" # Only run on push if this is the nf-core dev branch (merged PRs) - if: "{% raw %}${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/bamtofastq') }}{% endraw %}" + if: "{% raw %}${{{% endraw %} github.event_name != 'push' || (github.event_name == 'push' && github.repository == '{{ name }}') {% raw %}}}{% endraw %}" runs-on: ubuntu-latest strategy: matrix: @@ -35,13 +35,6 @@ jobs: - "singularity" test: - "test" - - "test_cram" - - "test_chr" - - "test_no_bai" - - "test_no_crai" - - "test_no_stats" - - "test_no_qc" - - "test_collate_fast" isMaster: - {% raw %}${{ github.base_ref == 'master' }}{% endraw %} exclude: From a1ff5e90367ba8e62b92d710d835bc0402f38c07 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 10 Oct 2024 16:25:19 +0200 Subject: [PATCH 05/15] add TODO --- nf_core/pipeline-template/.github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 35dcdfb47..a3843c7b8 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: - "docker" - "singularity" test: + # TODO nf-core: You can customise CI pipeline run tests as required - "test" isMaster: - {% raw %}${{ github.base_ref == 'master' }}{% endraw %} From 45012dde58d59af0b69f193eb3295ee8a07ab964 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 10 Oct 2024 16:52:12 +0200 Subject: [PATCH 06/15] Fix name --- nf_core/pipeline-template/.github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index a3843c7b8..44699ecb7 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -14,7 +14,7 @@ env: NXF_SINGULARITY_CACHEDIR: {% raw %}${{ github.workspace }}{% endraw %}/.singularity NXF_SINGULARITY_LIBRARYDIR: {% raw %}${{ github.workspace }}{% endraw %}/.singularity -concurrency: + concurrency: group: "{% raw %}${{ github.workflow }}{% endraw %}-{% raw %}${{ github.event.pull_request.number || github.ref }}{% endraw %}" cancel-in-progress: true @@ -47,12 +47,12 @@ jobs: - name: Check out pipeline code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4 - - name: Install Nextflow + - name: Set up Nextflow uses: nf-core/setup-nextflow@v2 with: version: "{% raw %}${{ matrix.NXF_VER }}{% endraw %}" - - name: Setup apptainer + - name: Set up Apptainer if: matrix.profile == 'singularity' uses: eWaterCycle/setup-apptainer@main @@ -62,7 +62,7 @@ jobs: mkdir -p $NXF_SINGULARITY_CACHEDIR mkdir -p $NXF_SINGULARITY_LIBRARYDIR - - name: Set up miniconda + - name: Set up Miniconda if: matrix.profile == 'conda' uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3 with: @@ -70,7 +70,7 @@ jobs: auto-update-conda: true channels: conda-forge,bioconda - - name: Conda setup + - name: Set up Conda if: matrix.profile == 'conda' run: | conda clean -a @@ -79,9 +79,9 @@ jobs: echo $(realpath $CONDA)/condabin >> $GITHUB_PATH echo $(realpath python) >> $GITHUB_PATH - - name: Disk space cleanup + - name: Clean up Disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - name: " Run pipeline with test data {% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.test }}{% endraw %} | {% raw %}${{ matrix.profile }}{% endraw %}" + - name: "Run pipeline with test data {% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.test }}{% endraw %} | {% raw %}${{ matrix.profile }}{% endraw %}" run: | nextflow run ${GITHUB_WORKSPACE} -profile {% raw %}${{ matrix.test }}{% endraw %},{% raw %}${{ matrix.profile }}{% endraw %} --outdir ./results From 27be623bda2b5287dc25a9284acf5abb541def25 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 10 Oct 2024 16:59:44 +0200 Subject: [PATCH 07/15] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipeline-template/.github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 44699ecb7..1947fc7b8 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -68,6 +68,7 @@ jobs: with: miniconda-version: "latest" auto-update-conda: true + conda-solver: libmamba channels: conda-forge,bioconda - name: Set up Conda From eb2be9ae43d60bbc3b53bff44bd875b49d483ad3 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 10 Oct 2024 17:00:05 +0200 Subject: [PATCH 08/15] Update nf_core/pipeline-template/.github/workflows/ci.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 1947fc7b8..be2232be7 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -14,7 +14,7 @@ env: NXF_SINGULARITY_CACHEDIR: {% raw %}${{ github.workspace }}{% endraw %}/.singularity NXF_SINGULARITY_LIBRARYDIR: {% raw %}${{ github.workspace }}{% endraw %}/.singularity - concurrency: +concurrency: group: "{% raw %}${{ github.workflow }}{% endraw %}-{% raw %}${{ github.event.pull_request.number || github.ref }}{% endraw %}" cancel-in-progress: true From 1c8d74b92816919629d7affa860bd26bf4097377 Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 10 Oct 2024 17:00:46 +0200 Subject: [PATCH 09/15] test -> test_name --- nf_core/pipeline-template/.github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index be2232be7..d521c91bd 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -20,7 +20,7 @@ concurrency: jobs: test: - name: "Run pipeline with test data ({% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.test }}{% endraw %} | {% raw %}${{ matrix.profile }}{% endraw %})" + name: "Run pipeline with test data ({% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.test_name }}{% endraw %} | {% raw %}${{ matrix.profile }}{% endraw %})" # Only run on push if this is the nf-core dev branch (merged PRs) if: "{% raw %}${{{% endraw %} github.event_name != 'push' || (github.event_name == 'push' && github.repository == '{{ name }}') {% raw %}}}{% endraw %}" runs-on: ubuntu-latest @@ -33,7 +33,7 @@ jobs: - "conda" - "docker" - "singularity" - test: + test_name: # TODO nf-core: You can customise CI pipeline run tests as required - "test" isMaster: @@ -83,6 +83,6 @@ jobs: - name: Clean up Disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - name: "Run pipeline with test data {% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.test }}{% endraw %} | {% raw %}${{ matrix.profile }}{% endraw %}" + - name: "Run pipeline with test data {% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.test_name }}{% endraw %} | {% raw %}${{ matrix.profile }}{% endraw %}" run: | - nextflow run ${GITHUB_WORKSPACE} -profile {% raw %}${{ matrix.test }}{% endraw %},{% raw %}${{ matrix.profile }}{% endraw %} --outdir ./results + nextflow run ${GITHUB_WORKSPACE} -profile {% raw %}${{ matrix.test_name }}{% endraw %},{% raw %}${{ matrix.profile }}{% endraw %} --outdir ./results From 7556c0ab4aa2c4c0ef730df7ad9067ad75dbd308 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 10 Oct 2024 17:10:22 +0200 Subject: [PATCH 10/15] Update nf_core/pipeline-template/.github/workflows/ci.yml --- nf_core/pipeline-template/.github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index d521c91bd..bbb5900fb 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -34,7 +34,6 @@ jobs: - "docker" - "singularity" test_name: - # TODO nf-core: You can customise CI pipeline run tests as required - "test" isMaster: - {% raw %}${{ github.base_ref == 'master' }}{% endraw %} From b51770ceae995933d1e1a944af711b111695077c Mon Sep 17 00:00:00 2001 From: maxulysse Date: Thu, 10 Oct 2024 17:18:06 +0200 Subject: [PATCH 11/15] more comments --- nf_core/pipeline-template/.github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index bbb5900fb..503ec64e6 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -37,6 +37,7 @@ jobs: - "test" isMaster: - {% raw %}${{ github.base_ref == 'master' }}{% endraw %} + # Exclude conda and singularity on dev exclude: - isMaster: false profile: "conda" From 42f1de46a885ed46731edda4e662c73bc0df5f81 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 10 Oct 2024 17:25:32 +0200 Subject: [PATCH 12/15] Update nf_core/pipeline-template/.github/workflows/ci.yml --- nf_core/pipeline-template/.github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 503ec64e6..e935f12d1 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -15,7 +15,7 @@ env: NXF_SINGULARITY_LIBRARYDIR: {% raw %}${{ github.workspace }}{% endraw %}/.singularity concurrency: - group: "{% raw %}${{ github.workflow }}{% endraw %}-{% raw %}${{ github.event.pull_request.number || github.ref }}{% endraw %}" + group: "{% raw %}${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}{% endraw %}" cancel-in-progress: true jobs: From ac3d57772d36b2bcaebb0410d41a3d14e8659eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BAlia=20Mir=20Pedrol?= Date: Thu, 10 Oct 2024 15:26:50 +0000 Subject: [PATCH 13/15] minimise raw and endraw blocks --- .../.github/workflows/ci.yml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index e935f12d1..59be65ed6 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -1,5 +1,5 @@ name: nf-core CI -# This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors +# {% raw %}This workflow runs the pipeline with the minimal test dataset to check that it completes without any syntax errors on: push: branches: @@ -11,18 +11,18 @@ on: env: NXF_ANSI_LOG: false - NXF_SINGULARITY_CACHEDIR: {% raw %}${{ github.workspace }}{% endraw %}/.singularity - NXF_SINGULARITY_LIBRARYDIR: {% raw %}${{ github.workspace }}{% endraw %}/.singularity + NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity + NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity concurrency: - group: "{% raw %}${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}{% endraw %}" + group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" cancel-in-progress: true jobs: test: - name: "Run pipeline with test data ({% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.test_name }}{% endraw %} | {% raw %}${{ matrix.profile }}{% endraw %})" + name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }})" # Only run on push if this is the nf-core dev branch (merged PRs) - if: "{% raw %}${{{% endraw %} github.event_name != 'push' || (github.event_name == 'push' && github.repository == '{{ name }}') {% raw %}}}{% endraw %}" + if: "${{{% endraw %} github.event_name != 'push' || (github.event_name == 'push' && github.repository == '{{ name }}') {% raw %}}}" runs-on: ubuntu-latest strategy: matrix: @@ -36,7 +36,7 @@ jobs: test_name: - "test" isMaster: - - {% raw %}${{ github.base_ref == 'master' }}{% endraw %} + - ${{ github.base_ref == 'master' }} # Exclude conda and singularity on dev exclude: - isMaster: false @@ -50,7 +50,7 @@ jobs: - name: Set up Nextflow uses: nf-core/setup-nextflow@v2 with: - version: "{% raw %}${{ matrix.NXF_VER }}{% endraw %}" + version: "${{ matrix.NXF_VER }}" - name: Set up Apptainer if: matrix.profile == 'singularity' @@ -83,6 +83,6 @@ jobs: - name: Clean up Disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - - name: "Run pipeline with test data {% raw %}${{ matrix.NXF_VER }}{% endraw %} | {% raw %}${{ matrix.test_name }}{% endraw %} | {% raw %}${{ matrix.profile }}{% endraw %}" + - name: "Run pipeline with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}" run: | - nextflow run ${GITHUB_WORKSPACE} -profile {% raw %}${{ matrix.test_name }}{% endraw %},{% raw %}${{ matrix.profile }}{% endraw %} --outdir ./results + nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results{% endraw %} From 71d348d9d94209010709897628a0af01c4cfb039 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Thu, 10 Oct 2024 18:21:27 +0200 Subject: [PATCH 14/15] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b215214c..2b23a2f9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ ### Template +- Parallelize pipeline GHA tests over docker/conda/singularity ([#3214](https://github.com/nf-core/tools/pull/3214)) + ### Download ### Linting From 3bd7105bb4199986db958409eb3494e09ad72420 Mon Sep 17 00:00:00 2001 From: Maxime U Garcia Date: Fri, 11 Oct 2024 08:33:27 +0200 Subject: [PATCH 15/15] Update nf_core/pipeline-template/.github/workflows/ci.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Matthias Hörtenhuber --- nf_core/pipeline-template/.github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/nf_core/pipeline-template/.github/workflows/ci.yml b/nf_core/pipeline-template/.github/workflows/ci.yml index 59be65ed6..61738a414 100644 --- a/nf_core/pipeline-template/.github/workflows/ci.yml +++ b/nf_core/pipeline-template/.github/workflows/ci.yml @@ -74,9 +74,6 @@ jobs: - name: Set up Conda if: matrix.profile == 'conda' run: | - conda clean -a - conda install -n base conda-libmamba-solver - conda config --set solver libmamba echo $(realpath $CONDA)/condabin >> $GITHUB_PATH echo $(realpath python) >> $GITHUB_PATH