[do not merge] stress testing PR #6286

.github/conda_skip.yml

@@ -0,0 +1,145 @@
+# TODO What do we do with these?
+# I think we can require Docker,modules,Shard 1..Singularity,subworkflows,3 and skip the condas now
+# - path: modules/nf-core/nf-test
+# - profile: conda
+#   path: modules/nf-core/angsd/gl
+# - profile: conda
+#   path: modules/nf-core/annotsv/installannotations
+# - profile: conda
+#   path: modules/nf-core/happy/sompy
+# - profile: conda
+#   path: modules/nf-core/backsub
+# - profile: conda
+#   path: modules/nf-core/bakta/bakta
+# - profile: conda
+#   path: modules/nf-core/bakta/baktadbdownload
+# - profile: conda
+#   path: modules/nf-core/bases2fastq
+# - profile: conda
+#   path: modules/nf-core/bcl2fastq
+# - profile: conda
+#   path: modules/nf-core/bclconvert
+# - profile: conda
+#   path: modules/nf-core/celesta
+# - profile: conda
+#   path: modules/nf-core/cellpose
+# - profile: conda
+#   path: modules/nf-core/cellranger/count
+# - profile: conda
+#   path: modules/nf-core/cellranger/mkfastq
+# - profile: conda
+#   path: modules/nf-core/cellranger/mkgtf
+# - profile: conda
+#   path: modules/nf-core/cellranger/mkref
+# - profile: conda
+#   path: modules/nf-core/cellranger/mkvdjref
+# - profile: conda
+#   path: modules/nf-core/cellranger/multi
+# - profile: conda
+#   path: modules/nf-core/cellranger/vdj
+# - profile: conda
+#   path: modules/nf-core/checkqc
+# - profile: conda
+#   path: modules/nf-core/custom/dumpsoftwareversions
+# - profile: conda
+#   path: modules/nf-core/deepcell/mesmer
+# - profile: conda
+#   path: modules/nf-core/deepvariant
+- profile: conda
+  path: modules/nf-core/deepvariant/callvariants
+- profile: conda
+  path: modules/nf-core/deepvariant/makeexamples
+- profile: conda
+  path: modules/nf-core/deepvariant/postprocessvariants
+- profile: conda
+  path: modules/nf-core/deepvariant/rundeepvariant
+# - profile: conda
+#   path: modules/nf-core/ensemblvep/vep
+# - profile: conda
+#   path: modules/nf-core/fastk/fastk
+# - profile: conda
+#   path: modules/nf-core/fastk/histex
+# - profile: conda
+#   path: modules/nf-core/fastk/merge
+# - profile: conda
+#   path: modules/nf-core/fcs/fcsadaptor
+# - profile: conda
+#   path: modules/nf-core/fcs/fcsgx
+# - profile: conda
+#   path: modules/nf-core/ganon/buildcustom
+# - profile: conda
+#   path: modules/nf-core/ganon/classify
+# - profile: conda
+#   path: modules/nf-core/ganon/report
+# - profile: conda
+#   path: modules/nf-core/ganon/table
+# - profile: conda
+#   path: modules/nf-core/gatk4/cnnscorevariants
+# - profile: conda
+#   path: modules/nf-core/gatk4/determinegermlinecontigploidy
+# - profile: conda
+#   path: modules/nf-core/genescopefk
+# - profile: conda
+#   path: modules/nf-core/ilastik/multicut
+# - profile: conda
+#   path: modules/nf-core/ilastik/pixelclassification
+# - profile: conda
+#   path: modules/nf-core/imputeme/vcftoprs
+# - profile: conda
+#   path: modules/nf-core/mcquant
+# - profile: conda
+#   path: modules/nf-core/merquryfk/katcomp
+# - profile: conda
+#   path: modules/nf-core/merquryfk/katgc
+# - profile: conda
+#   path: modules/nf-core/merquryfk/merquryfk
+# - profile: conda
+#   path: modules/nf-core/merquryfk/ploidyplot
+# - profile: conda
+#   path: modules/nf-core/molkartgarage/clahe
+# - profile: conda
+#   path: modules/nf-core/quartonotebook
+# - profile: conda
+#   path: modules/nf-core/scimap/spatiallda
+# - profile: conda
+#   path: modules/nf-core/sentieon/bwaindex
+# - profile: conda
+#   path: modules/nf-core/sentieon/bwamem
+- profile: conda
+  path: modules/nf-core/sentieon/datametrics
+# - profile: conda
+#   path: modules/nf-core/sentieon/dedup
+# - profile: conda
+#   path: modules/nf-core/sentieon/qualcal
+# - profile: conda
+#   path: modules/nf-core/spaceranger/mkgtf
+# - profile: conda
+#   path: modules/nf-core/spaceranger/mkref
+# - profile: conda
+#   path: modules/nf-core/spaceranger/count
+# - profile: conda
+#   path: modules/nf-core/spotiflow
+# - profile: conda
+#   path: modules/nf-core/svanalyzer/svbenchmark
+# - profile: conda
+#   path: modules/nf-core/universc
+# - profile: singularity
+#   path: modules/nf-core/universc
+# - profile: conda
+#   path: modules/nf-core/vt/decompose
+# - profile: singularity
+#   path: modules/nf-core/bases2fastq
+# - profile: conda
+#   path: modules/nf-core/wittyer
+# - profile: conda
+#   path: modules/nf-core/islandpath
+# - profile: conda
+#   path: subworkflows/nf-core/vcf_annotate_ensemblvep
+# - profile: conda
+#   path: subworkflows/nf-core/bcl_demultiplex
+- profile: conda
+  path: subworkflows/nf-core/deepvariant
+# - profile: conda
+#   path: subworkflows/nf-core/fastq_align_bamcmp_bwa
+# - profile: conda
+#   path: subworkflows/nf-core/fastq_align_bwa

.github/workflows/nf-test.yml

@@ -0,0 +1,162 @@
+name: Run nf-test
+on:
+  pull_request:
+    branches: [master]
+  merge_group:
+    types: [checks_requested]
+    branches: [master]
+  workflow_dispatch:
+    inputs:
+      runners:
+        description: "Runners to test on"
+        type: choice
+        options:
+          - "ubuntu-latest"
+          - "self-hosted"
+        default: "self-hosted"
+
+# Cancel if a newer run is started
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
+  NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+jobs:
+  nf-test:
+    runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
+    # NOTE I think this is the cleanest way to get them organized
+    # process | conda | 1
+    # process | conda | 2
+    # process | conda | 3
+    # process | docker_self_hosted | 1
+    # ...
+    # workflow | singularity | 3
+    name: ${{ matrix.filter }} | ${{ matrix.profile }} | ${{ matrix.shard }}
+    strategy:
+      fail-fast: false
+      matrix:
+        filter: [process, workflow]
+        profile: [conda, docker_self_hosted, singularity]
+        shard: [1, 2, 3]
+    env:
+      NXF_ANSI_LOG: false
+      NXF_VER: "24.04.4"
+      NFTEST_VER: "0.9.0"
+      TOTAL_SHARDS: 3
+      SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
+      SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
+
+    steps:
+      # TODO If conda-fail.yml exists and matrix = conda skip
+      # https://github.com/askimed/nf-test/issues/260
+
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-java@b36c23c0d998641eff861008f374ee103c25ac73 # v4
+        with:
+          distribution: "temurin"
+          java-version: "17"
+      - name: Setup Nextflow
+        uses: nf-core/setup-nextflow@v2
+        with:
+          version: ${{ env.NXF_VER }}
+
+      - name: Install nf-test
+        uses: nf-core/setup-nf-test@v1
+        with:
+          version: ${{ env.NFTEST_VER }}
+
+      - name: Setup apptainer
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-apptainer@main
+
+      - name: Set up Singularity
+        if: matrix.profile == 'singularity'
+        run: |
+          mkdir -p $NXF_SINGULARITY_CACHEDIR
+          mkdir -p $NXF_SINGULARITY_LIBRARYDIR
+
+      - name: Set up Python
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5
+        with:
+          python-version: "3.11"
+
+      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4
+        id: cache-pip-pdiff
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-pdiff
+          restore-keys: |
+            ${{ runner.os }}-pip-pdiff
+
+      - name: Install Python dependencies
+        run: python -m pip install --upgrade pip pdiff cryptography
+
+      - name: Set up miniconda
+        if: matrix.profile == 'conda'
+        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
+        with:
+          miniconda-version: "latest"
+          auto-update-conda: true
+          channels: conda-forge,bioconda,defaults
+
+      - name: Conda setup
+        if: matrix.profile == 'conda'
+        run: |
+          conda clean -a
+          conda install -n base conda-libmamba-solver
+          conda config --set solver libmamba
+          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
+          echo $(realpath python) >> $GITHUB_PATH
+
+      # Set up secrets
+      - name: Set up nextflow secrets
+        # TODO Only run if the tag includes `sentieon`
+        if: env.SENTIEON_ENCRYPTION_KEY != null && env.SENTIEON_LICENSE_MESSAGE != null
+        run: |
+          nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "${{ secrets.SENTIEON_ENCRYPTION_KEY }}" --message "${{ secrets.SENTIEON_LICENSE_MESSAGE }}")
+
+      # Test the module
+      - name: Run Tests (Shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }})
+        env:
+          NFT_DIFF: "pdiff"
+          NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2"
+          SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }}
+          SENTIEON_AUTH_MECH: "GitHub Actions - token"
+        run: |
+          # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker
+          if [ "${{ matrix.profile }}" == "docker" ]; then
+            PROFILE="docker_self_hosted"
+          else
+            PROFILE=${{ matrix.profile }}
+          fi
+
+          NFT_WORKDIR=~ \
+          nf-test test \
+            --profile=${{ matrix.profile }} \
+            --tap=test.tap \
+            --ci \
+            --verbose \
+            --changed-since HEAD^ \
+            --shard ${{ matrix.shard }}/${{ env.TOTAL_SHARDS }} \
+            --filter ${{ matrix.filter }} \
+            --follow-dependencies
+
+      # TODO If matrix == conda create a conda-fail.yml and commit it
+
+      # TODO If no test.tap, then make one to spoof?
+      # - uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1
+      #   with:
+      #     path: >-
+      #       test.tap
+
+      - name: Clean up
+        if: always()
+        run: |
+          sudo rm -rf /home/ubuntu/tests/