Small fixes & update to 2.3.1

.github/workflows/assemblies.yaml → .github/workflows/assemblies.yaml.bak

@@ -10,7 +10,6 @@ on:
     branches:
       - "*"
       - "!master"
-
 jobs:
   # Downloading_example_data:
   #   runs-on: ubuntu-latest

.github/workflows/conda.yaml → .github/workflows/conda.yaml.bak

@@ -66,6 +66,9 @@ jobs:
           snakefile: ./workflow/Snakefile
           args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config ashleys_pipeline=True use_light_data=True --conda-frontend mamba --report report.zip"
 
+<<<<<<< HEAD
+  # Testing_ashleys_multistep_norm:
+=======
   Testing_ashleys_multistep_norm:
     runs-on: ubuntu-latest
     # needs:
@@ -127,6 +130,7 @@ jobs:
           snakefile: ./workflow/Snakefile
           args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config ashleys_pipeline=True use_light_data=True hgsvc_based_normalized_counts=False --conda-frontend mamba --report report.zip"
   # Testing_ashleys_norm_enabled_multistep_normalisation:
+>>>>>>> origin/master
   #   runs-on: ubuntu-latest
   #   # needs:
   #   #   - Linting
@@ -139,14 +143,23 @@ jobs:
   #         directory: .
   #         snakefile: ./workflow/Snakefile
   #         stagein: "conda config --set channel_priority flexible"
+<<<<<<< HEAD
+  #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config ashleys_pipeline=True use_light_data=True multistep_normalisation=True --conda-frontend mamba -p --verbose --debug"
+=======
   #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config chromosomes=[chr17] ashleys_pipeline=True hgsvc_based_normalized_counts=True multistep_normalisation=True use_light_data=True  --conda-frontend mamba -p --verbose --debug"
+>>>>>>> origin/master
   #     - name: Testing report
   #       uses: snakemake/snakemake-github-action@v1.24.0
   #       with:
   #         directory: .
   #         snakefile: ./workflow/Snakefile
+<<<<<<< HEAD
+  #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config ashleys_pipeline=True use_light_data=True multistep_normalisation=True --conda-frontend mamba --report report.zip"
+  # Testing_ashleys_multistep_norm_for_SV_calling:
+=======
   #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config chromosomes=[chr17] ashleys_pipeline=True hgsvc_based_normalized_counts=True multistep_normalisation=True use_light_data=True  --conda-frontend mamba --report report.zip"
   # Testing_ashleys_jup_nb:
+>>>>>>> origin/master
   #   runs-on: ubuntu-latest
   #   # needs:
   #   #   - Linting
@@ -159,15 +172,156 @@ jobs:
   #         directory: .
   #         snakefile: ./workflow/Snakefile
   #         stagein: "conda config --set channel_priority flexible"
+<<<<<<< HEAD
+  #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config ashleys_pipeline=True use_light_data=True multistep_normalisation=True hgsvc_based_normalized_counts=False multistep_normalisation_for_SV_calling=True --conda-frontend mamba -p --verbose --debug"
+=======
   #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config ashleys_pipeline=True use_light_data=True hand_selection=True --conda-frontend mamba -p --verbose --debug"
+>>>>>>> origin/master
   #     - name: Testing report
   #       uses: snakemake/snakemake-github-action@v1.24.0
   #       with:
   #         directory: .
   #         snakefile: ./workflow/Snakefile
+<<<<<<< HEAD
+  #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config ashleys_pipeline=True use_light_data=True multistep_normalisation=True hgsvc_based_normalized_counts=False multistep_normalisation_for_SV_calling=True --conda-frontend mamba --report report.zip"
+  # Testing_ashleys_norm_counts_disabled:
+  #   runs-on: ubuntu-latest
+  #   # needs:
+  #   #   - Linting
+  #   #   - Formatting
+  #   steps:
+  #     - uses: actions/checkout@v3.3.0
+  #     - name: Testing data
+  #       uses: snakemake/snakemake-github-action@v1.24.0
+  #       with:
+  #         directory: .
+  #         snakefile: ./workflow/Snakefile
+  #         stagein: "conda config --set channel_priority flexible"
+  #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config ashleys_pipeline=True use_light_data=True hgsvc_based_normalized_counts=False --conda-frontend mamba -p --verbose --debug"
+  #     - name: Testing report
+  #       uses: snakemake/snakemake-github-action@v1.24.0
+  #       with:
+  #         directory: .
+  #         snakefile: ./workflow/Snakefile
+  #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config ashleys_pipeline=True use_light_data=True hgsvc_based_normalized_counts=False --conda-frontend mamba --report report.zip"
+  # # Testing_ashleys_norm_enabled_multistep_normalisation:
+  # #   runs-on: ubuntu-latest
+  # #   # needs:
+  # #   #   - Linting
+  # #   #   - Formatting
+  # #   steps:
+  # #     - uses: actions/checkout@v3.3.0
+  # #     - name: Testing data
+  # #       uses: snakemake/snakemake-github-action@v1.24.0
+  # #       with:
+  # #         directory: .
+  # #         snakefile: ./workflow/Snakefile
+  # #         stagein: "conda config --set channel_priority flexible"
+  # #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config chromosomes=[chr17] ashleys_pipeline=True hgsvc_based_normalized_counts=True multistep_normalisation=True use_light_data=True  --conda-frontend mamba -p --verbose --debug"
+  # #     - name: Testing report
+  # #       uses: snakemake/snakemake-github-action@v1.24.0
+  # #       with:
+  # #         directory: .
+  # #         snakefile: ./workflow/Snakefile
+  # #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config chromosomes=[chr17] ashleys_pipeline=True hgsvc_based_normalized_counts=True multistep_normalisation=True use_light_data=True  --conda-frontend mamba --report report.zip"
+  # # Testing_ashleys_jup_nb:
+  # #   runs-on: ubuntu-latest
+  # #   # needs:
+  # #   #   - Linting
+  # #   #   - Formatting
+  # #   steps:
+  # #     - uses: actions/checkout@v3.3.0
+  # #     - name: Testing data
+  # #       uses: snakemake/snakemake-github-action@v1.24.0
+  # #       with:
+  # #         directory: .
+  # #         snakefile: ./workflow/Snakefile
+  # #         stagein: "conda config --set channel_priority flexible"
+  # #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config ashleys_pipeline=True use_light_data=True hand_selection=True --conda-frontend mamba -p --verbose --debug"
+  # #     - name: Testing report
+  # #       uses: snakemake/snakemake-github-action@v1.24.0
+  # #       with:
+  # #         directory: .
+  # #         snakefile: ./workflow/Snakefile
+  # #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config ashleys_pipeline=True use_light_data=True hand_selection=True  --conda-frontend mamba --report report.zip"
+
+  # # Testing_ashleys_norm_enabled_hg19:
+  # #   runs-on: ubuntu-latest
+  # #   # needs:
+  # #   #   - Linting
+  # #   #   - Formatting
+  # #   steps:
+  # #     - uses: actions/checkout@v3.3.0
+  # #     - name: Testing data
+  # #       uses: snakemake/snakemake-github-action@v1.24.0
+  # #       with:
+  # #         directory: .
+  # #         snakefile: ./workflow/Snakefile
+  # #         stagein: "conda config --set channel_priority flexible"
+  # #         args: "--cores 1 --use-conda --latency-wait 60 --config ashleys_pipeline=True use_light_data=True chromosomes=[chr17] hgsvc_based_normalized_counts=True reference=hg19 --conda-frontend mamba -p --verbose --debug"
+  # #     - name: Testing report
+  # #       uses: snakemake/snakemake-github-action@v1.24.0
+  # #       with:
+  # #         directory: .
+  # #         snakefile: ./workflow/Snakefile
+  # #         args: "--cores 1 --use-conda --latency-wait 60 --config ashleys_pipeline=True use_light_data=True chromosomes=[chr17] hgsvc_based_normalized_counts=True reference=hg19 --conda-frontend mamba --report report.zip"
+  # # Testing_ashleys_norm_enabled_hg38_full:
+  # #   runs-on: ubuntu-latest
+  # #   # needs:
+  # #   #   - Linting
+  # #   #   - Formatting
+  # #   steps:
+  # #     - uses: actions/checkout@v3.3.0
+  # #     - name: Testing data
+  # #       uses: snakemake/snakemake-github-action@v1.24.0
+  # #       with:
+  # #         directory: .
+  # #         snakefile: ./workflow/Snakefile
+  # #         stagein: "conda config --set channel_priority flexible"
+  # #         args: "--cores 1 --use-conda --latency-wait 60 --config ashleys_pipeline=True use_light_data=True chromosomes=[chr17] hgsvc_based_normalized_counts=True reference=hg38 --conda-frontend mamba -p --verbose --debug"
+  # #     - name: Testing report
+  # #       uses: snakemake/snakemake-github-action@v1.24.0
+  # #       with:
+  # #         directory: .
+  # #         snakefile: ./workflow/Snakefile
+  # #         args: "--cores 1 --use-conda --latency-wait 60 --config ashleys_pipeline=True use_light_data=True chromosomes=[chr17] hgsvc_based_normalized_counts=True reference=hg38 --conda-frontend mamba --report report.zip"
+  # # Testing_ashleys_norm_enabled_T2T:
+  # #   runs-on: ubuntu-latest
+  # #   # needs:
+  # #   #   - Linting
+  # #   #   - Formatting
+  # #   steps:
+  # #     - uses: actions/checkout@v3.3.0
+  # #     - name: Testing data
+  # #       uses: snakemake/snakemake-github-action@v1.24.0
+  # #       with:
+  # #         directory: .
+  # #         snakefile: ./workflow/Snakefile
+  # #         stagein: "conda config --set channel_priority flexible"
+  # #         args: "--cores 1 --use-conda --latency-wait 60 --config ashleys_pipeline=True use_light_data=True chromosomes=[chr17] hgsvc_based_normalized_counts=True reference=T2T --conda-frontend mamba -p --verbose --debug"
+  # #     - name: Testing report
+  # #       uses: snakemake/snakemake-github-action@v1.24.0
+  # #       with:
+  # #         directory: .
+  # #         snakefile: ./workflow/Snakefile
+  # #         args: "--cores 1 --use-conda --latency-wait 60 --config ashleys_pipeline=True use_light_data=True chromosomes=[chr17] hgsvc_based_normalized_counts=True reference=T2T --conda-frontend mamba --report report.zip"
+  # # Testing_ashleys_norm_enabled_mmconda --latency-wait 60 --config ashleys_pipeline=True use_light_data=True chromosomes=[chr17] hgsvc_based_normalized_counts=True reference=mm10 --conda-frontend mamba --report report.zip"
+  # Linting_arbigent:
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v3.3.0
+  #     - name: Linting
+  #       uses: snakemake/snakemake-github-action@v1.24.0
+  #       with:
+  #         directory: .
+  #         snakefile: ./workflow/Snakefile
+  #         args: "--lint --config arbigent=True"
+  # Testing_arbigent:
+=======
   #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config ashleys_pipeline=True use_light_data=True hand_selection=True  --conda-frontend mamba --report report.zip"
 
   # Testing_ashleys_norm_enabled_hg19:
+>>>>>>> origin/master
   #   runs-on: ubuntu-latest
   #   # needs:
   #   #   - Linting
@@ -180,14 +334,24 @@ jobs:
   #         directory: .
   #         snakefile: ./workflow/Snakefile
   #         stagein: "conda config --set channel_priority flexible"
+<<<<<<< HEAD
+  #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config arbigent=True --conda-frontend mamba -p --verbose --debug"
+=======
   #         args: "--cores 1 --use-conda --latency-wait 60 --config ashleys_pipeline=True use_light_data=True chromosomes=[chr17] hgsvc_based_normalized_counts=True reference=hg19 --conda-frontend mamba -p --verbose --debug"
+>>>>>>> origin/master
   #     - name: Testing report
   #       uses: snakemake/snakemake-github-action@v1.24.0
   #       with:
   #         directory: .
   #         snakefile: ./workflow/Snakefile
+<<<<<<< HEAD
+  #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config arbigent=True --conda-frontend mamba -p --verbose --debug --report report.zip"
+
+  # Testing_publishdir:
+=======
   #         args: "--cores 1 --use-conda --latency-wait 60 --config ashleys_pipeline=True use_light_data=True chromosomes=[chr17] hgsvc_based_normalized_counts=True reference=hg19 --conda-frontend mamba --report report.zip"
   # Testing_ashleys_norm_enabled_hg38_full:
+>>>>>>> origin/master
   #   runs-on: ubuntu-latest
   #   # needs:
   #   #   - Linting
@@ -200,26 +364,44 @@ jobs:
   #         directory: .
   #         snakefile: ./workflow/Snakefile
   #         stagein: "conda config --set channel_priority flexible"
+<<<<<<< HEAD
+  #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config publishdir=.tests/data_chr17_publishdir --conda-frontend mamba -p --verbose --debug"
+=======
   #         args: "--cores 1 --use-conda --latency-wait 60 --config ashleys_pipeline=True use_light_data=True chromosomes=[chr17] hgsvc_based_normalized_counts=True reference=hg38 --conda-frontend mamba -p --verbose --debug"
+>>>>>>> origin/master
   #     - name: Testing report
   #       uses: snakemake/snakemake-github-action@v1.24.0
   #       with:
   #         directory: .
   #         snakefile: ./workflow/Snakefile
+<<<<<<< HEAD
+  #         args: "--cores 1 --use-conda --latency-wait 60 --configfile .tests/config/simple_config.yaml --config publishdir=.tests/data_chr17_publishdir --conda-frontend mamba -p --verbose --debug --report report.zip"
+  # Downloading_extracting_scNOVA_data:
+  #   runs-on: ubuntu-latest
+
+=======
   #         args: "--cores 1 --use-conda --latency-wait 60 --config ashleys_pipeline=True use_light_data=True chromosomes=[chr17] hgsvc_based_normalized_counts=True reference=hg38 --conda-frontend mamba --report report.zip"
   # Testing_ashleys_norm_enabled_T2T:
   #   runs-on: ubuntu-latest
+>>>>>>> origin/master
   #   # needs:
   #   #   - Linting
   #   #   - Formatting
   #   steps:
   #     - uses: actions/checkout@v3.3.0
+<<<<<<< HEAD
+  #     - name: Downloading & extracting scNOVA data
+=======
   #     - name: Testing data
+>>>>>>> origin/master
   #       uses: snakemake/snakemake-github-action@v1.24.0
   #       with:
   #         directory: .
   #         snakefile: ./workflow/Snakefile
   #         stagein: "conda config --set channel_priority flexible"
+<<<<<<< HEAD
+  #         args: "--cores 1 --use-conda --conda-frontend mamba --verbose --debug workflow/data/scNOVA/utils/bin_Genes_for_CNN_reshape_annot.txt -p --latency-wait 120"
+=======
   #         args: "--cores 1 --use-conda --latency-wait 60 --config ashleys_pipeline=True use_light_data=True chromosomes=[chr17] hgsvc_based_normalized_counts=True reference=T2T --conda-frontend mamba -p --verbose --debug"
   #     - name: Testing report
   #       uses: snakemake/snakemake-github-action@v1.24.0
@@ -294,3 +476,4 @@ jobs:
           snakefile: ./workflow/Snakefile
           stagein: "conda config --set channel_priority flexible"
           args: "--cores 1 --use-conda --conda-frontend mamba --verbose --debug workflow/data/scNOVA/utils/bin_Genes_for_CNN_reshape_annot.txt -p --latency-wait 120"
+>>>>>>> origin/master

.github/workflows/main_test.yaml

@@ -0,0 +1,116 @@
+name: MosaiCatcher basic checks
+
+on:
+  push:
+    branches:
+      - "**"
+
+jobs:
+
+  Formatting:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Formatting using Super-Linter
+        uses: github/super-linter@v4
+        env:
+          VALIDATE_ALL_CODEBASE: false
+          DEFAULT_BRANCH: master
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          VALIDATE_SNAKEMAKE_SNAKEFMT: true
+
+
+  Linting:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository with submodules
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'  # Ensures all submodules are checked out
+          lfs: true                # Ensures LFS objects for the main repo are pulled
+
+      - name: Pull LFS objects in submodules
+        run: |
+          git submodule foreach --recursive 'git lfs pull'
+
+      - name: List contents of .tests
+        run: |
+          tree -h .tests
+
+      - name: List contents of workflow/data
+        run: |
+          tree -h workflow/data
+
+      - name: Setup Python
+        uses: actions/setup-python@v5 
+        with:
+          python-version: '3.12' 
+
+      - name: Add Conda to system path
+        run: |
+          echo $CONDA/bin >> $GITHUB_PATH
+
+      - name: Install dependencies
+        run: |
+          conda install -c conda-forge mamba
+
+      - name: Setup snakemake
+        run: |
+          mamba install -c conda-forge -c bioconda snakemake==7.32.4
+
+      - name: Run Snakemake Lint
+        run: |
+          snakemake --lint
+
+  Testing-basic:
+    runs-on: ubuntu-latest
+    # needs: 
+    #   - Formatting
+    #   - Linting
+    steps:
+      - name: Checkout repository with submodules and LFS
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          lfs: true
+
+      - name: Pull LFS objects in submodules
+        run: git submodule foreach --recursive 'git lfs pull'
+
+      - name: Setup Python
+        uses: actions/setup-python@v5 
+        with:
+          python-version: '3.12' 
+
+      - name: List contents of .tests
+        run: |
+          tree -h .tests
+
+      - name: Show config file
+        run: |
+          cat .tests/config/simple_config_ashleys.yaml
+
+      - name: List contents of workflow/data
+        run: |
+          tree -h workflow/data
+
+      - name: Add Conda to system path
+        run: echo "$CONDA/bin" >> $GITHUB_PATH
+
+      - name: Install dependencies
+        run: conda install -c conda-forge mamba
+
+      - name: Setup Snakemake
+        run: mamba install -c conda-forge -c bioconda snakemake==7.32.4
+
+      - name: List options of workflow
+        run: |
+          snakemake --cores 1 --config list_commands=True --verbose --debug
+
+      - name: Run workflow
+        run: |
+          snakemake --cores 1 --use-conda --configfile .tests/config/simple_config_ashleys.yaml \
+          --conda-frontend mamba -p --verbose --debug --snakefile workflow/Snakefile
+