Merge pull request #4 from dbmi-bgm/v2

v2
dbmi-bgm · Sep 23, 2021 · 58d260d · 58d260d
2 parents b08bea3 + 132025b
commit 58d260d
Show file tree

Hide file tree

Showing 55 changed files with 1,256 additions and 182 deletions.
diff --git a/PIPELINE b/PIPELINE
@@ -0,0 +1 @@
+cnv
diff --git a/README.md b/README.md
@@ -2,7 +2,8 @@
 
 * This repo contains CGAP SV Pipeline components
   * CWL
-  * Docker sources - `cgap/cgap-manta:v1` for Manta and `cgap/cnv:v1` for annotation and filtering
+  * Public Docker sources - `cgap/cgap-manta:v2` for Manta and `cgap/cnv:v2` for annotation and filtering
+  * Private ECR sources created dynamically at deployment with `post_patch_to_portal.py`
   * Example Tibanna input jsons for individual steps
   * CGAP Portal Workflows and Metaworkflow
 
@@ -15,12 +16,36 @@ python post_patch_to_portal.py [--ff-env=<env_name>] [--del-prev-version]
                                [--skip-software]
                                [--skip-file-format] [--skip-file-reference]
                                [--skip-workflow] [--skip-metaworkflow]
+                               [--skip-cwl] [--skip-ecr] [--cwl-bucket=<cwl_s3_bucket>]
+                               [--account=<account_num>] [--region=<region>]
                                [--ugrp-unrelated] [--ignore-key-conflict]
+
 # env_name : fourfront-cgapwolf (default), fourfront-cgap
+# cwl_s3_bucket : '' (default); provide s3 cwl bucket name, required for cwl and workflow steps
+# account_num : '' (default); provide aws account number, required for cwl, workflow, and ecr steps
+# region : '' (default); provide aws account region, required for cwl, workflow, and ecr steps
 ```
 
 ### Version updates
 
+#### v2
+
+* The pipeline has been converted to work on private ECR images which are created from our public Docker images
+* Various updates throughout the CGAP SV Pipeline. The current pipeline is outlined below and updates are indicated. A new version of ``granite`` (v0.1.13) is being used for steps 2-13 of the pipeline.
+  * Step 1. Manta-based calling of SVs (**Update**: Manta now uses the `callRegions` flag instead of `regions`. We no longer use get_contigs.py from Parliament2 and have removed the Parliament2 github repo from the `cgap-manta:v2` Dockerfile)
+  * Step 2. Granite SVqcVCF is used to count DEL and DUP variants and provide a total number of DEL and DUP variants in each sample (**New Step**)
+  * Step 3. VEP/sansa annotation (**Update**: VEP now includes the `canonical` flag to identify the canonical transcript for each gene)
+  * Step 4. Granite SVqcVCF is used to count DEL and DUP variants and provide a total number of DEL and DUP variants in each sample (**New Step**)
+  * Step 5. Annotation filtering and SV type selection
+  * Step 6. 20 Unrelated filtering (**Update**: New 20 unrelated reference file resulting from UGRP samples re-mapped with v24 of the CGAP Pipeline including alt index)
+  * Step 7. Granite SVqcVCF is used to count DEL and DUP variants and provide a total number of DEL and DUP variants in each sample (**New Step**)
+  * Step 8. Cytoband annotation step adds the cytoband for each breakpoint; Cyto1 and Cyto2 (**New Step**)
+  * Step 9. Granite SVqcVCF is used to count DEL and DUP variants and provide a total number of DEL and DUP variants in each sample (**New Step**)
+  * Step 10. Length filtering
+  * Step 11. Granite SVqcVCF is used to count DEL and DUP variants and provide a total number of DEL and DUP variants in each sample (**New Step**)
+  * Step 12. Annotation cleaning to produce a vcf file that loads quickly in the Higlass genome browser
+  * Step 13. Granite SVqcVCF is used to count DEL and DUP variants and provide a total number of DEL and DUP variants in each sample (**New Step**)
+
 #### v1
 
 * Created entirely new pipeline - CGAP Structural Variant (SV) Pipeline

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-v1
+v2
diff --git a/cwl/20_unrelated_SV_filter.cwl b/cwl/20_unrelated_SV_filter.cwl
@@ -9,7 +9,7 @@ requirements:
 
 hints:
   - class: DockerRequirement
-    dockerPull: cgap/cnv:v1
+    dockerPull: ACCOUNT/cnv:VERSION
 
 baseCommand: [python3, /usr/local/bin/20_unrelated_SV_filter.py]
 

diff --git a/cwl/SV_annotation_VCF_cleaner.cwl b/cwl/SV_annotation_VCF_cleaner.cwl
@@ -9,7 +9,7 @@ requirements:
 
 hints:
   - class: DockerRequirement
-    dockerPull: cgap/cnv:v1
+    dockerPull: ACCOUNT/cnv:VERSION
 
 baseCommand: [python3, /usr/local/bin/SV_annotation_VCF_cleaner.py]
 

diff --git a/cwl/SV_cytoband.cwl b/cwl/SV_cytoband.cwl
@@ -0,0 +1,45 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+
+class: CommandLineTool
+
+requirements:
+  - class: InlineJavascriptRequirement
+
+hints:
+  - class: DockerRequirement
+    dockerPull: ACCOUNT/cnv:VERSION
+
+baseCommand: [python3, /usr/local/bin/SV_cytoband.py]
+
+inputs:
+  - id: input
+    type: File
+    inputBinding:
+      prefix: -i
+    doc: expect the path to the vcf file
+
+  - id: outputfile
+    type: string
+    default: "output.vcf"
+    inputBinding:
+      prefix: -o
+    doc: name of the output file
+
+  - id: cytoband
+    type: File
+    inputBinding:
+      prefix: -c
+    doc: expect the path to the cytoband reference file
+
+outputs:
+  - id: output
+    type: File
+    outputBinding:
+      glob: $(inputs.outputfile + ".gz")
+    secondaryFiles:
+      - .tbi
+
+doc: |
+  run SV_cytoband.py to add cytoband annotations for each SV breakpoint
diff --git a/cwl/SV_length_filter.cwl b/cwl/SV_length_filter.cwl
@@ -9,7 +9,7 @@ requirements:
 
 hints:
   - class: DockerRequirement
-    dockerPull: cgap/cnv:v1
+    dockerPull: ACCOUNT/cnv:VERSION
 
 baseCommand: [python3, /usr/local/bin/SV_length_filter.py]
 

diff --git a/cwl/SV_type_selector.cwl b/cwl/SV_type_selector.cwl
@@ -9,7 +9,7 @@ requirements:
 
 hints:
   - class: DockerRequirement
-    dockerPull: cgap/cnv:v1
+    dockerPull: ACCOUNT/cnv:VERSION
 
 baseCommand: [python3, /usr/local/bin/SV_type_selector.py]
 

diff --git a/cwl/combine_sansa_vep.cwl b/cwl/combine_sansa_vep.cwl
@@ -9,7 +9,7 @@ requirements:
 
 hints:
   - class: DockerRequirement
-    dockerPull: cgap/cnv:v1
+    dockerPull: ACCOUNT/cnv:VERSION
 
 baseCommand: [combine_sansa_and_VEP_vcf.py]
 

diff --git a/cwl/granite-SVqcVCF.cwl b/cwl/granite-SVqcVCF.cwl
@@ -0,0 +1,43 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+
+class: CommandLineTool
+
+requirements:
+  - class: InlineJavascriptRequirement
+
+hints:
+  - class: DockerRequirement
+    dockerPull: ACCOUNT/cnv:VERSION
+
+baseCommand: [granite, SVqcVCF]
+
+inputs:
+  - id: input_vcf
+    type: File
+    inputBinding:
+      prefix: -i
+    doc: expect the path to the vcf gz file
+
+  - id: outputfile
+    type: string
+    default: "output.json"
+    inputBinding:
+      prefix: -o
+    doc: name of the output file
+
+  - id: samples
+    type: string[]
+    inputBinding:
+      prefix: --samples
+    doc: samples to collect metrics for
+
+outputs:
+  - id: qc_json
+    type: File
+    outputBinding:
+      glob: $(inputs.outputfile)
+
+doc: |
+  run granite SVqcVCF
diff --git a/cwl/granite-blackList_SV.cwl b/cwl/granite-blackList_SV.cwl
@@ -9,7 +9,7 @@ requirements:
 
 hints:
   - class: DockerRequirement
-    dockerPull: cgap/cnv:v1
+    dockerPull: ACCOUNT/cnv:VERSION
 
 baseCommand: [granite, blackList]
 

diff --git a/cwl/granite-geneList_SV.cwl b/cwl/granite-geneList_SV.cwl
@@ -9,7 +9,7 @@ requirements:
 
 hints:
   - class: DockerRequirement
-    dockerPull: cgap/cnv:v1
+    dockerPull: ACCOUNT/cnv:VERSION
 
 baseCommand: [granite, geneList]
 

diff --git a/cwl/granite-whiteList_SV.cwl b/cwl/granite-whiteList_SV.cwl
@@ -9,7 +9,7 @@ requirements:
 
 hints:
   - class: DockerRequirement
-    dockerPull: cgap/cnv:v1
+    dockerPull: ACCOUNT/cnv:VERSION
 
 baseCommand: [granite, whiteList]
 

diff --git a/cwl/manta.cwl b/cwl/manta.cwl
@@ -24,6 +24,14 @@ inputs:
       position: 2
     secondaryFiles:
       - .fai
+  callRegions:
+    type: File
+    inputBinding:
+      prefix: -r
+      separate: false
+      position: 3
+    secondaryFiles:
+      - .tbi
 outputs:
   result:
     type: File
@@ -33,8 +41,8 @@ outputs:
     type: File
     outputBinding:
       glob: variants.vcf.gz
-      
+
 hints:
-  - dockerPull: cgap/cgap-manta:v1
+  - dockerPull: ACCOUNT/manta:VERSION
     class: DockerRequirement
 class: CommandLineTool
diff --git a/cwl/sansa.cwl b/cwl/sansa.cwl
@@ -9,7 +9,7 @@ requirements:
 
 hints:
   - class: DockerRequirement
-    dockerPull: cgap/cnv:v1
+    dockerPull: ACCOUNT/cnv:VERSION
 
 baseCommand: [sansa.sh]
 

diff --git a/cwl/vcf-integrity-check-manta.cwl b/cwl/vcf-integrity-check-manta.cwl
@@ -9,7 +9,7 @@ requirements:
 
 hints:
   - class: DockerRequirement
-    dockerPull: cgap/cgap-manta:v1
+    dockerPull: ACCOUNT/manta:VERSION
 
 baseCommand: [vcf-integrity-check.sh]
 

diff --git a/cwl/vcf-integrity-check.cwl b/cwl/vcf-integrity-check.cwl
@@ -9,7 +9,7 @@ requirements:
 
 hints:
   - class: DockerRequirement
-    dockerPull: cgap/cnv:v1
+    dockerPull: ACCOUNT/cnv:VERSION
 
 baseCommand: [vcf-integrity-check.sh]
 

diff --git a/cwl/vep-annot_SV.cwl b/cwl/vep-annot_SV.cwl
@@ -9,7 +9,7 @@ requirements:
 
 hints:
   - class: DockerRequirement
-    dockerPull: cgap/cnv:v1
+    dockerPull: ACCOUNT/cnv:VERSION
 
 baseCommand: [vep-annot_SV.sh]
 

diff --git a/cwl/workflow_SV_cytoband_plus_vcf-integrity-check.cwl b/cwl/workflow_SV_cytoband_plus_vcf-integrity-check.cwl
@@ -0,0 +1,52 @@
+cwlVersion: v1.0
+
+class: Workflow
+
+requirements:
+  MultipleInputFeatureRequirement: {}
+
+inputs:
+  - id: input_vcf
+    type: File
+    doc: expect the path to the sample vcf gz file
+
+  - id: output_vcf
+    type: string
+    default: "output.vcf"
+    doc: base name of output vcf gz file
+
+  - id: cytoband
+    type: File
+    doc: expect the path to the cytoband reference file
+
+outputs:
+  cytoband_SV_vcf:
+    type: File
+    outputSource: SV_cytoband/output
+
+  vcf-check:
+    type: File
+    outputSource: integrity-check/output
+
+steps:
+  SV_cytoband:
+    run: SV_cytoband.cwl
+    in:
+      input:
+        source: input_vcf
+      outputfile:
+        source: output_vcf
+      cytoband:
+        source: cytoband
+    out: [output]
+
+  integrity-check:
+    run: vcf-integrity-check.cwl
+    in:
+      input:
+        source: SV_cytoband/output
+    out: [output]
+
+doc: |
+  run SV_cytoband.py to add cytoband annotations for each SV breakpoint |
+  run an integrity check on the output vcf gz
diff --git a/cwl/workflow_granite-SVqcVCF.cwl b/cwl/workflow_granite-SVqcVCF.cwl
@@ -0,0 +1,34 @@
+cwlVersion: v1.0
+
+class: Workflow
+
+requirements:
+  MultipleInputFeatureRequirement: {}
+
+inputs:
+  - id: input_vcf
+    type: File
+    doc: expect the path to the vcf gz file
+
+  - id: samples
+    type: string[]
+    doc: samples to collect metrics for
+
+outputs:
+  qc_json:
+    type: File
+    outputSource: granite-SVqcVCF/qc_json
+
+steps:
+  granite-SVqcVCF:
+    run: granite-SVqcVCF.cwl
+    in:
+      input_vcf:
+        source: input_vcf
+      samples:
+        source: samples
+    out: [qc_json]
+
+
+doc: |
+  run granite SVqcVCF
diff --git a/cwl/workflow_manta_integrity-check.cwl b/cwl/workflow_manta_integrity-check.cwl
@@ -21,6 +21,12 @@ inputs:
       - .fai
     doc: expect the path to the reference fasta file
 
+  - id: callRegions
+    type: File
+    secondaryFiles:
+      - .tbi
+    doc: expect the path to the bed file for callRegions
+
 outputs:
   final_zip:
     type: File
@@ -42,6 +48,8 @@ steps:
         source: input_bams
       ref_fasta:
         source: ref_fasta
+      callRegions:
+        source: callRegions
 
     out: [result, variants]