🚧 rework hardfiltering

🐛 fix typos

🧹 cleanup unused tools

🧹 return old output name

Author: dmiller15

subworkflows/kfdrc-gatk-hardfiltering.cwl

@@ -12,13 +12,17 @@ doc: |-
   Finally the VCFs are merged back together using bcftools concat and returned.
 
 inputs:
-  input_vcf: {type: 'File', secondaryFiles: [.tbi], doc: "Input VCF containing INDEL and SNP variants"}
+  input_vcf: {type: 'File', secondaryFiles: [{pattern: '.tbi', required: true}], doc: "Input VCF containing INDEL and SNP variants"}
   output_basename: {type: 'string', doc: "String value to use as the base for the filename of the output"}
   snp_hardfilters: {type: 'string', doc: "String value of hardfilters to set for SNPs in input_vcf" }
   indel_hardfilters: {type: 'string', doc: "String value of hardfilters to set for INDELs in input_vcf" }
+  snp_filtration_extra_args: {type: 'string?', doc: "Any extra arguments for SNP VariantFiltration" }
+  indel_filtration_extra_args: {type: 'string?', doc: "Any extra arguments for INDEL VariantFiltration" }
+  filtration_cpu: { type: 'int?', doc: "CPUs to allocate to GATK VariantFiltration" }
+  filtration_ram: { type: 'int?', doc: "GB of RAM to allocate to GATK VariantFiltration" }
 
 outputs:
-  hardfiltered_vcf: {type: 'File', outputSource: bcftools_concat_snps_indels/output}
+  hardfiltered_vcf: {type: 'File', secondaryFiles: [{pattern: '.tbi', required: true}], outputSource: bcftools_concat_snps_indels/output}
 
 steps:
   gatk_selectvariants_snps:
@@ -39,15 +43,27 @@ steps:
     run: ../tools/gatk_variantfiltration.cwl
     in:
       input_vcf: gatk_selectvariants_snps/output
-      output_basename: output_basename
-      selection: {valueFrom: "SNP"}
+      output_basename:
+        source: output_basename
+        valueFrom: |
+          $(self).snp.filtered
+      variant_filters: snp_hardfilters
+      extra_args: snp_filtration_extra_args
+      max_memory: filtration_ram
+      cpu: filtration_cpu
     out: [output]
   gatk_variantfiltration_indels:
     run: ../tools/gatk_variantfiltration.cwl
     in:
       input_vcf: gatk_selectvariants_indels/output
-      output_basename: output_basename
-      selection: {valueFrom: "INDEL"}
+      output_basename:
+        source: output_basename
+        valueFrom: |
+          $(self).indel.filtered
+      variant_filters: indel_hardfilters
+      extra_args: indel_filtration_extra_args
+      max_memory: filtration_ram
+      cpu: filtration_cpu
     out: [output]
   bcftools_concat_snps_indels:
     run: ../tools/bcftools_concat.cwl

subworkflows/kfdrc-gatk-vqsr.cwl

@@ -6,6 +6,7 @@ doc: |-
 requirements:
 - class: ScatterFeatureRequirement
 - class: InlineJavascriptRequirement
+- class: StepInputExpressionRequirement
 
 inputs:
   genotyped_vcfs: {type: 'File[]', secondaryFiles: [{pattern: '.tbi', required: true}], doc: "Input VCF that has been jointly genotyped"}
@@ -61,20 +62,26 @@ outputs:
 
 steps:
   gatk_filter_excesshet:
-    run: ../tools/tools/gatk_variantfiltration2.cwl
+    run: ../tools/gatk_variantfiltration.cwl
     scatter: [input_vcf]
+    hints:
+    - class: 'sbg:AWSInstanceType'
+      value: m5.4xlarge
     in:
       input_vcf: genotyped_vcfs
-      output_filename:
-        valueFrom: 'variant_filtered.vcf.gz'
+      output_basename:
+        valueFrom: 'excesshet_filtered'
       variant_filters:
         valueFrom: '--filter-expression "ExcessHet > 54.69" --filter-name ExcessHet'
-    out: [filtered_vcf]
+    out: [output]
   gatk_makesitesonlyvcf:
     run: ../tools/gatk_makesitesonlyvcf.cwl
     scatter: [input_vcf]
+    hints:
+    - class: 'sbg:AWSInstanceType'
+      value: m5.4xlarge
     in:
-      input_vcf: gatk_filter_excesshet/filtered_vcf
+      input_vcf: gatk_filter_excesshet/output
       output_filename:
         valueFrom: 'sites_only.variant_filtered.vcf.gz'
     out: [sites_vcf]
@@ -92,7 +99,7 @@ steps:
       one_thousand_genomes_resource_vcf: one_thousand_genomes_resource_vcf
       sites_only_variant_filtered_vcf: gatk_gathervcfs/output
       max_gaussians: snp_max_gaussians
-      tranches: snp_tranches
+      tranche: snp_tranches
       annotations: snp_annotations
       cpu: snp_model_cpu
       ram: snp_model_ram
@@ -105,7 +112,7 @@ steps:
       mills_resource_vcf: mills_resource_vcf
       sites_only_variant_filtered_vcf: gatk_gathervcfs/output
       max_gaussians: indel_max_gaussians
-      tranches: indel_tranches
+      tranche: indel_tranches
       annotations: indel_annotations
       cpu: indel_recal_cpu
       ram: indel_recal_ram
@@ -115,22 +122,25 @@ steps:
     scatter: [sites_only_variant_filtered_vcf]
     hints:
     - class: 'sbg:AWSInstanceType'
-      value: r5.4xlarge
+      value: r5.2xlarge
     in:
-      sites_only_variant_filtered_vcf: gatk_filter_execesshet/filtered_vcf
+      sites_only_variant_filtered_vcf: gatk_filter_excesshet/output
       model_report: gatk_snpsvariantrecalibratorcreatemodel/model_report
       hapmap_resource_vcf: hapmap_resource_vcf
       omni_resource_vcf: omni_resource_vcf
       one_thousand_genomes_resource_vcf: one_thousand_genomes_resource_vcf
       dbsnp_resource_vcf: dbsnp_vcf
       max_gaussians: snp_max_gaussians
-      tranches: snp_tranches
+      tranche: snp_tranches
       annotations: snp_annotations
       cpu: snp_recal_cpu
       ram: snp_recal_ram
     out: [recalibration, tranches]
   gatk_gathertranches:
     run: ../tools/gatk_gathertranches.cwl
+    hints:
+    - class: 'sbg:AWSInstanceType'
+      value: r5.2xlarge
     in:
       tranches: gatk_snpsvariantrecalibratorscattered/tranches
       cpu: gathertranche_cpu
@@ -142,11 +152,11 @@ steps:
     scatterMethod: dotproduct
     hints:
     - class: 'sbg:AWSInstanceType'
-      value: r5.4xlarge
+      value: r5.2xlarge
     in:
       indels_recalibration: gatk_indelsvariantrecalibrator/recalibration
       indels_tranches: gatk_indelsvariantrecalibrator/tranches
-      input_vcf: variants_vcfs
+      input_vcf: gatk_filter_excesshet/output
       snps_recalibration: gatk_snpsvariantrecalibratorscattered/recalibration
       snps_tranches: gatk_gathertranches/output
       snp_ts_filter_level: snp_ts_filter_level

tools/bcftools_concat.cwl

@@ -1,4 +1,4 @@
-cwlVersion: v1.0
+cwlVersion: v1.2
 class: CommandLineTool
 id: bcftools_concat
 requirements:
@@ -27,8 +27,8 @@ arguments:
       tabix $(inputs.output_basename).merged.vcf.gz
 
 inputs:
-    indel_vcf: { type: 'File', secondaryFiles: ['.tbi'], doc: "VCF file containing INDELs" }
-    snp_vcf: { type: 'File', secondaryFiles: ['.tbi'], doc: "VCF file containing SNPs" }
+    indel_vcf: { type: 'File', secondaryFiles: [{pattern: '.tbi', required: true}], doc: "VCF file containing INDELs" }
+    snp_vcf: { type: 'File', secondaryFiles: [{pattern: '.tbi', required: true}], doc: "VCF file containing SNPs" }
     output_basename: { type: 'string', doc: "String value to use as the base of the output filename" }
     ram: { type: 'int?', default: 8, doc: "GB of memory to allocate to this task. default: 8; softcap" }
     cpu: { type: 'int?', default: 4, doc: "Number of CPUs to allocate to this task. default: 4" }
@@ -38,4 +38,4 @@ outputs:
     type: 'File'
     outputBinding:
       glob: '$(inputs.output_basename).merged.vcf.gz'
-    secondaryFiles: ['.tbi']
+    secondaryFiles: [{pattern: '.tbi', required: true}]

tools/filtering_defaults.cwl

@@ -16,7 +16,12 @@ requirements:
 - class: InlineJavascriptRequirement
 inputs:
   num_vcfs: int
-  experiment_type: string
+  experiment_type:
+    type:
+      - type: enum
+        name: experiment_type
+        symbols: ["WGS", "WXS", "Targeted Sequencing"]
+    doc: "Experimental strategy used to sequence the data in the input_vcfs"
 outputs:
   low_data: boolean
   snp_tranches: string[]?

tools/gatk_applyrecalibration.cwl

@@ -1,4 +1,4 @@
-cwlVersion: v1.0
+cwlVersion: v1.2
 class: CommandLineTool
 id: gatk_applyrecalibration
 requirements:
@@ -14,7 +14,7 @@ arguments:
   - position: 0
     shellQuote: false
     valueFrom: >-
-      /gatk --java-options "-Xmx5g -Xms5g"
+      /gatk --java-options "-Xms$(Math.floor((inputs.ram - 1)*1000/1.074-1))m -Xmx$(Math.floor(inputs.ram*1000/1.074-1))m"
       ApplyVQSR
       -O tmp.indel.recalibrated.vcf
       -V $(inputs.input_vcf.path)
@@ -24,7 +24,7 @@ arguments:
       --create-output-bam-index true
       -mode INDEL
 
-      /gatk --java-options "-Xmx5g -Xms5g"
+      /gatk --java-options "-Xms$(Math.floor((inputs.ram - 1)*1000/1.074-1))m -Xmx$(Math.floor(inputs.ram*1000/1.074-1))m"
       ApplyVQSR
       -O scatter.filtered.vcf.gz
       -V tmp.indel.recalibrated.vcf
@@ -36,23 +36,23 @@ arguments:
 inputs:
   input_vcf:
     type: File
-    secondaryFiles: [.tbi]
+    secondaryFiles: [{pattern: '.tbi', required: true}]
   indels_recalibration:
     type: File
-    secondaryFiles: [.idx]
+    secondaryFiles: [{pattern: '.idx', required: true}]
   indels_tranches: File
   snps_recalibration:
     type: File
-    secondaryFiles: [.idx]
+    secondaryFiles: [{pattern: '.idx', required: true}]
   snps_tranches: File
   snp_ts_filter_level: { type: 'float', doc: "The truth sensitivity level at which to start filtering SNP data" }
   indel_ts_filter_level: { type: 'float', doc: "The truth sensitivity level at which to start filtering INDEL data" }
-  cpu: { type: 'int?', default: 2, doc: "CPUs to allocate to this task." }
+  cpu: { type: 'int?', default: 1, doc: "CPUs to allocate to this task." }
   ram: { type: 'int?', default: 7, doc: "GB of RAM to allocate to this task." }
 
 outputs:
   recalibrated_vcf:
     type: File
     outputBinding:
       glob: scatter.filtered.vcf.gz
-    secondaryFiles: [.tbi]
+    secondaryFiles: [{pattern: '.tbi', required: true}]

tools/gatk_gatherfinalvcf.cwl

@@ -1,4 +1,4 @@
-cwlVersion: v1.0
+cwlVersion: v1.2
 class: CommandLineTool
 id: gatk_gathervcfs
 requirements:

tools/gatk_gathertranches.cwl

@@ -1,4 +1,4 @@
-cwlVersion: v1.0
+cwlVersion: v1.2
 class: CommandLineTool
 id: gatk_gathertranches
 requirements:
@@ -13,7 +13,7 @@ arguments:
   - position: 0
     shellQuote: false
     valueFrom: >-
-      /gatk --java-options "-Xmx6g -Xms6g"
+      /gatk --java-options "-Xmx$(Math.floor(inputs.ram*1000/1.074-1))m -Xms$(Math.floor((inputs.ram - 1)*1000/1.074-1))m"
       GatherTranches
       --output snps.gathered.tranches
 inputs:

tools/gatk_gathervcfs.cwl

@@ -1,4 +1,4 @@
-cwlVersion: v1.0
+cwlVersion: v1.2
 class: CommandLineTool
 id: gatk_gathervcfs
 requirements:
@@ -37,4 +37,4 @@ outputs:
     type: File
     outputBinding:
       glob: sites_only.vcf.gz
-    secondaryFiles: [.tbi]
+    secondaryFiles: [{pattern: '.tbi', required: true}]

tools/gatk_genomicsdbimport_genotypegvcfs.cwl

@@ -14,7 +14,7 @@ arguments:
   - position: 0
     shellQuote: false
     valueFrom: >-
-      /gatk --java-options "-Xms$(((inputs.max_memory - 5) * 500)m -Xmx$((inputs.max_memory - 5) * 1000)m"
+      /gatk --java-options "-Xms$((inputs.ram - 5) * 500)m -Xmx$((inputs.ram - 5) * 1000)m"
       GenomicsDBImport
       -L $(inputs.interval.path)
       --interval-padding 5
@@ -24,7 +24,7 @@ arguments:
   - position: 10
     shellQuote: false
     valueFrom: >-
-      && /gatk --java-options "-Xms$(Math.floor(inputs.max_memory*500/1.074-1))m -Xmx$(Math.floor(inputs.max_memory*1000/1.074-1))m"
+      && /gatk --java-options "-Xms$(Math.floor(inputs.ram*500/1.074-1))m -Xmx$(Math.floor(inputs.ram*1000/1.074-1))m"
       GenotypeGVCFs
       -R $(inputs.reference_fasta.path)
       -D $(inputs.dbsnp_vcf.path)
@@ -40,9 +40,9 @@ inputs:
   interval: File
   reference_fasta: { type: 'File', secondaryFiles: [{pattern: '^.dict', required: true}, {pattern: '.fai', required: true}]}
   dbsnp_vcf: { type: 'File', secondaryFiles: [{pattern: '.idx', required: true}]}
-  input_vcfs: { type: { type: array, items: 'File', inputBinding: { prefix: -V } }, inputBinding: { position: 1 }, secondaryFiles: [{pattern: '.tbi', requred: true}] }
+  input_vcfs: { type: { type: array, items: 'File', inputBinding: { prefix: '-V' } }, inputBinding: { position: 1 }, secondaryFiles: [{pattern: '.tbi', required: true}] }
   genomicsdbimport_extra_args: { type: 'string?', inputBinding: { position: 1, shellQuote: false } }
-  genotypegvcfs_extra_args: { type: 'string?' inputBinding: { position: 11, shellQuote: false } }
+  genotypegvcfs_extra_args: { type: 'string?', inputBinding: { position: 11, shellQuote: false } }
   cpu: { type: 'int?', default: 5, doc: "CPUs to allocate to this task" }
   ram: { type: 'int?', default: 10, doc: "GB of RAM to allocate to this task" }
 outputs: