From 6e77e130860017b26c48583afc4cc4018124f5fb Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Tue, 13 Apr 2021 22:37:03 -0400 Subject: [PATCH 01/13] guess instrument model in demux; pass to output --- pipes/WDL/tasks/tasks_assembly.wdl | 4 ++-- pipes/WDL/tasks/tasks_demux.wdl | 18 +++++++++++++----- pipes/WDL/tasks/tasks_interhost.wdl | 2 +- pipes/WDL/tasks/tasks_ncbi.wdl | 6 +++--- pipes/WDL/tasks/tasks_nextstrain.wdl | 4 ++-- pipes/WDL/tasks/tasks_read_utils.wdl | 10 +++++----- pipes/WDL/tasks/tasks_reports.wdl | 12 ++++++------ pipes/WDL/tasks/tasks_taxon_filter.wdl | 2 +- pipes/WDL/tasks/tasks_terra.wdl | 2 +- pipes/WDL/tasks/tasks_utils.wdl | 2 +- pipes/WDL/workflows/demux_deplete.wdl | 3 ++- pipes/WDL/workflows/demux_metag.wdl | 1 + pipes/WDL/workflows/demux_only.wdl | 1 + pipes/WDL/workflows/demux_plus.wdl | 1 + requirements-modules.txt | 2 +- 15 files changed, 41 insertions(+), 29 deletions(-) diff --git a/pipes/WDL/tasks/tasks_assembly.wdl b/pipes/WDL/tasks/tasks_assembly.wdl index 92897a747..54c6abdd9 100644 --- a/pipes/WDL/tasks/tasks_assembly.wdl +++ b/pipes/WDL/tasks/tasks_assembly.wdl @@ -291,7 +291,7 @@ task align_reads { Boolean? skip_mark_dupes=false Int? machine_mem_gb - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" String sample_name = basename(basename(basename(reads_unmapped_bam, ".bam"), ".taxfilt"), ".clean") } @@ -708,7 +708,7 @@ task run_discordance { String out_basename = "run" Int min_coverage=4 - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } command { diff --git a/pipes/WDL/tasks/tasks_demux.wdl b/pipes/WDL/tasks/tasks_demux.wdl index 5b086f449..fd24b62b3 100644 --- a/pipes/WDL/tasks/tasks_demux.wdl +++ b/pipes/WDL/tasks/tasks_demux.wdl @@ -6,7 +6,7 @@ task merge_tarballs { String out_filename Int? machine_mem_gb - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } command { @@ -100,7 +100,7 @@ task illumina_demux { Int? maxRecordsInRam Int? machine_mem_gb - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } parameter_meta { flowcell_tgz: { @@ -250,6 +250,12 @@ task illumina_demux { illumina.py guess_barcodes --expected_assigned_fraction=0 barcodes.txt metrics.txt barcodes_outliers.txt + illumina.py flowcell_metadata --inDir $FLOWCELL_DIR flowcellMetadataFile.tsv + + # output machine model and lane count + grep "machine" flowcellMetadataFile.tsv | cut -f2 | tee MACHINE_MODEL + grep "lane_count" flowcellMetadataFile.tsv | cut -f2 | tee LANE_COUNT + mkdir -p unmatched mv Unmatched.bam unmatched/ @@ -307,9 +313,11 @@ task illumina_demux { File unmatched_reads_bam = "unmatched/Unmatched.bam" Array[File] raw_reads_fastqc = glob("*_fastqc.html") Array[File] raw_reads_fastqc_zip = glob("*_fastqc.zip") - Int max_ram_gb = ceil(read_float("MEM_BYTES")/1000000000) - Int runtime_sec = ceil(read_float("UPTIME_SEC")) - Int cpu_load_15min = ceil(read_float("LOAD_15M")) + Int max_ram_gb = ceil(read_float("MEM_BYTES")/1000000000) + Int runtime_sec = ceil(read_float("UPTIME_SEC")) + Int cpu_load_15min = ceil(read_float("LOAD_15M")) + String instrument_model = read_string("MACHINE_MODEL") + String flowcell_lane_count = read_string("LANE_COUNT") String viralngs_version = read_string("VERSION") Map[String,Map[String,String]] meta_by_sample = read_json('meta_by_sample.json') diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl index 77c05a2b9..abb017a8f 100644 --- a/pipes/WDL/tasks/tasks_interhost.wdl +++ b/pipes/WDL/tasks/tasks_interhost.wdl @@ -142,7 +142,7 @@ task index_ref { File? novocraft_license Int? machine_mem_gb - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } command { diff --git a/pipes/WDL/tasks/tasks_ncbi.wdl b/pipes/WDL/tasks/tasks_ncbi.wdl index 5d6dd42d7..d31a82ff4 100644 --- a/pipes/WDL/tasks/tasks_ncbi.wdl +++ b/pipes/WDL/tasks/tasks_ncbi.wdl @@ -186,7 +186,7 @@ task structured_comments { File? filter_to_ids - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } String out_base = basename(assembly_stats_tsv, '.txt') command <<< @@ -264,7 +264,7 @@ task rename_fasta_header { String out_basename = basename(genome_fasta, ".fasta") - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } command { set -e @@ -402,7 +402,7 @@ task sra_meta_prep { Boolean paired String out_name = "sra_metadata.tsv" - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } parameter_meta { cleaned_bam_filepaths: { diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index 5e88cccdd..856b456d3 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -73,7 +73,7 @@ task derived_cols { String? lab_highlight_loc Array[File] table_map=[] - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } parameter_meta { lab_highlight_loc: { @@ -472,7 +472,7 @@ task filter_sequences_by_length { File sequences_fasta Int min_non_N = 1 - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } parameter_meta { sequences_fasta: { diff --git a/pipes/WDL/tasks/tasks_read_utils.wdl b/pipes/WDL/tasks/tasks_read_utils.wdl index 228b69374..d191414ea 100644 --- a/pipes/WDL/tasks/tasks_read_utils.wdl +++ b/pipes/WDL/tasks/tasks_read_utils.wdl @@ -79,7 +79,7 @@ task get_sample_meta { input { Array[File] samplesheets_extended - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } command <<< python3 << CODE @@ -137,7 +137,7 @@ task merge_and_reheader_bams { File? reheader_table String out_basename - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } command { @@ -197,7 +197,7 @@ task rmdup_ubam { String method="mvicuna" Int? machine_mem_gb - String? docker="quay.io/broadinstitute/viral-core:2.1.19" + String? docker="quay.io/broadinstitute/viral-core:2.1.21" } parameter_meta { @@ -251,7 +251,7 @@ task downsample_bams { Boolean? deduplicateAfter=false Int? machine_mem_gb - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } command { @@ -310,7 +310,7 @@ task FastqToUBAM { String? platform_name String? sequencing_center - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } parameter_meta { fastq_1: { description: "Unaligned read1 file in fastq format", patterns: ["*.fastq", "*.fastq.gz", "*.fq", "*.fq.gz"] } diff --git a/pipes/WDL/tasks/tasks_reports.wdl b/pipes/WDL/tasks/tasks_reports.wdl index 7a789c20f..966527b61 100644 --- a/pipes/WDL/tasks/tasks_reports.wdl +++ b/pipes/WDL/tasks/tasks_reports.wdl @@ -7,7 +7,7 @@ task alignment_metrics { File? primers_bed Int? machine_mem_gb - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } String out_basename = basename(aligned_bam, ".bam") @@ -77,7 +77,7 @@ task plot_coverage { Boolean bin_large_plots=false String? binning_summary_statistic="max" # max or min - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } command { @@ -152,7 +152,7 @@ task coverage_report { Array[File] mapped_bam_idx # optional.. speeds it up if you provide it, otherwise we auto-index String out_report_name="coverage_report.txt" - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } command { @@ -211,7 +211,7 @@ task fastqc { input { File reads_bam - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } String reads_basename=basename(reads_bam, ".bam") @@ -244,7 +244,7 @@ task align_and_count { Int topNHits = 3 Int? machine_mem_gb - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } String reads_basename=basename(reads_bam, ".bam") @@ -289,7 +289,7 @@ task align_and_count_summary { String output_prefix="count_summary" - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } command { diff --git a/pipes/WDL/tasks/tasks_taxon_filter.wdl b/pipes/WDL/tasks/tasks_taxon_filter.wdl index 3de90f735..000c56652 100644 --- a/pipes/WDL/tasks/tasks_taxon_filter.wdl +++ b/pipes/WDL/tasks/tasks_taxon_filter.wdl @@ -202,7 +202,7 @@ task merge_one_per_sample { Boolean? rmdup=false Int? machine_mem_gb - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } command { diff --git a/pipes/WDL/tasks/tasks_terra.wdl b/pipes/WDL/tasks/tasks_terra.wdl index 057e47124..b4f43588a 100644 --- a/pipes/WDL/tasks/tasks_terra.wdl +++ b/pipes/WDL/tasks/tasks_terra.wdl @@ -122,4 +122,4 @@ task download_entities_tsv { output { File tsv_file = '~{outname}' } -} \ No newline at end of file +} diff --git a/pipes/WDL/tasks/tasks_utils.wdl b/pipes/WDL/tasks/tasks_utils.wdl index 1b973531f..3a4bed9e6 100644 --- a/pipes/WDL/tasks/tasks_utils.wdl +++ b/pipes/WDL/tasks/tasks_utils.wdl @@ -204,7 +204,7 @@ task tsv_stack { input { Array[File]+ input_tsvs String out_basename - String docker="quay.io/broadinstitute/viral-core:2.1.19" + String docker="quay.io/broadinstitute/viral-core:2.1.21" } command { diff --git a/pipes/WDL/workflows/demux_deplete.wdl b/pipes/WDL/workflows/demux_deplete.wdl index c019c0d68..76e1ef510 100644 --- a/pipes/WDL/workflows/demux_deplete.wdl +++ b/pipes/WDL/workflows/demux_deplete.wdl @@ -115,7 +115,7 @@ workflow demux_deplete { platform = "ILLUMINA", paired = (illumina_demux.run_info[0]['indexes'] == '2'), out_name = "sra_metadata-~{basename(flowcell_tgz, '.tar.gz')}.tsv", - instrument_model = select_first([instrument_model]), + instrument_model = select_first(flatten([illumina_demux.instrument_model,[""]])), title = select_first([sra_title]) } } @@ -163,6 +163,7 @@ workflow demux_deplete { File spikein_counts = spike_summary.count_summary String run_date = illumina_demux.run_info[0]['run_start_date'] + String instrument_model = select_first(flatten([illumina_demux.instrument_model,[""]])) String demux_viral_core_version = illumina_demux.viralngs_version[0] } diff --git a/pipes/WDL/workflows/demux_metag.wdl b/pipes/WDL/workflows/demux_metag.wdl index 02bb802a0..1673924bf 100644 --- a/pipes/WDL/workflows/demux_metag.wdl +++ b/pipes/WDL/workflows/demux_metag.wdl @@ -115,6 +115,7 @@ workflow demux_metag { File demux_metrics = illumina_demux.metrics File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes + String instrument_model = select_first([illumina_demux.instrument_model,""]) File multiqc_report_raw = multiqc_raw.multiqc_report File multiqc_report_cleaned = multiqc_cleaned.multiqc_report diff --git a/pipes/WDL/workflows/demux_only.wdl b/pipes/WDL/workflows/demux_only.wdl index 36bafc703..053994b4c 100644 --- a/pipes/WDL/workflows/demux_only.wdl +++ b/pipes/WDL/workflows/demux_only.wdl @@ -23,6 +23,7 @@ workflow demux_only { File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes File multiqc_report_raw = MultiQC.multiqc_report + String instrument_model = select_first([illumina_demux.instrument_model,""]) String demux_viral_core_version = illumina_demux.viralngs_version } } diff --git a/pipes/WDL/workflows/demux_plus.wdl b/pipes/WDL/workflows/demux_plus.wdl index c768f5a31..7d7d04c59 100644 --- a/pipes/WDL/workflows/demux_plus.wdl +++ b/pipes/WDL/workflows/demux_plus.wdl @@ -85,6 +85,7 @@ workflow demux_plus { File demux_metrics = illumina_demux.metrics File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes + String instrument_model = select_first([illumina_demux.instrument_model,""]) File multiqc_report_raw = multiqc_raw.multiqc_report File multiqc_report_cleaned = multiqc_cleaned.multiqc_report diff --git a/requirements-modules.txt b/requirements-modules.txt index a5d5ee940..47891a905 100644 --- a/requirements-modules.txt +++ b/requirements-modules.txt @@ -1,4 +1,4 @@ -broadinstitute/viral-core=2.1.19 +broadinstitute/viral-core=2.1.21 broadinstitute/viral-assemble=2.1.16.1 broadinstitute/viral-classify=2.1.16.0 broadinstitute/viral-phylo=2.1.19.1 From 6b272f39c916ab9fea74232813778e35d96ca201 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Sat, 24 Apr 2021 23:58:12 -0400 Subject: [PATCH 02/13] incorporate changes following review by @dpark01 allow for override of instrument_model with instrument_model_user_specified --- pipes/WDL/workflows/demux_deplete.wdl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipes/WDL/workflows/demux_deplete.wdl b/pipes/WDL/workflows/demux_deplete.wdl index beaf27dc2..2d93b33d1 100644 --- a/pipes/WDL/workflows/demux_deplete.wdl +++ b/pipes/WDL/workflows/demux_deplete.wdl @@ -20,7 +20,7 @@ workflow demux_deplete { File? biosample_map Int min_reads_per_bam = 100 - String? instrument_model + String? instrument_model_user_specified String? sra_title File spikein_db @@ -116,7 +116,7 @@ workflow demux_deplete { platform = "ILLUMINA", paired = (illumina_demux.run_info[0]['indexes'] == '2'), out_name = "sra_metadata-~{flowcell_id}.tsv", - instrument_model = select_first(flatten([illumina_demux.instrument_model,[""]])), + instrument_model = select_first(flatten([[instrument_model_user_specified],illumina_demux.instrument_model])) title = select_first([sra_title]) } } @@ -164,7 +164,7 @@ workflow demux_deplete { File spikein_counts = spike_summary.count_summary String run_date = illumina_demux.run_info[0]['run_start_date'] - String instrument_model = select_first(flatten([illumina_demux.instrument_model,[""]])) + String instrument_model = select_first(flatten([[instrument_model_user_specified],illumina_demux.instrument_model])) String demux_viral_core_version = illumina_demux.viralngs_version[0] } From f3c5a615357f88bf7dfedac3428cf4a1423fcea2 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Sun, 25 Apr 2021 00:45:41 -0400 Subject: [PATCH 03/13] add missing comma --- pipes/WDL/workflows/demux_deplete.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/WDL/workflows/demux_deplete.wdl b/pipes/WDL/workflows/demux_deplete.wdl index 2d93b33d1..b1011b369 100644 --- a/pipes/WDL/workflows/demux_deplete.wdl +++ b/pipes/WDL/workflows/demux_deplete.wdl @@ -116,7 +116,7 @@ workflow demux_deplete { platform = "ILLUMINA", paired = (illumina_demux.run_info[0]['indexes'] == '2'), out_name = "sra_metadata-~{flowcell_id}.tsv", - instrument_model = select_first(flatten([[instrument_model_user_specified],illumina_demux.instrument_model])) + instrument_model = select_first(flatten([[instrument_model_user_specified],illumina_demux.instrument_model])), title = select_first([sra_title]) } } From 284522c65ceccacaeabc5ccb35869dc2dc2b0da1 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Sun, 25 Apr 2021 00:55:12 -0400 Subject: [PATCH 04/13] output instrument_model from demux_deplete in sarscov2_illumina_full assembly comment output instrument_model from demux_deplete in sarscov2_illumina_full assembly comment; pass in overrride to to demux_deplete via instrument_model_user_specified --- pipes/WDL/workflows/sarscov2_illumina_full.wdl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index 14b4a0a77..dbd2c4b56 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -42,7 +42,7 @@ workflow sarscov2_illumina_full { String amplicon_bed_prefix Array[File] biosample_attributes - String instrument_model + String? instrument_model String sra_title Int min_genome_bases = 24000 @@ -73,10 +73,10 @@ workflow sarscov2_illumina_full { ### demux, deplete, SRA submission prep, fastqc/multiqc call demux_deplete.demux_deplete { input: - flowcell_tgz = flowcell_tgz, - biosample_map = biosample_merge.out_tsv, - instrument_model = instrument_model, - sra_title = sra_title + flowcell_tgz = flowcell_tgz, + biosample_map = biosample_merge.out_tsv, + instrument_model_user_specified = instrument_model, + sra_title = sra_title } ### gather data by biosample @@ -129,7 +129,7 @@ workflow sarscov2_illumina_full { File passing_assemblies = rename_fasta_header.renamed_fasta String passing_assembly_ids = orig_name - Array[String] assembly_cmt = [orig_name, "Broad viral-ngs v. " + demux_deplete.demux_viral_core_version, assemble_refbased.assembly_mean_coverage, instrument_model] + Array[String] assembly_cmt = [orig_name, "Broad viral-ngs v. " + demux_deplete.demux_viral_core_version, assemble_refbased.assembly_mean_coverage, demux_deplete.instrument_model] # lineage assignment call sarscov2_lineages.sarscov2_lineages { From 2329b9ccfc7feadb3c033cb155fb8506a08e724f Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Sun, 25 Apr 2021 10:54:56 -0400 Subject: [PATCH 05/13] try flattening --- pipes/WDL/workflows/demux_metag.wdl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipes/WDL/workflows/demux_metag.wdl b/pipes/WDL/workflows/demux_metag.wdl index 56e4e7b0e..a7ffe6e7e 100644 --- a/pipes/WDL/workflows/demux_metag.wdl +++ b/pipes/WDL/workflows/demux_metag.wdl @@ -14,6 +14,7 @@ workflow demux_metag { Array[File]? bmtaggerDbs # .tar.gz, .tgz, .tar.bz2, .tar.lz4, .fasta, or .fasta.gz Array[File]? blastDbs # .tar.gz, .tgz, .tar.bz2, .tar.lz4, .fasta, or .fasta.gz Array[File]? bwaDbs + String? instrument_model_user_specified File kraken2_db_tgz File krona_taxonomy_db_kraken2_tgz @@ -116,7 +117,7 @@ workflow demux_metag { File demux_metrics = illumina_demux.metrics File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes - String instrument_model = select_first([illumina_demux.instrument_model,""]) + String instrument_model = select_first(flatten([[instrument_model_user_specified],[illumina_demux.instrument_model]])) File multiqc_report_raw = multiqc_raw.multiqc_report File multiqc_report_cleaned = multiqc_cleaned.multiqc_report From 8c7b57c5a6b37c0d9a3d884338c0c7103e73cca4 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Mon, 26 Apr 2021 12:34:23 -0400 Subject: [PATCH 06/13] rename instrument_model to instrument_model_inferred in outputs rename instrument_model to instrument_model_inferred in outputs to avoid collision with task-level output naming --- pipes/WDL/workflows/demux_deplete.wdl | 2 +- pipes/WDL/workflows/demux_metag.wdl | 2 +- pipes/WDL/workflows/demux_only.wdl | 14 +++++++------- pipes/WDL/workflows/demux_plus.wdl | 2 +- pipes/WDL/workflows/sarscov2_illumina_full.wdl | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pipes/WDL/workflows/demux_deplete.wdl b/pipes/WDL/workflows/demux_deplete.wdl index b1011b369..751238007 100644 --- a/pipes/WDL/workflows/demux_deplete.wdl +++ b/pipes/WDL/workflows/demux_deplete.wdl @@ -164,7 +164,7 @@ workflow demux_deplete { File spikein_counts = spike_summary.count_summary String run_date = illumina_demux.run_info[0]['run_start_date'] - String instrument_model = select_first(flatten([[instrument_model_user_specified],illumina_demux.instrument_model])) + String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],illumina_demux.instrument_model])) String demux_viral_core_version = illumina_demux.viralngs_version[0] } diff --git a/pipes/WDL/workflows/demux_metag.wdl b/pipes/WDL/workflows/demux_metag.wdl index a7ffe6e7e..095bd2460 100644 --- a/pipes/WDL/workflows/demux_metag.wdl +++ b/pipes/WDL/workflows/demux_metag.wdl @@ -117,7 +117,7 @@ workflow demux_metag { File demux_metrics = illumina_demux.metrics File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes - String instrument_model = select_first(flatten([[instrument_model_user_specified],[illumina_demux.instrument_model]])) + String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],[illumina_demux.instrument_model]])) File multiqc_report_raw = multiqc_raw.multiqc_report File multiqc_report_cleaned = multiqc_cleaned.multiqc_report diff --git a/pipes/WDL/workflows/demux_only.wdl b/pipes/WDL/workflows/demux_only.wdl index 053994b4c..617070d1c 100644 --- a/pipes/WDL/workflows/demux_only.wdl +++ b/pipes/WDL/workflows/demux_only.wdl @@ -18,12 +18,12 @@ workflow demux_only { } output { - Array[File] raw_reads_unaligned_bams = illumina_demux.raw_reads_unaligned_bams - File demux_metrics = illumina_demux.metrics - File demux_commonBarcodes = illumina_demux.commonBarcodes - File demux_outlierBarcodes = illumina_demux.outlierBarcodes - File multiqc_report_raw = MultiQC.multiqc_report - String instrument_model = select_first([illumina_demux.instrument_model,""]) - String demux_viral_core_version = illumina_demux.viralngs_version + Array[File] raw_reads_unaligned_bams = illumina_demux.raw_reads_unaligned_bams + File demux_metrics = illumina_demux.metrics + File demux_commonBarcodes = illumina_demux.commonBarcodes + File demux_outlierBarcodes = illumina_demux.outlierBarcodes + File multiqc_report_raw = MultiQC.multiqc_report + String instrument_model_inferred = select_first([illumina_demux.instrument_model,""]) + String demux_viral_core_version = illumina_demux.viralngs_version } } diff --git a/pipes/WDL/workflows/demux_plus.wdl b/pipes/WDL/workflows/demux_plus.wdl index 0b5fbd5a9..90eff1046 100644 --- a/pipes/WDL/workflows/demux_plus.wdl +++ b/pipes/WDL/workflows/demux_plus.wdl @@ -85,7 +85,7 @@ workflow demux_plus { File demux_metrics = illumina_demux.metrics File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes - String instrument_model = select_first([illumina_demux.instrument_model,""]) + String instrument_model_inferred = select_first([illumina_demux.instrument_model,""]) File multiqc_report_raw = multiqc_raw.multiqc_report File multiqc_report_cleaned = multiqc_cleaned.multiqc_report diff --git a/pipes/WDL/workflows/sarscov2_illumina_full.wdl b/pipes/WDL/workflows/sarscov2_illumina_full.wdl index dbd2c4b56..600ab2d12 100644 --- a/pipes/WDL/workflows/sarscov2_illumina_full.wdl +++ b/pipes/WDL/workflows/sarscov2_illumina_full.wdl @@ -129,7 +129,7 @@ workflow sarscov2_illumina_full { File passing_assemblies = rename_fasta_header.renamed_fasta String passing_assembly_ids = orig_name - Array[String] assembly_cmt = [orig_name, "Broad viral-ngs v. " + demux_deplete.demux_viral_core_version, assemble_refbased.assembly_mean_coverage, demux_deplete.instrument_model] + Array[String] assembly_cmt = [orig_name, "Broad viral-ngs v. " + demux_deplete.demux_viral_core_version, assemble_refbased.assembly_mean_coverage, demux_deplete.instrument_model_inferred] # lineage assignment call sarscov2_lineages.sarscov2_lineages { From b9d18a35ce5dbab325357722776aaf854cb88b74 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Wed, 28 Apr 2021 15:10:49 -0400 Subject: [PATCH 07/13] allow user override of instrument_model w/ instrument_model_user_specified --- pipes/WDL/workflows/demux_only.wdl | 6 +++++- pipes/WDL/workflows/demux_plus.wdl | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pipes/WDL/workflows/demux_only.wdl b/pipes/WDL/workflows/demux_only.wdl index 617070d1c..e7262532f 100644 --- a/pipes/WDL/workflows/demux_only.wdl +++ b/pipes/WDL/workflows/demux_only.wdl @@ -10,6 +10,10 @@ workflow demux_only { email: "viral-ngs@broadinstitute.org" } + input { + String? instrument_model_user_specified + } + call tasks_demux.illumina_demux call reports.MultiQC { @@ -23,7 +27,7 @@ workflow demux_only { File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes File multiqc_report_raw = MultiQC.multiqc_report - String instrument_model_inferred = select_first([illumina_demux.instrument_model,""]) + String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],[illumina_demux.instrument_model]])) String demux_viral_core_version = illumina_demux.viralngs_version } } diff --git a/pipes/WDL/workflows/demux_plus.wdl b/pipes/WDL/workflows/demux_plus.wdl index 90eff1046..c6ffc0dc2 100644 --- a/pipes/WDL/workflows/demux_plus.wdl +++ b/pipes/WDL/workflows/demux_plus.wdl @@ -20,6 +20,7 @@ workflow demux_plus { Array[File]? bmtaggerDbs # .tar.gz, .tgz, .tar.bz2, .tar.lz4, .fasta, or .fasta.gz Array[File]? blastDbs # .tar.gz, .tgz, .tar.bz2, .tar.lz4, .fasta, or .fasta.gz Array[File]? bwaDbs + String? instrument_model_user_specified } call demux.illumina_demux as illumina_demux @@ -85,7 +86,7 @@ workflow demux_plus { File demux_metrics = illumina_demux.metrics File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes - String instrument_model_inferred = select_first([illumina_demux.instrument_model,""]) + String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],[illumina_demux.instrument_model]])) File multiqc_report_raw = multiqc_raw.multiqc_report File multiqc_report_cleaned = multiqc_cleaned.multiqc_report From 8f4ec35ec3c9c7501808a4548c30ee8b13c17aca Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Thu, 29 Apr 2021 15:15:17 -0400 Subject: [PATCH 08/13] viral-core 2.1.24 -> 2.1.25 --- pipes/WDL/tasks/tasks_assembly.wdl | 4 ++-- pipes/WDL/tasks/tasks_demux.wdl | 4 ++-- pipes/WDL/tasks/tasks_interhost.wdl | 2 +- pipes/WDL/tasks/tasks_ncbi.wdl | 6 +++--- pipes/WDL/tasks/tasks_nextstrain.wdl | 4 ++-- pipes/WDL/tasks/tasks_read_utils.wdl | 10 +++++----- pipes/WDL/tasks/tasks_reports.wdl | 12 ++++++------ pipes/WDL/tasks/tasks_taxon_filter.wdl | 2 +- pipes/WDL/tasks/tasks_utils.wdl | 2 +- requirements-modules.txt | 2 +- 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pipes/WDL/tasks/tasks_assembly.wdl b/pipes/WDL/tasks/tasks_assembly.wdl index 8163f0729..4c0aac36c 100644 --- a/pipes/WDL/tasks/tasks_assembly.wdl +++ b/pipes/WDL/tasks/tasks_assembly.wdl @@ -291,7 +291,7 @@ task align_reads { Boolean? skip_mark_dupes = false Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" String sample_name = basename(basename(basename(reads_unmapped_bam, ".bam"), ".taxfilt"), ".clean") } @@ -708,7 +708,7 @@ task run_discordance { String out_basename = "run" Int min_coverage = 4 - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } command { diff --git a/pipes/WDL/tasks/tasks_demux.wdl b/pipes/WDL/tasks/tasks_demux.wdl index e0d9cfe7a..c7c0f4123 100644 --- a/pipes/WDL/tasks/tasks_demux.wdl +++ b/pipes/WDL/tasks/tasks_demux.wdl @@ -6,7 +6,7 @@ task merge_tarballs { String out_filename Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } command { @@ -100,7 +100,7 @@ task illumina_demux { Int? maxRecordsInRam Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } parameter_meta { flowcell_tgz: { diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl index eed4c33ad..6532bc4f8 100644 --- a/pipes/WDL/tasks/tasks_interhost.wdl +++ b/pipes/WDL/tasks/tasks_interhost.wdl @@ -142,7 +142,7 @@ task index_ref { File? novocraft_license Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } command { diff --git a/pipes/WDL/tasks/tasks_ncbi.wdl b/pipes/WDL/tasks/tasks_ncbi.wdl index 473a7bb07..a29be70f1 100644 --- a/pipes/WDL/tasks/tasks_ncbi.wdl +++ b/pipes/WDL/tasks/tasks_ncbi.wdl @@ -186,7 +186,7 @@ task structured_comments { File? filter_to_ids - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } String out_base = basename(assembly_stats_tsv, '.txt') command <<< @@ -264,7 +264,7 @@ task rename_fasta_header { String out_basename = basename(genome_fasta, ".fasta") - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } command { set -e @@ -447,7 +447,7 @@ task sra_meta_prep { Boolean paired String out_name = "sra_metadata.tsv" - String docker="quay.io/broadinstitute/viral-core:2.1.24" + String docker="quay.io/broadinstitute/viral-core:2.1.25" } parameter_meta { cleaned_bam_filepaths: { diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index f411aba7b..846d4de03 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -81,7 +81,7 @@ task derived_cols { String? lab_highlight_loc Array[File] table_map = [] - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } parameter_meta { lab_highlight_loc: { @@ -480,7 +480,7 @@ task filter_sequences_by_length { File sequences_fasta Int min_non_N = 1 - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } parameter_meta { sequences_fasta: { diff --git a/pipes/WDL/tasks/tasks_read_utils.wdl b/pipes/WDL/tasks/tasks_read_utils.wdl index ff32a48a9..238d302ec 100644 --- a/pipes/WDL/tasks/tasks_read_utils.wdl +++ b/pipes/WDL/tasks/tasks_read_utils.wdl @@ -79,7 +79,7 @@ task get_sample_meta { input { Array[File] samplesheets_extended - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } command <<< python3 << CODE @@ -137,7 +137,7 @@ task merge_and_reheader_bams { File? reheader_table String out_basename - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } command { @@ -197,7 +197,7 @@ task rmdup_ubam { String method = "mvicuna" Int? machine_mem_gb - String? docker = "quay.io/broadinstitute/viral-core:2.1.24" + String? docker = "quay.io/broadinstitute/viral-core:2.1.25" } parameter_meta { @@ -251,7 +251,7 @@ task downsample_bams { Boolean? deduplicateAfter = false Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } command { @@ -310,7 +310,7 @@ task FastqToUBAM { String? platform_name String? sequencing_center - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } parameter_meta { fastq_1: { description: "Unaligned read1 file in fastq format", patterns: ["*.fastq", "*.fastq.gz", "*.fq", "*.fq.gz"] } diff --git a/pipes/WDL/tasks/tasks_reports.wdl b/pipes/WDL/tasks/tasks_reports.wdl index 45ca73ec8..238973c13 100644 --- a/pipes/WDL/tasks/tasks_reports.wdl +++ b/pipes/WDL/tasks/tasks_reports.wdl @@ -7,7 +7,7 @@ task alignment_metrics { File? primers_bed Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } String out_basename = basename(aligned_bam, ".bam") @@ -77,7 +77,7 @@ task plot_coverage { Boolean bin_large_plots = false String? binning_summary_statistic = "max" # max or min - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } command { @@ -152,7 +152,7 @@ task coverage_report { Array[File] mapped_bam_idx # optional.. speeds it up if you provide it, otherwise we auto-index String out_report_name = "coverage_report.txt" - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } command { @@ -211,7 +211,7 @@ task fastqc { input { File reads_bam - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } String reads_basename=basename(reads_bam, ".bam") @@ -244,7 +244,7 @@ task align_and_count { Int topNHits = 3 Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } String reads_basename=basename(reads_bam, ".bam") @@ -289,7 +289,7 @@ task align_and_count_summary { String output_prefix = "count_summary" - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } command { diff --git a/pipes/WDL/tasks/tasks_taxon_filter.wdl b/pipes/WDL/tasks/tasks_taxon_filter.wdl index 0a84e6348..af2938334 100644 --- a/pipes/WDL/tasks/tasks_taxon_filter.wdl +++ b/pipes/WDL/tasks/tasks_taxon_filter.wdl @@ -202,7 +202,7 @@ task merge_one_per_sample { Boolean? rmdup = false Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } command { diff --git a/pipes/WDL/tasks/tasks_utils.wdl b/pipes/WDL/tasks/tasks_utils.wdl index 30f3fbd7d..bd78664f3 100644 --- a/pipes/WDL/tasks/tasks_utils.wdl +++ b/pipes/WDL/tasks/tasks_utils.wdl @@ -204,7 +204,7 @@ task tsv_stack { input { Array[File]+ input_tsvs String out_basename - String docker = "quay.io/broadinstitute/viral-core:2.1.24" + String docker = "quay.io/broadinstitute/viral-core:2.1.25" } command { diff --git a/requirements-modules.txt b/requirements-modules.txt index 995c93387..523ff1348 100644 --- a/requirements-modules.txt +++ b/requirements-modules.txt @@ -1,4 +1,4 @@ -broadinstitute/viral-core=2.1.24 +broadinstitute/viral-core=2.1.25 broadinstitute/viral-assemble=2.1.16.1 broadinstitute/viral-classify=2.1.16.0 broadinstitute/viral-phylo=2.1.19.1 From e9438194e91cb6ee5b4df866962a443981fd571b Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Thu, 29 Apr 2021 21:23:45 -0400 Subject: [PATCH 09/13] viral-core 2.1.25 -> 2.1.26 --- pipes/WDL/tasks/tasks_assembly.wdl | 4 ++-- pipes/WDL/tasks/tasks_demux.wdl | 4 ++-- pipes/WDL/tasks/tasks_interhost.wdl | 2 +- pipes/WDL/tasks/tasks_ncbi.wdl | 6 +++--- pipes/WDL/tasks/tasks_nextstrain.wdl | 4 ++-- pipes/WDL/tasks/tasks_read_utils.wdl | 10 +++++----- pipes/WDL/tasks/tasks_reports.wdl | 12 ++++++------ pipes/WDL/tasks/tasks_taxon_filter.wdl | 2 +- pipes/WDL/tasks/tasks_utils.wdl | 2 +- requirements-modules.txt | 2 +- 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pipes/WDL/tasks/tasks_assembly.wdl b/pipes/WDL/tasks/tasks_assembly.wdl index 4c0aac36c..eaffb72ae 100644 --- a/pipes/WDL/tasks/tasks_assembly.wdl +++ b/pipes/WDL/tasks/tasks_assembly.wdl @@ -291,7 +291,7 @@ task align_reads { Boolean? skip_mark_dupes = false Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" String sample_name = basename(basename(basename(reads_unmapped_bam, ".bam"), ".taxfilt"), ".clean") } @@ -708,7 +708,7 @@ task run_discordance { String out_basename = "run" Int min_coverage = 4 - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } command { diff --git a/pipes/WDL/tasks/tasks_demux.wdl b/pipes/WDL/tasks/tasks_demux.wdl index c7c0f4123..fe579725b 100644 --- a/pipes/WDL/tasks/tasks_demux.wdl +++ b/pipes/WDL/tasks/tasks_demux.wdl @@ -6,7 +6,7 @@ task merge_tarballs { String out_filename Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } command { @@ -100,7 +100,7 @@ task illumina_demux { Int? maxRecordsInRam Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } parameter_meta { flowcell_tgz: { diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl index 6532bc4f8..8652ee306 100644 --- a/pipes/WDL/tasks/tasks_interhost.wdl +++ b/pipes/WDL/tasks/tasks_interhost.wdl @@ -142,7 +142,7 @@ task index_ref { File? novocraft_license Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } command { diff --git a/pipes/WDL/tasks/tasks_ncbi.wdl b/pipes/WDL/tasks/tasks_ncbi.wdl index a29be70f1..eb996dd52 100644 --- a/pipes/WDL/tasks/tasks_ncbi.wdl +++ b/pipes/WDL/tasks/tasks_ncbi.wdl @@ -186,7 +186,7 @@ task structured_comments { File? filter_to_ids - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } String out_base = basename(assembly_stats_tsv, '.txt') command <<< @@ -264,7 +264,7 @@ task rename_fasta_header { String out_basename = basename(genome_fasta, ".fasta") - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } command { set -e @@ -447,7 +447,7 @@ task sra_meta_prep { Boolean paired String out_name = "sra_metadata.tsv" - String docker="quay.io/broadinstitute/viral-core:2.1.25" + String docker="quay.io/broadinstitute/viral-core:2.1.26" } parameter_meta { cleaned_bam_filepaths: { diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index 846d4de03..4ad05693e 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -81,7 +81,7 @@ task derived_cols { String? lab_highlight_loc Array[File] table_map = [] - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } parameter_meta { lab_highlight_loc: { @@ -480,7 +480,7 @@ task filter_sequences_by_length { File sequences_fasta Int min_non_N = 1 - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } parameter_meta { sequences_fasta: { diff --git a/pipes/WDL/tasks/tasks_read_utils.wdl b/pipes/WDL/tasks/tasks_read_utils.wdl index 238d302ec..fb994b4f8 100644 --- a/pipes/WDL/tasks/tasks_read_utils.wdl +++ b/pipes/WDL/tasks/tasks_read_utils.wdl @@ -79,7 +79,7 @@ task get_sample_meta { input { Array[File] samplesheets_extended - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } command <<< python3 << CODE @@ -137,7 +137,7 @@ task merge_and_reheader_bams { File? reheader_table String out_basename - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } command { @@ -197,7 +197,7 @@ task rmdup_ubam { String method = "mvicuna" Int? machine_mem_gb - String? docker = "quay.io/broadinstitute/viral-core:2.1.25" + String? docker = "quay.io/broadinstitute/viral-core:2.1.26" } parameter_meta { @@ -251,7 +251,7 @@ task downsample_bams { Boolean? deduplicateAfter = false Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } command { @@ -310,7 +310,7 @@ task FastqToUBAM { String? platform_name String? sequencing_center - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } parameter_meta { fastq_1: { description: "Unaligned read1 file in fastq format", patterns: ["*.fastq", "*.fastq.gz", "*.fq", "*.fq.gz"] } diff --git a/pipes/WDL/tasks/tasks_reports.wdl b/pipes/WDL/tasks/tasks_reports.wdl index 238973c13..7585d13e1 100644 --- a/pipes/WDL/tasks/tasks_reports.wdl +++ b/pipes/WDL/tasks/tasks_reports.wdl @@ -7,7 +7,7 @@ task alignment_metrics { File? primers_bed Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } String out_basename = basename(aligned_bam, ".bam") @@ -77,7 +77,7 @@ task plot_coverage { Boolean bin_large_plots = false String? binning_summary_statistic = "max" # max or min - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } command { @@ -152,7 +152,7 @@ task coverage_report { Array[File] mapped_bam_idx # optional.. speeds it up if you provide it, otherwise we auto-index String out_report_name = "coverage_report.txt" - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } command { @@ -211,7 +211,7 @@ task fastqc { input { File reads_bam - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } String reads_basename=basename(reads_bam, ".bam") @@ -244,7 +244,7 @@ task align_and_count { Int topNHits = 3 Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } String reads_basename=basename(reads_bam, ".bam") @@ -289,7 +289,7 @@ task align_and_count_summary { String output_prefix = "count_summary" - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } command { diff --git a/pipes/WDL/tasks/tasks_taxon_filter.wdl b/pipes/WDL/tasks/tasks_taxon_filter.wdl index af2938334..39fdb4297 100644 --- a/pipes/WDL/tasks/tasks_taxon_filter.wdl +++ b/pipes/WDL/tasks/tasks_taxon_filter.wdl @@ -202,7 +202,7 @@ task merge_one_per_sample { Boolean? rmdup = false Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } command { diff --git a/pipes/WDL/tasks/tasks_utils.wdl b/pipes/WDL/tasks/tasks_utils.wdl index bd78664f3..3c7a18cf3 100644 --- a/pipes/WDL/tasks/tasks_utils.wdl +++ b/pipes/WDL/tasks/tasks_utils.wdl @@ -204,7 +204,7 @@ task tsv_stack { input { Array[File]+ input_tsvs String out_basename - String docker = "quay.io/broadinstitute/viral-core:2.1.25" + String docker = "quay.io/broadinstitute/viral-core:2.1.26" } command { diff --git a/requirements-modules.txt b/requirements-modules.txt index 523ff1348..894510d59 100644 --- a/requirements-modules.txt +++ b/requirements-modules.txt @@ -1,4 +1,4 @@ -broadinstitute/viral-core=2.1.25 +broadinstitute/viral-core=2.1.26 broadinstitute/viral-assemble=2.1.16.1 broadinstitute/viral-classify=2.1.16.0 broadinstitute/viral-phylo=2.1.19.1 From 9b84127be4df4a1d4d25ff6fbc6a2e3d3394b9f1 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Fri, 30 Apr 2021 08:36:23 -0400 Subject: [PATCH 10/13] viral-core 2.1.26 -> 2.1.28 --- pipes/WDL/tasks/tasks_assembly.wdl | 4 ++-- pipes/WDL/tasks/tasks_demux.wdl | 4 ++-- pipes/WDL/tasks/tasks_interhost.wdl | 2 +- pipes/WDL/tasks/tasks_ncbi.wdl | 6 +++--- pipes/WDL/tasks/tasks_nextstrain.wdl | 4 ++-- pipes/WDL/tasks/tasks_read_utils.wdl | 10 +++++----- pipes/WDL/tasks/tasks_reports.wdl | 12 ++++++------ pipes/WDL/tasks/tasks_taxon_filter.wdl | 2 +- pipes/WDL/tasks/tasks_utils.wdl | 2 +- requirements-modules.txt | 2 +- 10 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pipes/WDL/tasks/tasks_assembly.wdl b/pipes/WDL/tasks/tasks_assembly.wdl index eaffb72ae..f13cede64 100644 --- a/pipes/WDL/tasks/tasks_assembly.wdl +++ b/pipes/WDL/tasks/tasks_assembly.wdl @@ -291,7 +291,7 @@ task align_reads { Boolean? skip_mark_dupes = false Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" String sample_name = basename(basename(basename(reads_unmapped_bam, ".bam"), ".taxfilt"), ".clean") } @@ -708,7 +708,7 @@ task run_discordance { String out_basename = "run" Int min_coverage = 4 - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } command { diff --git a/pipes/WDL/tasks/tasks_demux.wdl b/pipes/WDL/tasks/tasks_demux.wdl index b30054dbe..cf9578ecd 100644 --- a/pipes/WDL/tasks/tasks_demux.wdl +++ b/pipes/WDL/tasks/tasks_demux.wdl @@ -6,7 +6,7 @@ task merge_tarballs { String out_filename Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } command { @@ -142,7 +142,7 @@ task illumina_demux { Int? maxRecordsInRam Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } parameter_meta { flowcell_tgz: { diff --git a/pipes/WDL/tasks/tasks_interhost.wdl b/pipes/WDL/tasks/tasks_interhost.wdl index 8652ee306..3986fe74f 100644 --- a/pipes/WDL/tasks/tasks_interhost.wdl +++ b/pipes/WDL/tasks/tasks_interhost.wdl @@ -142,7 +142,7 @@ task index_ref { File? novocraft_license Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } command { diff --git a/pipes/WDL/tasks/tasks_ncbi.wdl b/pipes/WDL/tasks/tasks_ncbi.wdl index eb996dd52..7ab529ba1 100644 --- a/pipes/WDL/tasks/tasks_ncbi.wdl +++ b/pipes/WDL/tasks/tasks_ncbi.wdl @@ -186,7 +186,7 @@ task structured_comments { File? filter_to_ids - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } String out_base = basename(assembly_stats_tsv, '.txt') command <<< @@ -264,7 +264,7 @@ task rename_fasta_header { String out_basename = basename(genome_fasta, ".fasta") - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } command { set -e @@ -447,7 +447,7 @@ task sra_meta_prep { Boolean paired String out_name = "sra_metadata.tsv" - String docker="quay.io/broadinstitute/viral-core:2.1.26" + String docker="quay.io/broadinstitute/viral-core:2.1.28" } parameter_meta { cleaned_bam_filepaths: { diff --git a/pipes/WDL/tasks/tasks_nextstrain.wdl b/pipes/WDL/tasks/tasks_nextstrain.wdl index 4ad05693e..0cd7ba629 100644 --- a/pipes/WDL/tasks/tasks_nextstrain.wdl +++ b/pipes/WDL/tasks/tasks_nextstrain.wdl @@ -81,7 +81,7 @@ task derived_cols { String? lab_highlight_loc Array[File] table_map = [] - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } parameter_meta { lab_highlight_loc: { @@ -480,7 +480,7 @@ task filter_sequences_by_length { File sequences_fasta Int min_non_N = 1 - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } parameter_meta { sequences_fasta: { diff --git a/pipes/WDL/tasks/tasks_read_utils.wdl b/pipes/WDL/tasks/tasks_read_utils.wdl index fb994b4f8..865c49607 100644 --- a/pipes/WDL/tasks/tasks_read_utils.wdl +++ b/pipes/WDL/tasks/tasks_read_utils.wdl @@ -79,7 +79,7 @@ task get_sample_meta { input { Array[File] samplesheets_extended - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } command <<< python3 << CODE @@ -137,7 +137,7 @@ task merge_and_reheader_bams { File? reheader_table String out_basename - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } command { @@ -197,7 +197,7 @@ task rmdup_ubam { String method = "mvicuna" Int? machine_mem_gb - String? docker = "quay.io/broadinstitute/viral-core:2.1.26" + String? docker = "quay.io/broadinstitute/viral-core:2.1.28" } parameter_meta { @@ -251,7 +251,7 @@ task downsample_bams { Boolean? deduplicateAfter = false Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } command { @@ -310,7 +310,7 @@ task FastqToUBAM { String? platform_name String? sequencing_center - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } parameter_meta { fastq_1: { description: "Unaligned read1 file in fastq format", patterns: ["*.fastq", "*.fastq.gz", "*.fq", "*.fq.gz"] } diff --git a/pipes/WDL/tasks/tasks_reports.wdl b/pipes/WDL/tasks/tasks_reports.wdl index 7585d13e1..03d4953bb 100644 --- a/pipes/WDL/tasks/tasks_reports.wdl +++ b/pipes/WDL/tasks/tasks_reports.wdl @@ -7,7 +7,7 @@ task alignment_metrics { File? primers_bed Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } String out_basename = basename(aligned_bam, ".bam") @@ -77,7 +77,7 @@ task plot_coverage { Boolean bin_large_plots = false String? binning_summary_statistic = "max" # max or min - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } command { @@ -152,7 +152,7 @@ task coverage_report { Array[File] mapped_bam_idx # optional.. speeds it up if you provide it, otherwise we auto-index String out_report_name = "coverage_report.txt" - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } command { @@ -211,7 +211,7 @@ task fastqc { input { File reads_bam - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } String reads_basename=basename(reads_bam, ".bam") @@ -244,7 +244,7 @@ task align_and_count { Int topNHits = 3 Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } String reads_basename=basename(reads_bam, ".bam") @@ -289,7 +289,7 @@ task align_and_count_summary { String output_prefix = "count_summary" - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } command { diff --git a/pipes/WDL/tasks/tasks_taxon_filter.wdl b/pipes/WDL/tasks/tasks_taxon_filter.wdl index 39fdb4297..6e919ec54 100644 --- a/pipes/WDL/tasks/tasks_taxon_filter.wdl +++ b/pipes/WDL/tasks/tasks_taxon_filter.wdl @@ -202,7 +202,7 @@ task merge_one_per_sample { Boolean? rmdup = false Int? machine_mem_gb - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } command { diff --git a/pipes/WDL/tasks/tasks_utils.wdl b/pipes/WDL/tasks/tasks_utils.wdl index 3c7a18cf3..2009abe54 100644 --- a/pipes/WDL/tasks/tasks_utils.wdl +++ b/pipes/WDL/tasks/tasks_utils.wdl @@ -204,7 +204,7 @@ task tsv_stack { input { Array[File]+ input_tsvs String out_basename - String docker = "quay.io/broadinstitute/viral-core:2.1.26" + String docker = "quay.io/broadinstitute/viral-core:2.1.28" } command { diff --git a/requirements-modules.txt b/requirements-modules.txt index 894510d59..19be2b62c 100644 --- a/requirements-modules.txt +++ b/requirements-modules.txt @@ -1,4 +1,4 @@ -broadinstitute/viral-core=2.1.26 +broadinstitute/viral-core=2.1.28 broadinstitute/viral-assemble=2.1.16.1 broadinstitute/viral-classify=2.1.16.0 broadinstitute/viral-phylo=2.1.19.1 From 1e16691feee64fd47f4a52713349b85a0f600e7f Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Fri, 30 Apr 2021 12:05:08 -0400 Subject: [PATCH 11/13] in workflow outputs, access sequencer_model from run_info json rather than direct output --- pipes/WDL/workflows/demux_metag.wdl | 2 +- pipes/WDL/workflows/demux_only.wdl | 2 +- pipes/WDL/workflows/demux_plus.wdl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipes/WDL/workflows/demux_metag.wdl b/pipes/WDL/workflows/demux_metag.wdl index 095bd2460..e093b4a5b 100644 --- a/pipes/WDL/workflows/demux_metag.wdl +++ b/pipes/WDL/workflows/demux_metag.wdl @@ -117,7 +117,7 @@ workflow demux_metag { File demux_metrics = illumina_demux.metrics File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes - String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],[illumina_demux.instrument_model]])) + String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],illumina_demux.run_info['sequencer_model']])) File multiqc_report_raw = multiqc_raw.multiqc_report File multiqc_report_cleaned = multiqc_cleaned.multiqc_report diff --git a/pipes/WDL/workflows/demux_only.wdl b/pipes/WDL/workflows/demux_only.wdl index e7262532f..d40dda561 100644 --- a/pipes/WDL/workflows/demux_only.wdl +++ b/pipes/WDL/workflows/demux_only.wdl @@ -27,7 +27,7 @@ workflow demux_only { File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes File multiqc_report_raw = MultiQC.multiqc_report - String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],[illumina_demux.instrument_model]])) + String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],illumina_demux.run_info['sequencer_model']])) String demux_viral_core_version = illumina_demux.viralngs_version } } diff --git a/pipes/WDL/workflows/demux_plus.wdl b/pipes/WDL/workflows/demux_plus.wdl index c6ffc0dc2..e27f9a11c 100644 --- a/pipes/WDL/workflows/demux_plus.wdl +++ b/pipes/WDL/workflows/demux_plus.wdl @@ -86,7 +86,7 @@ workflow demux_plus { File demux_metrics = illumina_demux.metrics File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes - String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],[illumina_demux.instrument_model]])) + String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],illumina_demux.run_info['sequencer_model']])) File multiqc_report_raw = multiqc_raw.multiqc_report File multiqc_report_cleaned = multiqc_cleaned.multiqc_report From 052a146d3123c962c3f0a09d03aa1ff5ca2ed2f1 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Fri, 30 Apr 2021 12:57:09 -0400 Subject: [PATCH 12/13] [illumina_demux.run_info[0]['sequencer_model']] --- pipes/WDL/workflows/demux_deplete.wdl | 4 ++-- pipes/WDL/workflows/demux_metag.wdl | 2 +- pipes/WDL/workflows/demux_only.wdl | 2 +- pipes/WDL/workflows/demux_plus.wdl | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pipes/WDL/workflows/demux_deplete.wdl b/pipes/WDL/workflows/demux_deplete.wdl index 9c39ad91a..78554a36b 100644 --- a/pipes/WDL/workflows/demux_deplete.wdl +++ b/pipes/WDL/workflows/demux_deplete.wdl @@ -115,7 +115,7 @@ workflow demux_deplete { paired = (illumina_demux.run_info[0]['indexes'] == '2'), out_name = "sra_metadata-~{illumina_demux.run_info[0]['run_id']}.tsv", - instrument_model = select_first(flatten([[instrument_model_user_specified],illumina_demux.run_info[0]['sequencer_model']])), + instrument_model = select_first(flatten([[instrument_model_user_specified],[illumina_demux.run_info[0]['sequencer_model']]])), title = select_first([sra_title]) } } @@ -162,7 +162,7 @@ workflow demux_deplete { File multiqc_report_cleaned = multiqc_cleaned.multiqc_report File spikein_counts = spike_summary.count_summary - String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],illumina_demux.run_info[0]['sequencer_model']])) + String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],[illumina_demux.run_info[0]['sequencer_model']]])) String run_date = illumina_demux.run_info[0]['run_start_date'] Map[String,String] run_info = illumina_demux.run_info[0] diff --git a/pipes/WDL/workflows/demux_metag.wdl b/pipes/WDL/workflows/demux_metag.wdl index e093b4a5b..575602c4c 100644 --- a/pipes/WDL/workflows/demux_metag.wdl +++ b/pipes/WDL/workflows/demux_metag.wdl @@ -117,7 +117,7 @@ workflow demux_metag { File demux_metrics = illumina_demux.metrics File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes - String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],illumina_demux.run_info['sequencer_model']])) + String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],[illumina_demux.run_info['sequencer_model']]])) File multiqc_report_raw = multiqc_raw.multiqc_report File multiqc_report_cleaned = multiqc_cleaned.multiqc_report diff --git a/pipes/WDL/workflows/demux_only.wdl b/pipes/WDL/workflows/demux_only.wdl index d40dda561..bbbd96340 100644 --- a/pipes/WDL/workflows/demux_only.wdl +++ b/pipes/WDL/workflows/demux_only.wdl @@ -27,7 +27,7 @@ workflow demux_only { File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes File multiqc_report_raw = MultiQC.multiqc_report - String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],illumina_demux.run_info['sequencer_model']])) + String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],[illumina_demux.run_info['sequencer_model']]])) String demux_viral_core_version = illumina_demux.viralngs_version } } diff --git a/pipes/WDL/workflows/demux_plus.wdl b/pipes/WDL/workflows/demux_plus.wdl index e27f9a11c..2931d08c5 100644 --- a/pipes/WDL/workflows/demux_plus.wdl +++ b/pipes/WDL/workflows/demux_plus.wdl @@ -86,7 +86,7 @@ workflow demux_plus { File demux_metrics = illumina_demux.metrics File demux_commonBarcodes = illumina_demux.commonBarcodes File demux_outlierBarcodes = illumina_demux.outlierBarcodes - String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],illumina_demux.run_info['sequencer_model']])) + String instrument_model_inferred = select_first(flatten([[instrument_model_user_specified],[illumina_demux.run_info['sequencer_model']]])) File multiqc_report_raw = multiqc_raw.multiqc_report File multiqc_report_cleaned = multiqc_cleaned.multiqc_report From 4916e8e49da5a519d02ceb7c1b6f8b7de1129c44 Mon Sep 17 00:00:00 2001 From: Christopher Tomkins-Tinch Date: Fri, 30 Apr 2021 22:43:39 -0400 Subject: [PATCH 13/13] simplification following review by @dpark01 --- pipes/WDL/tasks/tasks_demux.wdl | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pipes/WDL/tasks/tasks_demux.wdl b/pipes/WDL/tasks/tasks_demux.wdl index cf9578ecd..d981ca5b4 100644 --- a/pipes/WDL/tasks/tasks_demux.wdl +++ b/pipes/WDL/tasks/tasks_demux.wdl @@ -294,10 +294,6 @@ task illumina_demux { illumina.py flowcell_metadata --inDir $FLOWCELL_DIR flowcellMetadataFile.tsv - # output machine model and lane count - grep "machine" flowcellMetadataFile.tsv | cut -f2 | tee MACHINE_MODEL - grep "lane_count" flowcellMetadataFile.tsv | cut -f2 | tee LANE_COUNT - mkdir -p unmatched mv Unmatched.bam unmatched/ @@ -359,8 +355,8 @@ task illumina_demux { Int runtime_sec = ceil(read_float("UPTIME_SEC")) Int cpu_load_15min = ceil(read_float("LOAD_15M")) - String instrument_model = read_string("MACHINE_MODEL") - String flowcell_lane_count = read_string("LANE_COUNT") + String instrument_model = read_json("~{out_base}-runinfo.json")["sequencer_model"] + String flowcell_lane_count = read_json("~{out_base}-runinfo.json")["lane_count"] String viralngs_version = read_string("VERSION")