Provenance (#11)

* Add hashing of inputs and unicycler version

* Collect provenance

* Describe provenance output in README

* newline

* newline

Author: dfornika

README.md

@@ -51,6 +51,7 @@ If we have `sample-01_R{1,2}.fastq.gz`, the output directory will be:
 
 ```
 sample-01
+├── sample-01_20211125165316_provenance.yml
 ├── sample-01_fastp.csv
 ├── sample-01_fastp.json
 ├── sample-01_prokka.gbk
@@ -65,6 +66,8 @@ Including the tool name suffixes to output files allows re-analysis of the same
 
 ```
 sample-01
+├── sample-01_20211125165316_provenance.yml
+├── sample-01_20211128122118_provenance.yml
 ├── sample-01_bakta.gbk
 ├── sample-01_bakta.gff
 ├── sample-01_bakta.json
@@ -81,3 +84,26 @@ sample-01
 ├── sample-01_unicycler.gfa
 └── sample-01_unicycler.log
 ```
+
+### Provenance files
+For each pipeline invocation, each sample will produce a `provenance.yml` file with the following contents:
+
+```yml
+- tool_name: fastp
+  tool_version: 0.23.1
+- tool_name: shovill
+  tool_version: 1.1.0
+- tool_name: prokka
+  tool_version: 1.14.5
+- tool_name: quast
+  tool_version: v5.0.2
+- input_filename: sample-01_R1.fastq.gz
+  sha256: 4ac3055ac5f03114a005aff033e7018ea98486cbebdae669880e3f0511ed21bb
+- input_filename: sample-01_R2.fastq.gz
+  sha256: 8db388f56a51920752319c67b5308c7e99f2a566ca83311037a425f8d6bb1ecc
+- pipeline_name: BCCDC-PHL/routine-assembly
+  pipeline_version: 0.1.0
+- timestamp_analysis_start: 2021-11-25T16:53:10.549863
+```
+
+The filename of the provenance file includes a timestamp with format `YYYYMMDDHHMMSS` to ensure that re-analysis of the same sample will create a unique `provenance.yml` file.

environments/environment.yml

@@ -5,6 +5,7 @@ channels:
   - defaults
 dependencies:
   - python=3
+  - perl-digest-sha=5.88
   - fastp=0.23.1
   - shovill=1.1.0
   - prokka=1.14.5

main.nf

@@ -1,7 +1,10 @@
 #!/usr/bin/env nextflow
 
+import java.time.LocalDateTime
+
 nextflow.enable.dsl = 2
 
+include { hash_files } from './modules/hash_files.nf'
 include { fastp } from './modules/fastp.nf'
 include { fastp_json_to_csv } from './modules/fastp.nf'
 include { shovill } from './modules/shovill.nf'
@@ -10,32 +13,66 @@ include { prokka } from './modules/prokka.nf'
 include { bakta } from './modules/bakta.nf'
 include { quast } from './modules/quast.nf'
 include { parse_quast_report } from './modules/quast.nf'
+include { pipeline_provenance } from './modules/provenance.nf'
+include { collect_provenance } from './modules/provenance.nf'
 
 
 workflow {
+  ch_start_time = Channel.of(LocalDateTime.now())
+  ch_pipeline_name = Channel.of(workflow.manifest.name)
+  ch_pipeline_version = Channel.of(workflow.manifest.version)
+
+  ch_pipeline_provenance = pipeline_provenance(ch_pipeline_name.combine(ch_pipeline_version).combine(ch_start_time))
+
   if (params.samplesheet_input != 'NO_FILE') {
-    ch_fastq = Channel.fromPath(params.samplesheet_input).splitCsv(header: true)
+    ch_fastq = Channel.fromPath(params.samplesheet_input).splitCsv(header: true).map{ it -> [it['ID'], it['R1'], it['R2']] }
   } else {
     ch_fastq = Channel.fromFilePairs( params.fastq_search_path, flat: true ).map{ it -> [it[0].split('_')[0], it[1], it[2]] }.unique{ it -> it[0] }
   }
+
   run_shovill = params.unicycler ? false : true
   run_unicycler = run_shovill ? false : true
   run_prokka = params.bakta ? false : true
   run_bakta = run_prokka ? false : true
 
   main:
+    hash_files(ch_fastq.map{ it -> [it[0], [it[1], it[2]]] }.combine(Channel.of("fastq-input")))
+
     fastp(ch_fastq)
     fastp_json_to_csv(fastp.out.json)
+
     if (run_shovill) {
       ch_assembly = shovill(fastp.out.trimmed_reads).assembly
     } else {
       ch_assembly = unicycler(fastp.out.trimmed_reads).assembly
     }
+
     if (run_prokka) {
       prokka(ch_assembly)
     } else if (run_bakta) {
       bakta(ch_assembly)
     }
+
     quast(ch_assembly)
-    parse_quast_report(quast.out)
+    parse_quast_report(quast.out.tsv)
+    
+    ch_provenance = fastp.out.provenance
+    
+    if (run_shovill) {
+      ch_provenance = ch_provenance.join(shovill.out.provenance).map{ it -> [it[0], [it[1], it[2]]] }
+    }
+    if (run_unicycler) {
+      ch_provenance = ch_provenance.join(unicycler.out.provenance).map{ it -> [it[0], [it[1], it[2]]] }
+    }
+    if (run_prokka) {
+      ch_provenance = ch_provenance.join(prokka.out.provenance).map{ it -> [it[0], it[1] << it[2]] }
+    }
+    if (run_bakta) {
+      ch_provenance = ch_provenance.join(bakta.out.provenance).map{ it -> [it[0], it[1] << it[2]] }
+    }
+    ch_provenance = ch_provenance.join(quast.out.provenance).map{ it -> [it[0], it[1] << it[2]] }
+
+    ch_provenance = ch_provenance.join(hash_files.out.provenance).map{ it -> [it[0], it[1] << it[2]] }
+    ch_provenance = ch_provenance.join(ch_fastq.map{ it -> it[0] }.combine(ch_pipeline_provenance)).map{ it -> [it[0], it[1] << it[2]] }
+    collect_provenance(ch_provenance)
 }

modules/bakta.nf

@@ -8,13 +8,15 @@ process bakta {
       tuple val(sample_id), path(assembly)
 
     output:
-      tuple val(sample_id), path("${sample_id}_bakta.gbk"), emit: genbank
+      tuple val(sample_id), path("${sample_id}_bakta.gbk"), emit: gbk
       tuple val(sample_id), path("${sample_id}_bakta.gff"), emit: gff
       tuple val(sample_id), path("${sample_id}_bakta.json"), emit: json
       tuple val(sample_id), path("${sample_id}_bakta.log"), emit: log
+      tuple val(sample_id), path("${sample_id}_bakta_provenance.yml"), emit: provenance
 
     script:
       """
+      printf -- "- tool_name: bakta\\n  tool_version: \$(bakta --version | cut -d ' ' -f 2)\\n" > ${sample_id}_bakta_provenance.yml
       bakta --db ${params.bakta_db} --threads ${task.cpus} --compliant --keep-contig-headers --locus-tag ${sample_id} --prefix "${sample_id}" ${assembly}
       cp ${sample_id}.gff3 ${sample_id}_bakta.gff
       cp ${sample_id}.gbff ${sample_id}_bakta.gbk

modules/fastp.nf

@@ -9,10 +9,12 @@ process fastp {
     output:
     tuple val(sample_id), path("${sample_id}_R1.trim.fastq.gz"), path("${sample_id}_R2.trim.fastq.gz"), emit: trimmed_reads
     tuple val(sample_id), path("${sample_id}_fastp.json"), emit: json
+    tuple val(sample_id), path("${sample_id}_fastp_provenance.yml"), emit: provenance
 
 
     script:
     """
+    printf -- "- tool_name: fastp\\n  tool_version: \$(fastp --version 2>&1 | cut -d ' ' -f 2)\\n" > ${sample_id}_fastp_provenance.yml
     fastp \
       -t ${task.cpus} \
       -i ${reads_1} \

modules/hash_files.nf

@@ -0,0 +1,19 @@
+process hash_files {
+
+    tag { sample_id + " / " + file_type }
+
+    input:
+    tuple  val(sample_id), path(files_to_hash), val(file_type)
+
+    output:
+    tuple  val(sample_id), path("${sample_id}_${file_type}.sha256.csv"), emit: csv
+    tuple  val(sample_id), path("${sample_id}_${file_type}_provenance.yml"), emit: provenance
+
+    script:
+    """
+    shasum -a 256 ${files_to_hash} | tr -s ' ' ',' > ${sample_id}_${file_type}.sha256.csv
+    while IFS=',' read -r hash filename; do
+      printf -- "- input_filename: \$filename\\n  sha256: \$hash\\n" >> ${sample_id}_${file_type}_provenance.yml
+    done < ${sample_id}_${file_type}.sha256.csv
+    """
+}

modules/prokka.nf

@@ -8,10 +8,13 @@ process prokka {
       tuple val(sample_id), path(assembly)
 
     output:
-      tuple val(sample_id), path("${sample_id}_prokka.gbk"), path("${sample_id}_prokka.gff")
+      tuple val(sample_id), path("${sample_id}_prokka.gbk"), emit: gbk
+      tuple val(sample_id), path("${sample_id}_prokka.gff"), emit: gff
+      tuple val(sample_id), path("${sample_id}_prokka_provenance.yml"), emit: provenance
 
     script:
       """
+      printf -- "- tool_name: prokka\\n  tool_version: \$(prokka --version 2>&1 | cut -d ' ' -f 2)\\n" > ${sample_id}_prokka_provenance.yml
       prokka --cpus ${task.cpus} --compliant --locustag ${sample_id} --centre "BCCDC-PHL" --prefix "${sample_id}" ${assembly}
       cp ${sample_id}/${sample_id}.gbk ${sample_id}_prokka.gbk
       cp ${sample_id}/${sample_id}.gff ${sample_id}_prokka.gff

modules/provenance.nf

@@ -0,0 +1,35 @@
+process collect_provenance {
+  tag { sample_id }
+
+  executor 'local'
+
+  publishDir "${params.outdir}/${sample_id}", pattern: "${sample_id}_*_provenance.yml", mode: 'copy'
+
+  input:
+  tuple val(sample_id), path(provenance_files)
+
+  output:
+  tuple val(sample_id), file("${sample_id}_*_provenance.yml")
+
+  script:
+  """
+  cat ${provenance_files} > ${sample_id}_\$(date +%Y%m%d%H%M%S)_provenance.yml
+  """
+}
+
+process pipeline_provenance {
+  tag { pipeline_name + " / " + pipeline_version }
+
+  executor 'local'
+
+  input:
+  tuple val(pipeline_name), val(pipeline_version), val(analysis_start)
+
+  output:
+  file("pipeline_provenance.yml")
+
+  script:
+  """
+  printf -- "- pipeline_name: ${pipeline_name}\\n  pipeline_version: ${pipeline_version}\\n- timestamp_analysis_start: ${analysis_start}\\n" > pipeline_provenance.yml
+  """
+}

modules/quast.nf

@@ -10,10 +10,12 @@ process quast {
       tuple val(sample_id), path(assembly)
 
     output:
-      tuple val(sample_id), path("${sample_id}_quast.tsv")
+      tuple val(sample_id), path("${sample_id}_quast.tsv"), emit: tsv
+      tuple val(sample_id), path("${sample_id}_quast_provenance.yml"), emit: provenance
 
     script:
       """
+      printf -- "- tool_name: quast\\n  tool_version: \$(quast --version | cut -d ' ' -f 2)\\n" > ${sample_id}_quast_provenance.yml
       quast --threads ${task.cpus} ${assembly} --space-efficient --fast --output-dir ${sample_id}
       mv ${sample_id}/transposed_report.tsv ${sample_id}_quast.tsv
       """

modules/shovill.nf

@@ -4,17 +4,18 @@ process shovill {
 
     publishDir "${params.outdir}/${sample_id}", pattern: "${sample_id}_shovill.{fa,log}", mode: 'copy'
 
-    cpus 8
 
     input:
       tuple val(sample_id), path(reads_1), path(reads_2)
 
     output:
       tuple val(sample_id), path("${sample_id}_shovill.fa"), emit: assembly
       tuple val(sample_id), path("${sample_id}_shovill.log"), emit: log
+      tuple val(sample_id), path("${sample_id}_shovill_provenance.yml"), emit: provenance
 
     script:
       """
+      printf -- "- tool_name: shovill\\n  tool_version: \$(shovill --version | cut -d ' ' -f 2)\\n" > ${sample_id}_shovill_provenance.yml
       shovill --cpus ${task.cpus} --trim --namefmt \"${sample_id}_contig%0d\" --outdir ${sample_id}_assembly --R1 ${reads_1} --R2 ${reads_2}
       cp ${sample_id}_assembly/contigs.fa ${sample_id}_shovill.fa
       cp ${sample_id}_assembly/shovill.log ${sample_id}_shovill.log

modules/unicycler.nf

@@ -11,9 +11,11 @@ process unicycler {
       tuple val(sample_id), path("${sample_id}_unicycler.fa"), emit: assembly
       tuple val(sample_id), path("${sample_id}_unicycler.gfa"), emit: assembly_graph
       tuple val(sample_id), path("${sample_id}_unicycler.log"), emit: log
+      tuple val(sample_id), path("${sample_id}_unicycler_provenance.yml"), emit: provenance
 
     script:
       """
+      printf -- "- tool_name: unicycler\\n  tool_version: \$(unicycler --version | cut -d ' ' -f 2)\\n" > ${sample_id}_unicycler_provenance.yml
       unicycler --threads ${task.cpus} -1 ${reads_1} -2 ${reads_2} -o ${sample_id}_assembly
       sed 's/^>/>${sample_id}_/' ${sample_id}_assembly/assembly.fasta > ${sample_id}_unicycler.fa
       cp ${sample_id}_assembly/assembly.gfa ${sample_id}_unicycler.gfa

nextflow.config

@@ -56,9 +56,10 @@ process {
 
 manifest {
   author = 'Dan Fornika'
+  name = 'BCCDC-PHL/routine-assembly'
+  version = '0.2.0'
   description = 'BCCDC-PHL Routine Assembly'
   mainScript = 'main.nf'
   nextflowVersion = '>=20.01.0'
-  version = '0.1.0'
 }