Deprecate the use of cg_pipeline for nanoplot stats

tasks/assembly/task_dragonflye.wdl

@@ -17,6 +17,7 @@ task dragonflye {
     String docker = "us-docker.pkg.dev/general-theiagen/biocontainers/dragonflye:1.0.14--hdfd78af_0"
     Int disk_size = 100
     Int cpu = 4
+    Int memory = 32
   }
   command <<<
     # get version information
@@ -46,6 +47,9 @@ task dragonflye {
       ILLUMINA_POLISHER=""
     fi
 
+    # reduce requested memory to prevent out of memory errors
+    mem=$(( ~{memory}-1 ))
+
     # run dragonflye
     # --reads for input nanopore fastq
     # --depth 0 disables sub-sampling of reads (performed with rasusa already)
@@ -64,7 +68,7 @@ task dragonflye {
       --outdir dragonflye \
       ~{'--gsize ' + genome_size} \
       --cpus ~{cpu} \
-      --ram 8 \
+      --ram ${mem} \
       --nofilter \
       ~{'--assembler ' + assembler} \
       ~{'--opts "' + assembler_options + '"'} \
@@ -73,14 +77,16 @@ task dragonflye {
 
     # rename final output file to have .fasta ending instead of .fa
     mv dragonflye/contigs.fa ~{samplename}.fasta
+    mv dragonflye/*.gfa ~{samplename}_contigs.gfa
   >>>
   output {
     File assembly_fasta = "~{samplename}.fasta"
+    File contigs_gfa = "~{samplename}_contigs.gfa"
     String dragonflye_version = read_string("VERSION")
   }
   runtime {
     docker: "~{docker}"
-    memory: "16 GB"
+    memory: "~{memory} GB"
     cpu: cpu
     disks: "local-disk " + disk_size + " SSD"
     disk: disk_size + " GB"

tasks/gene_typing/task_tiptoft.wdl

@@ -34,7 +34,9 @@ task tiptoft {
 
     # gzip output filtered FASTQ; output filename will be ~{samplename}.tiptoft.fastq.gz
     # could change this to pigz if we include it in the docker image
-    gzip ~{samplename}.tiptoft.fastq
+    if [ -f ~{samplename}.tiptoft.fastq ]; then
+      gzip ~{samplename}.tiptoft.fastq
+    fi
 
     # parse tiptoft output TSV
     # parse out gene names into list of strings, comma-separated, final comma at end removed by sed
@@ -44,8 +46,8 @@ task tiptoft {
   >>>
   output {
     File tiptoft_tsv = "~{samplename}.tiptoft.tsv"
-    File tiptoft_plasmid_replicon_fastq = "~{samplename}.tiptoft.fastq.gz"
-    String plasmid_replicon_genes = read_string("PLASMID_REPLICON_GENES.txt")
+    File? tiptoft_plasmid_replicon_fastq = "~{samplename}.tiptoft.fastq.gz"
+    String? plasmid_replicon_genes = read_string("PLASMID_REPLICON_GENES.txt")
     String tiptoft_version = read_string("VERSION")
     String tiptoft_docker = docker
   }

tasks/quality_control/task_nanoplot.wdl

@@ -5,7 +5,12 @@ task nanoplot {
     File read1 # intended for ONT data only
     String samplename
     Int max_length = 100000
+    Int est_genome_size
+
     Int disk_size = 100
+    String docker = "us-docker.pkg.dev/general-theiagen/staphb/nanoplot:1.40.0"
+    Int memory = 16
+    Int cpu = 4
   }
   command <<<
     # get version
@@ -26,16 +31,31 @@ task nanoplot {
       --loglength \
       --tsv_stats \
       --maxlength ~{max_length}
-
+
+    # grep read statistics from tsv stats file
+    grep "number_of_reads" ~{samplename}_NanoStats.txt | cut -f 2 | tee NUMBER_OF_READS
+    NUM_BASES=$(grep "number_of_bases" ~{samplename}_NanoStats.txt | cut -f 2 | tee NUMBER_OF_BASES)
+    grep "mean_read_length" ~{samplename}_NanoStats.txt | cut -f 2 | tee MEAN_READ_LENGTH
+    grep "mean_qual" ~{samplename}_NanoStats.txt | cut -f 2 | tee MEAN_QUAL
+
+    # estimate coverage
+    # using math: C = N / G where N is number of bases, and G is estimated genome size 
+    python3 -c "print(round(${NUM_BASES} / ~{est_genome_size}, 2))" | tee EST_COVERAGE
   >>>
   output {
     File nanoplot_html = "~{samplename}_NanoPlot-report.html"
+    File nanoplot_tsv = "~{samplename}_NanoStats.txt"
+    Int num_reads = read_int("NUMBER_OF_READS")
+    Float mean_readlength = read_float("MEAN_READ_LENGTH")
+    Float mean_q = read_float("MEAN_QUAL")
+    Float est_coverage = read_float("EST_COVERAGE")
     String nanoplot_version = read_string("VERSION")
+    String nanoplot_docker = docker
   }
   runtime {
-    docker: "us-docker.pkg.dev/general-theiagen/staphb/nanoplot:1.40.0"
-    memory: "16 GB"
-    cpu: 4
+    docker: docker
+    memory: "~{memory} GB"
+    cpu: cpu
     disks: "local-disk " + disk_size + " SSD"
     disk: disk_size + " GB"
     preemptible: 0

tasks/quality_control/task_nanoq.wdl

@@ -6,7 +6,7 @@ task nanoq {
     String samplename
     String docker = "us-docker.pkg.dev/general-theiagen/biocontainers/nanoq:0.9.0--hec16e2b_1"
     Int disk_size = 100
-    Int max_read_length = 10000
+    Int max_read_length = 100000
     Int min_read_length = 500
     Int max_read_qual = 40
     Int min_read_qual = 10

tasks/quality_control/task_screen.wdl

@@ -136,7 +136,7 @@ task check_reads {
           else # they provided 0 for estimated_genome_size, nice
             estimated_coverage=0
           fi
-        else # workflow series was not provided; default to fail
+        else # workflow series was not provided or no est genome size was provided; default to fail
           estimated_genome_size=0
           estimated_coverage=0
         fi
@@ -184,6 +184,7 @@ task check_reads_se {
     Int max_genome_size 
     Int min_coverage
     Boolean skip_screen 
+    Boolean skip_mash
     String workflow_series = "theiaprok" # default to theiaprok so we don't have to change those workflows
     String? organism
     Int? expected_genome_size
@@ -235,7 +236,7 @@ task check_reads_se {
       fi
 
       #checks four and five: estimated genome size and coverage
-      if [ "${flag}" = "PASS" ]; then
+      if [ "${flag}" == "PASS" ] && [ "~{skip_mash}" == "false" ]; then
         # estimate genome size if theiaprok AND expected_genome_size was not provided
         if [ "~{workflow_series}" == "theiaprok" ] && [[ -z "~{expected_genome_size}" ]]; then            
           # First Pass; assuming average depth

tasks/utilities/task_broad_terra_tools.wdl

@@ -50,6 +50,8 @@ task export_taxon_tables {
     Float? r1_mean_readlength_clean
     String? nanoq_version
     File? nanoplot_html
+    File? nanoplot_tsv
+    String? nanoplot_docker
     String? nanoplot_version
     String? kmc_est_genome_size
     File? kmc_kmer_stats
@@ -398,6 +400,8 @@ task export_taxon_tables {
       "r1_mean_readlength_clean": "~{r1_mean_readlength_clean}",
       "nanoq_version": "~{nanoq_version}",
       "nanoplot_html": "~{nanoplot_html}",
+      "nanoplot_tsv": "~{nanoplot_tsv}",
+      "nanoplot_docker": "~{nanoplot_docker}",
       "nanoplot_version": "~{nanoplot_version}",
       "kmc_est_genome_size": "~{kmc_est_genome_size}",
       "kmc_kmer_stats": "~{kmc_kmer_stats}",

tests/workflows/theiacov/test_wf_theiacov_illumina_pe.yml

@@ -141,7 +141,7 @@
       md5sum: 897316929176464ebc9ad085f31e7284
     # clean read screen
     - path: miniwdl_run/call-clean_check_reads/command
-      md5sum: 7b889616aa2220b8c7f8413cc59a8c73
+      md5sum: 38e0fccd1a5689a1c19c4dfe3826f854
     - path: miniwdl_run/call-clean_check_reads/inputs.json
       contains: ["read1", "read2", "organism"]
     - path: miniwdl_run/call-clean_check_reads/outputs.json

tests/workflows/theiacov/test_wf_theiacov_illumina_se.yml

@@ -96,7 +96,7 @@
       md5sum: 7e4fc05efbbc3937b99420e6193be061
     # clean read screen
     - path: miniwdl_run/call-clean_check_reads/command
-      md5sum: cfcacb6a4fc055fcd794ed60e161db3f
+      md5sum: cc696fd41c7d4c73e329e153c6b5b285
     - path: miniwdl_run/call-clean_check_reads/inputs.json
       contains: ["read1", "organism"]
     - path: miniwdl_run/call-clean_check_reads/outputs.json

tests/workflows/theiacov/test_wf_theiacov_ont.yml

@@ -18,7 +18,7 @@
     - wf_theiacov_ont_miniwdl
   files:
     - path: miniwdl_run/call-clean_check_reads/command
-      md5sum: 1330a5740e4ecab6d65e1d54983d0ed7
+      md5sum: 3b1ec8ea1879fc101d233f552fa4419f
     - path: miniwdl_run/call-clean_check_reads/inputs.json
     - path: miniwdl_run/call-clean_check_reads/outputs.json
     - path: miniwdl_run/call-clean_check_reads/stderr.txt
@@ -119,111 +119,6 @@
       md5sum: 1684062540bab8897921ed5e40c747cf
     - path: miniwdl_run/call-consensus_qc/work/_miniwdl_inputs/0/ont.medaka.consensus.fasta
       md5sum: d41d8cd98f00b204e9800998ecf8427e
-    - path: miniwdl_run/call-fastq_scan_clean_reads/command
-      md5sum: be9f6914525f199ada50840fcfbac009
-    - path: miniwdl_run/call-fastq_scan_clean_reads/inputs.json
-      contains: ["read1", "ont"]
-    - path: miniwdl_run/call-fastq_scan_clean_reads/outputs.json
-      contains: ["fastq_scan_se", "pipeline_date", "read1_seq"]
-    - path: miniwdl_run/call-fastq_scan_clean_reads/stderr.txt
-    - path: miniwdl_run/call-fastq_scan_clean_reads/stderr.txt.offset
-    - path: miniwdl_run/call-fastq_scan_clean_reads/stdout.txt
-    - path: miniwdl_run/call-fastq_scan_clean_reads/task.log
-      contains: ["wdl", "theiacov_ont", "fastq_scan_clean_reads", "done"]
-    - path: miniwdl_run/call-fastq_scan_clean_reads/work/DATE
-    - path: miniwdl_run/call-fastq_scan_clean_reads/work/READ1_SEQS
-      md5sum: b5f484e6f67b3526a3e8c96e99fa90fa
-    - path: miniwdl_run/call-fastq_scan_clean_reads/work/VERSION
-      md5sum: 8e4e9cdfbacc9021a3175ccbbbde002b
-    - path: miniwdl_run/call-fastq_scan_clean_reads/work/_miniwdl_inputs/0/artic_ncov2019_ont.fastq
-      md5sum: d41d8cd98f00b204e9800998ecf8427e
-    - path: miniwdl_run/call-fastq_scan_clean_reads/work/artic_ncov2019_ont_fastq-scan.json
-      md5sum: b99beb5bf5001d6809503e464110194d
-    - path: miniwdl_run/call-fastq_scan_raw_reads/command
-      md5sum: 73ed9325f9e455c6a917931f636c79bb
-    - path: miniwdl_run/call-fastq_scan_raw_reads/inputs.json
-      contains: ["read1", "ont"]
-    - path: miniwdl_run/call-fastq_scan_raw_reads/outputs.json
-      contains: ["fastq_scan_se", "pipeline_date", "read1_seq"]
-    - path: miniwdl_run/call-fastq_scan_raw_reads/stderr.txt
-    - path: miniwdl_run/call-fastq_scan_raw_reads/stderr.txt.offset
-    - path: miniwdl_run/call-fastq_scan_raw_reads/stdout.txt
-    - path: miniwdl_run/call-fastq_scan_raw_reads/task.log
-      contains: ["wdl", "theiacov_ont", "fastq_scan_raw_reads", "done"]
-    - path: miniwdl_run/call-fastq_scan_raw_reads/work/DATE
-    - path: miniwdl_run/call-fastq_scan_raw_reads/work/READ1_SEQS
-      md5sum: 36ae80653fd0a8bd44e62985beaa5f59
-    - path: miniwdl_run/call-fastq_scan_raw_reads/work/VERSION
-      md5sum: 8e4e9cdfbacc9021a3175ccbbbde002b
-    - path: miniwdl_run/call-fastq_scan_raw_reads/work/_miniwdl_inputs/0/ont.fastq.gz
-    - path: miniwdl_run/call-fastq_scan_raw_reads/work/ont_fastq-scan.json
-      md5sum: e17ebdd15b2abaf9d000780559e86b48
-    - path: miniwdl_run/call-kraken2_dehosted/command
-      md5sum: 4e16d6131c1b4cf9a9c305f9c46dee0f
-    - path: miniwdl_run/call-kraken2_dehosted/inputs.json
-      contains: ["read1", "samplename"]
-    - path: miniwdl_run/call-kraken2_dehosted/outputs.json
-      contains: ["kraken2", "percent_human", "percent_sc2"]
-    - path: miniwdl_run/call-kraken2_dehosted/stderr.txt
-    - path: miniwdl_run/call-kraken2_dehosted/stderr.txt.offset
-    - path: miniwdl_run/call-kraken2_dehosted/stdout.txt
-    - path: miniwdl_run/call-kraken2_dehosted/task.log
-      contains: ["wdl", "theiacov_ont", "kraken2_dehosted", "done"]
-    - path: miniwdl_run/call-kraken2_dehosted/work/DATE
-    - path: miniwdl_run/call-kraken2_dehosted/work/PERCENT_HUMAN
-      md5sum: 14673621d9ed27dfa4fd8c2f42a43cd4
-    - path: miniwdl_run/call-kraken2_dehosted/work/PERCENT_SC2
-      md5sum: 37c88efe14bd0bb0f52f6ef3a0b4266b
-    - path: miniwdl_run/call-kraken2_dehosted/work/PERCENT_TARGET_ORG
-      md5sum: 68b329da9893e34099c7d8ad5cb9c940
-    - path: miniwdl_run/call-kraken2_dehosted/work/VERSION
-      md5sum: 379b99c23325315c502e74614c035e7d
-    - path: miniwdl_run/call-kraken2_dehosted/work/_miniwdl_inputs/0/ont_R1_dehosted.fastq.gz
-    - path: miniwdl_run/call-kraken2_dehosted/work/ont_kraken2_report.txt
-      md5sum: 5780753ae61523a3621fbe2635b04b70
-    - path: miniwdl_run/call-kraken2_raw/command
-      md5sum: c6d2471fe35fd293789895b69aec7b79
-    - path: miniwdl_run/call-kraken2_raw/inputs.json
-      contains: ["read1", "samplename"]
-    - path: miniwdl_run/call-kraken2_raw/outputs.json
-      contains: ["kraken2", "percent_human", "percent_sc2"]
-    - path: miniwdl_run/call-kraken2_raw/stderr.txt
-    - path: miniwdl_run/call-kraken2_raw/stderr.txt.offset
-    - path: miniwdl_run/call-kraken2_raw/stdout.txt
-    - path: miniwdl_run/call-kraken2_raw/task.log
-      contains: ["wdl", "theiacov_ont", "kraken2_raw", "done"]
-    - path: miniwdl_run/call-kraken2_raw/work/DATE
-    - path: miniwdl_run/call-kraken2_raw/work/PERCENT_HUMAN
-      md5sum: 00846d3d5c7e0303a4c3f387972912fc
-    - path: miniwdl_run/call-kraken2_raw/work/PERCENT_SC2
-      md5sum: 368eeeca39d309477cdf423f85e59e8d
-    - path: miniwdl_run/call-kraken2_raw/work/PERCENT_TARGET_ORG
-      md5sum: 68b329da9893e34099c7d8ad5cb9c940
-    - path: miniwdl_run/call-kraken2_raw/work/VERSION
-      md5sum: 379b99c23325315c502e74614c035e7d
-    - path: miniwdl_run/call-kraken2_raw/work/_miniwdl_inputs/0/ont.fastq.gz
-    - path: miniwdl_run/call-kraken2_raw/work/ont_kraken2_report.txt
-      md5sum: f2c6f26b1ef2786d124eae2ab3fe80c1
-    - path: miniwdl_run/call-ncbi_scrub_se/command
-      md5sum: c1f43152f2c0d9ff55d7216d3b63abdd
-    - path: miniwdl_run/call-ncbi_scrub_se/inputs.json
-      contains: ["read1", "samplename", "ont"]
-    - path: miniwdl_run/call-ncbi_scrub_se/outputs.json
-      contains: ["ncbi_scrub_se", "read1_dehosted", "read1_human_spots_removed"]
-    - path: miniwdl_run/call-ncbi_scrub_se/stderr.txt
-    - path: miniwdl_run/call-ncbi_scrub_se/stderr.txt.offset
-    - path: miniwdl_run/call-ncbi_scrub_se/stdout.txt
-    - path: miniwdl_run/call-ncbi_scrub_se/task.log
-      contains: ["wdl", "theiacov_ont", "ncbi_scrub_se", "done"]
-    - path: miniwdl_run/call-ncbi_scrub_se/work/DATE
-    - path: miniwdl_run/call-ncbi_scrub_se/work/FWD_SPOTS_REMOVED
-      md5sum: e760668b6273d38c832c153fde5725da
-    - path: miniwdl_run/call-ncbi_scrub_se/work/_miniwdl_inputs/0/ont.fastq.gz
-    - path: miniwdl_run/call-ncbi_scrub_se/work/ont_R1_dehosted.fastq.gz
-    - path: miniwdl_run/call-ncbi_scrub_se/work/r1.fastq
-      md5sum: 3b2d349ed5f0d1234ce4b8a5b2646300
-    - path: miniwdl_run/call-ncbi_scrub_se/work/r1.fastq.clean
-      md5sum: 2cf5f8defc05395fecd10c68865f9ae3
     - path: miniwdl_run/call-nextclade/command
     - path: miniwdl_run/call-nextclade/inputs.json
       contains: ["dataset_name", "dataset_tag", "genome_fasta"]
@@ -336,7 +231,7 @@
       md5sum: d41d8cd98f00b204e9800998ecf8427e
     - path: miniwdl_run/call-pangolin4/work/ont.pangolin_report.csv
     - path: miniwdl_run/call-raw_check_reads/command
-      md5sum: 539522601b488d7fd02ca529d869abb7
+      md5sum: 5a9ed8f4856b36fd75d3e83dbde4ecfd
     - path: miniwdl_run/call-raw_check_reads/inputs.json
     - path: miniwdl_run/call-raw_check_reads/outputs.json
     - path: miniwdl_run/call-raw_check_reads/stderr.txt
@@ -348,20 +243,6 @@
     - path: miniwdl_run/call-raw_check_reads/work/FLAG
       md5sum: 32c0be4fb7f3030bf9c74c0a836d4f2e
     - path: miniwdl_run/call-raw_check_reads/work/_miniwdl_inputs/0/ont.fastq.gz
-    - path: miniwdl_run/call-read_filtering/command
-    - path: miniwdl_run/call-read_filtering/inputs.json
-      contains: ["demultiplexed_reads", "max_length", "samplename"]
-    - path: miniwdl_run/call-read_filtering/outputs.json
-      contains: ["read_filtering", "filtered_reads", "fastq"]
-    - path: miniwdl_run/call-read_filtering/stderr.txt
-    - path: miniwdl_run/call-read_filtering/stderr.txt.offset
-    - path: miniwdl_run/call-read_filtering/stdout.txt
-    - path: miniwdl_run/call-read_filtering/task.log
-      contains: ["wdl", "theiacov_ont", "read_filtering", "done"]
-    - path: miniwdl_run/call-read_filtering/work/_miniwdl_inputs/0/ont_R1_dehosted.fastq.gz
-    - path: miniwdl_run/call-read_filtering/work/artic_ncov2019_ont.fastq
-      md5sum: 167eb433eaa9063c20b37661b2ef793c
-    - path: miniwdl_run/call-read_filtering/work/ont/ont_R1_dehosted.fastq.gz
     - path: miniwdl_run/call-sc2_gene_coverage/command
       md5sum: da1bb1103606f2f9d8e2211e53ea3cbb
     - path: miniwdl_run/call-sc2_gene_coverage/inputs.json
@@ -479,8 +360,6 @@
       contains: ["stat", "command", "output", "runtime"]
     - path: miniwdl_run/wdl/tasks/quality_control/task_consensus_qc.wdl
       contains: ["qc", "command", "output", "runtime"]
-    - path: miniwdl_run/wdl/tasks/quality_control/task_fastq_scan.wdl
-      contains: ["fastq", "command", "output", "runtime"]
     - path: miniwdl_run/wdl/tasks/quality_control/task_ncbi_scrub.wdl
       contains: ["scrubber", "command", "output", "runtime"]
     - path: miniwdl_run/wdl/tasks/quality_control/task_screen.wdl

tests/workflows/theiaprok/test_wf_theiaprok_illumina_pe.yml

@@ -202,7 +202,7 @@
     - path: miniwdl_run/call-cg_pipeline_raw/work/test_readMetrics.tsv
       contains: ["File", "fastq", "coverage"]
     - path: miniwdl_run/call-clean_check_reads/command
-      md5sum: 6b7e661ab2b1278fa43f7264c53a832a
+      md5sum: aab0121590d0bd1e244e932bfcb39852
     - path: miniwdl_run/call-clean_check_reads/inputs.json
       contains: ["read1", "fastq", "skip_screen", "true"]
     - path: miniwdl_run/call-clean_check_reads/outputs.json
@@ -365,7 +365,7 @@
     - path: miniwdl_run/call-quast/work/transposed_report.txt
       contains: ["Assembly", "length", "contigs", "test"]
     - path: miniwdl_run/call-raw_check_reads/command
-      md5sum: 9c72950c964c4898dde51d0d575c7452
+      md5sum: b0cdeab40636fd679a3d79a3e831dde1
     - path: miniwdl_run/call-raw_check_reads/inputs.json
       contains: ["read1", "fastq", "skip_screen", "true"]
     - path: miniwdl_run/call-raw_check_reads/outputs.json
@@ -582,7 +582,7 @@
     - path: miniwdl_run/wdl/tasks/quality_control/task_quast.wdl
       md5sum: 884d99106ba32d6508d30dec4b78c04c
     - path: miniwdl_run/wdl/tasks/quality_control/task_screen.wdl
-      md5sum: 19fe4b646274e39d9e2cf48a06f5da92
+      md5sum: c1f2f5cef67a0bd0cbc9da2a834d40e4
     - path: miniwdl_run/wdl/tasks/quality_control/task_trimmomatic.wdl
       md5sum: 6efbb44a314c5d7604031ebfd774e10f
     - path: miniwdl_run/wdl/tasks/species_typing/task_cauris_cladetyper.wdl
@@ -632,7 +632,7 @@
     - path: miniwdl_run/wdl/tasks/taxon_id/task_midas.wdl
       md5sum: 024971d1439dff7d59c0a26a824bd2c6
     - path: miniwdl_run/wdl/tasks/utilities/task_broad_terra_tools.wdl
-      md5sum: 499eb797a809306edf7e97005f67f8f2
+      md5sum: 0236363c7f0694cd3f96416aa43e2f91
     - path: miniwdl_run/wdl/workflows/theiaprok/wf_theiaprok_illumina_pe.wdl
       md5sum: a76d59109075ce8b861e63ffe70d7c77
     - path: miniwdl_run/wdl/workflows/utilities/wf_merlin_magic.wdl