From d22af39a354554a9e8f5c6017c7763dfc97b08ad Mon Sep 17 00:00:00 2001 From: kapsakcj Date: Thu, 10 Aug 2023 15:18:10 -0400 Subject: [PATCH 1/6] fastq-dl changes: update default docker to v2.0.3; default options are to only download from SRA (instead of ENA); capture date and output as string; fastq-dl is always verbose; added --cpus option to cmd; added string outputs for fastq-dl version, docker image, and date it was run; added maxRetries to runtime block. --- tasks/utilities/task_sra_fetch.wdl | 25 ++++++++++++++++--- .../utilities/data_import/wf_sra_fetch.wdl | 3 +++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/tasks/utilities/task_sra_fetch.wdl b/tasks/utilities/task_sra_fetch.wdl index a9a22d615..89875ba00 100644 --- a/tasks/utilities/task_sra_fetch.wdl +++ b/tasks/utilities/task_sra_fetch.wdl @@ -4,15 +4,30 @@ version 1.0 task fastq_dl_sra { input { String sra_accession - String docker = "us-docker.pkg.dev/general-theiagen/biocontainers/fastq-dl:2.0.1--pyhdfd78af_0" + String docker = "us-docker.pkg.dev/general-theiagen/biocontainers/fastq-dl:2.0.3--pyhdfd78af_0" Int disk_size = 100 Int cpus = 2 Int memory = 8 - String? fastq_dl_opts + # default set to force the use of SRA instead of ENA due to SRA Lite FASTQ file format issues + String fastq_dl_opts = "--provider sra --only-provider" + } + meta { + # so that call caching is always turned off + volatile: true } command <<< + # capture version fastq-dl --version | tee VERSION - fastq-dl -a ~{sra_accession} ~{fastq_dl_opts} + + # capture date in UTC timezone + date -u | tee DATE + + # download fastq files + fastq-dl \ + --verbose \ + -a ~{sra_accession} \ + --cpus ~{cpus} \ + ~{fastq_dl_opts} # tag single-end reads with _1 if [ -f "~{sra_accession}.fastq.gz" ] && [ ! -f "~{sra_accession}_1.fastq.gz" ]; then @@ -22,6 +37,9 @@ task fastq_dl_sra { output { File read1 = "~{sra_accession}_1.fastq.gz" File? read2 = "~{sra_accession}_2.fastq.gz" + String fastq_dl_version = read_string("VERSION") + String fastq_dl_docker = docker + String fastq_dl_date = read_string("DATE") } runtime { docker: docker @@ -30,5 +48,6 @@ task fastq_dl_sra { disks: "local-disk " + disk_size + " SSD" disk: disk_size + " GB" # TES preemptible: 1 + maxRetries: 3 } } diff --git a/workflows/utilities/data_import/wf_sra_fetch.wdl b/workflows/utilities/data_import/wf_sra_fetch.wdl index ba6a9906b..b557ffb7e 100644 --- a/workflows/utilities/data_import/wf_sra_fetch.wdl +++ b/workflows/utilities/data_import/wf_sra_fetch.wdl @@ -23,5 +23,8 @@ workflow fetch_sra_to_fastq { output { File read1 = fastq_dl_sra.read1 File? read2 = fastq_dl_sra.read2 + String fastq_dl_version = fastq_dl_sra.fastq_dl_version + String fastq_dl_docker = fastq_dl_sra.fastq_dl_docker + String fastq_dl_date = fastq_dl_sra.fastq_dl_date } } \ No newline at end of file From 44ec461645278bf6a5657784afca3aa5a24da60b Mon Sep 17 00:00:00 2001 From: kapsakcj Date: Thu, 10 Aug 2023 15:40:49 -0400 Subject: [PATCH 2/6] added new file output to fastq-dl task and sra_fetch wf: fastq metadata TSV --- tasks/utilities/task_sra_fetch.wdl | 4 ++++ workflows/utilities/data_import/wf_sra_fetch.wdl | 1 + 2 files changed, 5 insertions(+) diff --git a/tasks/utilities/task_sra_fetch.wdl b/tasks/utilities/task_sra_fetch.wdl index 89875ba00..2eaddb5cf 100644 --- a/tasks/utilities/task_sra_fetch.wdl +++ b/tasks/utilities/task_sra_fetch.wdl @@ -33,10 +33,14 @@ task fastq_dl_sra { if [ -f "~{sra_accession}.fastq.gz" ] && [ ! -f "~{sra_accession}_1.fastq.gz" ]; then mv "~{sra_accession}.fastq.gz" "~{sra_accession}_1.fastq.gz" fi + + # rename FASTQ metadata file to include SRR accession + mv -v fastq-run-info.tsv ~{sra_accession}.fastq-run-info.tsv >>> output { File read1 = "~{sra_accession}_1.fastq.gz" File? read2 = "~{sra_accession}_2.fastq.gz" + File fastq_metadata = "~{sra_accession}.fastq-run-info.tsv" String fastq_dl_version = read_string("VERSION") String fastq_dl_docker = docker String fastq_dl_date = read_string("DATE") diff --git a/workflows/utilities/data_import/wf_sra_fetch.wdl b/workflows/utilities/data_import/wf_sra_fetch.wdl index b557ffb7e..015e5fca0 100644 --- a/workflows/utilities/data_import/wf_sra_fetch.wdl +++ b/workflows/utilities/data_import/wf_sra_fetch.wdl @@ -23,6 +23,7 @@ workflow fetch_sra_to_fastq { output { File read1 = fastq_dl_sra.read1 File? read2 = fastq_dl_sra.read2 + File fastq_dl_fastq_metadata = fastq_dl_sra.fastq_metadata String fastq_dl_version = fastq_dl_sra.fastq_dl_version String fastq_dl_docker = fastq_dl_sra.fastq_dl_docker String fastq_dl_date = fastq_dl_sra.fastq_dl_date From 3fbb063faa5b948d4002449ae134c377a0aee16b Mon Sep 17 00:00:00 2001 From: kapsakcj Date: Fri, 11 Aug 2023 09:35:39 -0400 Subject: [PATCH 3/6] update fastq-dl docker to 2.0.4 due to version mismatch issue with v2.0.3 --- tasks/utilities/task_sra_fetch.wdl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tasks/utilities/task_sra_fetch.wdl b/tasks/utilities/task_sra_fetch.wdl index 2eaddb5cf..adcfa2034 100644 --- a/tasks/utilities/task_sra_fetch.wdl +++ b/tasks/utilities/task_sra_fetch.wdl @@ -4,7 +4,7 @@ version 1.0 task fastq_dl_sra { input { String sra_accession - String docker = "us-docker.pkg.dev/general-theiagen/biocontainers/fastq-dl:2.0.3--pyhdfd78af_0" + String docker = "us-docker.pkg.dev/general-theiagen/biocontainers/fastq-dl:2.0.4--pyhdfd78af_0" Int disk_size = 100 Int cpus = 2 Int memory = 8 @@ -36,6 +36,7 @@ task fastq_dl_sra { # rename FASTQ metadata file to include SRR accession mv -v fastq-run-info.tsv ~{sra_accession}.fastq-run-info.tsv + >>> output { File read1 = "~{sra_accession}_1.fastq.gz" From 101c915765dcab2433ccfadfbe4102f8526a60bf Mon Sep 17 00:00:00 2001 From: kapsakcj Date: Mon, 21 Aug 2023 12:32:09 -0400 Subject: [PATCH 4/6] added fastq-dl --prefix flag; adjusted output filename to match --- tasks/utilities/task_sra_fetch.wdl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks/utilities/task_sra_fetch.wdl b/tasks/utilities/task_sra_fetch.wdl index adcfa2034..e82964ffd 100644 --- a/tasks/utilities/task_sra_fetch.wdl +++ b/tasks/utilities/task_sra_fetch.wdl @@ -27,6 +27,7 @@ task fastq_dl_sra { --verbose \ -a ~{sra_accession} \ --cpus ~{cpus} \ + --prefix ~{sra_accession} \ ~{fastq_dl_opts} # tag single-end reads with _1 @@ -35,13 +36,13 @@ task fastq_dl_sra { fi # rename FASTQ metadata file to include SRR accession - mv -v fastq-run-info.tsv ~{sra_accession}.fastq-run-info.tsv + #mv -v fastq-run-info.tsv ~{sra_accession}.fastq-run-info.tsv >>> output { File read1 = "~{sra_accession}_1.fastq.gz" File? read2 = "~{sra_accession}_2.fastq.gz" - File fastq_metadata = "~{sra_accession}.fastq-run-info.tsv" + File fastq_metadata = "~{sra_accession}-run-info.tsv" String fastq_dl_version = read_string("VERSION") String fastq_dl_docker = docker String fastq_dl_date = read_string("DATE") From efd062e65c2a2d42816f91c02541e253069923e0 Mon Sep 17 00:00:00 2001 From: kapsakcj Date: Mon, 21 Aug 2023 12:36:00 -0400 Subject: [PATCH 5/6] remove "--only-provider" from fastq-dl default options --- tasks/utilities/task_sra_fetch.wdl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/utilities/task_sra_fetch.wdl b/tasks/utilities/task_sra_fetch.wdl index e82964ffd..019839196 100644 --- a/tasks/utilities/task_sra_fetch.wdl +++ b/tasks/utilities/task_sra_fetch.wdl @@ -9,7 +9,7 @@ task fastq_dl_sra { Int cpus = 2 Int memory = 8 # default set to force the use of SRA instead of ENA due to SRA Lite FASTQ file format issues - String fastq_dl_opts = "--provider sra --only-provider" + String fastq_dl_opts = "--provider sra" } meta { # so that call caching is always turned off From 1abb78191987d317e9101dbdb26ade8ab29b6d94 Mon Sep 17 00:00:00 2001 From: kapsakcj Date: Mon, 21 Aug 2023 13:19:35 -0400 Subject: [PATCH 6/6] deleted old code --- tasks/utilities/task_sra_fetch.wdl | 2 -- 1 file changed, 2 deletions(-) diff --git a/tasks/utilities/task_sra_fetch.wdl b/tasks/utilities/task_sra_fetch.wdl index 019839196..c3d281c26 100644 --- a/tasks/utilities/task_sra_fetch.wdl +++ b/tasks/utilities/task_sra_fetch.wdl @@ -35,8 +35,6 @@ task fastq_dl_sra { mv "~{sra_accession}.fastq.gz" "~{sra_accession}_1.fastq.gz" fi - # rename FASTQ metadata file to include SRR accession - #mv -v fastq-run-info.tsv ~{sra_accession}.fastq-run-info.tsv >>> output {