diff --git a/example-params.local-qa.json b/example-params.local-qa.json index 60aab11..af534c3 100644 --- a/example-params.local-qa.json +++ b/example-params.local-qa.json @@ -16,14 +16,14 @@ "cleanup": true, "lane_qc": false, "aln_qc": false, - "song_url": "https://song.rdpc-qa.cancercollaboratory.org", - "score_url": "https://score.rdpc-qa.cancercollaboratory.org", + "song_url": "https://song.rdpc-qa.cumulus.genomeinformatics.org", + "score_url": "https://score.rdpc-qa.cumulus.genomeinformatics.org", "download": { "score_mem": 4, "song_mem": 2, "score_cpus": 2, - "score_url": "https://submission-score.rdpc-qa.cancercollaboratory.org", - "song_url": "https://submission-song.rdpc-qa.cancercollaboratory.org", + "score_url": "https://submission-score.rdpc-qa.cumulus.genomeinformatics.org", + "song_url": "https://submission-song.rdpc-qa.cumulus.genomeinformatics.org", "song_cpus": 1 } } \ No newline at end of file diff --git a/example-params.rdpc-dev.json b/example-params.rdpc-dev.json index f06a4a0..1593e2c 100644 --- a/example-params.rdpc-dev.json +++ b/example-params.rdpc-dev.json @@ -4,8 +4,8 @@ "mem":16, "cpus":4 }, - "ribosomal_interval_list":"/nfs-dev-1-vol-dev-1/reference/rna-seq-references/GRCh38_Verily_v1.Picard_CollectRnaSeqMetrics/GRCh38_Verily_v1.rRNA.interval_list", - "ref_genome_gtf":"/nfs-dev-1-vol-dev-1/reference/rna-seq-references/GRCh38_Verily_v1.annotation/gencode.v40.chr_patch_hapl_scaff.annotation.gtf", + "ribosomal_interval_list":"/nfs-dev-vol-qa-3/reference/rna-seq-references/GRCh38_Verily_v1.Picard_CollectRnaSeqMetrics/GRCh38_Verily_v1.rRNA.interval_list", + "ref_genome_gtf":"/nfs-dev-vol-qa-3/reference/rna-seq-references/GRCh38_Verily_v1.annotation/gencode.v40.chr_patch_hapl_scaff.annotation.gtf", "aln_qc":true, "upload":{ "score_mem":10, @@ -14,13 +14,14 @@ "song_cpus":2 }, "study_id":"TCRB-CA", - "song_url":"https://song.rdpc-dev.cancercollaboratory.org", + "song_url": "https://song.rdpc-dev.cumulus.genomeinformatics.org", + "score_url": "https://score.rdpc-dev.cumulus.genomeinformatics.org", "download":{ "score_mem":10, "song_mem":2, "score_cpus":4, - "score_url":"https://submission-score.rdpc-dev.cancercollaboratory.org", - "song_url":"https://submission-song.rdpc-dev.cancercollaboratory.org", + "score_url": "https://submission-score.rdpc-dev.cumulus.genomeinformatics.org", + "song_url": "https://submission-song.rdpc-dev.cumulus.genomeinformatics.org", "song_cpus":2 }, "mem":4, @@ -33,16 +34,15 @@ "mem":18, "cpus":4 }, - "ref_genome_index_star":"/nfs-dev-1-vol-dev-1/reference/rna-seq-references/GRCh38_Verily_v1.STARindex.sjdbOverhang_75", + "ref_genome_index_star":"/nfs-dev-vol-qa-3/reference/rna-seq-references/GRCh38_Verily_v1.STARindex.sjdbOverhang_75", "lane_qc":false, - "ref_genome_fa":"/nfs-dev-1-vol-dev-1/reference/rna-seq-references/GRCh38_Verily_v1.genome/GRCh38_Verily_v1.genome.fa", + "ref_genome_fa":"/nfs-dev-vol-qa-3/reference/rna-seq-references/GRCh38_Verily_v1.genome/GRCh38_Verily_v1.genome.fa", "aligned-seq-QC":{ "mem":16, "cpus":4 }, "analysis_metadata":"", "cpus":1, - "score_url":"https://score.rdpc-dev.cancercollaboratory.org", "payloadGen":{ "mem":8, "cpus":2 @@ -53,9 +53,9 @@ "mem":80, "cpus":12 }, - "ref_flat":"/nfs-dev-1-vol-dev-1/reference/rna-seq-references/GRCh38_Verily_v1.Picard_CollectRnaSeqMetrics/GRCh38_Verily_v1.refFlat.txt.gz", + "ref_flat":"/nfs-dev-vol-qa-3/reference/rna-seq-references/GRCh38_Verily_v1.Picard_CollectRnaSeqMetrics/GRCh38_Verily_v1.refFlat.txt.gz", "cleanup":true, - "ref_genome_index_hisat2":"/nfs-dev-1-vol-dev-1/reference/rna-seq-references/GRCh38_Verily_v1.HISAT2index/GRCh38_Verily_v1", + "ref_genome_index_hisat2":"/nfs-dev-vol-qa-3/reference/rna-seq-references/GRCh38_Verily_v1.HISAT2index/GRCh38_Verily_v1", "sequencing_files":[ ], diff --git a/example-params.rdpc-qa.json b/example-params.rdpc-qa.json index 12f201a..be2220e 100644 --- a/example-params.rdpc-qa.json +++ b/example-params.rdpc-qa.json @@ -2,28 +2,28 @@ "study_id": "TCRB-CA", "analysis_id": "c62cee87-04ae-4988-acee-8704aec988d4", "analysis_metadata": "", - "ref_genome_index_star": "/nfs-dev-1-vol-qa-1/reference/rna-seq-references/GRCh38_Verily_v1.STARindex.sjdbOverhang_75", - "ref_genome_index_hisat2": "/nfs-dev-1-vol-qa-1/reference/rna-seq-references/GRCh38_Verily_v1.HISAT2index/GRCh38_Verily_v1", - "ref_genome_gtf": "/nfs-dev-1-vol-qa-1/reference/rna-seq-references/GRCh38_Verily_v1.annotation/gencode.v40.chr_patch_hapl_scaff.annotation.gtf", - "ref_genome_fa": "/nfs-dev-1-vol-qa-1/reference/rna-seq-references/GRCh38_Verily_v1.genome/GRCh38_Verily_v1.genome.fa", + "ref_genome_index_star": "/nfs-dev-vol-qa-3/reference/rna-seq-references/GRCh38_Verily_v1.STARindex.sjdbOverhang_75", + "ref_genome_index_hisat2": "/nfs-dev-vol-qa-3/reference/rna-seq-references/GRCh38_Verily_v1.HISAT2index/GRCh38_Verily_v1", + "ref_genome_gtf": "/nfs-dev-vol-qa-3/reference/rna-seq-references/GRCh38_Verily_v1.annotation/gencode.v40.chr_patch_hapl_scaff.annotation.gtf", + "ref_genome_fa": "/nfs-dev-vol-qa-3/reference/rna-seq-references/GRCh38_Verily_v1.genome/GRCh38_Verily_v1.genome.fa", "sequencing_files": [], "sjdboverhang": 75, - "ref_flat": "/nfs-dev-1-vol-qa-1/reference/rna-seq-references/GRCh38_Verily_v1.Picard_CollectRnaSeqMetrics/GRCh38_Verily_v1.refFlat.txt.gz", - "ribosomal_interval_list": "/nfs-dev-1-vol-qa-1/reference/rna-seq-references/GRCh38_Verily_v1.Picard_CollectRnaSeqMetrics/GRCh38_Verily_v1.rRNA.interval_list", + "ref_flat": "/nfs-dev-vol-qa-3/reference/rna-seq-references/GRCh38_Verily_v1.Picard_CollectRnaSeqMetrics/GRCh38_Verily_v1.refFlat.txt.gz", + "ribosomal_interval_list": "/nfs-dev-vol-qa-3/reference/rna-seq-references/GRCh38_Verily_v1.Picard_CollectRnaSeqMetrics/GRCh38_Verily_v1.rRNA.interval_list", "cpus": 1, "mem": 4, "tempdir": "/icgc-argo-scratch", "cleanup": true, "lane_qc": false, "aln_qc": false, - "song_url": "https://song.rdpc-qa.cancercollaboratory.org", - "score_url": "https://score.rdpc-qa.cancercollaboratory.org", + "song_url": "https://song.rdpc-qa.cumulus.genomeinformatics.org", + "score_url": "https://score.rdpc-qa.cumulus.genomeinformatics.org", "download": { "score_mem": 10, "song_mem": 2, "score_cpus": 4, - "score_url": "https://submission-score.rdpc-qa.cancercollaboratory.org", - "song_url": "https://submission-song.rdpc-qa.cancercollaboratory.org", + "score_url": "https://submission-score.rdpc-qa.cumulus.genomeinformatics.org", + "song_url": "https://submission-song.rdpc-qa.cumulus.genomeinformatics.org", "song_cpus": 2 }, "seqDataToLaneBam": { diff --git a/rna-seq-alignment-wf/main.nf b/rna-seq-alignment-wf/main.nf index 99cb0ff..1fd03be 100755 --- a/rna-seq-alignment-wf/main.nf +++ b/rna-seq-alignment-wf/main.nf @@ -175,7 +175,7 @@ upload_params = [ *:(params.upload ?: [:]) ] -include { SongScoreDownload as dnld } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.8.0/main.nf' params(download_params) +include { SongScoreDownload as dnld } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/main.nf' params(download_params) include { seqDataToLaneBam as toLaneBam } from "./modules/raw.githubusercontent.com/icgc-argo-workflows/dna-seq-processing-tools/seq-data-to-lane-bam.0.3.3.0/tools/seq-data-to-lane-bam/seq-data-to-lane-bam.nf" params(seqDataToLaneBam_params) include { icgcArgoRnaSeqAlignmentSTAR as star } from "./wfpr_modules/github.com/icgc-argo-workflows/rna-seq-alignment/genome-alignment-star@0.2.6/alignSTAR.nf" params(starAligner_params) include { icgcArgoRnaSeqAlignmentHISAT2 as hisat2 } from "./wfpr_modules/github.com/icgc-argo-workflows/rna-seq-alignment/genome-alignment-hisat2@0.2.3/alignHISAT2.nf" params(hisat2Aligner_params) @@ -193,12 +193,12 @@ include { payloadGenRnaAlignment as pGenAlnStarSj; payloadGenRnaAlignment as pG include { payloadGenRnaAlignment as pGenQcStar; payloadGenRnaAlignment as pGenQcHisat2 } from './wfpr_modules/github.com/icgc-argo-workflows/data-processing-utility-tools/payload-gen-rna-alignment@0.2.0/main.nf' params(payloadGen_params) include { payloadGenRnaAlignment as pGenSuppStar; payloadGenRnaAlignment as pGenSuppHisat2 } from './wfpr_modules/github.com/icgc-argo-workflows/data-processing-utility-tools/payload-gen-rna-alignment@0.2.0/main.nf' params(payloadGen_params) include { payloadGenRnaAlignment as pGenQcLane } from './wfpr_modules/github.com/icgc-argo-workflows/data-processing-utility-tools/payload-gen-rna-alignment@0.2.0/main.nf' params(payloadGen_params) -include { SongScoreUpload as upAlnStar; SongScoreUpload as upAlnHisat2} from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.0/main.nf' params(upload_params) -include { SongScoreUpload as upAlnTxStar } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.0/main.nf' params(upload_params) -include { SongScoreUpload as upAlnStarSj; SongScoreUpload as upAlnHisat2Sj} from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.0/main.nf' params(upload_params) -include { SongScoreUpload as upQcStar; SongScoreUpload as upQcHisat2} from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.0/main.nf' params(upload_params) -include { SongScoreUpload as upSuppStar; SongScoreUpload as upSuppHisat2} from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.0/main.nf' params(upload_params) -include { SongScoreUpload as upQcLane} from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.0/main.nf' params(upload_params) +include { SongScoreUpload as upAlnStar; SongScoreUpload as upAlnHisat2} from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/main.nf' params(upload_params) +include { SongScoreUpload as upAlnTxStar } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/main.nf' params(upload_params) +include { SongScoreUpload as upAlnStarSj; SongScoreUpload as upAlnHisat2Sj} from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/main.nf' params(upload_params) +include { SongScoreUpload as upQcStar; SongScoreUpload as upQcHisat2} from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/main.nf' params(upload_params) +include { SongScoreUpload as upSuppStar; SongScoreUpload as upSuppHisat2} from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/main.nf' params(upload_params) +include { SongScoreUpload as upQcLane} from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/main.nf' params(upload_params) // please update workflow code as needed diff --git a/rna-seq-alignment-wf/pkg.json b/rna-seq-alignment-wf/pkg.json index f1b76b3..a3f6760 100644 --- a/rna-seq-alignment-wf/pkg.json +++ b/rna-seq-alignment-wf/pkg.json @@ -14,8 +14,8 @@ "url": "https://github.com/icgc-argo-workflows/rna-seq-alignment.git" }, "dependencies": [ - "github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.8.0", - "github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.0", + "github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0", + "github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3", "github.com/icgc-argo-workflows/rna-seq-alignment/genome-alignment-hisat2@0.2.3", "github.com/icgc-argo-workflows/rna-seq-alignment/genome-alignment-star@0.2.6", "github.com/icgc-argo-workflows/argo-qc-tools/fastqc@0.1.0.1", diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/local_modules/score-download.nf b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/local_modules/score-download.nf new file mode 100644 index 0000000..5829acf --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/local_modules/score-download.nf @@ -0,0 +1,71 @@ +#!/usr/bin/env nextflow +nextflow.enable.dsl=2 + +// processes resources +params.cpus = 8 +params.mem = 20 + +params.publish_dir = "" + +params.max_retries = 5 // set to 0 will disable retry +params.first_retry_wait_time = 1 // in seconds + +// required params w/ default +params.container = "ghcr.io/overture-stack/score" +params.container_version = "5.8.1" +params.transport_mem = 2 // Transport memory is in number of GBs + +// optional if secret mounted from pod else required +params.api_token = "" // song/score API token for download process + +// required params, no default +// --song_url song url for download process +// --score_url score url for download process + +// TODO: Replace with score container once it can download files via analysis_id +process scoreDownload { + maxRetries params.max_retries + errorStrategy { + sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long); // backoff time increases exponentially before each retry + return params.max_retries ? 'retry' : 'finish' + } + + pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"] + + cpus params.cpus + memory "${params.mem} GB" + + container "${ params.score_container ?: params.container}:${params.score_container_version ?: params.container_version}" + publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir ? true : false + + label "scoreDownload" + tag "${analysis_id}" + + if (workflow.containerEngine == "singularity") { + containerOptions "--bind \$(pwd):/score-client/logs" + } else if (workflow.containerEngine == "docker") { + containerOptions "-v \$(pwd):/score-client/logs" + } + + input: + path analysis + val study_id + val analysis_id + + output: + path analysis, emit: analysis_json + path 'out/*', emit: files + + + script: + accessToken = params.api_token ? params.api_token : "`cat /tmp/rdpc_secret/secret`" + """ + export METADATA_URL=${params.song_url} + export STORAGE_URL=${params.score_url} + export TRANSPORT_PARALLEL=${params.cpus} + export TRANSPORT_MEMORY=${params.transport_mem} + export ACCESSTOKEN=${accessToken} + + score-client download --analysis-id ${analysis_id} --study-id ${study_id} --output-dir ./out + """ +} diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/local_modules/song-get-analysis.nf b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/local_modules/song-get-analysis.nf new file mode 100644 index 0000000..596dbe9 --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/local_modules/song-get-analysis.nf @@ -0,0 +1,64 @@ +#!/usr/bin/env nextflow +nextflow.enable.dsl=2 + +// processes resources +params.cpus = 1 +params.mem = 1 + +params.publish_dir = "" + +params.max_retries = 5 // set to 0 will disable retry +params.first_retry_wait_time = 1 // in seconds + +// required params w/ default +params.container = "ghcr.io/overture-stack/song-client" +params.container_version = "5.0.2" + +// optional if secret mounted from pod else required +params.api_token = "" // song/score API token for download process + +// required params, no default +// --song_url song url for download process +// --score_url score url for download process + +process songGetAnalysis { + maxRetries params.max_retries + errorStrategy { + sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long); // backoff time increases exponentially before each retry + return params.max_retries ? 'retry' : 'finish' + } + + pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"] + + cpus params.cpus + memory "${params.mem} GB" + + container "${ params.song_container ?: params.container}:${params.song_container_version ?: params.container_version}" + publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir ? true : false + + tag "${analysis_id}" + + if (workflow.containerEngine == "singularity") { + containerOptions "--bind \$(pwd):/song-client/logs" + } else if (workflow.containerEngine == "docker") { + containerOptions "-v \$(pwd):/song-client/logs" + } + + input: + val study_id + val analysis_id + + output: + path "*.analysis.json", emit: json + + + script: + accessToken = params.api_token ? params.api_token : "`cat /tmp/rdpc_secret/secret`" + """ + export CLIENT_SERVER_URL=${params.song_url} + export CLIENT_STUDY_ID=${study_id} + export CLIENT_ACCESS_TOKEN=${accessToken} + + sing search -a ${analysis_id} > ${analysis_id}.analysis.json + """ +} diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/main.nf b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/main.nf new file mode 100755 index 0000000..d07709b --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/main.nf @@ -0,0 +1,104 @@ +#!/usr/bin/env nextflow + +/* + Copyright (c) 2020-2021, Ontario Institute for Cancer Research + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Authors: + Alex Lepsa + Junjun Zhang +*/ + +nextflow.enable.dsl = 2 +version = '2.9.0' + +// universal params go here, change default value as needed +params.publish_dir = "" // set to empty string will disable publishDir + +params.max_retries = 5 // set to 0 will disable retry +params.first_retry_wait_time = 1 // in seconds + +// tool specific parmas go here, add / change as needed +params.study_id = "" +params.analysis_id = "" + +params.api_token = "" + +params.song_cpus = 1 +params.song_mem = 1 // GB +params.song_url = "https://song.rdpc-qa.cancercollaboratory.org" +params.song_api_token = "" +params.song_container = "ghcr.io/overture-stack/song-client" +params.song_container_version = "5.0.2" + +params.score_cpus = 1 +params.score_mem = 1 // GB +params.score_transport_mem = 1 // GB +params.score_url = "https://score.rdpc-qa.cancercollaboratory.org" +params.score_api_token = "" +params.score_container = "ghcr.io/overture-stack/score" +params.score_container_version = "5.8.1" + + +song_params = [ + *:params, + 'cpus': params.song_cpus, + 'mem': params.song_mem, + 'song_url': params.song_url, + 'song_container': params.song_container, + 'song_container_version': params.song_container_version, + 'api_token': params.song_api_token ?: params.api_token +] + +score_params = [ + *:params, + 'cpus': params.score_cpus, + 'mem': params.score_mem, + 'transport_mem': params.score_transport_mem, + 'song_url': params.song_url, + 'score_url': params.score_url, + 'score_container': params.score_container, + 'score_container_version': params.score_container_version, + 'api_token': params.score_api_token ?: params.api_token +] + + +include { songGetAnalysis as songGet } from './local_modules/song-get-analysis' params(song_params) +include { scoreDownload as scoreDn } from './local_modules/score-download' params(score_params) + + +// please update workflow code as needed +workflow SongScoreDownload { + take: // update as needed + study_id + analysis_id + + main: + songGet(study_id, analysis_id) + scoreDn(songGet.out.json, study_id, analysis_id) + + emit: + analysis_json = songGet.out.json + files = scoreDn.out.files +} + + +// this provides an entry point for this main script, so it can be run directly without clone the repo +// using this command: nextflow run ///.nf -r .v --params-file xxx +workflow { + SongScoreDownload( + params.study_id, + params.analysis_id + ) +} diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/nextflow.config b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/nextflow.config new file mode 100644 index 0000000..f2cd1e3 --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/nextflow.config @@ -0,0 +1,4 @@ +docker { + enabled = true + runOptions = '-u \$(id -u):\$(id -g)' +} diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/pkg.json b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/pkg.json new file mode 100644 index 0000000..35bb9f3 --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/pkg.json @@ -0,0 +1,33 @@ +{ + "name": "song-score-download", + "version": "2.9.0", + "description": "SONG/SCORE download", + "main": "main.nf", + "deprecated": false, + "keywords": [ + "bioinformatics", + "data", + "metadata", + "download" + ], + "repository": { + "type": "git", + "url": "https://github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools.git" + }, + "dependencies": [], + "devDependencies": [], + "contributors": [ + { + "name": "Alex Lepsa" + }, + { + "name": "Junjun Zhang" + }, + { + "name": "Linda Xiang" + } + ], + "license": "Apache License 2.0", + "bugReport": "https://github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/issues", + "homepage": "https://github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools#readme" +} \ No newline at end of file diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/wfpr_modules b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/wfpr_modules new file mode 120000 index 0000000..1cc74ba --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/wfpr_modules @@ -0,0 +1 @@ +../../../../../wfpr_modules \ No newline at end of file diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/score-upload.nf b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/score-upload.nf new file mode 100644 index 0000000..61ef9be --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/score-upload.nf @@ -0,0 +1,65 @@ +#!/usr/bin/env nextflow +nextflow.enable.dsl=2 + +// processes resources +params.cpus = 8 +params.mem = 20 + +params.max_retries = 5 // set to 0 will disable retry +params.first_retry_wait_time = 1 // in seconds + +// required params w/ default +params.container = "ghcr.io/overture-stack/score" +params.container_version = "latest" +params.transport_mem = 2 // Transport memory is in number of GBs + +// optional if secret mounted from pod else required +params.api_token = "" // song/score API token for download process +params.score_force = false +// required params, no default +// --song_url song url for download process +// --score_url score url for download process + +process scoreUpload { + maxRetries params.max_retries + errorStrategy { + sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long); // backoff time doubles before each retry + return params.max_retries ? 'retry' : 'finish' + } + + pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"] + + cpus params.cpus + memory "${params.mem} GB" + + container "${ params.score_container ?: params.container}:${params.score_container_version ?: params.container_version}" + + if (workflow.containerEngine == "singularity") { + containerOptions "--bind \$(pwd):/score-client/logs" + } else if (workflow.containerEngine == "docker") { + containerOptions "-v \$(pwd):/score-client/logs" + } + + tag "${analysis_id}" + + input: + val analysis_id + path manifest + path upload + + output: + val analysis_id, emit: ready_to_publish + + script: + accessToken = params.api_token ? params.api_token : "`cat /tmp/rdpc_secret/secret`" + forceFlag = params.score_force ? "--force" : "" + """ + export METADATA_URL=${params.song_url} + export STORAGE_URL=${params.score_url} + export TRANSPORT_PARALLEL=${params.cpus} + export TRANSPORT_MEMORY=${params.transport_mem} + export ACCESSTOKEN=${accessToken} + + score-client upload --manifest ${manifest} ${forceFlag} + """ +} diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/song-manifest.nf b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/song-manifest.nf new file mode 100644 index 0000000..43e45ad --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/song-manifest.nf @@ -0,0 +1,63 @@ +#!/usr/bin/env nextflow +nextflow.enable.dsl=2 + +// processes resources +params.cpus = 1 +params.mem = 1 + +params.publish_dir = "" + +params.max_retries = 5 // set to 0 will disable retry +params.first_retry_wait_time = 1 // in seconds + +// required params w/ default +params.container = "ghcr.io/overture-stack/song-client" +params.container_version = "latest" + +// optional if secret mounted from pod else required +params.api_token = "" // song/score API token for download process + +// required params, no default +// --song_url song url for download process +// --score_url score url for download process + +process songManifest { + maxRetries params.max_retries + errorStrategy { + sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long); // backoff time doubles before each retry + return params.max_retries ? 'retry' : 'finish' + } + + container "${ params.song_container ?: params.container}:${params.song_container_version ?: params.container_version}" + publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir ? true : false + + pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"] + + cpus params.cpus + memory "${params.mem} GB" + tag "${analysis_id}" + + if (workflow.containerEngine == "singularity") { + containerOptions "--bind \$(pwd):/song-client/logs" + } else if (workflow.containerEngine == "docker") { + containerOptions "-v \$(pwd):/song-client/logs" + } + + input: + val study_id + val analysis_id + path upload + + output: + path "out/manifest.txt" + + script: + accessToken = params.api_token ? params.api_token : "`cat /tmp/rdpc_secret/secret`" + """ + export CLIENT_SERVER_URL=${params.song_url} + export CLIENT_STUDY_ID=${study_id} + export CLIENT_ACCESS_TOKEN=${accessToken} + + sing manifest -a ${analysis_id} -d . -f ./out/manifest.txt + """ +} diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/song-publish.nf b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/song-publish.nf new file mode 100644 index 0000000..7d1f77e --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/song-publish.nf @@ -0,0 +1,60 @@ +#!/usr/bin/env nextflow +nextflow.enable.dsl=2 + +// processes resources +params.cpus = 1 +params.mem = 1 + +params.max_retries = 5 // set to 0 will disable retry +params.first_retry_wait_time = 1 // in seconds + +// required params w/ default +params.container = "ghcr.io/overture-stack/song-client" +params.container_version = "latest" + +// optional if secret mounted from pod else required +params.api_token = "" // song/score API token for download process + +// required params, no default +// --song_url song url for download process +// --score_url score url for download process + +process songPublish { + maxRetries params.max_retries + errorStrategy { + sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long); // backoff time doubles before each retry + return params.max_retries ? 'retry' : 'finish' + } + + pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"] + + cpus params.cpus + memory "${params.mem} GB" + + container "${ params.song_container ?: params.container}:${params.song_container_version ?: params.container_version}" + + if (workflow.containerEngine == "singularity") { + containerOptions "--bind \$(pwd):/song-client/logs" + } else if (workflow.containerEngine == "docker") { + containerOptions "-v \$(pwd):/song-client/logs" + } + + tag "${analysis_id}" + + input: + val study_id + val analysis_id + + output: + val analysis_id, emit: analysis_id + + script: + accessToken = params.api_token ? params.api_token : "`cat /tmp/rdpc_secret/secret`" + """ + export CLIENT_SERVER_URL=${params.song_url} + export CLIENT_STUDY_ID=${study_id} + export CLIENT_ACCESS_TOKEN=${accessToken} + + sing publish -a ${analysis_id} + """ +} diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/song-submit.nf b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/song-submit.nf new file mode 100644 index 0000000..4d322d6 --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/local_modules/song-submit.nf @@ -0,0 +1,62 @@ +#!/usr/bin/env nextflow +nextflow.enable.dsl=2 + +// processes resources +params.cpus = 1 +params.mem = 1 + +params.max_retries = 5 // set to 0 will disable retry +params.first_retry_wait_time = 1 // in seconds + +// required params w/ default +params.container = "ghcr.io/overture-stack/song-client" +params.container_version = "latest" + +// optional if secret mounted from pod else required +params.api_token = "" // song/score API token for download process + +// required params, no default +// --song_url song url for download process +// --score_url score url for download process + +process songSubmit { + maxRetries params.max_retries + errorStrategy { + sleep(Math.pow(2, task.attempt) * params.first_retry_wait_time * 1000 as long); // backoff time doubles before each retry + return params.max_retries ? 'retry' : 'finish' + } + + pod = [secret: workflow.runName + "-secret", mountPath: "/tmp/rdpc_secret"] + + cpus params.cpus + memory "${params.mem} GB" + + container "${ params.song_container ?: params.container}:${params.song_container_version ?: params.container_version}" + + if (workflow.containerEngine == "singularity") { + containerOptions "--bind \$(pwd):/song-client/logs" + } else if (workflow.containerEngine == "docker") { + containerOptions "-v \$(pwd):/song-client/logs" + } + + tag "${study_id}" + label "songSubmit" + + input: + val study_id + path payload + + output: + stdout() + + script: + accessToken = params.api_token ? params.api_token : "`cat /tmp/rdpc_secret/secret`" + """ + export CLIENT_SERVER_URL=${params.song_url} + export CLIENT_STUDY_ID=${study_id} + export CLIENT_ACCESS_TOKEN=${accessToken} + + set -euxo pipefail + sing submit -f ${payload} | jq -er .analysisId | tr -d '\\n' + """ +} diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/main.nf b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/main.nf new file mode 100755 index 0000000..87e6b55 --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/main.nf @@ -0,0 +1,123 @@ +#!/usr/bin/env nextflow + +/* + Copyright (c) 2020-2021, Ontario Institute for Cancer Research + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + Authors: + Alex Lepsa + Junjun Zhang +*/ + +nextflow.enable.dsl = 2 +version = '2.9.3' + +// universal params go here, change default value as needed +params.publish_dir = "" // set to empty string will disable publishDir + +params.max_retries = 5 // set to 0 will disable retry +params.first_retry_wait_time = 1 // in seconds + +// tool specific parmas go here, add / change as needed +params.study_id = "TEST-PR" +params.payload = "NO_FILE" +params.upload = [] +params.analysis_id = "" // optional, analysis must already exist and in UNPUBLISHED state if analysis_id provided + +params.api_token = "" + +params.song_cpus = 1 +params.song_mem = 1 // GB +params.song_url = "https://song.rdpc-qa.cancercollaboratory.org" +params.song_api_token = "" +params.song_container = "ghcr.io/overture-stack/song-client" +params.song_container_version = "latest" + +params.score_cpus = 1 +params.score_mem = 1 // GB +params.score_transport_mem = 1 // GB +params.score_url = "https://score.rdpc-qa.cancercollaboratory.org" +params.score_api_token = "" +params.score_container = "ghcr.io/overture-stack/score" +params.score_container_version = "latest" +params.score_force = false + +song_params = [ + *:params, + 'cpus': params.song_cpus, + 'mem': params.song_mem, + 'song_url': params.song_url, + 'song_container': params.song_container, + 'song_container_version': params.song_container_version, + 'api_token': params.song_api_token ?: params.api_token +] + +score_params = [ + *:params, + 'cpus': params.score_cpus, + 'mem': params.score_mem, + 'transport_mem': params.score_transport_mem, + 'song_url': params.song_url, + 'score_url': params.score_url, + 'score_container': params.score_container, + 'score_container_version': params.score_container_version, + 'api_token': params.score_api_token ?: params.api_token, + 'score_force': params.score_force +] + +include { songSubmit as songSub } from './local_modules/song-submit' params(song_params) +include { songManifest as songMan } from './local_modules/song-manifest' params(song_params) +include { scoreUpload as scoreUp } from './local_modules/score-upload' params(score_params) +include { songPublish as songPub } from './local_modules/song-publish' params(song_params) + + +workflow SongScoreUpload { + take: + study_id + payload + upload + analysis_id + + main: + if (!analysis_id) { + // Create new analysis + songSub(study_id, payload) + analysis_id = songSub.out + } + + // Generate file manifest for upload + songMan(study_id, analysis_id, upload.collect()) + + // Upload to SCORE + scoreUp(analysis_id, songMan.out, upload.collect()) + + // Publish the analysis + songPub(study_id, scoreUp.out.ready_to_publish) + + emit: + analysis_id = songPub.out.analysis_id +} + + +// this provides an entry point for this main script, so it can be run directly without clone the repo +// using this command: nextflow run ///.nf -r .v --params-file xxx + +workflow { + SongScoreUpload( + params.study_id, + file(params.payload), + Channel.fromPath(params.upload), + params.analysis_id + ) +} diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/nextflow.config b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/nextflow.config new file mode 100644 index 0000000..f2cd1e3 --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/nextflow.config @@ -0,0 +1,4 @@ +docker { + enabled = true + runOptions = '-u \$(id -u):\$(id -g)' +} diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/pkg.json b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/pkg.json new file mode 100644 index 0000000..be5281e --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/pkg.json @@ -0,0 +1,33 @@ +{ + "name": "song-score-upload", + "version": "2.9.3", + "description": "SONG/SCORE upload", + "main": "main.nf", + "deprecated": false, + "keywords": [ + "bioinformatics", + "data", + "metadata", + "upload" + ], + "repository": { + "type": "git", + "url": "https://github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools.git" + }, + "dependencies": [], + "devDependencies": [], + "contributors": [ + { + "name": "Alex Lepsa" + }, + { + "name": "Junjun Zhang" + }, + { + "name": "Linda Xiang" + } + ], + "license": "Apache License 2.0", + "bugReport": "https://github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/issues", + "homepage": "https://github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools#readme" +} \ No newline at end of file diff --git a/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/wfpr_modules b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/wfpr_modules new file mode 120000 index 0000000..1cc74ba --- /dev/null +++ b/wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/wfpr_modules @@ -0,0 +1 @@ +../../../../../wfpr_modules \ No newline at end of file