Skip to content

Commit

Permalink
Merge pull request #5 from icgc-argo-workflows/open-access-variant-fi…
Browse files Browse the repository at this point in the history
[email protected]

[release]
  • Loading branch information
lindaxiang authored Jun 29, 2023
2 parents 90670cb + 19186a1 commit f843d1d
Show file tree
Hide file tree
Showing 17 changed files with 163 additions and 70 deletions.
20 changes: 14 additions & 6 deletions .github/workflows/build-test-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,13 @@ jobs:
steps:
- uses: actions/checkout@v2

- name: Set up Python 3.6
uses: actions/setup-python@v2
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: 3.6
python-version: "3.10"
architecture: "x64"
env:
AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache

- name: Extract package name and version from branch name
id: get_pkg_info
Expand Down Expand Up @@ -80,10 +83,13 @@ jobs:
steps:
- uses: actions/checkout@v2

- name: Set up Python 3.6
uses: actions/setup-python@v2
- name: Set up Python 3.10
uses: actions/setup-python@v3
with:
python-version: 3.6
python-version: "3.10"
architecture: "x64"
env:
AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache

- name: Install dependencies
run: |
Expand Down Expand Up @@ -177,6 +183,8 @@ jobs:
body: |
* Release `${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }}` (${{ github.sha }})
* Package `${{ steps.prep_assets.outputs.pkg_tar }}` (sha256: `${{ steps.prep_assets.outputs.pkg_tar_sha }}`)
* Package URI `github.com/${{ needs.build.outputs.repo_lowercase }}/${{ steps.get_pkg_info.outputs.pkg_name }}@${{ steps.get_pkg_info.outputs.pkg_ver }}`
* Run the package: `nextflow run ${{ needs.build.outputs.repo_lowercase }}/${{ steps.get_pkg_info.outputs.pkg_name }}/main.nf -r ${{ steps.get_pkg_info.outputs.pkg_name }}.v${{ steps.get_pkg_info.outputs.pkg_ver }} -params-file <params-json-file>`
draft: false
prerelease: false

Expand Down
54 changes: 39 additions & 15 deletions open-access-variant-filtering-wf/main.nf
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,15 @@
nextflow.enable.dsl = 2
name = 'open-access-variant-filtering'
short_name = 'open-filter'
version = '0.4.0'
version = '0.5.0'

// universal params go here, change default value as needed
params.container = ""
params.container_registry = ""
params.container_version = ""
params.cpus = 1
params.mem = 1 // GB
params.publish_dir = "" // set to empty string will disable publishDir
params.cleanup = true
params.max_retries = 5 // set to 0 will disable retry
params.first_retry_wait_time = 1 // in seconds

// tool specific parmas go here, add / change as needed
params.study_id = ""
Expand Down Expand Up @@ -59,33 +59,57 @@ params.open = true
params.analysis_metadata = "NO_FILE_metadata"
params.vcf_file = "NO_FILE_vcf"

params.api_token = ""
// song/score setting
params.song_url = ""
params.song_container = "ghcr.io/overture-stack/song-client"
params.song_container_version = "5.0.2"
params.score_url = ""
params.cleanup = true
params.score_container = "ghcr.io/overture-stack/score"
params.score_container_version = "5.9.0"
params.score_mem = 20
params.score_cpus = 8
params.score_force = false
params.api_token = ""

params.download = [:]
params.upload = [:]
params.filter = [:]
params.payloadGenVcf = [:]

download_params = [
'max_retries': params.max_retries,
'first_retry_wait_time': params.first_retry_wait_time,
'song_url': params.song_url,
'song_container': params.song_container,
'song_container_version': params.song_container_version,
'song_cpus': params.cpus,
'song_mem': params.mem,
'score_cpus': params.cpus,
'score_mem': params.mem,
'song_url': params.song_url,
'score_url': params.score_url,
'score_container': params.score_container,
'score_container_version': params.score_container_version,
'score_cpus' : params.score_cpus,
'score_mem' : params.score_mem,
'score_transport_mem' : params.score_mem,
'api_token': params.api_token,
'publish_dir': params.publish_dir,
*:(params.download ?: [:])
]

upload_params = [
'cpus': params.cpus,
'mem': params.mem,
'max_retries': params.max_retries,
'first_retry_wait_time': params.first_retry_wait_time,
'song_url': params.song_url,
'song_container': params.song_container,
'song_container_version': params.song_container_version,
'song_cpus': params.cpus,
'song_mem': params.mem,
'score_url': params.score_url,
'score_container': params.score_container,
'score_container_version': params.score_container_version,
'score_force' : params.score_force,
'score_cpus' : params.score_cpus,
'score_mem' : params.score_mem,
'score_transport_mem' : params.score_mem,
'api_token': params.api_token,
'publish_dir': params.publish_dir,
*:(params.upload ?: [:])
Expand All @@ -94,7 +118,7 @@ upload_params = [
filter_params = [
'cpus': params.cpus,
'mem': params.mem,
'publish_dir': '',
'publish_dir': params.publish_dir,
'regions_file': params.regions_file,
'apply_filters': params.apply_filters,
'output_type': params.output_type,
Expand All @@ -110,11 +134,11 @@ payloadGenVcf_params = [
*:(params.payloadGenVcf ?: [:])
]

include { SongScoreDownload as dnVcf } from './wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1/main.nf' params(download_params)
include { SongScoreDownload as dnVcf } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0/main.nf' params(download_params)
include { metadataParser as mParser } from "./modules/raw.githubusercontent.com/icgc-argo/data-processing-utility-tools/metadata-parser.0.2.0.0/tools/metadata-parser/metadata-parser.nf"
include { variantFilter as vFilter } from './wfpr_modules/github.com/icgc-argo/variant-calling-tools/[email protected]/main.nf' params(filter_params)
include { payloadGenVariantProcessing as pGenVar } from "./wfpr_modules/github.com/icgc-argo-workflows/data-processing-utility-tools/[email protected]/main.nf" params(payloadGenVcf_params)
include { SongScoreUpload as upVcf } from './wfpr_modules/github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-upload@2.6.1/main.nf' params(upload_params)
include { SongScoreUpload as upVcf } from './wfpr_modules/github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3/main.nf' params(upload_params)
include { getSecondaryFiles as getSec } from './wfpr_modules/github.com/icgc-argo/data-processing-utility-tools/[email protected]/main.nf'
include { cleanupWorkdir as cleanup } from './wfpr_modules/github.com/icgc-argo/data-processing-utility-tools/[email protected]/main.nf'

Expand Down Expand Up @@ -175,7 +199,7 @@ workflow OpenFilterWf {
// skip upload if in local_mode
if (!local_mode) {
// uploadVariant
upVcf(study_id, pGenVar.out.payload, pGenVar.out.files_to_upload)
upVcf(study_id, pGenVar.out.payload, pGenVar.out.files_to_upload, '')
}

// cleanup, skip cleanup when running in local mode
Expand Down
6 changes: 3 additions & 3 deletions open-access-variant-filtering-wf/pkg.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "open-access-variant-filtering-wf",
"version": "0.4.0",
"version": "0.5.0",
"description": "ICGC-ARGO Open-access Somatic Variant Filtering Workflow",
"main": "main.nf",
"deprecated": false,
Expand All @@ -18,8 +18,8 @@
"github.com/icgc-argo/data-processing-utility-tools/[email protected]",
"github.com/icgc-argo-workflows/data-processing-utility-tools/[email protected]",
"github.com/icgc-argo/variant-calling-tools/[email protected]",
"github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-download@2.6.1",
"github.com/icgc-argo/nextflow-data-processing-utility-tools/song-score-upload@2.6.1"
"github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-download@2.9.0",
"github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/song-score-upload@2.9.3"
],
"devDependencies": [],
"contributors": [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ params.max_retries = 5 // set to 0 will disable retry
params.first_retry_wait_time = 1 // in seconds

// required params w/ default
params.container_version = "5.0.0"
params.container = "ghcr.io/overture-stack/score"
params.container_version = "5.8.1"
params.transport_mem = 2 // Transport memory is in number of GBs

// optional if secret mounted from pod else required
Expand All @@ -34,12 +35,18 @@ process scoreDownload {
cpus params.cpus
memory "${params.mem} GB"

container "overture/score:${params.container_version}"
publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir
container "${ params.score_container ?: params.container}:${params.score_container_version ?: params.container_version}"
publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir ? true : false

label "scoreDownload"
tag "${analysis_id}"

if (workflow.containerEngine == "singularity") {
containerOptions "--bind \$(pwd):/score-client/logs"
} else if (workflow.containerEngine == "docker") {
containerOptions "-v \$(pwd):/score-client/logs"
}

input:
path analysis
val study_id
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ params.max_retries = 5 // set to 0 will disable retry
params.first_retry_wait_time = 1 // in seconds

// required params w/ default
params.container_version = "4.2.1"
params.container = "ghcr.io/overture-stack/song-client"
params.container_version = "5.0.2"

// optional if secret mounted from pod else required
params.api_token = "" // song/score API token for download process
Expand All @@ -31,12 +32,18 @@ process songGetAnalysis {

cpus params.cpus
memory "${params.mem} GB"

container "overture/song-client:${params.container_version}"
publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir
container "${ params.song_container ?: params.container}:${params.song_container_version ?: params.container_version}"
publishDir "${params.publish_dir}/${task.process.replaceAll(':', '_')}", mode: "copy", enabled: params.publish_dir ? true : false

tag "${analysis_id}"

if (workflow.containerEngine == "singularity") {
containerOptions "--bind \$(pwd):/song-client/logs"
} else if (workflow.containerEngine == "docker") {
containerOptions "-v \$(pwd):/song-client/logs"
}

input:
val study_id
val analysis_id
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
*/

nextflow.enable.dsl = 2
version = '2.6.1'
version = '2.9.0'

// universal params go here, change default value as needed
params.publish_dir = "" // set to empty string will disable publishDir
Expand All @@ -30,33 +30,35 @@ params.max_retries = 5 // set to 0 will disable retry
params.first_retry_wait_time = 1 // in seconds

// tool specific parmas go here, add / change as needed
params.study_id = "TEST-PR"
params.analysis_id = "9940db0f-c100-496a-80db-0fc100d96ac1"
params.study_id = ""
params.analysis_id = ""

params.api_token = ""

params.song_cpus = 1
params.song_mem = 1 // GB
params.song_url = "https://song.rdpc-qa.cancercollaboratory.org"
params.song_api_token = ""
params.song_container_version = "4.2.1"
params.song_container = "ghcr.io/overture-stack/song-client"
params.song_container_version = "5.0.2"

params.score_cpus = 1
params.score_mem = 1 // GB
params.score_transport_mem = 1 // GB
params.score_url = "https://score.rdpc-qa.cancercollaboratory.org"
params.score_api_token = ""
params.score_container_version = "5.0.0"
params.score_container = "ghcr.io/overture-stack/score"
params.score_container_version = "5.8.1"


song_params = [
*:params,
'cpus': params.song_cpus,
'mem': params.song_mem,
'song_url': params.song_url,
'song_container': params.song_container,
'song_container_version': params.song_container_version,
'api_token': params.song_api_token ?: params.api_token,
'publish_dir': ''
'api_token': params.song_api_token ?: params.api_token
]

score_params = [
Expand All @@ -66,6 +68,7 @@ score_params = [
'transport_mem': params.score_transport_mem,
'song_url': params.song_url,
'score_url': params.score_url,
'score_container': params.score_container,
'score_container_version': params.score_container_version,
'api_token': params.score_api_token ?: params.api_token
]
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "song-score-download",
"version": "2.6.1",
"version": "2.9.0",
"description": "SONG/SCORE download",
"main": "main.nf",
"deprecated": false,
Expand All @@ -12,7 +12,7 @@
],
"repository": {
"type": "git",
"url": "https://github.com/icgc-argo/nextflow-data-processing-utility-tools.git"
"url": "https://github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools.git"
},
"dependencies": [],
"devDependencies": [],
Expand All @@ -22,9 +22,12 @@
},
{
"name": "Junjun Zhang"
},
{
"name": "Linda Xiang"
}
],
"license": "Apache License 2.0",
"bugReport": "https://github.com/icgc-argo/nextflow-data-processing-utility-tools/issues",
"homepage": "https://github.com/icgc-argo/nextflow-data-processing-utility-tools#readme"
"bugReport": "https://github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools/issues",
"homepage": "https://github.com/icgc-argo-workflows/nextflow-data-processing-utility-tools#readme"
}
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@ params.max_retries = 5 // set to 0 will disable retry
params.first_retry_wait_time = 1 // in seconds

// required params w/ default
params.container_version = "5.0.0"
params.container = "ghcr.io/overture-stack/score"
params.container_version = "latest"
params.transport_mem = 2 // Transport memory is in number of GBs

// optional if secret mounted from pod else required
params.api_token = "" // song/score API token for download process

params.score_force = false
// required params, no default
// --song_url song url for download process
// --score_url score url for download process
Expand All @@ -30,8 +31,14 @@ process scoreUpload {

cpus params.cpus
memory "${params.mem} GB"

container "overture/score:${params.container_version}"

container "${ params.score_container ?: params.container}:${params.score_container_version ?: params.container_version}"

if (workflow.containerEngine == "singularity") {
containerOptions "--bind \$(pwd):/score-client/logs"
} else if (workflow.containerEngine == "docker") {
containerOptions "-v \$(pwd):/score-client/logs"
}

tag "${analysis_id}"

Expand All @@ -45,13 +52,14 @@ process scoreUpload {

script:
accessToken = params.api_token ? params.api_token : "`cat /tmp/rdpc_secret/secret`"
forceFlag = params.score_force ? "--force" : ""
"""
export METADATA_URL=${params.song_url}
export STORAGE_URL=${params.score_url}
export TRANSPORT_PARALLEL=${params.cpus}
export TRANSPORT_MEMORY=${params.transport_mem}
export ACCESSTOKEN=${accessToken}
score-client upload --manifest ${manifest}
score-client upload --manifest ${manifest} ${forceFlag}
"""
}
Loading

0 comments on commit f843d1d

Please sign in to comment.