Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Implement test_full #79

Merged
merged 10 commits into from
Oct 2, 2023
Merged

Implement test_full #79

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d19cf26
Add proper test_full configuration
nictru Sep 26, 2023
e434d38
Fix problem with bwt index path in ciriquant
nictru Sep 26, 2023
a5b7dd1
Increase resource limits
nictru Sep 26, 2023
7343497
Update modules
nictru Sep 27, 2023
2d2c032
Simplify full test definition
nictru Oct 1, 2023
87a1acd
Add check for species parameter
nictru Oct 1, 2023
b2ed0da
Fix problems with channel structure
nictru Oct 1, 2023
a30d07a
Fix DCC not executed bug
nictru Oct 1, 2023
06b4898
Fix problems with unpredictable failures
nictru Oct 1, 2023
36d2772
Add test_full file
nictru Oct 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions conf/base.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,14 @@ process {
withName:CUSTOM_DUMPSOFTWAREVERSIONS {
cache = false
}
withName:HISAT2_ALIGN {
// Workaround: This step sometimes failes irreproduceably
errorStrategy = 'retry'
maxRetries = 2
}
withName:CIRIQUANT {
// Workaround: This step sometimes failes irreproduceably
errorStrategy = 'retry'
maxRetries = 2
}
}
17 changes: 17 additions & 0 deletions conf/full.config
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Nextflow config for full-size tests
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Defines parameters so that mimimal tests are converted to full size pipeline test.

Use as follows:
nextflow run nf-core/circrna -profile test,full,<docker/singularity> --outdir <OUTDIR>
nextflow run nf-core/circrna -profile test_igenomes,full,<docker/singularity> --outdir <OUTDIR>

----------------------------------------------------------------------------------------
*/

params {
tool = 'circexplorer2,ciriquant,find_circ,circrna_finder,mapsplice,dcc,segemehl'
tool_filter = 2
}
12 changes: 6 additions & 6 deletions conf/modules.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,7 +418,7 @@ if (!params.skip_trimming) {
}

withName: DCC_MATE1_1ST_PASS {
ext.when = { !meta.single_end && params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
ext.when = { params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
ext.prefix = { "${meta.id}_mate1" }
ext.args = [ "",
"--chimOutType Junctions WithinBAM",
Expand All @@ -439,7 +439,7 @@ if (!params.skip_trimming) {
}

withName: DCC_MATE1_SJDB {
ext.when = { !meta.single_end && params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
ext.when = { params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
publishDir = [
path: { "${params.outdir}/circrna_discovery/dcc/intermediates/mate1/sjdb" },
mode: params.publish_dir_mode,
Expand All @@ -449,7 +449,7 @@ if (!params.skip_trimming) {
}

withName: DCC_MATE1_2ND_PASS {
ext.when = { !meta.single_end && params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
ext.when = { params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
ext.prefix = { "${meta.id}_mate1" }
ext.args = [ "",
"--chimOutType Junctions WithinBAM",
Expand All @@ -471,7 +471,7 @@ if (!params.skip_trimming) {
}

withName: DCC_MATE2_1ST_PASS {
ext.when = { !meta.single_end && params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
ext.when = { params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
ext.prefix = { "${meta.id}_mate2" }
ext.args = [ "",
"--chimOutType Junctions WithinBAM",
Expand All @@ -492,7 +492,7 @@ if (!params.skip_trimming) {
}

withName: DCC_MATE2_SJDB {
ext.when = { !meta.single_end && params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
ext.when = { params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
publishDir = [
path: { "${params.outdir}/circrna_discovery/dcc/intermediates/mate2/sjdb" },
mode: params.publish_dir_mode,
Expand All @@ -502,7 +502,7 @@ if (!params.skip_trimming) {
}

withName: DCC_MATE2_2ND_PASS {
ext.when = { !meta.single_end && params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
ext.when = { params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
ext.prefix = { "${meta.id}_mate2" }
ext.args = [ "",
"--chimOutType Junctions WithinBAM",
Expand Down
2 changes: 1 addition & 1 deletion conf/test.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@ params {
fasta = 'https://raw.githubusercontent.com/nf-core/test-datasets/circrna/reference/chrI.fa'
gtf = 'https://raw.githubusercontent.com/nf-core/test-datasets/circrna/reference/chrI.gtf'
mature = 'https://raw.githubusercontent.com/nf-core/test-datasets/circrna/reference/mature.fa'
species = 'cel'
tool = 'circexplorer2'
phenotype = 'https://raw.githubusercontent.com/nf-core/test-datasets/circrna/phenotype.csv'
skip_trimming = false
module = 'circrna_discovery,mirna_prediction,differential_expression'
outdir = 'results/'
bsj_reads = 2
species = 'cel'
}
26 changes: 2 additions & 24 deletions conf/test_full.config
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,24 +1,2 @@
/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Nextflow config file for running full-size tests
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Defines input files and everything required to run a full size pipeline test.

Use as follows:
nextflow run nf-core/circrna -profile test_full,<docker/singularity> --outdir <OUTDIR>

----------------------------------------------------------------------------------------
*/

params {
config_profile_name = 'Full test profile'
config_profile_description = 'Full test dataset to check pipeline function'

// Input data for full size test
// TODO nf-core: Specify the paths to your full test data ( on nf-core/test-datasets or directly in repositories, e.g. SRA)
// TODO nf-core: Give any required params for the test so that command line flags are not needed
input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv'

// Genome references
genome = 'R64-1-1'
}
includeConfig 'test.config'
includeConfig 'full.config'
28 changes: 28 additions & 0 deletions conf/test_igenomes.config
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
/*
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Nextflow config file for running minimal tests using igenomes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Defines input files and everything required to run a minimal pipeline test.

Use as follows:
nextflow run nf-core/circrna -profile test_full,<docker/singularity> --outdir <OUTDIR>

----------------------------------------------------------------------------------------
*/

params {
config_profile_name = 'Minimal igenomes profile'
config_profile_description = 'Minimal igenomes test dataset to check pipeline function'

// Input data for minima test using igenomes
input = 'https://raw.githubusercontent.com/nf-core/test-datasets/circrna/samples.csv'

genome = 'ce10'
tool = 'circexplorer2'
phenotype = 'https://raw.githubusercontent.com/nf-core/test-datasets/circrna/phenotype.csv'
skip_trimming = false
star = null // igenomes STAR version is not compatible
module = 'circrna_discovery,mirna_prediction,differential_expression'
outdir = 'results/'
bsj_reads = 2
}
4 changes: 4 additions & 0 deletions lib/WorkflowMain.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,10 @@ class WorkflowMain {
if (!params.input) {
Nextflow.error("Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'")
}

if (!params.species) {
Nextflow.error("Please provide a species to the pipeline e.g. '--species hsa'")
}
}
//
// Get attribute from genome config file e.g. fasta
Expand Down
62 changes: 31 additions & 31 deletions modules.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,137 +7,137 @@
"nf-core": {
"bowtie/align": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "603ecbd9f45300c9788f197d2a15a005685b4220",
"installed_by": ["modules"]
},
"bowtie/build": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
"installed_by": ["modules"]
},
"bowtie2/align": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "fe54581f8bed20e4c4a51c616c93fd3379d89820",
"installed_by": ["modules"]
},
"bowtie2/build": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "6a24fbe314bb2e6fe6306c29a63076ea87e8eb3c",
"installed_by": ["modules"]
},
"bwa/index": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "bfed129da5134b4439b1821c917972570d44d39c",
"installed_by": ["modules"]
},
"cat/fastq": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "5c460c5a4736974abde2843294f35307ee2b0e5e",
"installed_by": ["modules"]
},
"circexplorer2/annotate": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
"installed_by": ["modules"]
},
"circexplorer2/parse": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
"installed_by": ["modules"]
},
"custom/dumpsoftwareversions": {
"branch": "master",
"git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
"git_sha": "05c280924b6c768d484c7c443dad5e605c4ff4b4",
"installed_by": ["modules"]
},
"fastqc": {
"branch": "master",
"git_sha": "bd8092b67b5103bdd52e300f75889442275c3117",
"git_sha": "9a4517e720bc812e95b56d23d15a1653b6db4f53",
"installed_by": ["modules"]
},
"hisat2/align": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "a1881f6374506f9e031b7af814768cdb44a6a7d3",
"installed_by": ["modules"]
},
"hisat2/build": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "f2f48836bf5c59434966a6c3b2211b29363f31ab",
"installed_by": ["modules"]
},
"hisat2/extractsplicesites": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "735e1e04e7e01751d2d6e97055bbdb6f70683cc1",
"installed_by": ["modules"]
},
"miranda": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
"installed_by": ["modules"]
},
"multiqc": {
"branch": "master",
"git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
"git_sha": "a6e11ac655e744f7ebc724be669dd568ffdc0e80",
"installed_by": ["modules"]
},
"samtools/flagstat": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "570ec5bcfe19c49e16c9ca35a7a116563af6cc1c",
"installed_by": ["bam_stats_samtools"]
},
"samtools/idxstats": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "e662ab16e0c11f1e62983e21de9871f59371a639",
"installed_by": ["bam_stats_samtools"]
},
"samtools/index": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"installed_by": ["bam_sort_stats_samtools", "modules"]
"git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
"installed_by": ["modules", "bam_sort_stats_samtools"]
},
"samtools/sort": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"installed_by": ["bam_sort_stats_samtools", "modules"]
"git_sha": "a0f7be95788366c1923171e358da7d049eb440f9",
"installed_by": ["modules", "bam_sort_stats_samtools"]
},
"samtools/stats": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "735e1e04e7e01751d2d6e97055bbdb6f70683cc1",
"installed_by": ["bam_stats_samtools"]
},
"samtools/view": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "3ffae3598260a99e8db3207dead9f73f87f90d1f",
"installed_by": ["modules"]
},
"segemehl/align": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
"installed_by": ["modules"]
},
"segemehl/index": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
"installed_by": ["modules"]
},
"star/align": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "cc08a888069f67cab8120259bddab8032d4c0fe3",
"installed_by": ["modules"]
},
"star/genomegenerate": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "cc08a888069f67cab8120259bddab8032d4c0fe3",
"installed_by": ["modules"]
},
"stringtie/stringtie": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
"installed_by": ["modules"]
},
"trimgalore": {
"branch": "master",
"git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
"git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
"installed_by": ["modules"]
}
}
Expand All @@ -146,12 +146,12 @@
"nf-core": {
"bam_sort_stats_samtools": {
"branch": "master",
"git_sha": "3911652a6b24249358f79e8b8466338d63efb2a2",
"git_sha": "dedc0e31087f3306101c38835d051bf49789445a",
"installed_by": ["subworkflows"]
},
"bam_stats_samtools": {
"branch": "master",
"git_sha": "92eb5091ae5368a60cda58b3a0ced8b36d715b0f",
"git_sha": "dedc0e31087f3306101c38835d051bf49789445a",
"installed_by": ["bam_sort_stats_samtools"]
}
}
Expand Down
7 changes: 5 additions & 2 deletions modules/local/ciriquant/yml/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ process CIRIQUANT_YML {
task.ext.when == null || task.ext.when

script:
bwa_prefix = fasta.toString() == 'genome.fa' ? fasta.toString() : fasta.toString() - ~/.(fa|fasta)$/
hisat2_prefix = fasta.toString() - ~/.(fa|fasta)$/
fasta_path = fasta.toRealPath()
gtf_path = gtf.toRealPath()
Expand All @@ -31,7 +30,11 @@ process CIRIQUANT_YML {
STRINGTIE=`which stringtie`
SAMTOOLS=`which samtools`

# Get first file in bwa index directory
BWA_FILE=`ls ${bwa_path}/*.bwt`
BWA_FILE=`basename \$BWA_FILE .bwt`

touch travis.yml
printf "name: ciriquant\ntools:\n bwa: \$BWA\n hisat2: \$HISAT2\n stringtie: \$STRINGTIE\n samtools: \$SAMTOOLS\n\nreference:\n fasta: ${fasta_path}\n gtf: ${gtf_path}\n bwa_index: ${bwa_path}/${bwa_prefix}\n hisat_index: ${hisat2_path}/${hisat2_prefix}" >> travis.yml
printf "name: ciriquant\ntools:\n bwa: \$BWA\n hisat2: \$HISAT2\n stringtie: \$STRINGTIE\n samtools: \$SAMTOOLS\n\nreference:\n fasta: ${fasta_path}\n gtf: ${gtf_path}\n bwa_index: ${bwa_path}/\$BWA_FILE\n hisat_index: ${hisat2_path}/${hisat2_prefix}" >> travis.yml
"""
}
5 changes: 3 additions & 2 deletions modules/local/star/sjdb/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
process SJDB {
tag "$meta.id"
label 'process_single'

conda "conda-forge::sed=4.7"
Expand All @@ -7,11 +8,11 @@ process SJDB {
'nf-core/ubuntu:20.04' }"

input:
path(sjdb)
tuple val(meta), path(sjdb)
val(bsj_reads)

output:
path("dataset.SJ.out.tab"), emit: sjtab
tuple val(meta), path("dataset.SJ.out.tab"), emit: sjtab
path "versions.yml" , emit: versions

when:
Expand Down
2 changes: 1 addition & 1 deletion modules/nf-core/bowtie/align/main.nf
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading