Merge pull request #79 from nf-core/fix-test_full

Implement test_full
nf-core · Oct 2, 2023 · 905efe4 · 905efe4
2 parents 0eacfcf + 36d2772
commit 905efe4
Show file tree

Hide file tree

Showing 62 changed files with 562 additions and 274 deletions.
diff --git a/conf/base.config b/conf/base.config
@@ -60,4 +60,14 @@ process {
     withName:CUSTOM_DUMPSOFTWAREVERSIONS {
         cache = false
     }
+    withName:HISAT2_ALIGN {
+        // Workaround: This step sometimes failes irreproduceably
+        errorStrategy = 'retry'
+        maxRetries    = 2
+    }
+    withName:CIRIQUANT {
+        // Workaround: This step sometimes failes irreproduceably
+        errorStrategy = 'retry'
+        maxRetries    = 2
+    }
 }
diff --git a/conf/full.config b/conf/full.config
@@ -0,0 +1,17 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config for full-size tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines parameters so that mimimal tests are converted to full size pipeline test.
+
+    Use as follows:
+        nextflow run nf-core/circrna -profile test,full,<docker/singularity> --outdir <OUTDIR>
+        nextflow run nf-core/circrna -profile test_igenomes,full,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+
+params {
+    tool                       = 'circexplorer2,ciriquant,find_circ,circrna_finder,mapsplice,dcc,segemehl'
+    tool_filter                = 2
+}
diff --git a/conf/modules.config b/conf/modules.config
@@ -418,7 +418,7 @@ if (!params.skip_trimming) {
     }
 
     withName: DCC_MATE1_1ST_PASS {
-        ext.when = { !meta.single_end && params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
+        ext.when = { params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
         ext.prefix = { "${meta.id}_mate1" }
         ext.args = [ "",
             "--chimOutType Junctions WithinBAM",
@@ -439,7 +439,7 @@ if (!params.skip_trimming) {
     }
 
     withName: DCC_MATE1_SJDB {
-        ext.when = { !meta.single_end && params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
+        ext.when = { params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
         publishDir = [
             path: { "${params.outdir}/circrna_discovery/dcc/intermediates/mate1/sjdb" },
             mode: params.publish_dir_mode,
@@ -449,7 +449,7 @@ if (!params.skip_trimming) {
     }
 
     withName: DCC_MATE1_2ND_PASS {
-        ext.when = { !meta.single_end && params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
+        ext.when = { params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
         ext.prefix = { "${meta.id}_mate1" }
         ext.args = [ "",
             "--chimOutType Junctions WithinBAM",
@@ -471,7 +471,7 @@ if (!params.skip_trimming) {
     }
 
     withName: DCC_MATE2_1ST_PASS {
-        ext.when = { !meta.single_end && params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
+        ext.when = { params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
         ext.prefix = { "${meta.id}_mate2" }
         ext.args = [ "",
             "--chimOutType Junctions WithinBAM",
@@ -492,7 +492,7 @@ if (!params.skip_trimming) {
     }
 
     withName: DCC_MATE2_SJDB {
-        ext.when = { !meta.single_end && params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
+        ext.when = { params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
         publishDir = [
             path: { "${params.outdir}/circrna_discovery/dcc/intermediates/mate2/sjdb" },
             mode: params.publish_dir_mode,
@@ -502,7 +502,7 @@ if (!params.skip_trimming) {
     }
 
     withName: DCC_MATE2_2ND_PASS {
-        ext.when = { !meta.single_end && params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
+        ext.when = { params.module.split(',').contains('circrna_discovery') && params.tool.split(',').contains('dcc') }
         ext.prefix = { "${meta.id}_mate2" }
         ext.args = [ "",
             "--chimOutType Junctions WithinBAM",

diff --git a/conf/test.config b/conf/test.config
@@ -24,11 +24,11 @@ params {
     fasta                      = 'https://raw.githubusercontent.com/nf-core/test-datasets/circrna/reference/chrI.fa'
     gtf                        = 'https://raw.githubusercontent.com/nf-core/test-datasets/circrna/reference/chrI.gtf'
     mature                     = 'https://raw.githubusercontent.com/nf-core/test-datasets/circrna/reference/mature.fa'
+    species                    = 'cel'
     tool                       = 'circexplorer2'
     phenotype                  = 'https://raw.githubusercontent.com/nf-core/test-datasets/circrna/phenotype.csv'
     skip_trimming              = false
     module                     = 'circrna_discovery,mirna_prediction,differential_expression'
     outdir                     = 'results/'
     bsj_reads                  = 2
-    species                    = 'cel'
 }
diff --git a/conf/test_full.config b/conf/test_full.config
@@ -1,24 +1,2 @@
-/*
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    Nextflow config file for running full-size tests
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    Defines input files and everything required to run a full size pipeline test.
-
-    Use as follows:
-        nextflow run nf-core/circrna -profile test_full,<docker/singularity> --outdir <OUTDIR>
-
-----------------------------------------------------------------------------------------
-*/
-
-params {
-    config_profile_name        = 'Full test profile'
-    config_profile_description = 'Full test dataset to check pipeline function'
-
-    // Input data for full size test
-    // TODO nf-core: Specify the paths to your full test data ( on nf-core/test-datasets or directly in repositories, e.g. SRA)
-    // TODO nf-core: Give any required params for the test so that command line flags are not needed
-    input = 'https://raw.githubusercontent.com/nf-core/test-datasets/viralrecon/samplesheet/samplesheet_full_illumina_amplicon.csv'
-
-    // Genome references
-    genome = 'R64-1-1'
-}
+includeConfig 'test.config'
+includeConfig 'full.config'
diff --git a/conf/test_igenomes.config b/conf/test_igenomes.config
@@ -0,0 +1,28 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests using igenomes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a minimal pipeline test.
+
+    Use as follows:
+        nextflow run nf-core/circrna -profile test_full,<docker/singularity> --outdir <OUTDIR>
+
+----------------------------------------------------------------------------------------
+*/
+
+params {
+    config_profile_name        = 'Minimal igenomes profile'
+    config_profile_description = 'Minimal igenomes test dataset to check pipeline function'
+
+    // Input data for minima test using igenomes
+    input                      = 'https://raw.githubusercontent.com/nf-core/test-datasets/circrna/samples.csv'
+
+    genome                     = 'ce10'
+    tool                       = 'circexplorer2'
+    phenotype                  = 'https://raw.githubusercontent.com/nf-core/test-datasets/circrna/phenotype.csv'
+    skip_trimming              = false
+    star                       = null // igenomes STAR version is not compatible
+    module                     = 'circrna_discovery,mirna_prediction,differential_expression'
+    outdir                     = 'results/'
+    bsj_reads                  = 2
+}
diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy
@@ -48,6 +48,10 @@ class WorkflowMain {
         if (!params.input) {
             Nextflow.error("Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'")
         }
+
+        if (!params.species) {
+            Nextflow.error("Please provide a species to the pipeline e.g. '--species hsa'")
+        }
     }
     //
     // Get attribute from genome config file e.g. fasta

diff --git a/modules.json b/modules.json
@@ -7,137 +7,137 @@
                 "nf-core": {
                     "bowtie/align": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "603ecbd9f45300c9788f197d2a15a005685b4220",
                         "installed_by": ["modules"]
                     },
                     "bowtie/build": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
                         "installed_by": ["modules"]
                     },
                     "bowtie2/align": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "fe54581f8bed20e4c4a51c616c93fd3379d89820",
                         "installed_by": ["modules"]
                     },
                     "bowtie2/build": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "6a24fbe314bb2e6fe6306c29a63076ea87e8eb3c",
                         "installed_by": ["modules"]
                     },
                     "bwa/index": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "bfed129da5134b4439b1821c917972570d44d39c",
                         "installed_by": ["modules"]
                     },
                     "cat/fastq": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "5c460c5a4736974abde2843294f35307ee2b0e5e",
                         "installed_by": ["modules"]
                     },
                     "circexplorer2/annotate": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
                         "installed_by": ["modules"]
                     },
                     "circexplorer2/parse": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
                         "installed_by": ["modules"]
                     },
                     "custom/dumpsoftwareversions": {
                         "branch": "master",
-                        "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
+                        "git_sha": "05c280924b6c768d484c7c443dad5e605c4ff4b4",
                         "installed_by": ["modules"]
                     },
                     "fastqc": {
                         "branch": "master",
-                        "git_sha": "bd8092b67b5103bdd52e300f75889442275c3117",
+                        "git_sha": "9a4517e720bc812e95b56d23d15a1653b6db4f53",
                         "installed_by": ["modules"]
                     },
                     "hisat2/align": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "a1881f6374506f9e031b7af814768cdb44a6a7d3",
                         "installed_by": ["modules"]
                     },
                     "hisat2/build": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "f2f48836bf5c59434966a6c3b2211b29363f31ab",
                         "installed_by": ["modules"]
                     },
                     "hisat2/extractsplicesites": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "735e1e04e7e01751d2d6e97055bbdb6f70683cc1",
                         "installed_by": ["modules"]
                     },
                     "miranda": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
                         "installed_by": ["modules"]
                     },
                     "multiqc": {
                         "branch": "master",
-                        "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
+                        "git_sha": "a6e11ac655e744f7ebc724be669dd568ffdc0e80",
                         "installed_by": ["modules"]
                     },
                     "samtools/flagstat": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "570ec5bcfe19c49e16c9ca35a7a116563af6cc1c",
                         "installed_by": ["bam_stats_samtools"]
                     },
                     "samtools/idxstats": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "e662ab16e0c11f1e62983e21de9871f59371a639",
                         "installed_by": ["bam_stats_samtools"]
                     },
                     "samtools/index": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
-                        "installed_by": ["bam_sort_stats_samtools", "modules"]
+                        "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
+                        "installed_by": ["modules", "bam_sort_stats_samtools"]
                     },
                     "samtools/sort": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
-                        "installed_by": ["bam_sort_stats_samtools", "modules"]
+                        "git_sha": "a0f7be95788366c1923171e358da7d049eb440f9",
+                        "installed_by": ["modules", "bam_sort_stats_samtools"]
                     },
                     "samtools/stats": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "735e1e04e7e01751d2d6e97055bbdb6f70683cc1",
                         "installed_by": ["bam_stats_samtools"]
                     },
                     "samtools/view": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "3ffae3598260a99e8db3207dead9f73f87f90d1f",
                         "installed_by": ["modules"]
                     },
                     "segemehl/align": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
                         "installed_by": ["modules"]
                     },
                     "segemehl/index": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
                         "installed_by": ["modules"]
                     },
                     "star/align": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "cc08a888069f67cab8120259bddab8032d4c0fe3",
                         "installed_by": ["modules"]
                     },
                     "star/genomegenerate": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "cc08a888069f67cab8120259bddab8032d4c0fe3",
                         "installed_by": ["modules"]
                     },
                     "stringtie/stringtie": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
                         "installed_by": ["modules"]
                     },
                     "trimgalore": {
                         "branch": "master",
-                        "git_sha": "c8e35eb2055c099720a75538d1b8adb3fb5a464c",
+                        "git_sha": "911696ea0b62df80e900ef244d7867d177971f73",
                         "installed_by": ["modules"]
                     }
                 }
@@ -146,12 +146,12 @@
                 "nf-core": {
                     "bam_sort_stats_samtools": {
                         "branch": "master",
-                        "git_sha": "3911652a6b24249358f79e8b8466338d63efb2a2",
+                        "git_sha": "dedc0e31087f3306101c38835d051bf49789445a",
                         "installed_by": ["subworkflows"]
                     },
                     "bam_stats_samtools": {
                         "branch": "master",
-                        "git_sha": "92eb5091ae5368a60cda58b3a0ced8b36d715b0f",
+                        "git_sha": "dedc0e31087f3306101c38835d051bf49789445a",
                         "installed_by": ["bam_sort_stats_samtools"]
                     }
                 }

diff --git a/modules/local/ciriquant/yml/main.nf b/modules/local/ciriquant/yml/main.nf
@@ -19,7 +19,6 @@ process CIRIQUANT_YML {
     task.ext.when == null || task.ext.when
 
     script:
-    bwa_prefix = fasta.toString() == 'genome.fa' ? fasta.toString() : fasta.toString() - ~/.(fa|fasta)$/
     hisat2_prefix = fasta.toString() - ~/.(fa|fasta)$/
     fasta_path = fasta.toRealPath()
     gtf_path = gtf.toRealPath()
@@ -31,7 +30,11 @@ process CIRIQUANT_YML {
     STRINGTIE=`which stringtie`
     SAMTOOLS=`which samtools`
 
+    # Get first file in bwa index directory
+    BWA_FILE=`ls ${bwa_path}/*.bwt`
+    BWA_FILE=`basename \$BWA_FILE .bwt`
+
     touch travis.yml
-    printf "name: ciriquant\ntools:\n  bwa: \$BWA\n  hisat2: \$HISAT2\n  stringtie: \$STRINGTIE\n  samtools: \$SAMTOOLS\n\nreference:\n  fasta: ${fasta_path}\n  gtf: ${gtf_path}\n  bwa_index: ${bwa_path}/${bwa_prefix}\n  hisat_index: ${hisat2_path}/${hisat2_prefix}" >> travis.yml
+    printf "name: ciriquant\ntools:\n  bwa: \$BWA\n  hisat2: \$HISAT2\n  stringtie: \$STRINGTIE\n  samtools: \$SAMTOOLS\n\nreference:\n  fasta: ${fasta_path}\n  gtf: ${gtf_path}\n  bwa_index: ${bwa_path}/\$BWA_FILE\n  hisat_index: ${hisat2_path}/${hisat2_prefix}" >> travis.yml
     """
 }
diff --git a/modules/local/star/sjdb/main.nf b/modules/local/star/sjdb/main.nf
@@ -1,4 +1,5 @@
 process SJDB {
+    tag "$meta.id"
     label 'process_single'
 
     conda "conda-forge::sed=4.7"
@@ -7,11 +8,11 @@ process SJDB {
         'nf-core/ubuntu:20.04' }"
 
     input:
-    path(sjdb)
+    tuple val(meta), path(sjdb)
     val(bsj_reads)
 
     output:
-    path("dataset.SJ.out.tab"), emit: sjtab
+    tuple val(meta), path("dataset.SJ.out.tab"), emit: sjtab
     path "versions.yml"       , emit: versions
 
     when:

diff --git a/modules/nf-core/bowtie/align/main.nf b/modules/nf-core/bowtie/align/main.nf