Merge pull request #140 from Aratz/dev

Make it possible to skip MultiQC
nf-core · Aug 26, 2023 · 69ca385 · 69ca385
2 parents aefefde + c09b873
commit 69ca385
Show file tree

Hide file tree

Showing 5 changed files with 100 additions and 36 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### `Fixed`
 
 - [#127](https://github.com/nf-core/demultiplex/pull/127) Add `singularity.registry = 'quay.io'` and bump NF version to 23.04.0
+- [#140](https://github.com/nf-core/demultiplex/pull/140) Make it possible to skip MultiQC, fix error raising
 
 ## `Removed`
 

diff --git a/nextflow_schema.json b/nextflow_schema.json
@@ -18,7 +18,8 @@
                 },
                 "skip_tools": {
                     "type": "string",
-                    "default": null
+                    "default": null,
+                    "description": "Comma-separated list of tools to skip (fastp,falco,multiqc)"
                 }
             }
         },

diff --git a/tests/pipeline/skip_tools.nf.test b/tests/pipeline/skip_tools.nf.test
@@ -22,6 +22,7 @@ nextflow_pipeline {
                 { assert workflow.success },
                 { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions_skip_trimming") },
                 { assert workflow.trace.succeeded().size() == 6 },
+                { assert path("$outputDir/multiqc/multiqc_report.html").exists() },
                 { assert snapshot(
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz.md5"),
@@ -49,6 +50,7 @@ nextflow_pipeline {
                 { assert workflow.success },
                 { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions_skip_fastp") },
                 { assert workflow.trace.succeeded().size() == 5 },
+                { assert path("$outputDir/multiqc/multiqc_report.html").exists() },
                 { assert snapshot(
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz.md5"),
@@ -76,10 +78,10 @@ nextflow_pipeline {
                 { assert workflow.success },
                 { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions_skip_fastqc") },
                 { assert workflow.trace.succeeded().size() == 6 },
+                { assert path("$outputDir/multiqc/multiqc_report.html").exists() },
                 { assert snapshot(
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
-                        // FIXME
-                        // path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz.md5"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001.fastp.fastq.gz.md5"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Undetermined_S0_L001_R1_001.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
                     ).match("skip_fastqc") }
@@ -104,6 +106,7 @@ nextflow_pipeline {
                 { assert workflow.success },
                 { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions_skip_fastp_fastqc") },
                 { assert workflow.trace.succeeded().size() == 5 },
+                { assert path("$outputDir/multiqc/multiqc_report.html").exists() },
                 { assert snapshot(
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
                         path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz.md5"),
@@ -115,4 +118,32 @@ nextflow_pipeline {
 
     }
 
+    test("Skip MultiQC") {
+
+        when {
+            params {
+                input         = 'https://raw.githubusercontent.com/nf-core/test-datasets/demultiplex/samplesheet/1.3.0/flowcell_input.csv'
+                demultiplexer = 'bclconvert'
+                outdir        = "$outputDir"
+                skip_tools    = "multiqc"
+            }
+        }
+
+        then {
+            assertAll(
+                { assert workflow.success },
+                { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions_skip_multiqc") },
+                { assert workflow.trace.succeeded().size() == 5 },
+                { assert !path("$outputDir/multiqc/multiqc_report.html").exists() },
+                { assert snapshot(
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Sample1_S1_L001.fastp.fastq.gz.md5"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/Undetermined_S0_L001_R1_001.fastq.gz"),
+                        path("$outputDir/220422_M11111_0222_000000000-K9H97/L001/Reports/").list(),
+                    ).match("skip_multiqc") }
+            )
+        }
+
+    }
+
 }
diff --git a/tests/pipeline/skip_tools.nf.test.snap b/tests/pipeline/skip_tools.nf.test.snap
@@ -3,11 +3,12 @@
         "content": [
             "{BCLCONVERT={bclconvert=00.000.000.4.0.3}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FALCO={falco=1.2.1}, FASTP={fastp=0.23.4}, Workflow={nf-core/demultiplex=1.4.0dev}}"
         ],
-        "timestamp": "2023-06-15T00:02:11+0000"
+        "timestamp": "2023-08-24T07:43:09+0000"
     },
     "skip_fastqc": {
         "content": [
             "Sample1_S1_L001_R1_001.fastq.gz:md5,0a0341e2990b4fa1d9ad4b4c603144c1",
+            "Sample1_S1_L001.fastp.fastq.gz.md5:md5,c3cd96f3a22fb6afbaa1df324814d54c",
             "Undetermined_S0_L001_R1_001.fastq.gz:md5,febef808ae5397ea4ee7ee40e5fd39e0",
             [
                 "Adapter_Cycle_Metrics.csv:md5,5a0c88793b4a0885fe3dda16609b576e",
@@ -24,7 +25,7 @@
                 "fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
             ]
         ],
-        "timestamp": "2023-06-15T00:02:11+0000"
+        "timestamp": "2023-08-24T07:43:09+0000"
     },
     "skip_fastp": {
         "content": [
@@ -46,7 +47,29 @@
                 "fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
             ]
         ],
-        "timestamp": "2023-06-15T00:02:11+0000"
+        "timestamp": "2023-08-24T07:43:09+0000"
+    },
+    "skip_multiqc": {
+        "content": [
+            "Sample1_S1_L001_R1_001.fastq.gz:md5,0a0341e2990b4fa1d9ad4b4c603144c1",
+            "Sample1_S1_L001.fastp.fastq.gz.md5:md5,c3cd96f3a22fb6afbaa1df324814d54c",
+            "Undetermined_S0_L001_R1_001.fastq.gz:md5,febef808ae5397ea4ee7ee40e5fd39e0",
+            [
+                "Adapter_Cycle_Metrics.csv:md5,5a0c88793b4a0885fe3dda16609b576e",
+                "Adapter_Metrics.csv:md5,989240b8840b2169ac1061f952c90f6c",
+                "Demultiplex_Stats.csv:md5,93949a8cd96f907d83e0808c1ec2a04b",
+                "Demultiplex_Tile_Stats.csv:md5,83120160b0f22a1303fa1db31c19f6e9",
+                "IndexMetricsOut.bin:md5,9e688c58a5487b8eaf69c9e1005ad0bf",
+                "Index_Hopping_Counts.csv:md5,1059369e375fd8f8423c0f6c934be978",
+                "Quality_Metrics.csv:md5,6614accb1bb414fe312b17b81f5521f7",
+                "Quality_Tile_Metrics.csv:md5,cdc89fd2962bdd4a24f71e186112118a",
+                "RunInfo.xml:md5,03038959f4dd181c86bc97ae71fe270a",
+                "SampleSheet.csv:md5,2df2e405991814571c021dc8749c2a89",
+                "Top_Unknown_Barcodes.csv:md5,2e2faba761137f228e56bd3428453ccc",
+                "fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
+            ]
+        ],
+        "timestamp": "2023-08-24T07:43:09+0000"
     },
     "skip_fastp_fastqc": {
         "content": [
@@ -68,25 +91,31 @@
                 "fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
             ]
         ],
-        "timestamp": "2023-06-15T00:02:11+0000"
+        "timestamp": "2023-08-24T07:43:09+0000"
     },
     "software_versions_skip_fastqc": {
         "content": [
             "{BCLCONVERT={bclconvert=00.000.000.4.0.3}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FALCO={falco=1.2.1}, FASTP={fastp=0.23.4}, Workflow={nf-core/demultiplex=1.4.0dev}}"
         ],
-        "timestamp": "2023-06-15T00:02:11+0000"
+        "timestamp": "2023-08-24T07:43:09+0000"
     },
     "software_versions_skip_fastp_fastqc": {
         "content": [
             "{BCLCONVERT={bclconvert=00.000.000.4.0.3}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FALCO={falco=1.2.1}, Workflow={nf-core/demultiplex=1.4.0dev}}"
         ],
-        "timestamp": "2023-06-15T00:02:11+0000"
+        "timestamp": "2023-08-24T07:43:09+0000"
+    },
+    "software_versions_skip_multiqc": {
+        "content": [
+            "{BCLCONVERT={bclconvert=00.000.000.4.0.3}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FALCO={falco=1.2.1}, FASTP={fastp=0.23.4}, Workflow={nf-core/demultiplex=1.4.0dev}}"
+        ],
+        "timestamp": "2023-08-24T07:43:09+0000"
     },
     "software_versions_skip_fastp": {
         "content": [
             "{BCLCONVERT={bclconvert=00.000.000.4.0.3}, CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.0, yaml=6.0}, FALCO={falco=1.2.1}, Workflow={nf-core/demultiplex=1.4.0dev}}"
         ],
-        "timestamp": "2023-06-15T00:02:11+0000"
+        "timestamp": "2023-08-24T07:43:09+0000"
     },
     "skip_trimming": {
         "content": [
@@ -108,6 +137,6 @@
                 "fastq_list.csv:md5,05bc84f51840f5754cfb8381b36f2cb0"
             ]
         ],
-        "timestamp": "2023-06-15T00:02:11+0000"
+        "timestamp": "2023-08-24T07:43:09+0000"
     }
 }
diff --git a/workflows/demultiplex.nf b/workflows/demultiplex.nf
@@ -184,7 +184,7 @@ workflow DEMULTIPLEX {
             ch_versions = ch_versions.mix(SINGULAR_DEMULTIPLEX.out.versions)
             break
         default:
-            Nextflow.error "Unknown demultiplexer: ${demultiplexer}"
+            error "Unknown demultiplexer: ${demultiplexer}"
     }
     ch_raw_fastq.dump(tag: "DEMULTIPLEX::Demultiplexed Fastq",{FormattingService.prettyFormat(it)})
 
@@ -221,25 +221,27 @@ workflow DEMULTIPLEX {
     )
 
     // MODULE: MultiQC
-    workflow_summary    = WorkflowDemultiplex.paramsSummaryMultiqc(workflow, summary_params)
-    ch_workflow_summary = Channel.value(workflow_summary)
-
-    methods_description    = WorkflowDemultiplex.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description, params)
-    ch_methods_description = Channel.value(methods_description)
-
-    ch_multiqc_files = Channel.empty()
-    ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml'))
-    ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml'))
-    ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect())
-    ch_multiqc_files.collect().dump(tag: "DEMULTIPLEX::MultiQC files",{FormattingService.prettyFormat(it)})
-
-    MULTIQC (
-        ch_multiqc_files.collect(),
-        ch_multiqc_config.toList(),
-        ch_multiqc_custom_config.toList(),
-        ch_multiqc_logo.toList()
-    )
-    multiqc_report = MULTIQC.out.report.toList()
+    if (!("multiqc" in skip_tools)){
+        workflow_summary    = WorkflowDemultiplex.paramsSummaryMultiqc(workflow, summary_params)
+        ch_workflow_summary = Channel.value(workflow_summary)
+
+        methods_description    = WorkflowDemultiplex.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description, params)
+        ch_methods_description = Channel.value(methods_description)
+
+        ch_multiqc_files = Channel.empty()
+        ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml'))
+        ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml'))
+        ch_multiqc_files = ch_multiqc_files.mix(CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect())
+        ch_multiqc_files.collect().dump(tag: "DEMULTIPLEX::MultiQC files",{FormattingService.prettyFormat(it)})
+
+        MULTIQC (
+            ch_multiqc_files.collect(),
+            ch_multiqc_config.toList(),
+            ch_multiqc_custom_config.toList(),
+            ch_multiqc_logo.toList()
+        )
+        multiqc_report = MULTIQC.out.report.toList()
+    }
 }
 
 /*
@@ -314,14 +316,14 @@ def extract_csv(input_csv, input_schema=null) {
                     diff in all_columns ? missing_columns.add(diff) : wrong_columns.add(diff)
                 }
                 if(missing_columns.size() > 0){
-                    Nextflow.error "[Samplesheet Error] The column(s) $missing_columns is/are not present. The header should look like: $all_columns"
+                    error "[Samplesheet Error] The column(s) $missing_columns is/are not present. The header should look like: $all_columns"
                 }
                 else {
-                    Nextflow.error "[Samplesheet Error] The column(s) $wrong_columns should not be in the header. The header should look like: $all_columns"
+                    error "[Samplesheet Error] The column(s) $wrong_columns should not be in the header. The header should look like: $all_columns"
                 }
             }
             else {
-                Nextflow.error "[Samplesheet Error] The columns $row are not in the right order. The header should look like: $all_columns"
+                error "[Samplesheet Error] The columns $row are not in the right order. The header should look like: $all_columns"
             }
 
         }
@@ -338,7 +340,7 @@ def extract_csv(input_csv, input_schema=null) {
             row[column] ?: missing_mandatory_columns.add(column)
         }
         if(missing_mandatory_columns.size > 0){
-            Nextflow.error "[Samplesheet Error] The mandatory column(s) $missing_mandatory_columns is/are empty on line $row_count"
+            error "[Samplesheet Error] The mandatory column(s) $missing_mandatory_columns is/are empty on line $row_count"
         }
 
         def output = []
@@ -348,7 +350,7 @@ def extract_csv(input_csv, input_schema=null) {
             content = row[key]
 
             if(!(content ==~ col.value['pattern']) && col.value['pattern'] != '' && content != '') {
-                Nextflow.error "[Samplesheet Error] The content of column '$key' on line $row_count does not match the pattern '${col.value['pattern']}'"
+                error "[Samplesheet Error] The content of column '$key' on line $row_count does not match the pattern '${col.value['pattern']}'"
             }
 
             if(col.value['content'] == 'path'){