Refactor how versions are handled in pipeline

.devcontainer/devcontainer.json

@@ -18,11 +18,11 @@
                 "python.linting.flake8Path": "/opt/conda/bin/flake8",
                 "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle",
                 "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle",
-                "python.linting.pylintPath": "/opt/conda/bin/pylint"
+                "python.linting.pylintPath": "/opt/conda/bin/pylint",
             },
 
             // Add the IDs of extensions you want installed when the container is created.
-            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"]
-        }
-    }
+            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"],
+        },
+    },
 }

.nf-core.yml

@@ -9,4 +9,6 @@ lint:
     - lib/nfcore_external_java_deps.jar
   files_unchanged:
     - .gitignore
-    - assets/sendmail_template.txt
+    - assets/email_template.html
+    - assets/email_template.txt
+    - assets/sendmail_template.txt

CHANGELOG.md

@@ -9,18 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 Special thanks to the following for their contributions to the release:
 
+- [Adam Talbot](https://github.com/adamrtalbot)
 - [Alexandru Mizeranschi](https://github.com/nicolae06)
 - [Alexander Blaessle](https://github.com/alexblaessle)
+- [Maxime Garcia](https://github.com/maxulysse)
 - [Sebastian Uhrig](https://github.com/suhrig)
 
 Thank you to everyone else that has contributed by reporting bugs, enhancements or in any other way, shape or form.
 
 ### Enhancements & fixes
 
 - [PR #238](https://github.com/nf-core/fetchngs/pull/238) - Resolved bug when prefetching large studies ([#236](https://github.com/nf-core/fetchngs/issues/236))
-- [PR #241](https://github.com/nf-core/fetchngs/pull/241) - Download of big files time out and don't resume ([#169](https://github.com/nf-core/fetchngs/issues/169))
+- [PR #241](https://github.com/nf-core/fetchngs/pull/241) - Use wget instead of curl to download files from FTP ([#169](https://github.com/nf-core/fetchngs/issues/169), [#194](https://github.com/nf-core/fetchngs/issues/194))
 - [PR #242](https://github.com/nf-core/fetchngs/pull/242) - Template update for nf-core/tools v2.11
 - [PR #243](https://github.com/nf-core/fetchngs/pull/243) - Fixes for [PR #238](https://github.com/nf-core/fetchngs/pull/238)
+- [PR #245](https://github.com/nf-core/fetchngs/pull/246) - Refactor nf-test CI and test and other pre-release fixes ([#233](https://github.com/nf-core/fetchngs/issues/233))
 - [PR #246](https://github.com/nf-core/fetchngs/pull/246) - Handle dark/light mode for logo in GitHub README properly
 - [PR #248](https://github.com/nf-core/fetchngs/pull/248) - Update pipeline level test data path to use mirror on s3
 - [PR #249](https://github.com/nf-core/fetchngs/pull/249) - Update modules which includes absolute paths for test data, making module level test compatible within the pipeline.

assets/email_template.html

@@ -12,7 +12,7 @@
 
 <img src="cid:nfcorepipelinelogo">
 
-<h1>nf-core/fetchngs v${version}</h1>
+<h1>nf-core/fetchngs ${version}</h1>
 <h2>Run Name: $runName</h2>
 
 <% if (!success){

assets/email_template.txt

@@ -4,7 +4,7 @@
   |\\ | |__  __ /  ` /  \\ |__) |__         }  {
   | \\| |       \\__, \\__/ |  \\ |___     \\`-._,-`-,
                                         `._,._,'
-  nf-core/fetchngs v${version}
+  nf-core/fetchngs ${version}
 ----------------------------------------------------
 Run Name: $runName
 

assets/schema_input.json

@@ -7,7 +7,7 @@
     "items": {
         "type": "object",
         "properties": {
-            "id": {
+            "": {
                 "type": "string",
                 "pattern": "^(((SR|ER|DR)[APRSX])|(SAM(N|EA|D))|(PRJ(NA|EB|DB))|(GS[EM])|(syn))(\\d+)$",
                 "errorMessage": "Please provide a valid SRA, ENA, DDBJ or GEO identifier"

main.nf

@@ -30,25 +30,19 @@ workflow NFCORE_FETCHNGS {
 
     main:
 
-    ch_versions = Channel.empty()
-
     //
     // WORKFLOW: Download FastQ files for SRA / ENA / GEO / DDBJ ids
     //
     if (params.input_type == 'sra') {
         SRA ( ids )
-        ch_versions = SRA.out.versions
 
     //
     // WORKFLOW: Download FastQ files for Synapse ids
     //
     } else if (params.input_type == 'synapse') {
         SYNAPSE ( ids )
-        ch_versions = SYNAPSE.out.versions
     }
 
-    emit:
-    versions = ch_versions
 }
 
 /*
@@ -81,12 +75,13 @@ workflow {
     // SUBWORKFLOW: Run completion tasks
     //
     PIPELINE_COMPLETION (
-        NFCORE_FETCHNGS.out.versions,
         params.input_type,
         params.email,
         params.email_on_fail,
-        params.hook_url,
-        PIPELINE_INITIALISATION.out.summary_params
+        params.plaintext_email,
+        params.outdir,
+        params.monochrome_logs,
+        params.hook_url
     )
 }
 

subworkflows/local/utils_nfcore_fetchngs_pipeline/main.nf

@@ -8,17 +8,17 @@
 ========================================================================================
 */
 
-include { UTILS_NEXTFLOW_PIPELINE; getWorkflowVersion } from '../../nf-core/utils_nextflow_pipeline'
-include { UTILS_NFVALIDATION_PLUGIN                   } from '../../nf-core/utils_nfvalidation_plugin'
-include { 
-    UTILS_NFCORE_PIPELINE; 
-    workflowCitation; 
-    nfCoreLogo; 
-    dashedLine; 
-    completionEmail; 
-    completionSummary; 
-    imNotification 
-} from '../../nf-core/utils_nfcore_pipeline'
+include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin'
+include { fromSamplesheet           } from 'plugin/nf-validation'
+include { paramsSummaryMap          } from 'plugin/nf-validation'
+include { UTILS_NEXTFLOW_PIPELINE   } from '../../nf-core/utils_nextflow_pipeline'
+include { completionEmail           } from '../../nf-core/utils_nfcore_pipeline'
+include { completionSummary         } from '../../nf-core/utils_nfcore_pipeline'
+include { dashedLine                } from '../../nf-core/utils_nfcore_pipeline'
+include { nfCoreLogo                } from '../../nf-core/utils_nfcore_pipeline'
+include { imNotification            } from '../../nf-core/utils_nfcore_pipeline'
+include { UTILS_NFCORE_PIPELINE     } from '../../nf-core/utils_nfcore_pipeline'
+include { workflowCitation          } from '../../nf-core/utils_nfcore_pipeline'
 
 /*
 ========================================================================================
@@ -43,8 +43,8 @@ workflow PIPELINE_INITIALISATION {
     //
     // Validate parameters and generate parameter summary to stdout
     //
-    def pre_help_text = nfCoreLogo(getWorkflowVersion())
-    def post_help_text = '\n' + workflowCitation() + '\n' + dashedLine()
+    def pre_help_text = nfCoreLogo(params.monochrome_logs)
+    def post_help_text = '\n' + workflowCitation() + '\n' + dashedLine(params.monochrome_logs)
     def String workflow_command = "nextflow run ${workflow.manifest.name} -profile <docker/singularity/.../institute> --input ids.csv --outdir <OUTDIR>"
     UTILS_NFVALIDATION_PLUGIN (
         params.help,
@@ -58,9 +58,7 @@ workflow PIPELINE_INITIALISATION {
     //
     // Check config provided to the pipeline
     //
-    UTILS_NFCORE_PIPELINE (
-        params.monochrome_logs
-    )
+    UTILS_NFCORE_PIPELINE ()
 
     //
     // Auto-detect input id type
@@ -89,8 +87,7 @@ workflow PIPELINE_INITIALISATION {
         .set { ch_ids }
 
     emit:
-    ids            = ch_ids
-    summary_params = UTILS_NFVALIDATION_PLUGIN.out.summary_params
+    ids = ch_ids
 }
 
 /*
@@ -102,38 +99,30 @@ workflow PIPELINE_INITIALISATION {
 workflow PIPELINE_COMPLETION {
 
     take:
-    versions       // channel: software tools versions
-    input_type     //  string: 'sra' or 'synapse'
-    email          //  string: email address
-    email_on_fail  //  string: email address sent on pipeline failure
-    hook_url       //  string: hook URL for notifications
-    summary_params //     map: Groovy map of the parameters used in the pipeline
+    input_type      //  string: 'sra' or 'synapse'
+    email           //  string: email address
+    email_on_fail   //  string: email address sent on pipeline failure
+    plaintext_email // boolean: Send plain-text email instead of HTML
+    outdir          //    path: Path to output directory where results will be published
+    monochrome_logs // boolean: Disable ANSI colour codes in log output
+    hook_url        //  string: hook URL for notifications
 
     main:
 
-    //
-    // MODULE: Dump software versions for all tools used in the workflow
-    //
-    pipeline_version_info = Channel.of("""\"workflow\":
-        nextflow: ${workflow.nextflow.version}
-        ${workflow.manifest.name}: ${workflow.manifest.version}
-    """.stripIndent())
-
-    versions = versions.mix(pipeline_version_info)
-    versions.collectFile(name: 'fetchngs_mqc_versions.yml', storeDir: "${params.outdir}/pipeline_info")
+    summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json")
 
     //
     // Completion email and summary
     //
     workflow.onComplete {
         if (email || email_on_fail) {
-            completionEmail(summary_params)
+            completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs)
         }
 
-        completionSummary()
+        completionSummary(monochrome_logs)
 
         if (hook_url) {
-            imNotification(summary_params)
+            imNotification(summary_params, hook_url)
         }
 
         if (input_type == 'sra') {