Merge pull request #252 from drpatelh/fixes

Refactor how versions are handled in pipeline

.devcontainer/devcontainer.json

@@ -18,11 +18,11 @@
                 "python.linting.flake8Path": "/opt/conda/bin/flake8",
                 "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle",
                 "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle",
-                "python.linting.pylintPath": "/opt/conda/bin/pylint"
+                "python.linting.pylintPath": "/opt/conda/bin/pylint",
             },
 
             // Add the IDs of extensions you want installed when the container is created.
-            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"]
-        }
-    }
+            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"],
+        },
+    },
 }

.nf-core.yml

@@ -9,4 +9,6 @@ lint:
     - lib/nfcore_external_java_deps.jar
   files_unchanged:
     - .gitignore
+    - assets/email_template.html
+    - assets/email_template.txt
     - assets/sendmail_template.txt

CHANGELOG.md

@@ -9,18 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 Special thanks to the following for their contributions to the release:
 
+- [Adam Talbot](https://github.com/adamrtalbot)
 - [Alexandru Mizeranschi](https://github.com/nicolae06)
 - [Alexander Blaessle](https://github.com/alexblaessle)
+- [Maxime Garcia](https://github.com/maxulysse)
 - [Sebastian Uhrig](https://github.com/suhrig)
 
 Thank you to everyone else that has contributed by reporting bugs, enhancements or in any other way, shape or form.
 
 ### Enhancements & fixes
 
 - [PR #238](https://github.com/nf-core/fetchngs/pull/238) - Resolved bug when prefetching large studies ([#236](https://github.com/nf-core/fetchngs/issues/236))
-- [PR #241](https://github.com/nf-core/fetchngs/pull/241) - Download of big files time out and don't resume ([#169](https://github.com/nf-core/fetchngs/issues/169))
+- [PR #241](https://github.com/nf-core/fetchngs/pull/241) - Use wget instead of curl to download files from FTP ([#169](https://github.com/nf-core/fetchngs/issues/169), [#194](https://github.com/nf-core/fetchngs/issues/194))
 - [PR #242](https://github.com/nf-core/fetchngs/pull/242) - Template update for nf-core/tools v2.11
 - [PR #243](https://github.com/nf-core/fetchngs/pull/243) - Fixes for [PR #238](https://github.com/nf-core/fetchngs/pull/238)
+- [PR #245](https://github.com/nf-core/fetchngs/pull/246) - Refactor nf-test CI and test and other pre-release fixes ([#233](https://github.com/nf-core/fetchngs/issues/233))
 - [PR #246](https://github.com/nf-core/fetchngs/pull/246) - Handle dark/light mode for logo in GitHub README properly
 - [PR #248](https://github.com/nf-core/fetchngs/pull/248) - Update pipeline level test data path to use mirror on s3
 - [PR #249](https://github.com/nf-core/fetchngs/pull/249) - Update modules which includes absolute paths for test data, making module level test compatible within the pipeline.

assets/email_template.html

@@ -12,7 +12,7 @@
 
 <img src="cid:nfcorepipelinelogo">
 
-<h1>nf-core/fetchngs v${version}</h1>
+<h1>nf-core/fetchngs ${version}</h1>
 <h2>Run Name: $runName</h2>
 
 <% if (!success){

assets/email_template.txt

@@ -4,7 +4,7 @@
   |\\ | |__  __ /  ` /  \\ |__) |__         }  {
   | \\| |       \\__, \\__/ |  \\ |___     \\`-._,-`-,
                                         `._,._,'
-  nf-core/fetchngs v${version}
+  nf-core/fetchngs ${version}
 ----------------------------------------------------
 Run Name: $runName
 

assets/schema_input.json

@@ -7,7 +7,7 @@
     "items": {
         "type": "object",
         "properties": {
-            "id": {
+            "": {
                 "type": "string",
                 "pattern": "^(((SR|ER|DR)[APRSX])|(SAM(N|EA|D))|(PRJ(NA|EB|DB))|(GS[EM])|(syn))(\\d+)$",
                 "errorMessage": "Please provide a valid SRA, ENA, DDBJ or GEO identifier"

main.nf

@@ -30,25 +30,19 @@ workflow NFCORE_FETCHNGS {
 
     main:
 
-    ch_versions = Channel.empty()
-
     //
     // WORKFLOW: Download FastQ files for SRA / ENA / GEO / DDBJ ids
     //
     if (params.input_type == 'sra') {
         SRA ( ids )
-        ch_versions = SRA.out.versions
 
     //
     // WORKFLOW: Download FastQ files for Synapse ids
     //
     } else if (params.input_type == 'synapse') {
         SYNAPSE ( ids )
-        ch_versions = SYNAPSE.out.versions
     }
 
-    emit:
-    versions = ch_versions
 }
 
 /*
@@ -68,7 +62,16 @@ workflow {
     //
     // SUBWORKFLOW: Run initialisation tasks
     //
-    PIPELINE_INITIALISATION ()
+    PIPELINE_INITIALISATION (
+        params.version,
+        params.help,
+        params.validate_params,
+        params.monochrome_logs,
+        params.outdir,
+        params.input,
+        params.input_type,
+        params.ena_metadata_fields
+    )
 
     //
     // WORKFLOW: Run primary workflows for the pipeline
@@ -81,12 +84,13 @@ workflow {
     // SUBWORKFLOW: Run completion tasks
     //
     PIPELINE_COMPLETION (
-        NFCORE_FETCHNGS.out.versions,
         params.input_type,
         params.email,
         params.email_on_fail,
-        params.hook_url,
-        PIPELINE_INITIALISATION.out.summary_params
+        params.plaintext_email,
+        params.outdir,
+        params.monochrome_logs,
+        params.hook_url
     )
 }
 

subworkflows/local/utils_nfcore_fetchngs_pipeline/main.nf

@@ -8,17 +8,17 @@
 ========================================================================================
 */
 
-include { UTILS_NEXTFLOW_PIPELINE; getWorkflowVersion } from '../../nf-core/utils_nextflow_pipeline'
-include { UTILS_NFVALIDATION_PLUGIN                   } from '../../nf-core/utils_nfvalidation_plugin'
-include { 
-    UTILS_NFCORE_PIPELINE; 
-    workflowCitation; 
-    nfCoreLogo; 
-    dashedLine; 
-    completionEmail; 
-    completionSummary; 
-    imNotification 
-} from '../../nf-core/utils_nfcore_pipeline'
+include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin'
+include { fromSamplesheet           } from 'plugin/nf-validation'
+include { paramsSummaryMap          } from 'plugin/nf-validation'
+include { UTILS_NEXTFLOW_PIPELINE   } from '../../nf-core/utils_nextflow_pipeline'
+include { completionEmail           } from '../../nf-core/utils_nfcore_pipeline'
+include { completionSummary         } from '../../nf-core/utils_nfcore_pipeline'
+include { dashedLine                } from '../../nf-core/utils_nfcore_pipeline'
+include { nfCoreLogo                } from '../../nf-core/utils_nfcore_pipeline'
+include { imNotification            } from '../../nf-core/utils_nfcore_pipeline'
+include { UTILS_NFCORE_PIPELINE     } from '../../nf-core/utils_nfcore_pipeline'
+include { workflowCitation          } from '../../nf-core/utils_nfcore_pipeline'
 
 /*
 ========================================================================================
@@ -28,56 +28,64 @@ include {
 
 workflow PIPELINE_INITIALISATION {
 
+    take:
+    version             // boolean: Display version and exit
+    help                // boolean: Display help text
+    validate_params     // boolean: Boolean whether to validate parameters against the schema at runtime 
+    monochrome_logs     // boolean: Do not use coloured log outputs
+    outdir              //  string: The output directory where the results will be saved
+    input               //  string: File containing SRA/ENA/GEO/DDBJ identifiers one per line to download their associated metadata and FastQ files
+    input_type          //  string: Specifies the type of identifier provided via `--input` - available options are 'sra' and 'synapse'
+    ena_metadata_fields //  string: Comma-separated list of ENA metadata fields to fetch before downloading data
+
     main:
 
     //
     // Print version and exit if required and dump pipeline parameters to JSON file
     //
     UTILS_NEXTFLOW_PIPELINE (
-        params.version,
+        version,
         true,
-        params.outdir,
+        outdir,
         workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1
     )
 
     //
     // Validate parameters and generate parameter summary to stdout
     //
-    def pre_help_text = nfCoreLogo(getWorkflowVersion())
-    def post_help_text = '\n' + workflowCitation() + '\n' + dashedLine()
+    def pre_help_text = nfCoreLogo(monochrome_logs)
+    def post_help_text = '\n' + workflowCitation() + '\n' + dashedLine(monochrome_logs)
     def String workflow_command = "nextflow run ${workflow.manifest.name} -profile <docker/singularity/.../institute> --input ids.csv --outdir <OUTDIR>"
     UTILS_NFVALIDATION_PLUGIN (
-        params.help,
+        help,
         workflow_command,
         pre_help_text,
         post_help_text,
-        params.validate_params,
+        validate_params,
         "nextflow_schema.json"
     )
 
     //
     // Check config provided to the pipeline
     //
-    UTILS_NFCORE_PIPELINE (
-        params.monochrome_logs
-    )
+    UTILS_NFCORE_PIPELINE ()
 
     //
     // Auto-detect input id type
     //
-    ch_input = file(params.input)
-    def input_type = ''
+    ch_input = file(input)
+    def inferred_input_type = ''
     if (isSraId(ch_input)) {
-        input_type = 'sra'
-        sraCheckENAMetadataFields()
+        inferred_input_type = 'sra'
+        sraCheckENAMetadataFields(ena_metadata_fields)
     } else if (isSynapseId(ch_input)) {
-        input_type = 'synapse'
+        inferred_input_type = 'synapse'
     } else {
         error('Ids provided via --input not recognised please make sure they are either SRA / ENA / GEO / DDBJ or Synapse ids!')
     }
 
-    if (params.input_type != input_type) {
-        error("Ids auto-detected as ${input_type}. Please provide '--input_type ${input_type}' as a parameter to the pipeline!")
+    if (input_type != inferred_input_type) {
+        error("Ids auto-detected as ${inferred_input_type}. Please provide '--input_type ${inferred_input_type}' as a parameter to the pipeline!")
     }
 
     // Read in ids from --input file
@@ -89,8 +97,7 @@ workflow PIPELINE_INITIALISATION {
         .set { ch_ids }
 
     emit:
-    ids            = ch_ids
-    summary_params = UTILS_NFVALIDATION_PLUGIN.out.summary_params
+    ids = ch_ids
 }
 
 /*
@@ -102,38 +109,30 @@ workflow PIPELINE_INITIALISATION {
 workflow PIPELINE_COMPLETION {
 
     take:
-    versions       // channel: software tools versions
-    input_type     //  string: 'sra' or 'synapse'
-    email          //  string: email address
-    email_on_fail  //  string: email address sent on pipeline failure
-    hook_url       //  string: hook URL for notifications
-    summary_params //     map: Groovy map of the parameters used in the pipeline
+    input_type      //  string: 'sra' or 'synapse'
+    email           //  string: email address
+    email_on_fail   //  string: email address sent on pipeline failure
+    plaintext_email // boolean: Send plain-text email instead of HTML
+    outdir          //    path: Path to output directory where results will be published
+    monochrome_logs // boolean: Disable ANSI colour codes in log output
+    hook_url        //  string: hook URL for notifications
 
     main:
 
-    //
-    // MODULE: Dump software versions for all tools used in the workflow
-    //
-    pipeline_version_info = Channel.of("""\"workflow\":
-        nextflow: ${workflow.nextflow.version}
-        ${workflow.manifest.name}: ${workflow.manifest.version}
-    """.stripIndent())
-
-    versions = versions.mix(pipeline_version_info)
-    versions.collectFile(name: 'fetchngs_mqc_versions.yml', storeDir: "${params.outdir}/pipeline_info")
+    summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json")
 
     //
     // Completion email and summary
     //
     workflow.onComplete {
         if (email || email_on_fail) {
-            completionEmail(summary_params)
+            completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs)
         }
 
-        completionSummary()
+        completionSummary(monochrome_logs)
 
         if (hook_url) {
-            imNotification(summary_params)
+            imNotification(summary_params, hook_url)
         }
 
         if (input_type == 'sra') {
@@ -205,12 +204,12 @@ def isSynapseId(input) {
 //
 // Check and validate parameters
 //
-def sraCheckENAMetadataFields() {
+def sraCheckENAMetadataFields(ena_metadata_fields) {
     // Check minimal ENA fields are provided to download FastQ files
     def valid_ena_metadata_fields = ['run_accession', 'experiment_accession', 'library_layout', 'fastq_ftp', 'fastq_md5']
-    def ena_metadata_fields = params.ena_metadata_fields ? params.ena_metadata_fields.split(',').collect{ it.trim().toLowerCase() } : valid_ena_metadata_fields
-    if (!ena_metadata_fields.containsAll(valid_ena_metadata_fields)) {
-        error("Invalid option: '${params.ena_metadata_fields}'. Minimally required fields for '--ena_metadata_fields': '${valid_ena_metadata_fields.join(',')}'")
+    def actual_ena_metadata_fields = ena_metadata_fields ? ena_metadata_fields.split(',').collect{ it.trim().toLowerCase() } : valid_ena_metadata_fields
+    if (!actual_ena_metadata_fields.containsAll(valid_ena_metadata_fields)) {
+        error("Invalid option: '${ena_metadata_fields}'. Minimally required fields for '--ena_metadata_fields': '${valid_ena_metadata_fields.join(',')}'")
     }
 }