From 127e3945293d55758a7fd9456f064d5cb796b956 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 4 Jul 2024 08:41:40 +0200 Subject: [PATCH 01/27] Post release bump --- CHANGELOG.md | 12 ++++++++++++ assets/multiqc_config.yml | 4 ++-- nextflow.config | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e36b245..93022c43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## dev [unreleased] + +### `Added` + +### `Changed` + +### `Fixed` + +### `Dependencies` + +### `Deprecated` + ## 3.0.2 [2024-07-04] ### `Added` diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index 47f9932a..9f4e5e7e 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -1,7 +1,7 @@ report_comment: > - This report has been generated by the nf-core/mag + This report has been generated by the nf-core/mag analysis pipeline. For information about how to interpret these results, please see the - documentation. + documentation. report_section_order: "nf-core-mag-methods-description": order: -1000 diff --git a/nextflow.config b/nextflow.config index e2a4398c..c03b0c49 100644 --- a/nextflow.config +++ b/nextflow.config @@ -390,7 +390,7 @@ manifest { description = """Assembly, binning and annotation of metagenomes""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = '3.0.2' + version = 'dev' doi = '10.1093/nargab/lqac007' } From 3b950359630b9455ded5be5e8758fc457f819359 Mon Sep 17 00:00:00 2001 From: Danielle Callan Date: Thu, 18 Jul 2024 13:10:10 -0400 Subject: [PATCH 02/27] only make bin depths plots for cases w >2 samples and >2 bins --- subworkflows/local/depths.nf | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/subworkflows/local/depths.nf b/subworkflows/local/depths.nf index 87fc21cc..ffe89daa 100644 --- a/subworkflows/local/depths.nf +++ b/subworkflows/local/depths.nf @@ -10,6 +10,14 @@ def getColNo(filename) { return lines[0].split('\t').size() } +/* + * Get number of rows in a file + */ +def getRowNo(filename) { + lines = file(filename).readLines() + return lines.size() +} + workflow DEPTHS { take: bins_unbins //channel: val(meta), [ path(bins) ] @@ -53,9 +61,10 @@ workflow DEPTHS { .collectFile(name:'sample_groups.tsv'){ meta, reads -> meta.id + '\t' + meta.group + '\n' } // Filter MAG depth files: use only those for plotting that contain depths for > 2 samples + // as well as > 2 bins ch_mag_depths_plot = MAG_DEPTHS.out.depths .map { meta, bin_depths_file -> - if (getColNo(bin_depths_file) > 2) [ meta, bin_depths_file ] + if (getColNo(bin_depths_file) > 2 && getRowNo(bin_depths_file) > 2) [ meta, bin_depths_file ] } MAG_DEPTHS_PLOT ( ch_mag_depths_plot, ch_sample_groups.collect() ) From 80829728af9e890057a2aba78ddecbc59f102c2a Mon Sep 17 00:00:00 2001 From: Jim Downie <19718667+prototaxites@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:05:19 +0100 Subject: [PATCH 03/27] Update gtdbtk.nf --- subworkflows/local/gtdbtk.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/subworkflows/local/gtdbtk.nf b/subworkflows/local/gtdbtk.nf index 95e343c8..aae4f9ae 100644 --- a/subworkflows/local/gtdbtk.nf +++ b/subworkflows/local/gtdbtk.nf @@ -76,6 +76,7 @@ workflow GTDBTK { .of(gtdb_dir) .map{['gtdb', it]} .groupTuple() + .first() } else { error("Unsupported object given to --gtdb, database must be supplied as either a directory or a .tar.gz file!") } From deab0076bf6c58272aafbcf9f0bae1b6473e41d5 Mon Sep 17 00:00:00 2001 From: Jim Downie <19718667+prototaxites@users.noreply.github.com> Date: Thu, 1 Aug 2024 17:33:49 +0100 Subject: [PATCH 04/27] Update gtdbtk.nf --- subworkflows/local/gtdbtk.nf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/subworkflows/local/gtdbtk.nf b/subworkflows/local/gtdbtk.nf index aae4f9ae..21ef25b2 100644 --- a/subworkflows/local/gtdbtk.nf +++ b/subworkflows/local/gtdbtk.nf @@ -74,9 +74,8 @@ workflow GTDBTK { gtdb_dir = gtdb.listFiles() ch_db_for_gtdbtk = Channel .of(gtdb_dir) - .map{['gtdb', it]} - .groupTuple() - .first() + .collect() + .map { ["gtdb", it] } } else { error("Unsupported object given to --gtdb, database must be supplied as either a directory or a .tar.gz file!") } From 5b9c4f1386f800e9c0021eed7722522246bbbd65 Mon Sep 17 00:00:00 2001 From: Jim Downie <19718667+prototaxites@users.noreply.github.com> Date: Fri, 2 Aug 2024 09:23:23 +0100 Subject: [PATCH 05/27] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93022c43..06bd879f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` +- [#646](https://github.com/nf-core/mag/pull/646) - GTDB-Tk directory input now creates a value channel so it runs for all entries to the process and not just the first (by @prototaxites). + ### `Dependencies` ### `Deprecated` From 8f4147fc7aeacafc759284fc5a22efa1838fef18 Mon Sep 17 00:00:00 2001 From: prototaxites Date: Fri, 2 Aug 2024 09:14:53 +0000 Subject: [PATCH 06/27] Fix code that combines assembly and sample IDs for checking --- subworkflows/local/utils_nfcore_mag_pipeline/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/utils_nfcore_mag_pipeline/main.nf b/subworkflows/local/utils_nfcore_mag_pipeline/main.nf index 592d7a09..2abde4b5 100644 --- a/subworkflows/local/utils_nfcore_mag_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_mag_pipeline/main.nf @@ -150,7 +150,7 @@ workflow PIPELINE_INITIALISATION { .toList() .sort() - ch_read_ids.cross(ch_assembly_ids) + ch_read_ids.concat(ch_assembly_ids).collect(flat: false) .map { ids1, ids2 -> if (ids1.sort() != ids2.sort()) { exit 1, "[nf-core/mag] ERROR: supplied IDs or Groups in read and assembly CSV files do not match!" From 86412f06f7dbe1f62f73219cc8fdc46d2adf5bd4 Mon Sep 17 00:00:00 2001 From: prototaxites Date: Fri, 2 Aug 2024 09:17:44 +0000 Subject: [PATCH 07/27] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93022c43..6e4d3390 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` +- [#648](https://github.com/nf-core/mag/pull/648) - Fix sample ID/assembly ID check failure when no IDs match (by @prototaxites) + ### `Dependencies` ### `Deprecated` From f143707dd653c07952ca332092ef727f566ae8c4 Mon Sep 17 00:00:00 2001 From: Jim Downie <19718667+prototaxites@users.noreply.github.com> Date: Tue, 6 Aug 2024 10:32:39 +0100 Subject: [PATCH 08/27] Update CHANGELOG.md Co-authored-by: James A. Fellows Yates --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71a6d826..3182454d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` -- [#648](https://github.com/nf-core/mag/pull/648) - Fix sample ID/assembly ID check failure when no IDs match (by @prototaxites) +- [#648](https://github.com/nf-core/mag/pull/648) - Fix sample ID/assembly ID check failure when no IDs match (reported by @zackhenny, fix by @prototaxites) - [#646](https://github.com/nf-core/mag/pull/646) - GTDB-Tk directory input now creates a value channel so it runs for all entries to the process and not just the first (by @prototaxites). ### `Dependencies` From 098c192f5a95959ea784adc6337eaab6e413ff8f Mon Sep 17 00:00:00 2001 From: Jim Downie <19718667+prototaxites@users.noreply.github.com> Date: Tue, 6 Aug 2024 10:32:46 +0100 Subject: [PATCH 09/27] Update subworkflows/local/utils_nfcore_mag_pipeline/main.nf Co-authored-by: James A. Fellows Yates --- subworkflows/local/utils_nfcore_mag_pipeline/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/utils_nfcore_mag_pipeline/main.nf b/subworkflows/local/utils_nfcore_mag_pipeline/main.nf index 2abde4b5..29806112 100644 --- a/subworkflows/local/utils_nfcore_mag_pipeline/main.nf +++ b/subworkflows/local/utils_nfcore_mag_pipeline/main.nf @@ -150,7 +150,7 @@ workflow PIPELINE_INITIALISATION { .toList() .sort() - ch_read_ids.concat(ch_assembly_ids).collect(flat: false) + ch_read_ids.concat(ch_assembly_ids).collect(flat: false) // need flat:false to ensure the two lists of IDs in the channels don't get smushed into a single list (and thus no ids1 and ids2 lists to compare) .map { ids1, ids2 -> if (ids1.sort() != ids2.sort()) { exit 1, "[nf-core/mag] ERROR: supplied IDs or Groups in read and assembly CSV files do not match!" From 3b509edd4e2a9c7a058d0fe3e2da591c4a9824a4 Mon Sep 17 00:00:00 2001 From: Jim Downie <19718667+prototaxites@users.noreply.github.com> Date: Tue, 6 Aug 2024 10:32:52 +0100 Subject: [PATCH 10/27] Update CHANGELOG.md Co-authored-by: James A. Fellows Yates --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3182454d..9f2ca659 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Fixed` - [#648](https://github.com/nf-core/mag/pull/648) - Fix sample ID/assembly ID check failure when no IDs match (reported by @zackhenny, fix by @prototaxites) -- [#646](https://github.com/nf-core/mag/pull/646) - GTDB-Tk directory input now creates a value channel so it runs for all entries to the process and not just the first (by @prototaxites). +- [#646](https://github.com/nf-core/mag/pull/646) - GTDB-Tk directory input now creates a value channel so it runs for all entries to the process and not just the first (reported by @amizeranschi, fix by @prototaxites). ### `Dependencies` From 66a0af4a4922c1b23dcea63037a86feb7db337e0 Mon Sep 17 00:00:00 2001 From: Danielle Callan Date: Fri, 9 Aug 2024 13:52:06 -0400 Subject: [PATCH 11/27] clarify when clustered heatmaps will be produced as outputs --- docs/output.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/output.md b/docs/output.md index d044e544..96d2f82c 100644 --- a/docs/output.md +++ b/docs/output.md @@ -454,7 +454,7 @@ For each bin or refined bin the median sequencing depth is computed based on the - `GenomeBinning/depths/bins/` - `bin_depths_summary.tsv`: Summary of bin sequencing depths for all samples. Depths are available for samples mapped against the corresponding assembly, i.e. according to the mapping strategy specified with `--binning_map_mode`. Only for short reads. - `bin_refined_depths_summary.tsv`: Summary of sequencing depths for refined bins for all samples, if refinement was performed. Depths are available for samples mapped against the corresponding assembly, i.e. according to the mapping strategy specified with `--binning_map_mode`. Only for short reads. - - `[assembler]-[binner]-[sample/group]-binDepths.heatmap.png`: Clustered heatmap showing bin abundances of the assembly across samples. Bin depths are transformed to centered log-ratios and bins as well as samples are clustered by Euclidean distance. Again, sample depths are available according to the mapping strategy specified with `--binning_map_mode`. + - `[assembler]-[binner]-[sample/group]-binDepths.heatmap.png`: Clustered heatmap showing bin abundances of the assembly across samples. Bin depths are transformed to centered log-ratios and bins as well as samples are clustered by Euclidean distance. Again, sample depths are available according to the mapping strategy specified with `--binning_map_mode`. If a sample produces only a single bin, a heatmap will not be provided. From f7c8acae66a94cb91a2dcc7f7615c2bd8d3bfb86 Mon Sep 17 00:00:00 2001 From: Danielle Callan Date: Fri, 9 Aug 2024 13:52:20 -0400 Subject: [PATCH 12/27] add bug fix 639 to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 93022c43..04a1b2ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` ### `Fixed` +- [#639](https://github.com/nf-core/mag/pull/639) - Fix pipeline failure when a sample produces only a single bin (fix by @d-callan) ### `Dependencies` From bd3e81309ef18976adf72862bd24114279982c9b Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Sat, 10 Aug 2024 11:02:22 +0000 Subject: [PATCH 13/27] [automated] Fix code linting --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04a1b2ba..aefb1372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### `Changed` ### `Fixed` + - [#639](https://github.com/nf-core/mag/pull/639) - Fix pipeline failure when a sample produces only a single bin (fix by @d-callan) ### `Dependencies` From 83f1fe79ad8e0e08a131c889438b1f4785c0a7d6 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Aug 2024 16:23:48 +0200 Subject: [PATCH 14/27] Add @d-callan and other recent code contributors --- README.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b2a723eb..f92e844f 100644 --- a/README.md +++ b/README.md @@ -77,14 +77,25 @@ When group-wise co-assembly is enabled, `SPAdes` is run on accordingly pooled re ## Credits -nf-core/mag was written by [Hadrien Gourlé](https://hadriengourle.com) at [SLU](https://slu.se), [Daniel Straub](https://github.com/d4straub) and [Sabrina Krakau](https://github.com/skrakau) at the [Quantitative Biology Center (QBiC)](http://qbic.life). [James A. Fellows Yates](https://github.com/jfy133) and [Maxime Borry](https://github.com/maxibor) at the [Max Planck Institute for Evolutionary Anthropology](https://www.eva.mpg.de) joined in version 2.2.0. More recent contributors include [Jim Downie](https://github.com/prototaxites) and [Carson Miller](https://github.com/CarsonJM). +nf-core/mag was written by [Hadrien Gourlé](https://hadriengourle.com) at [SLU](https://slu.se), [Daniel Straub](https://github.com/d4straub) and [Sabrina Krakau](https://github.com/skrakau) at the [Quantitative Biology Center (QBiC)](http://qbic.life). [James A. Fellows Yates](https://github.com/jfy133) and [Maxime Borry](https://github.com/maxibor) at the [Max Planck Institute for Evolutionary Anthropology](https://www.eva.mpg.de) joined in version 2.2.0. + +Other code contributors include: + +- [Antonia Schuster](https://github.com/AntoniaSchuster) +- [Alexander Ramos](https://github.com/alxndrdiaz) +- [Carson Miller](https://github.com/CarsonJM) +- [Daniel Lundin](https://github.com/erikrikarddaniel) +- [Danielle Callan](https://github.com/d-callan) +- [Gregory Sprenger](https://github.com/gregorysprenger) +- [Jim Downie](https://github.com/prototaxites) +- [Phil Palmer](https://github.com/PhilPalmer) +- [@willros](https://github.com/willros) Long read processing was inspired by [caspargross/HybridAssembly](https://github.com/caspargross/HybridAssembly) written by Caspar Gross [@caspargross](https://github.com/caspargross) We thank the following people for their extensive assistance in the development of this pipeline: - [Alexander Peltzer](https://github.com/apeltzer) -- [Antonia Schuster](https://github.com/antoniaschuster) - [Phil Ewels](https://github.com/ewels) - [Gisela Gabernet](https://github.com/ggabernet) - [Harshil Patel](https://github.com/drpatelh) From f79135caa96f424e667ee33af30e68d436bc525f Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Aug 2024 16:24:40 +0200 Subject: [PATCH 15/27] Add other third wave of developer leads to authors --- nextflow.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow.config b/nextflow.config index c03b0c49..9d2b2f44 100644 --- a/nextflow.config +++ b/nextflow.config @@ -385,7 +385,7 @@ dag { manifest { name = 'nf-core/mag' - author = """Hadrien Gourlé, Daniel Straub, Sabrina Krakau""" + author = """Hadrien Gourlé, Daniel Straub, Sabrina Krakau, James A. Fellows Yates, Maxime Borry""" homePage = 'https://github.com/nf-core/mag' description = """Assembly, binning and annotation of metagenomes""" mainScript = 'main.nf' From b708160407d569b19ad387d60cc5b1be3a02b53d Mon Sep 17 00:00:00 2001 From: nf-core-bot Date: Thu, 15 Aug 2024 14:25:34 +0000 Subject: [PATCH 16/27] [automated] Fix code linting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f92e844f..405e298a 100644 --- a/README.md +++ b/README.md @@ -77,12 +77,12 @@ When group-wise co-assembly is enabled, `SPAdes` is run on accordingly pooled re ## Credits -nf-core/mag was written by [Hadrien Gourlé](https://hadriengourle.com) at [SLU](https://slu.se), [Daniel Straub](https://github.com/d4straub) and [Sabrina Krakau](https://github.com/skrakau) at the [Quantitative Biology Center (QBiC)](http://qbic.life). [James A. Fellows Yates](https://github.com/jfy133) and [Maxime Borry](https://github.com/maxibor) at the [Max Planck Institute for Evolutionary Anthropology](https://www.eva.mpg.de) joined in version 2.2.0. +nf-core/mag was written by [Hadrien Gourlé](https://hadriengourle.com) at [SLU](https://slu.se), [Daniel Straub](https://github.com/d4straub) and [Sabrina Krakau](https://github.com/skrakau) at the [Quantitative Biology Center (QBiC)](http://qbic.life). [James A. Fellows Yates](https://github.com/jfy133) and [Maxime Borry](https://github.com/maxibor) at the [Max Planck Institute for Evolutionary Anthropology](https://www.eva.mpg.de) joined in version 2.2.0. Other code contributors include: - [Antonia Schuster](https://github.com/AntoniaSchuster) -- [Alexander Ramos](https://github.com/alxndrdiaz) +- [Alexander Ramos](https://github.com/alxndrdiaz) - [Carson Miller](https://github.com/CarsonJM) - [Daniel Lundin](https://github.com/erikrikarddaniel) - [Danielle Callan](https://github.com/d-callan) From e7ebbfabfb1972fd5b4ec4ab6123bb7e62213216 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Aug 2024 21:29:48 +0200 Subject: [PATCH 17/27] Update subworkflows/local/depths.nf --- subworkflows/local/depths.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/local/depths.nf b/subworkflows/local/depths.nf index ffe89daa..a2b69c95 100644 --- a/subworkflows/local/depths.nf +++ b/subworkflows/local/depths.nf @@ -10,7 +10,7 @@ def getColNo(filename) { return lines[0].split('\t').size() } -/* +/* * Get number of rows in a file */ def getRowNo(filename) { From 36cfd422de071513b576086f87564a286113ecef Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 15 Aug 2024 21:30:46 +0200 Subject: [PATCH 18/27] Update LICENSE to include more recent contributors --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index fa60ad5d..d90d555c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) Hadrien Gourlé, Daniel Straub, Sabrina Krakau +Copyright (c) Hadrien Gourlé, Daniel Straub, Sabrina Krakau, James A. Fellows Yates, Maxime Borry Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From 5851a1d1b0e675366def8dc5cb23e3181a536d4a Mon Sep 17 00:00:00 2001 From: Greg Date: Fri, 16 Aug 2024 11:29:25 -0700 Subject: [PATCH 19/27] replaced old MultiQC containers with nf-core/ubuntu:20.04 --- modules/local/adjust_maxbin2_ext.nf | 7 +++---- modules/local/rename_postdastool.nf | 7 +++---- modules/local/rename_predastool.nf | 7 +++---- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/modules/local/adjust_maxbin2_ext.nf b/modules/local/adjust_maxbin2_ext.nf index 4d7fecd0..70eae99c 100644 --- a/modules/local/adjust_maxbin2_ext.nf +++ b/modules/local/adjust_maxbin2_ext.nf @@ -2,11 +2,10 @@ process ADJUST_MAXBIN2_EXT { tag "${meta.assembler}-${meta.id}" label 'process_low' - // Using container from multiqc since it'll be included anyway - conda "bioconda::multiqc=1.12" + conda "conda-forge::sed=4.7" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.12--pyhdfd78af_0' : - 'biocontainers/multiqc:1.12--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : + 'nf-core/ubuntu:20.04' }" input: tuple val(meta), path(bins) diff --git a/modules/local/rename_postdastool.nf b/modules/local/rename_postdastool.nf index 7d5a325e..6129dfbe 100644 --- a/modules/local/rename_postdastool.nf +++ b/modules/local/rename_postdastool.nf @@ -2,11 +2,10 @@ process RENAME_POSTDASTOOL { tag "${meta.assembler}-${meta.id}" label 'process_low' - // Using container from multiqc since it'll be included anyway - conda "bioconda::multiqc=1.12" + conda "conda-forge::sed=4.7" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.12--pyhdfd78af_0' : - 'biocontainers/multiqc:1.12--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : + 'nf-core/ubuntu:20.04' }" input: tuple val(meta), path(bins) diff --git a/modules/local/rename_predastool.nf b/modules/local/rename_predastool.nf index cc3bab18..3d9373b8 100644 --- a/modules/local/rename_predastool.nf +++ b/modules/local/rename_predastool.nf @@ -2,11 +2,10 @@ process RENAME_PREDASTOOL { tag "${meta.assembler}-${meta.binner}-${meta.id}" label 'process_low' - // Using container from multiqc since it'll be included anyway - conda "bioconda::multiqc=1.12" + conda "conda-forge::sed=4.7" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.12--pyhdfd78af_0' : - 'biocontainers/multiqc:1.12--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : + 'nf-core/ubuntu:20.04' }" input: tuple val(meta), path(bins) From bd0ef3b44a18fca004314563de7edc15de6b71a1 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 20 Aug 2024 08:35:22 +0200 Subject: [PATCH 20/27] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a260063b..129da60e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#648](https://github.com/nf-core/mag/pull/648) - Fix sample ID/assembly ID check failure when no IDs match (reported by @zackhenny, fix by @prototaxites) - [#646](https://github.com/nf-core/mag/pull/646) - GTDB-Tk directory input now creates a value channel so it runs for all entries to the process and not just the first (reported by @amizeranschi, fix by @prototaxites). - [#639](https://github.com/nf-core/mag/pull/639) - Fix pipeline failure when a sample produces only a single bin (fix by @d-callan) +- [#652](https://github.com/nf-core/mag/pull/651) - Replace base container for bash only modules to reduce number of containers in pipeline (reported and fixed by @harper357) ### `Dependencies` From 993ac6d429056c5c3a505d56f47519052ab52cad Mon Sep 17 00:00:00 2001 From: amizeranschi Date: Tue, 20 Aug 2024 12:50:47 +0300 Subject: [PATCH 21/27] Update nextflow_schema.json Remove "=" sign from parameter value assignment --- nextflow_schema.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nextflow_schema.json b/nextflow_schema.json index b2847b0c..35e85825 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -605,12 +605,12 @@ "spades_options": { "type": "string", "description": "Additional custom options for SPAdes.", - "help_text": "An example is adjusting k-mers (\"-k 21,33,55,77\") or adding [advanced options](https://github.com/ablab/spades#advanced-options). But not -t, -m, -o or --out-prefix, because these are already in use. Must be used like this: --spades_options=\"-k 21,33,55,77\")" + "help_text": "An example is adjusting k-mers (\"-k 21,33,55,77\") or adding [advanced options](https://github.com/ablab/spades#advanced-options). But not -t, -m, -o or --out-prefix, because these are already in use. Must be used like this: --spades_options \"-k 21,33,55,77\")" }, "megahit_options": { "type": "string", "description": "Additional custom options for MEGAHIT.", - "help_text": "An example is adjusting presets (e.g. \"--presets meta-large\"), k-mers (e.g. \"-k 21,33,55,77\") or adding other [advanced options](https://github.com/voutcn/megahit#advanced-usage). For example, increase the minimum k-mer in the event of an error message such as \"Too many vertices in the unitig graph, you may increase the kmer size to remove tons of erroneous kmers.\" in the MEGAHIT log file. But not --threads, --memory, -o or input read files, because these are already in use. Must be used like this: --megahit_options=\"--presets meta-large\"" + "help_text": "An example is adjusting presets (e.g. \"--presets meta-large\"), k-mers (e.g. \"-k 21,33,55,77\") or adding other [advanced options](https://github.com/voutcn/megahit#advanced-usage). For example, increase the minimum k-mer in the event of an error message such as \"Too many vertices in the unitig graph, you may increase the kmer size to remove tons of erroneous kmers.\" in the MEGAHIT log file. But not --threads, --memory, -o or input read files, because these are already in use. Must be used like this: --megahit_options \"--presets meta-large\"" }, "skip_spades": { "type": "boolean", @@ -735,7 +735,7 @@ "bowtie2_mode": { "type": "string", "description": "Bowtie2 alignment mode", - "help_text": "Bowtie2 alignment mode options, for example: `--very-fast` , `--very-sensitive-local -N 1` , ... Must be used like this: --bowtie2_mode=\"--very-sensitive\"" + "help_text": "Bowtie2 alignment mode options, for example: `--very-fast` , `--very-sensitive-local -N 1` , ... Must be used like this: --bowtie2_mode \"--very-sensitive\"" }, "save_assembly_mapped_reads": { "type": "boolean", From 364b3eefd005bf8e7debdf30e325abc7cf16f986 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Wed, 21 Aug 2024 21:42:17 +0200 Subject: [PATCH 22/27] Fix GUNC overwriting of raw output files --- CHANGELOG.md | 4 +++- conf/modules.config | 4 ++-- docs/output.md | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 129da60e..ccb1decf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - [#648](https://github.com/nf-core/mag/pull/648) - Fix sample ID/assembly ID check failure when no IDs match (reported by @zackhenny, fix by @prototaxites) - [#646](https://github.com/nf-core/mag/pull/646) - GTDB-Tk directory input now creates a value channel so it runs for all entries to the process and not just the first (reported by @amizeranschi, fix by @prototaxites). - [#639](https://github.com/nf-core/mag/pull/639) - Fix pipeline failure when a sample produces only a single bin (fix by @d-callan) -- [#652](https://github.com/nf-core/mag/pull/651) - Replace base container for bash only modules to reduce number of containers in pipeline (reported and fixed by @harper357) +- [#651](https://github.com/nf-core/mag/pull/651) - Replace base container for bash only modules to reduce number of containers in pipeline (reported and fixed by @harper357) +- [#652](https://github.com/nf-core/mag/pull/652) - Fix documentation typo in using user-defined assembly parameters (reported and fixed by @amizeranschi) +- [#653](https://github.com/nf-core/mag/pull/653) - Fix overwriting of per-bin 'raw' GUNC RUN output files (multi-bin summary tables not affected) (reported by @zackhenny and fixed by @jfy133) ### `Dependencies` diff --git a/conf/modules.config b/conf/modules.config index 93a82d83..81df5bc8 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -447,7 +447,7 @@ process { // Make sure to keep directory in sync with gunc_qc.nf withName: 'GUNC_RUN' { publishDir = [ - path: { "${params.outdir}/GenomeBinning/QC/GUNC/raw/${meta.assembler}-${meta.binner}-${meta.domain}-${meta.refinement}-${meta.id}" }, + path: { "${params.outdir}/GenomeBinning/QC/GUNC/raw/${meta.assembler}-${meta.binner}-${meta.domain}-${meta.refinement}-${meta.id}/${fasta.baseName}/" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] @@ -456,7 +456,7 @@ process { // Make sure to keep directory in sync with gunc_qc.nf withName: 'GUNC_MERGECHECKM' { publishDir = [ - path: { "${params.outdir}/GenomeBinning/QC/GUNC/checkmmerged/${meta.assembler}-${meta.binner}-${meta.domain}-${meta.refinement}-${meta.id}" }, + path: { "${params.outdir}/GenomeBinning/QC/GUNC/checkmmerged/${meta.assembler}-${meta.binner}-${meta.domain}-${meta.refinement}-${meta.id}/${checkm_file.baseName}" }, mode: params.publish_dir_mode, saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] diff --git a/docs/output.md b/docs/output.md index 96d2f82c..5f889056 100644 --- a/docs/output.md +++ b/docs/output.md @@ -565,9 +565,9 @@ If the parameter `--save_checkm_reference` is set, additionally the used the Che - `[gunc-database].dmnd` - `GUNC/` - `raw/` - - `[assembler]-[binner]-[domain]-[refinement]-[sample/group]/GUNC_checkM.merged.tsv`: Per sample GUNC [output](https://grp-bork.embl-community.io/gunc/output.html) containing with taxonomic and completeness QC statistics. + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group]/[fasta input file name]/GUNC_checkM.merged.tsv`: Per sample GUNC [output](https://grp-bork.embl-community.io/gunc/output.html) containing with taxonomic and completeness QC statistics. - `checkmmerged/` - - `[assembler]-[binner]-[domain]-[refinement]-[sample/group]/GUNC.progenomes_2.1.maxCSS_level.tsv`: Per sample GUNC output merged with output from [CheckM](#checkm) + - `[assembler]-[binner]-[domain]-[refinement]-[sample/group]/[checkm input file name]/GUNC.progenomes_2.1.maxCSS_level.tsv`: Per sample GUNC output merged with output from [CheckM](#checkm) From 2971376a4253bf6713a03a1bcfb8e3962efd4fbb Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 22 Aug 2024 14:18:53 +0200 Subject: [PATCH 23/27] Remove unused module --- modules.json | 215 +++++++++++++----- .../dumpsoftwareversions/environment.yml | 7 - .../custom/dumpsoftwareversions/main.nf | 24 -- .../custom/dumpsoftwareversions/meta.yml | 37 --- .../templates/dumpsoftwareversions.py | 101 -------- .../dumpsoftwareversions/tests/main.nf.test | 43 ---- .../tests/main.nf.test.snap | 33 --- .../dumpsoftwareversions/tests/tags.yml | 2 - 8 files changed, 157 insertions(+), 305 deletions(-) delete mode 100644 modules/nf-core/custom/dumpsoftwareversions/environment.yml delete mode 100644 modules/nf-core/custom/dumpsoftwareversions/main.nf delete mode 100644 modules/nf-core/custom/dumpsoftwareversions/meta.yml delete mode 100755 modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py delete mode 100644 modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test delete mode 100644 modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap delete mode 100644 modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml diff --git a/modules.json b/modules.json index 16a805b8..5dab95c1 100644 --- a/modules.json +++ b/modules.json @@ -8,249 +8,340 @@ "adapterremoval": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "aria2": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"], + "installed_by": [ + "modules" + ], "patch": "modules/nf-core/aria2/aria2.diff" }, "bbmap/bbnorm": { "branch": "master", "git_sha": "603ecbd9f45300c9788f197d2a15a005685b4220", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "bcftools/consensus": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "bcftools/index": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "bcftools/view": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "cat/fastq": { "branch": "master", "git_sha": "5c460c5a4736974abde2843294f35307ee2b0e5e", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "centrifuge/centrifuge": { "branch": "master", "git_sha": "9a07a1293d9b818d1e06d0f7b58152f74d462012", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "centrifuge/kreport": { "branch": "master", "git_sha": "9a07a1293d9b818d1e06d0f7b58152f74d462012", - "installed_by": ["modules"], + "installed_by": [ + "modules" + ], "patch": "modules/nf-core/centrifuge/kreport/centrifuge-kreport.diff" }, "checkm/lineagewf": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "checkm/qa": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "concoct/concoct": { "branch": "master", "git_sha": "baa30accc6c50ea8a98662417d4f42ed18966353", - "installed_by": ["fasta_binning_concoct"] + "installed_by": [ + "fasta_binning_concoct" + ] }, "concoct/concoctcoveragetable": { "branch": "master", "git_sha": "baa30accc6c50ea8a98662417d4f42ed18966353", - "installed_by": ["fasta_binning_concoct"] + "installed_by": [ + "fasta_binning_concoct" + ] }, "concoct/cutupfasta": { "branch": "master", "git_sha": "73a6d7e6077b88aba1c5d6805635d79d6718270c", - "installed_by": ["fasta_binning_concoct"] + "installed_by": [ + "fasta_binning_concoct" + ] }, "concoct/extractfastabins": { "branch": "master", "git_sha": "baa30accc6c50ea8a98662417d4f42ed18966353", - "installed_by": ["fasta_binning_concoct"] + "installed_by": [ + "fasta_binning_concoct" + ] }, "concoct/mergecutupclustering": { "branch": "master", "git_sha": "baa30accc6c50ea8a98662417d4f42ed18966353", - "installed_by": ["fasta_binning_concoct"] - }, - "custom/dumpsoftwareversions": { - "branch": "master", - "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93", - "installed_by": ["modules"] + "installed_by": [ + "fasta_binning_concoct" + ] }, "dastool/dastool": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "dastool/fastatocontig2bin": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "fastp": { "branch": "master", "git_sha": "d497a4868ace3302016ea8ed4b395072d5e833cd", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "fastqc": { "branch": "master", "git_sha": "285a50500f9e02578d90b3ce6382ea3c30216acd", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "freebayes": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "genomad/download": { "branch": "master", "git_sha": "ca813f3f73adedf3547a5a677e992d9d43a71870", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "genomad/endtoend": { "branch": "master", "git_sha": "ca813f3f73adedf3547a5a677e992d9d43a71870", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "gtdbtk/classifywf": { "branch": "master", "git_sha": "0735b6d2b509cbb5cf71d15fda819cd7392722fe", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "gunc/downloaddb": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "gunc/mergecheckm": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "gunc/run": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "gunzip": { "branch": "master", "git_sha": "e06548bfa36ee31869b81041879dd6b3a83b1d57", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "krakentools/kreport2krona": { "branch": "master", "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "krona/kronadb": { "branch": "master", "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "krona/ktimporttaxonomy": { "branch": "master", "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "maxbin2": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "metabat2/jgisummarizebamcontigdepths": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "metabat2/metabat2": { "branch": "master", "git_sha": "d2e220fdec3aa2f4482c70017df4cdf8a4c94f27", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "metaeuk/easypredict": { "branch": "master", "git_sha": "30d06da5bd7ae67be32758bf512cd75a4325d386", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "mmseqs/databases": { "branch": "master", "git_sha": "699e078133f580548aeb43114f93ac29928c6143", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "multiqc": { "branch": "master", "git_sha": "b7ebe95761cd389603f9cc0e0dc384c0f663815a", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "nanolyse": { "branch": "master", "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "nanoplot": { "branch": "master", "git_sha": "3135090b46f308a260fc9d5991d7d2f9c0785309", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "porechop/porechop": { "branch": "master", "git_sha": "1d68c7f248d1a480c5959548a9234602b771199e", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "prodigal": { "branch": "master", "git_sha": "603ecbd9f45300c9788f197d2a15a005685b4220", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "prokka": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "pydamage/analyze": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "pydamage/filter": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "samtools/faidx": { "branch": "master", "git_sha": "fd742419940e01ba1c5ecb172c3e32ec840662fe", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "seqtk/mergepe": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "tiara/tiara": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] }, "untar": { "branch": "master", "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", - "installed_by": ["modules"] + "installed_by": [ + "modules" + ] } } }, @@ -259,25 +350,33 @@ "fasta_binning_concoct": { "branch": "master", "git_sha": "c60c14b285b89bdd0607e371417dadb80385ad6e", - "installed_by": ["subworkflows"] + "installed_by": [ + "subworkflows" + ] }, "utils_nextflow_pipeline": { "branch": "master", "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", - "installed_by": ["subworkflows"] + "installed_by": [ + "subworkflows" + ] }, "utils_nfcore_pipeline": { "branch": "master", "git_sha": "92de218a329bfc9a9033116eb5f65fd270e72ba3", - "installed_by": ["subworkflows"] + "installed_by": [ + "subworkflows" + ] }, "utils_nfvalidation_plugin": { "branch": "master", "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", - "installed_by": ["subworkflows"] + "installed_by": [ + "subworkflows" + ] } } } } } -} +} \ No newline at end of file diff --git a/modules/nf-core/custom/dumpsoftwareversions/environment.yml b/modules/nf-core/custom/dumpsoftwareversions/environment.yml deleted file mode 100644 index 9b3272bc..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/environment.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: custom_dumpsoftwareversions -channels: - - conda-forge - - bioconda - - defaults -dependencies: - - bioconda::multiqc=1.19 diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf deleted file mode 100644 index f2187611..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ /dev/null @@ -1,24 +0,0 @@ -process CUSTOM_DUMPSOFTWAREVERSIONS { - label 'process_single' - - // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.19--pyhdfd78af_0' : - 'biocontainers/multiqc:1.19--pyhdfd78af_0' }" - - input: - path versions - - output: - path "software_versions.yml" , emit: yml - path "software_versions_mqc.yml", emit: mqc_yml - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - template 'dumpsoftwareversions.py' -} diff --git a/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/custom/dumpsoftwareversions/meta.yml deleted file mode 100644 index 5f15a5fd..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ /dev/null @@ -1,37 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json -name: custom_dumpsoftwareversions -description: Custom module used to dump software versions within the nf-core pipeline template -keywords: - - custom - - dump - - version -tools: - - custom: - description: Custom module used to dump software versions within the nf-core pipeline template - homepage: https://github.com/nf-core/tools - documentation: https://github.com/nf-core/tools - licence: ["MIT"] -input: - - versions: - type: file - description: YML file containing software versions - pattern: "*.yml" -output: - - yml: - type: file - description: Standard YML file containing software versions - pattern: "software_versions.yml" - - mqc_yml: - type: file - description: MultiQC custom content YML file containing software versions - pattern: "software_versions_mqc.yml" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@drpatelh" - - "@grst" -maintainers: - - "@drpatelh" - - "@grst" diff --git a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py deleted file mode 100755 index da033408..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ /dev/null @@ -1,101 +0,0 @@ -#!/usr/bin/env python - - -"""Provide functions to merge multiple versions.yml files.""" - - -import yaml -import platform -from textwrap import dedent - - -def _make_versions_html(versions): - """Generate a tabular HTML output of all versions for MultiQC.""" - html = [ - dedent( - """\\ - - - - - - - - - - """ - ) - ] - for process, tmp_versions in sorted(versions.items()): - html.append("") - for i, (tool, version) in enumerate(sorted(tmp_versions.items())): - html.append( - dedent( - f"""\\ - - - - - - """ - ) - ) - html.append("") - html.append("
Process Name Software Version
{process if (i == 0) else ''}{tool}{version}
") - return "\\n".join(html) - - -def main(): - """Load all version files and generate merged output.""" - versions_this_module = {} - versions_this_module["${task.process}"] = { - "python": platform.python_version(), - "yaml": yaml.__version__, - } - - with open("$versions") as f: - versions_by_process = yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module - - # aggregate versions by the module name (derived from fully-qualified process name) - versions_by_module = {} - for process, process_versions in versions_by_process.items(): - module = process.split(":")[-1] - try: - if versions_by_module[module] != process_versions: - raise AssertionError( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) - except KeyError: - versions_by_module[module] = process_versions - - versions_by_module["Workflow"] = { - "Nextflow": "$workflow.nextflow.version", - "$workflow.manifest.name": "$workflow.manifest.version", - } - - versions_mqc = { - "id": "software_versions", - "section_name": "${workflow.manifest.name} Software Versions", - "section_href": "https://github.com/${workflow.manifest.name}", - "plot_type": "html", - "description": "are collected at run time from the software output.", - "data": _make_versions_html(versions_by_module), - } - - with open("software_versions.yml", "w") as f: - yaml.dump(versions_by_module, f, default_flow_style=False) - with open("software_versions_mqc.yml", "w") as f: - yaml.dump(versions_mqc, f, default_flow_style=False) - - with open("versions.yml", "w") as f: - yaml.dump(versions_this_module, f, default_flow_style=False) - - -if __name__ == "__main__": - main() diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test deleted file mode 100644 index b1e1630b..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test +++ /dev/null @@ -1,43 +0,0 @@ -nextflow_process { - - name "Test Process CUSTOM_DUMPSOFTWAREVERSIONS" - script "../main.nf" - process "CUSTOM_DUMPSOFTWAREVERSIONS" - tag "modules" - tag "modules_nfcore" - tag "custom" - tag "dumpsoftwareversions" - tag "custom/dumpsoftwareversions" - - test("Should run without failures") { - when { - process { - """ - def tool1_version = ''' - TOOL1: - tool1: 0.11.9 - '''.stripIndent() - - def tool2_version = ''' - TOOL2: - tool2: 1.9 - '''.stripIndent() - - input[0] = Channel.of(tool1_version, tool2_version).collectFile() - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot( - process.out.versions, - file(process.out.mqc_yml[0]).readLines()[0..10], - file(process.out.yml[0]).readLines()[0..7] - ).match() - } - ) - } - } -} diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap deleted file mode 100644 index 5f59a936..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap +++ /dev/null @@ -1,33 +0,0 @@ -{ - "Should run without failures": { - "content": [ - [ - "versions.yml:md5,76d454d92244589d32455833f7c1ba6d" - ], - [ - "data: \"\\n\\n \\n \\n \\n \\n \\n \\n \\n\\", - " \\n\\n\\n \\n \\n\\", - " \\ \\n\\n\\n\\n \\n \\", - " \\ \\n \\n\\n\\n\\n\\", - " \\n\\n \\n \\n\\", - " \\ \\n\\n\\n\\n\\n\\n \\n\\", - " \\ \\n \\n\\n\\n\\n\\", - " \\n\\n \\n \\n\\" - ], - [ - "CUSTOM_DUMPSOFTWAREVERSIONS:", - " python: 3.11.7", - " yaml: 5.4.1", - "TOOL1:", - " tool1: 0.11.9", - "TOOL2:", - " tool2: '1.9'", - "Workflow:" - ] - ], - "timestamp": "2024-01-09T23:01:18.710682" - } -} \ No newline at end of file diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml b/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml deleted file mode 100644 index 405aa24a..00000000 --- a/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -custom/dumpsoftwareversions: - - modules/nf-core/custom/dumpsoftwareversions/** From 25ce9b1ca92f390d297d28b24a9752b2a46eb9b9 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 22 Aug 2024 14:22:51 +0200 Subject: [PATCH 24/27] Fix linting --- modules.json | 210 +++++++++++++-------------------------------------- 1 file changed, 53 insertions(+), 157 deletions(-) diff --git a/modules.json b/modules.json index 5dab95c1..0cab4e4e 100644 --- a/modules.json +++ b/modules.json @@ -8,340 +8,244 @@ "adapterremoval": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "aria2": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ], + "installed_by": ["modules"], "patch": "modules/nf-core/aria2/aria2.diff" }, "bbmap/bbnorm": { "branch": "master", "git_sha": "603ecbd9f45300c9788f197d2a15a005685b4220", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "bcftools/consensus": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "bcftools/index": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "bcftools/view": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "cat/fastq": { "branch": "master", "git_sha": "5c460c5a4736974abde2843294f35307ee2b0e5e", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "centrifuge/centrifuge": { "branch": "master", "git_sha": "9a07a1293d9b818d1e06d0f7b58152f74d462012", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "centrifuge/kreport": { "branch": "master", "git_sha": "9a07a1293d9b818d1e06d0f7b58152f74d462012", - "installed_by": [ - "modules" - ], + "installed_by": ["modules"], "patch": "modules/nf-core/centrifuge/kreport/centrifuge-kreport.diff" }, "checkm/lineagewf": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "checkm/qa": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "concoct/concoct": { "branch": "master", "git_sha": "baa30accc6c50ea8a98662417d4f42ed18966353", - "installed_by": [ - "fasta_binning_concoct" - ] + "installed_by": ["fasta_binning_concoct"] }, "concoct/concoctcoveragetable": { "branch": "master", "git_sha": "baa30accc6c50ea8a98662417d4f42ed18966353", - "installed_by": [ - "fasta_binning_concoct" - ] + "installed_by": ["fasta_binning_concoct"] }, "concoct/cutupfasta": { "branch": "master", "git_sha": "73a6d7e6077b88aba1c5d6805635d79d6718270c", - "installed_by": [ - "fasta_binning_concoct" - ] + "installed_by": ["fasta_binning_concoct"] }, "concoct/extractfastabins": { "branch": "master", "git_sha": "baa30accc6c50ea8a98662417d4f42ed18966353", - "installed_by": [ - "fasta_binning_concoct" - ] + "installed_by": ["fasta_binning_concoct"] }, "concoct/mergecutupclustering": { "branch": "master", "git_sha": "baa30accc6c50ea8a98662417d4f42ed18966353", - "installed_by": [ - "fasta_binning_concoct" - ] + "installed_by": ["fasta_binning_concoct"] }, "dastool/dastool": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "dastool/fastatocontig2bin": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "fastp": { "branch": "master", "git_sha": "d497a4868ace3302016ea8ed4b395072d5e833cd", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "fastqc": { "branch": "master", "git_sha": "285a50500f9e02578d90b3ce6382ea3c30216acd", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "freebayes": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "genomad/download": { "branch": "master", "git_sha": "ca813f3f73adedf3547a5a677e992d9d43a71870", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "genomad/endtoend": { "branch": "master", "git_sha": "ca813f3f73adedf3547a5a677e992d9d43a71870", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gtdbtk/classifywf": { "branch": "master", "git_sha": "0735b6d2b509cbb5cf71d15fda819cd7392722fe", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gunc/downloaddb": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gunc/mergecheckm": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gunc/run": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "gunzip": { "branch": "master", "git_sha": "e06548bfa36ee31869b81041879dd6b3a83b1d57", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "krakentools/kreport2krona": { "branch": "master", "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "krona/kronadb": { "branch": "master", "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "krona/ktimporttaxonomy": { "branch": "master", "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "maxbin2": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "metabat2/jgisummarizebamcontigdepths": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "metabat2/metabat2": { "branch": "master", "git_sha": "d2e220fdec3aa2f4482c70017df4cdf8a4c94f27", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "metaeuk/easypredict": { "branch": "master", "git_sha": "30d06da5bd7ae67be32758bf512cd75a4325d386", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "mmseqs/databases": { "branch": "master", "git_sha": "699e078133f580548aeb43114f93ac29928c6143", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "multiqc": { "branch": "master", "git_sha": "b7ebe95761cd389603f9cc0e0dc384c0f663815a", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "nanolyse": { "branch": "master", "git_sha": "3f5420aa22e00bd030a2556dfdffc9e164ec0ec5", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "nanoplot": { "branch": "master", "git_sha": "3135090b46f308a260fc9d5991d7d2f9c0785309", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "porechop/porechop": { "branch": "master", "git_sha": "1d68c7f248d1a480c5959548a9234602b771199e", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "prodigal": { "branch": "master", "git_sha": "603ecbd9f45300c9788f197d2a15a005685b4220", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "prokka": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "pydamage/analyze": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "pydamage/filter": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "samtools/faidx": { "branch": "master", "git_sha": "fd742419940e01ba1c5ecb172c3e32ec840662fe", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "seqtk/mergepe": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "tiara/tiara": { "branch": "master", "git_sha": "911696ea0b62df80e900ef244d7867d177971f73", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] }, "untar": { "branch": "master", "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", - "installed_by": [ - "modules" - ] + "installed_by": ["modules"] } } }, @@ -350,33 +254,25 @@ "fasta_binning_concoct": { "branch": "master", "git_sha": "c60c14b285b89bdd0607e371417dadb80385ad6e", - "installed_by": [ - "subworkflows" - ] + "installed_by": ["subworkflows"] }, "utils_nextflow_pipeline": { "branch": "master", "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", - "installed_by": [ - "subworkflows" - ] + "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { "branch": "master", "git_sha": "92de218a329bfc9a9033116eb5f65fd270e72ba3", - "installed_by": [ - "subworkflows" - ] + "installed_by": ["subworkflows"] }, "utils_nfvalidation_plugin": { "branch": "master", "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", - "installed_by": [ - "subworkflows" - ] + "installed_by": ["subworkflows"] } } } } } -} \ No newline at end of file +} From 70e647cd0ace2c0bd2de2e55cc67bf3993febb9a Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 22 Aug 2024 14:38:23 +0200 Subject: [PATCH 25/27] Remove reference to CUSTOM_DUMPSOFTWAREVERSONS in base.config --- conf/base.config | 4 ---- 1 file changed, 4 deletions(-) diff --git a/conf/base.config b/conf/base.config index 4cbf14f0..2928a99b 100644 --- a/conf/base.config +++ b/conf/base.config @@ -58,10 +58,6 @@ process { maxRetries = 2 } - withName:CUSTOM_DUMPSOFTWAREVERSIONS { - cache = false - } - withName: BOWTIE2_HOST_REMOVAL_BUILD { cpus = { check_max (10 * task.attempt, 'cpus' ) } memory = { check_max (20.GB * task.attempt, 'memory' ) } From 5972c67372de806b28348808c4a3ffa4626ccf10 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Thu, 22 Aug 2024 15:05:50 +0200 Subject: [PATCH 26/27] Dump versions for release --- CHANGELOG.md | 2 +- assets/multiqc_config.yml | 4 ++-- nextflow.config | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccb1decf..d0eff648 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## dev [unreleased] +## 3.0.3 [2024-08-23] ### `Added` diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index 9f4e5e7e..10c24150 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -1,7 +1,7 @@ report_comment: > - This report has been generated by the nf-core/mag + This report has been generated by the nf-core/mag analysis pipeline. For information about how to interpret these results, please see the - documentation. + documentation. report_section_order: "nf-core-mag-methods-description": order: -1000 diff --git a/nextflow.config b/nextflow.config index 9d2b2f44..9306ae99 100644 --- a/nextflow.config +++ b/nextflow.config @@ -390,7 +390,7 @@ manifest { description = """Assembly, binning and annotation of metagenomes""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = 'dev' + version = '3.0.3' doi = '10.1093/nargab/lqac007' } From f629418c9137c4cc9642fbeb6c5c09c5a4266a40 Mon Sep 17 00:00:00 2001 From: "James A. Fellows Yates" Date: Tue, 27 Aug 2024 09:51:59 +0200 Subject: [PATCH 27/27] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0eff648..6089039e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## 3.0.3 [2024-08-23] +## 3.0.3 [2024-08-27] ### `Added`
Process Name \\", - " \\ Software Version
CUSTOM_DUMPSOFTWAREVERSIONSpython3.11.7
yaml5.4.1
TOOL1tool10.11.9
TOOL2tool21.9
WorkflowNextflow