From 104bb763c82e18cc3768abed10b251ebba321c0e Mon Sep 17 00:00:00 2001 From: Felix Lenner <52530259+fellen31@users.noreply.github.com> Date: Tue, 7 Jan 2025 09:30:49 +0100 Subject: [PATCH 1/4] Add missing file input to bcftools/annotate (#6942) Add input to bcftools/annotate --- modules/nf-core/bcftools/annotate/main.nf | 3 + modules/nf-core/bcftools/annotate/meta.yml | 3 + .../bcftools/annotate/tests/main.nf.test | 67 +++- .../bcftools/annotate/tests/main.nf.test.snap | 315 ++++++++++-------- 4 files changed, 231 insertions(+), 157 deletions(-) diff --git a/modules/nf-core/bcftools/annotate/main.nf b/modules/nf-core/bcftools/annotate/main.nf index 9ffdfdf0a49..b6b73309b89 100644 --- a/modules/nf-core/bcftools/annotate/main.nf +++ b/modules/nf-core/bcftools/annotate/main.nf @@ -10,6 +10,7 @@ process BCFTOOLS_ANNOTATE { input: tuple val(meta), path(input), path(index), path(annotations), path(annotations_index) path(header_lines) + path(rename_chrs) output: tuple val(meta), path("*.{vcf,vcf.gz,bcf,bcf.gz}"), emit: vcf @@ -25,6 +26,7 @@ process BCFTOOLS_ANNOTATE { def prefix = task.ext.prefix ?: "${meta.id}" def header_file = header_lines ? "--header-lines ${header_lines}" : '' def annotations_file = annotations ? "--annotations ${annotations}" : '' + def rename_chrs_file = rename_chrs ? "--rename-chrs ${rename_chrs}" : '' def extension = args.contains("--output-type b") || args.contains("-Ob") ? "bcf.gz" : args.contains("--output-type u") || args.contains("-Ou") ? "bcf" : args.contains("--output-type z") || args.contains("-Oz") ? "vcf.gz" : @@ -40,6 +42,7 @@ process BCFTOOLS_ANNOTATE { annotate \\ $args \\ $annotations_file \\ + $rename_chrs_file \\ $header_file \\ --output ${prefix}.${extension} \\ --threads $task.cpus \\ diff --git a/modules/nf-core/bcftools/annotate/meta.yml b/modules/nf-core/bcftools/annotate/meta.yml index 5bfccd2bd8f..fd2c91c706a 100644 --- a/modules/nf-core/bcftools/annotate/meta.yml +++ b/modules/nf-core/bcftools/annotate/meta.yml @@ -35,6 +35,9 @@ input: - - header_lines: type: file description: Contains lines to append to the output VCF header + - - rename_chrs: + type: file + description: Rename annotations according to this file containing "old_name new_name\n" pairs separated by whitespaces, each on a separate line. output: - vcf: - meta: diff --git a/modules/nf-core/bcftools/annotate/tests/main.nf.test b/modules/nf-core/bcftools/annotate/tests/main.nf.test index 3a5c493314f..fd6d2cd37f6 100644 --- a/modules/nf-core/bcftools/annotate/tests/main.nf.test +++ b/modules/nf-core/bcftools/annotate/tests/main.nf.test @@ -9,7 +9,7 @@ nextflow_process { tag "bcftools" tag "bcftools/annotate" - test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_output") { + test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_output") { config "./vcf.config" @@ -24,6 +24,7 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi', checkIfExists: true) ] input[1] = [] + input[2] = [] """ } } @@ -40,7 +41,7 @@ nextflow_process { } - test("sarscov2 - [vcf, [], annotation, annotation_tbi], [] - vcf_output") { + test("sarscov2 - [vcf, [], annotation, annotation_tbi], [], [] - vcf_output") { config "./vcf.config" @@ -55,6 +56,7 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi', checkIfExists: true) ] input[1] = [] + input[2] = [] """ } } @@ -70,7 +72,7 @@ nextflow_process { } } - test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_gz_index") { + test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_gz_index") { config "./vcf_gz_index.config" @@ -85,6 +87,7 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi', checkIfExists: true) ] input[1] = [] + input[2] = [] """ } } @@ -104,7 +107,7 @@ nextflow_process { } - test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_gz_index_csi") { + test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_gz_index_csi") { config "./vcf_gz_index_csi.config" @@ -119,6 +122,7 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi', checkIfExists: true) ] input[1] = [] + input[2] = [] """ } } @@ -138,7 +142,7 @@ nextflow_process { } - test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_gz_index_tbi") { + test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_gz_index_tbi") { config "./vcf_gz_index_tbi.config" @@ -153,6 +157,7 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi', checkIfExists: true) ] input[1] = [] + input[2] = [] """ } } @@ -171,7 +176,8 @@ nextflow_process { } } - test("sarscov2 - [vcf, [], annotation, annotation_tbi], header - bcf_output") { + + test("sarscov2 - [vcf, [], annotation, annotation_tbi], header, [] - bcf_output") { config "./bcf.config" @@ -189,6 +195,7 @@ nextflow_process { '##INFO=', '##INFO=' ).collectFile(name:"headers.vcf", newLine:true) + input[2] = [] """ } } @@ -205,7 +212,43 @@ nextflow_process { } - test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - stub") { + test("sarscov2 - [vcf, [], annotation, annotation_tbi], header, rename_chrs - vcf_gz_index") { + + config "./vcf_gz_index.config" + + when { + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test.vcf.gz', checkIfExists: true), + [], + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test2.vcf.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi', checkIfExists: true) + ] + input[1] = Channel.of( + '##INFO=', + '##INFO=' + ).collectFile(name:"headers.vcf", newLine:true) + input[2] = Channel.of('MT192765.1 renamed').collectFile(name:"rename.txt", newLine:true) + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert path(process.out.vcf.get(0).get(1)).LinesGzip.contains("##contig=")}, + { assert snapshot( + process.out.vcf.collect { it.collect { it instanceof Map ? it : file(it).name }}, + process.out.versions + ).match() } + ) + } + + } + + test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - stub") { config "./vcf.config" options "-stub" @@ -221,6 +264,7 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi', checkIfExists: true) ] input[1] = [] + input[2] = [] """ } } @@ -234,7 +278,7 @@ nextflow_process { } - test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_gz_index - stub") { + test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_gz_index - stub") { config "./vcf_gz_index.config" options "-stub" @@ -250,6 +294,7 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi', checkIfExists: true) ] input[1] = [] + input[2] = [] """ } } @@ -264,7 +309,7 @@ nextflow_process { } - test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_gz_index_csi - stub") { + test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_gz_index_csi - stub") { config "./vcf_gz_index_csi.config" options "-stub" @@ -280,6 +325,7 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi', checkIfExists: true) ] input[1] = [] + input[2] = [] """ } } @@ -294,7 +340,7 @@ nextflow_process { } - test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_gz_index_tbi - stub") { + test("sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_gz_index_tbi - stub") { config "./vcf_gz_index_tbi.config" options "-stub" @@ -310,6 +356,7 @@ nextflow_process { file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/vcf/test2.vcf.gz.tbi', checkIfExists: true) ] input[1] = [] + input[2] = [] """ } } diff --git a/modules/nf-core/bcftools/annotate/tests/main.nf.test.snap b/modules/nf-core/bcftools/annotate/tests/main.nf.test.snap index bac2224a3b5..16828b6ce04 100644 --- a/modules/nf-core/bcftools/annotate/tests/main.nf.test.snap +++ b/modules/nf-core/bcftools/annotate/tests/main.nf.test.snap @@ -1,4 +1,37 @@ { + "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_gz_index_tbi": { + "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + "test_vcf.vcf.gz" + ] + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test_vcf.vcf.gz.tbi" + ] + ], + [ + + ], + [ + "versions.yml:md5,ea53f98610d42597cf384ff1fa3eb204" + ] + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-06T14:02:05.809043894" + }, "bcf": { "content": [ [ @@ -20,7 +53,7 @@ }, "timestamp": "2024-06-12T16:39:33.331888" }, - "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_gz_index": { + "sarscov2 - [vcf, [], annotation, annotation_tbi], [], [] - vcf_output": { "content": [ [ [ @@ -32,15 +65,24 @@ ] ], [ - - ], + "versions.yml:md5,ea53f98610d42597cf384ff1fa3eb204" + ] + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-06T14:01:32.326248064" + }, + "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_output": { + "content": [ [ [ { "id": "test", "single_end": false }, - "test_vcf.vcf.gz.csi" + "test_vcf.vcf.gz" ] ], [ @@ -48,12 +90,12 @@ ] ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.1", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-15T10:07:59.658031137" + "timestamp": "2024-11-06T14:01:17.821142466" }, - "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_gz_index_csi - stub": { + "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_gz_index_tbi - stub": { "content": [ { "0": [ @@ -66,31 +108,31 @@ ] ], "1": [ - - ], - "2": [ [ { "id": "test", "single_end": false }, - "test_vcf.vcf.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + "test_vcf.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" ] + ], + "2": [ + ], "3": [ "versions.yml:md5,ea53f98610d42597cf384ff1fa3eb204" ], "csi": [ + + ], + "tbi": [ [ { "id": "test", "single_end": false }, - "test_vcf.vcf.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" + "test_vcf.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" ] - ], - "tbi": [ - ], "vcf": [ [ @@ -107,45 +149,12 @@ } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.1", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-15T10:09:05.096883418" + "timestamp": "2024-11-06T14:03:12.574479421" }, - "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_gz_index_csi": { - "content": [ - [ - [ - { - "id": "test", - "single_end": false - }, - "test_vcf.vcf.gz" - ] - ], - [ - - ], - [ - [ - { - "id": "test", - "single_end": false - }, - "test_vcf.vcf.gz.csi" - ] - ], - [ - "versions.yml:md5,ea53f98610d42597cf384ff1fa3eb204" - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" - }, - "timestamp": "2024-08-15T10:08:10.581301219" - }, - "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - stub": { + "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - stub": { "content": [ { "0": [ @@ -187,87 +196,12 @@ } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" - }, - "timestamp": "2024-08-15T10:08:43.975017625" - }, - "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_gz_index_tbi": { - "content": [ - [ - [ - { - "id": "test", - "single_end": false - }, - "test_vcf.vcf.gz" - ] - ], - [ - [ - { - "id": "test", - "single_end": false - }, - "test_vcf.vcf.gz.tbi" - ] - ], - [ - - ], - [ - "versions.yml:md5,ea53f98610d42597cf384ff1fa3eb204" - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.1", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-15T10:08:21.354059092" + "timestamp": "2024-11-06T14:02:38.89910979" }, - "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_output": { - "content": [ - [ - [ - { - "id": "test", - "single_end": false - }, - "test_vcf.vcf.gz" - ] - ], - [ - "versions.yml:md5,ea53f98610d42597cf384ff1fa3eb204" - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" - }, - "timestamp": "2024-08-15T10:07:37.788393317" - }, - "sarscov2 - [vcf, [], annotation, annotation_tbi], [] - vcf_output": { - "content": [ - [ - [ - { - "id": "test", - "single_end": false - }, - "test_vcf.vcf.gz" - ] - ], - [ - "versions.yml:md5,ea53f98610d42597cf384ff1fa3eb204" - ] - ], - "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" - }, - "timestamp": "2024-08-15T10:07:48.500746325" - }, - "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_gz_index_tbi - stub": { + "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_gz_index - stub": { "content": [ { "0": [ @@ -280,31 +214,31 @@ ] ], "1": [ + + ], + "2": [ [ { "id": "test", "single_end": false }, - "test_vcf.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + "test_vcf.vcf.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" ] - ], - "2": [ - ], "3": [ "versions.yml:md5,ea53f98610d42597cf384ff1fa3eb204" ], "csi": [ - - ], - "tbi": [ [ { "id": "test", "single_end": false }, - "test_vcf.vcf.gz.tbi:md5,d41d8cd98f00b204e9800998ecf8427e" + "test_vcf.vcf.gz.csi:md5,d41d8cd98f00b204e9800998ecf8427e" ] + ], + "tbi": [ + ], "vcf": [ [ @@ -321,12 +255,12 @@ } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.1", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-15T10:09:16.094918834" + "timestamp": "2024-11-06T14:02:49.803045339" }, - "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [] - vcf_gz_index - stub": { + "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_gz_index_csi - stub": { "content": [ { "0": [ @@ -380,9 +314,96 @@ } ], "meta": { - "nf-test": "0.8.4", - "nextflow": "24.04.2" + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-06T14:03:00.654715777" + }, + "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_gz_index": { + "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + "test_vcf.vcf.gz" + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test_vcf.vcf.gz.csi" + ] + ], + [ + "versions.yml:md5,ea53f98610d42597cf384ff1fa3eb204" + ] + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-06T14:01:45.446114465" + }, + "sarscov2 - [vcf, [], annotation, annotation_tbi], header, rename_chrs - vcf_gz_index": { + "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + "test_vcf.vcf.gz" + ] + ], + [ + "versions.yml:md5,ea53f98610d42597cf384ff1fa3eb204" + ] + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" + }, + "timestamp": "2024-11-06T13:58:14.280394724" + }, + "sarscov2 - [vcf, tbi, annotation, annotation_tbi], [], [] - vcf_gz_index_csi": { + "content": [ + [ + [ + { + "id": "test", + "single_end": false + }, + "test_vcf.vcf.gz" + ] + ], + [ + + ], + [ + [ + { + "id": "test", + "single_end": false + }, + "test_vcf.vcf.gz.csi" + ] + ], + [ + "versions.yml:md5,ea53f98610d42597cf384ff1fa3eb204" + ] + ], + "meta": { + "nf-test": "0.9.1", + "nextflow": "24.10.0" }, - "timestamp": "2024-08-15T10:08:54.366358502" + "timestamp": "2024-11-06T14:01:55.70445216" } } \ No newline at end of file From 09681763f3d9c772b26599d75b669957a04b1d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20H=C3=B6rtenhuber?= Date: Tue, 7 Jan 2025 17:16:53 +0100 Subject: [PATCH 2/4] Fix docker ownserhip issue and remove userEmulation (#7272) * fix docker ownership issue * remove deprecated docker.userEmulation * add stub test * move setup step to beginning of process * remove stub test again --- .../checkm2/predict/tests/main.nf.test | 7 ++--- .../checkm2/predict/tests/main.nf.test.snap | 29 +++++++++++++++++++ tests/config/nf-test.config | 3 +- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/modules/nf-core/checkm2/predict/tests/main.nf.test b/modules/nf-core/checkm2/predict/tests/main.nf.test index de2a3c85910..1e64ff4dde3 100644 --- a/modules/nf-core/checkm2/predict/tests/main.nf.test +++ b/modules/nf-core/checkm2/predict/tests/main.nf.test @@ -9,9 +9,7 @@ nextflow_process { script "modules/nf-core/checkm2/predict/main.nf" process "CHECKM2_PREDICT" - test("Test CheckM2 Predict") { - - setup { + setup { run("CHECKM2_DATABASEDOWNLOAD") { script "../../databasedownload/main.nf" process { @@ -22,6 +20,8 @@ nextflow_process { } } + test("Test CheckM2 Predict") { + when { params { outdir = "${launchDir}/tests/results" @@ -40,7 +40,6 @@ nextflow_process { { assert snapshot(process.out.checkm2_tsv, process.out.versions).match() } ) } - } } \ No newline at end of file diff --git a/modules/nf-core/checkm2/predict/tests/main.nf.test.snap b/modules/nf-core/checkm2/predict/tests/main.nf.test.snap index 4364c12f41b..9702543c337 100644 --- a/modules/nf-core/checkm2/predict/tests/main.nf.test.snap +++ b/modules/nf-core/checkm2/predict/tests/main.nf.test.snap @@ -1,4 +1,33 @@ { + "Test CheckM2 Predict - stub": { + "content": [ + { + "0": [ + + ], + "1": [ + + ], + "2": [ + + ], + "checkm2_output": [ + + ], + "checkm2_tsv": [ + + ], + "versions": [ + + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.3" + }, + "timestamp": "2025-01-07T15:20:27.358984" + }, "Test CheckM2 Predict": { "content": [ [ diff --git a/tests/config/nf-test.config b/tests/config/nf-test.config index daaf2ffe2ff..d8350fafaea 100644 --- a/tests/config/nf-test.config +++ b/tests/config/nf-test.config @@ -27,9 +27,8 @@ profiles { } docker_self_hosted { docker.enabled = true - docker.userEmulation = false docker.fixOwnership = true - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + docker.runOptions = '--platform=linux/amd64' } arm { docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' From cb8e4c926f1e8de22aa1f1375e188bd844c693f8 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 7 Jan 2025 17:30:42 +0100 Subject: [PATCH 3/4] Update GitHub Actions (#7239) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/actions/nf-test-action/action.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/pytest-workflow.yml | 4 ++-- .github/workflows/wave.yml | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/nf-test-action/action.yml b/.github/actions/nf-test-action/action.yml index a6f2532f926..a2539a6ba2f 100644 --- a/.github/actions/nf-test-action/action.yml +++ b/.github/actions/nf-test-action/action.yml @@ -17,7 +17,7 @@ inputs: runs: using: "composite" steps: - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + - uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4 with: distribution: "temurin" java-version: "17" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c617c10ccae..a743b62a57a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -120,7 +120,7 @@ jobs: - name: Install pip run: python -m pip install --upgrade pip - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + - uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4 with: distribution: "temurin" java-version: "17" diff --git a/.github/workflows/pytest-workflow.yml b/.github/workflows/pytest-workflow.yml index 7c0d70c77c2..ec33e8be0dd 100644 --- a/.github/workflows/pytest-workflow.yml +++ b/.github/workflows/pytest-workflow.yml @@ -258,7 +258,7 @@ jobs: - name: Install Python dependencies run: python -m pip install --upgrade pip pytest-workflow cryptography - - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4 + - uses: actions/setup-java@7a6d8a8234af8eb26422e24e3006232cccaa061b # v4 with: distribution: "temurin" java-version: "17" @@ -308,7 +308,7 @@ jobs: - name: Upload logs on failure if: failure() - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4 + uses: actions/upload-artifact@6f51ac03b9356f520e9adb1b1b7802705f340c2b # v4 with: name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }} path: | diff --git a/.github/workflows/wave.yml b/.github/workflows/wave.yml index b56ed4795d6..f4557eec329 100644 --- a/.github/workflows/wave.yml +++ b/.github/workflows/wave.yml @@ -31,7 +31,7 @@ jobs: - name: Find conda differences id: conda-diff - uses: tj-actions/changed-files@bab30c2299617f6615ec02a68b9a40d10bd21366 # v45 + uses: tj-actions/changed-files@d6e91a2266cdb9d62096cebf1e8546899c6aa18f # v45 with: json: true escape_json: false @@ -40,7 +40,7 @@ jobs: - name: Find Dockerfile differences id: docker-diff - uses: tj-actions/changed-files@bab30c2299617f6615ec02a68b9a40d10bd21366 # v45 + uses: tj-actions/changed-files@d6e91a2266cdb9d62096cebf1e8546899c6aa18f # v45 with: json: true escape_json: false From 70538021b9b92958afa73c697d0d5decea7aa9ca Mon Sep 17 00:00:00 2001 From: MazzaLab Date: Tue, 7 Jan 2025 19:54:45 +0100 Subject: [PATCH 4/4] Added BBMAP/REPAIR submodule (#7249) * New submodule: BBMAP/REPAIR * BBMAP/repair snapshot added * removed cpu setting in nextflow.config --------- Co-authored-by: irongraft --- modules/nf-core/bbmap/repair/environment.yml | 7 + modules/nf-core/bbmap/repair/main.nf | 57 +++++++ modules/nf-core/bbmap/repair/meta.yml | 67 ++++++++ .../nf-core/bbmap/repair/tests/main.nf.test | 115 +++++++++++++ .../bbmap/repair/tests/main.nf.test.snap | 156 ++++++++++++++++++ .../bbmap/repair/tests/nextflow.config | 5 + modules/nf-core/bbmap/repair/tests/tags.yml | 2 + 7 files changed, 409 insertions(+) create mode 100644 modules/nf-core/bbmap/repair/environment.yml create mode 100644 modules/nf-core/bbmap/repair/main.nf create mode 100644 modules/nf-core/bbmap/repair/meta.yml create mode 100644 modules/nf-core/bbmap/repair/tests/main.nf.test create mode 100644 modules/nf-core/bbmap/repair/tests/main.nf.test.snap create mode 100644 modules/nf-core/bbmap/repair/tests/nextflow.config create mode 100644 modules/nf-core/bbmap/repair/tests/tags.yml diff --git a/modules/nf-core/bbmap/repair/environment.yml b/modules/nf-core/bbmap/repair/environment.yml new file mode 100644 index 00000000000..4e65bfe6a99 --- /dev/null +++ b/modules/nf-core/bbmap/repair/environment.yml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - "bioconda::bbmap=39.13" diff --git a/modules/nf-core/bbmap/repair/main.nf b/modules/nf-core/bbmap/repair/main.nf new file mode 100644 index 00000000000..9c12be2cf53 --- /dev/null +++ b/modules/nf-core/bbmap/repair/main.nf @@ -0,0 +1,57 @@ +process BBMAP_REPAIR { + tag "$meta.id" + label 'process_single' + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/bbmap:39.13--he5f24ec_1': + 'biocontainers/bbmap:39.13--he5f24ec_1' }" + + input: + tuple val(meta), path(reads) + val(interleave) + + output: + tuple val(meta), path("*_repaired.fastq.gz") , emit: repaired + tuple val(meta), path("${prefix}_singleton.fastq.gz"), emit: singleton + path "versions.yml" , emit: versions + path "*.log" , emit: log + + when: + task.ext.when == null || task.ext.when + + script: + def args = task.ext.args ?: '' + prefix = task.ext.prefix ?: "${meta.id}" + in_reads = ( interleave ) ? "in=${reads[0]}" : "in=${reads[0]} in2=${reads[1]}" + out_reads = ( interleave ) ? "out=${prefix}_repaired.fastq.gz outs=${prefix}_singleton.fastq.gz" + : "out=${prefix}_1_repaired.fastq.gz out2=${prefix}_2_repaired.fastq.gz outs=${prefix}_singleton.fastq.gz" + """ + maxmem=\$(echo \"$task.memory\"| sed 's/ GB/g/g') + repair.sh \\ + -Xmx\$maxmem \\ + $in_reads \\ + $out_reads \\ + threads=${task.cpus} + ${args} \\ + &> ${prefix}.repair.sh.log + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + repair.sh: \$(repair.sh --version |& sed '2!d ; s/BBMap version //') + END_VERSIONS + """ + + stub: + prefix = task.ext.prefix ?: "${meta.id}" + """ + echo "" | gzip > ${prefix}_1_repaired.fastq.gz + echo "" | gzip > ${prefix}_2_repaired.fastq.gz + echo "" | gzip > ${prefix}_singleton.fastq.gz + touch ${prefix}.repair.sh.log + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + repair.sh: \$(repair.sh --version |& sed '2!d ; s/BBMap version //') + END_VERSIONS + """ +} diff --git a/modules/nf-core/bbmap/repair/meta.yml b/modules/nf-core/bbmap/repair/meta.yml new file mode 100644 index 00000000000..9190cd99c5e --- /dev/null +++ b/modules/nf-core/bbmap/repair/meta.yml @@ -0,0 +1,67 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json +name: "bbmap_repair" +description: Re-pairs reads that became disordered or had some mates eliminated. +keywords: + - paired reads re-pairing + - fastq + - preprocessing +tools: + - repair: + description: Repair.sh is a tool that re-pairs reads that became disordered or had some mates eliminated + tools. + homepage: https://jgi.doe.gov/data-and-tools/software-tools/bbtools/ + documentation: https://jgi.doe.gov/data-and-tools/software-tools/bbtools/bb-tools-user-guide/ + licence: ["UC-LBL license (see package)"] + identifier: biotools:bbmap + +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - reads: + type: file + description: | + List of input paired end fastq files + pattern: "*.{fastq,fq}.gz" + - - interleave: + type: boolean + description: | + Indicates whether the input paired reads are interleaved or not +output: + - repaired: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - "*_repaired.fastq.gz": + type: file + description: re-paired reads + pattern: "*_repaired.fastq.gz" + - singleton: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]` + - "${prefix}_singleton.fastq.gz": + type: file + description: singleton reads + pattern: "*singleton.fastq.gz" + - versions: + - versions.yml: + type: file + description: File containing software versions + pattern: "versions.yml" + - log: + - "*.log": + type: file + description: log file containing stdout and stderr from repair.sh + pattern: "*.log" +authors: + - "@mazzalab" +maintainers: + - "@mazzalab" + - "@tm4zza" diff --git a/modules/nf-core/bbmap/repair/tests/main.nf.test b/modules/nf-core/bbmap/repair/tests/main.nf.test new file mode 100644 index 00000000000..ea6ff0f3664 --- /dev/null +++ b/modules/nf-core/bbmap/repair/tests/main.nf.test @@ -0,0 +1,115 @@ +nextflow_process { + + name "Test Process BBMAP_REPAIR" + script "../main.nf" + process "BBMAP_REPAIR" + + tag "modules" + tag "modules_nfcore" + tag "bbmap" + tag "bbmap/repair" + + test("sarscov2_illumina_paired - fastq_gz") { + config "./nextflow.config" + + when { + params { + module_args = 'qin=33' + } + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + [file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true)] + ] + input[1] = false + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.versions).match() } + ) + } + } + + test("sarscov2_illumina_interleaved - fastq_gz") { + config "./nextflow.config" + + when { + params { + module_args = 'qin=33' + } + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + [file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_interleaved.fastq.gz', checkIfExists: true)] + ] + input[1] = true + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out.versions).match() } + ) + } + } + + test("sarscov2_illumina_paired - fastq_gz - stub") { + config "./nextflow.config" + options "-stub" + + when { + params { + module_args = 'qin=33' + } + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + [file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_1.fastq.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_2.fastq.gz', checkIfExists: true)] + ] + input[1] = false + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } + + test("sarscov2_illumina_interleaved - fastq_gz - stub") { + config "./nextflow.config" + options "-stub" + + when { + params { + module_args = 'qin=33' + } + process { + """ + input[0] = [ + [ id:'test', single_end:false ], // meta map + [file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/fastq/test_interleaved.fastq.gz', checkIfExists: true)] + ] + input[1] = true + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/nf-core/bbmap/repair/tests/main.nf.test.snap b/modules/nf-core/bbmap/repair/tests/main.nf.test.snap new file mode 100644 index 00000000000..54e929b0611 --- /dev/null +++ b/modules/nf-core/bbmap/repair/tests/main.nf.test.snap @@ -0,0 +1,156 @@ +{ + "sarscov2_illumina_paired - fastq_gz": { + "content": [ + [ + "versions.yml:md5,006e8f2167df95a778b428413ed9d859" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.2" + }, + "timestamp": "2024-12-19T11:01:18.284624712" + }, + "sarscov2_illumina_interleaved - fastq_gz - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1_repaired.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "test_2_repaired.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test_singleton.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "2": [ + "versions.yml:md5,006e8f2167df95a778b428413ed9d859" + ], + "3": [ + "test.repair.sh.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + "log": [ + "test.repair.sh.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + "repaired": [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1_repaired.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "test_2_repaired.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ] + ], + "singleton": [ + [ + { + "id": "test", + "single_end": false + }, + "test_singleton.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "versions": [ + "versions.yml:md5,006e8f2167df95a778b428413ed9d859" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.2" + }, + "timestamp": "2024-12-19T11:02:01.385727451" + }, + "sarscov2_illumina_interleaved - fastq_gz": { + "content": [ + [ + "versions.yml:md5,006e8f2167df95a778b428413ed9d859" + ] + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.2" + }, + "timestamp": "2024-12-19T11:01:32.752620818" + }, + "sarscov2_illumina_paired - fastq_gz - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1_repaired.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "test_2_repaired.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ] + ], + "1": [ + [ + { + "id": "test", + "single_end": false + }, + "test_singleton.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "2": [ + "versions.yml:md5,006e8f2167df95a778b428413ed9d859" + ], + "3": [ + "test.repair.sh.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + "log": [ + "test.repair.sh.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ], + "repaired": [ + [ + { + "id": "test", + "single_end": false + }, + [ + "test_1_repaired.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940", + "test_2_repaired.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ] + ], + "singleton": [ + [ + { + "id": "test", + "single_end": false + }, + "test_singleton.fastq.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] + ], + "versions": [ + "versions.yml:md5,006e8f2167df95a778b428413ed9d859" + ] + } + ], + "meta": { + "nf-test": "0.9.2", + "nextflow": "24.10.2" + }, + "timestamp": "2024-12-19T11:01:47.341193946" + } +} \ No newline at end of file diff --git a/modules/nf-core/bbmap/repair/tests/nextflow.config b/modules/nf-core/bbmap/repair/tests/nextflow.config new file mode 100644 index 00000000000..6b9f0bf755b --- /dev/null +++ b/modules/nf-core/bbmap/repair/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: BBMAP_REPAIR { + ext.args = params.module_args + } +} diff --git a/modules/nf-core/bbmap/repair/tests/tags.yml b/modules/nf-core/bbmap/repair/tests/tags.yml new file mode 100644 index 00000000000..a6e491df28f --- /dev/null +++ b/modules/nf-core/bbmap/repair/tests/tags.yml @@ -0,0 +1,2 @@ +bbmap/repair: + - "modules/nf-core/bbmap/repair/**"