From c4ca9761f51f9a3d505e573ec777bc53e1a8fe1a Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Fri, 5 Apr 2024 22:50:24 +0000 Subject: [PATCH 01/14] Added ability to parse setup runs in nf-tests --- .github/python/find_changed_files.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py index c0c6175..7997eeb 100644 --- a/.github/python/find_changed_files.py +++ b/.github/python/find_changed_files.py @@ -185,7 +185,7 @@ def detect_nf_test_files(changed_files: list[Path]) -> list[Path]: return result -def process_files(files: list[Path]) -> list[str]: +def process_nf_test_files(files: list[Path]) -> list[str]: """ Process the files and return lines that begin with 'workflow', 'process', or 'function' and have a single string afterwards. @@ -239,6 +239,13 @@ def convert_nf_test_files_to_test_types( name = words[1].strip("'\"") # Strip both single and double quotes if keyword in types: result[keyword].append(name) + elif "run" in line: + run_words = words[0].strip(')').split('(') + if len(run_words) == 2 and re.match(r'^".*"$', run_words[1]): + keyword = run_words[0] + name = run_words[1].strip("'\"") # Strip both single and double quotes + if keyword in types: + result[keyword].append(name) return result @@ -266,8 +273,8 @@ def find_changed_dependencies(paths: list[Path], tags: list[str]) -> list[Path]: # Make case insensitive with .casefold() tags_in_nf_test_file = [ tag.casefold().replace("/", "_") - for tag in convert_nf_test_files_to_test_types(lines, types=["tag"])[ - "tag" + for tag in convert_nf_test_files_to_test_types(lines, types=["run"])[ + "run" ] ] # Check if tag in nf-test file appears in a tag. @@ -296,7 +303,7 @@ def find_changed_dependencies(paths: list[Path], tags: list[str]) -> list[Path]: changed_files, include_files ) nf_test_files = detect_nf_test_files(changed_files) - lines = process_files(nf_test_files) + lines = process_nf_test_files(nf_test_files) result = convert_nf_test_files_to_test_types(lines) # Get only relevant results (specified by -t) From 72366a2678872150022597d35861c8b23b78aaab Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Fri, 5 Apr 2024 22:52:32 +0000 Subject: [PATCH 02/14] Test changes --- modules/local/viromeqc/install/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/local/viromeqc/install/main.nf b/modules/local/viromeqc/install/main.nf index 7e682a2..a49dadb 100644 --- a/modules/local/viromeqc/install/main.nf +++ b/modules/local/viromeqc/install/main.nf @@ -1,6 +1,7 @@ process VIROMEQC_INSTALL { label 'process_low' + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/mulled-v2-b28a1a551d380ce8d57f9d83894ccb9559b44404:08a4cf815fcef0080ede5b3633202cbda8edf59b-0': From a0b657d544e97cc7f4524e378e8365deba98e1d6 Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 8 Apr 2024 16:21:08 +0000 Subject: [PATCH 03/14] Automatically find nf-files with changed dependencies via include statements --- .github/python/find_changed_files.py | 73 ++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py index 7997eeb..4e91751 100644 --- a/.github/python/find_changed_files.py +++ b/.github/python/find_changed_files.py @@ -158,12 +158,13 @@ def detect_include_files( return new_changed_files -def detect_nf_test_files(changed_files: list[Path]) -> list[Path]: +def detect_files(changed_files: list[Path], suffix: str) -> list[Path]: """ Detects and returns a list of nf-test files from the given list of changed files. Args: changed_files (list[Path]): A list of file paths. + suffix (str): File suffix to detect Returns: list[Path]: A list of nf-test file paths. @@ -171,7 +172,7 @@ def detect_nf_test_files(changed_files: list[Path]) -> list[Path]: result: list[Path] = [] for path in changed_files: # If Path is the exact nf-test file add to list: - if path.match("*.nf.test") and path.exists(): + if path.match(suffix) and path.exists(): result.append(path) # Else recursively search for nf-test files: else: @@ -180,7 +181,7 @@ def detect_nf_test_files(changed_files: list[Path]) -> list[Path]: # dir/ # ├─ main.nf # ├─ main.nf.test - for file in path.parent.rglob("*.nf.test"): + for file in path.parent.rglob(suffix): result.append(file) return result @@ -246,12 +247,19 @@ def convert_nf_test_files_to_test_types( name = run_words[1].strip("'\"") # Strip both single and double quotes if keyword in types: result[keyword].append(name) + elif "include {" in line and "include" in types: + keyword = words[0] + name = words[2].strip("'\"") # Strip both single and double quotes + if keyword in types: + result[keyword].append(name) + return result -def find_changed_dependencies(paths: list[Path], tags: list[str]) -> list[Path]: +def find_nf_tests_with_changed_dependencies(paths: list[Path], tags: list[str]) -> list[Path]: """ - Find all *.nf.test files with changed dependencies from a list of paths. + Find all *.nf.test files with changed dependencies + (identified as modules loaded in the via setup { run("") } from a list of paths. Args: paths (list): List of directories or files to scan. @@ -263,7 +271,7 @@ def find_changed_dependencies(paths: list[Path], tags: list[str]) -> list[Path]: result: list[Path] = [] - nf_test_files = detect_nf_test_files(paths) + nf_test_files = detect_files(paths, "*.nf.test") # find nf-test files with changed dependencies for nf_test_file in nf_test_files: @@ -287,6 +295,48 @@ def find_changed_dependencies(paths: list[Path], tags: list[str]) -> list[Path]: return result +def find_nf_files_with_changed_dependencies(paths: list[Path], tags: list[str]) -> list[Path]: + """ + Find all *.nf.test files with where the *.nf file uses changed dependencies + (identified via include { }) in *.nf files from a list of paths. + + Args: + paths (list): List of directories or files to scan. + tags (list): List of tags identified as having changes. + + Returns: + list: List of *.nf.test files from *.nf files with changed dependencies. + """ + + nf_files_w_changed_dependencies: list[Path] = [] + + nf_files = detect_files(paths, "*.nf") + + # find nf files with changed dependencies + for nf_file in nf_files: + with open(nf_file, "r") as f: + lines = f.readlines() + # Get all include statements from nf file + # Make case insensitive with .casefold() + includes_in_nf_file = [ + tag.casefold().replace("/", "_") + for tag in convert_nf_test_files_to_test_types(lines, types=["include"])[ + "include" + ] + ] + # Check if include in nf file appears in a tag. + # Use .casefold() to be case insensitive + if any( + tag.casefold().replace("/", "_") in includes_in_nf_file for tag in tags + ): + nf_files_w_changed_dependencies.append(nf_file) + + # find nf-test for nf files with changed dependencies + nf_test_files_for_changed_dependencies = detect_files(nf_files_w_changed_dependencies, "*.nf.test") + + return nf_test_files_for_changed_dependencies + + if __name__ == "__main__": # Utility stuff @@ -302,7 +352,7 @@ def find_changed_dependencies(paths: list[Path], tags: list[str]) -> list[Path]: changed_files = changed_files + detect_include_files( changed_files, include_files ) - nf_test_files = detect_nf_test_files(changed_files) + nf_test_files = detect_files(changed_files, "*.nf.test") lines = process_nf_test_files(nf_test_files) result = convert_nf_test_files_to_test_types(lines) @@ -312,12 +362,15 @@ def find_changed_dependencies(paths: list[Path], tags: list[str]) -> list[Path]: {item for sublist in map(result.get, args.types) for item in sublist} ) - # Parse files to identify nf-tests with changed dependencies - changed_dep_files = find_changed_dependencies([Path(".")], target_results) + # Parse nf-test files to identify nf-tests containing "setup" with changed module/subworkflow/workflow + nf_test_changed_setup = find_nf_tests_with_changed_dependencies([Path(".")], target_results) + + # Parse *.nf files to identify nf-files containing include with changed module/subworkflow/workflow + nf_files_changed_include = find_nf_files_with_changed_dependencies([Path(".")], target_results) # Combine target nf-test files and nf-test files with changed dependencies all_nf_tests = [ - str(test_path) for test_path in set(changed_dep_files + nf_test_files) + str(test_path) for test_path in set(nf_test_files + nf_test_changed_setup + nf_files_changed_include) ] # Print to stdout From c64be70b14112659cc3ae4bd478bde6263024b4f Mon Sep 17 00:00:00 2001 From: "Carson J. Miller" Date: Mon, 8 Apr 2024 21:38:28 +0000 Subject: [PATCH 04/14] Extra line to test changes --- modules/nf-core/mash/sketch/main.nf | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/nf-core/mash/sketch/main.nf b/modules/nf-core/mash/sketch/main.nf index 77e3d4b..aee3af0 100644 --- a/modules/nf-core/mash/sketch/main.nf +++ b/modules/nf-core/mash/sketch/main.nf @@ -6,6 +6,7 @@ process MASH_SKETCH { 'https://depot.galaxyproject.org/singularity/mash:2.3--he348c14_1' : 'biocontainers/mash:2.3--he348c14_1' }" + input: tuple val(meta), path(reads) From 46f9b5c10c258b091f6bfe874b9f1f0e2f59b9f4 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Tue, 16 Apr 2024 17:11:40 +0000 Subject: [PATCH 05/14] Updated workflow to use Adam's GH action --- .github/workflows/ci.yml | 72 +++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46ebe58..0cb57e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,44 +23,44 @@ concurrency: cancel-in-progress: true jobs: - changes: - name: Check for changes + nf-test-changes: + name: nf-test-changes runs-on: ubuntu-latest outputs: - nf_test_files: ${{ steps.list.outputs.nf_test_files }} - steps: - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - architecture: "x64" + # Expose detected tags as 'modules' and 'workflows' output variables + paths: ${{ steps.list.outputs.components }} + modules: ${{ steps.outputs.outputs.modules }} + subworkflows: ${{ steps.outputs.outputs.subworkflows}} + # Prod for version bumping - - uses: actions/checkout@v3 + steps: + - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Install gitpython find changed files - run: | - python -m pip install --upgrade pip - pip install gitpython pyyaml - - - name: nf-test list nf_test_files + - name: List nf-test files id: list + uses: adamrtalbot/detect-nf-test-changes@main + with: + head: ${{ github.sha }} + base: origin/${{ github.base_ref }} + n_parents: 2 + + - name: Separate modules and subworkflows + id: outputs run: | - echo nf_test_files=$(python \ - .github/python/find_changed_files.py \ - -t pipeline workflow process \ - --head_ref ${{ github.sha }} \ - --base_ref origin/${{ github.base_ref }} \ - ) >> $GITHUB_OUTPUT + echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT + echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT - name: debug run: | - echo ${{ steps.list.outputs.nf_test_files }} + echo ${{ steps.outputs.outputs.modules }} + echo ${{ steps.outputs.outputs.subworkflows }} - test: + nf-test: name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }} - needs: [changes] - if: needs.changes.outputs.nf_test_files != '[]' + needs: [nf-test-changes] + if: ( needs.nf-test-changes.outputs.paths != '[]' ) runs-on: ubuntu-latest strategy: fail-fast: false @@ -110,8 +110,23 @@ jobs: uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - name: Run nf-test + env: + NFT_DIFF: "pdiff" + NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" run: | - nf-test test ${{ matrix.nf_test_files }} --verbose --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker + if [ "${{ matrix.profile }}" == "docker" ]; then + PROFILE="docker_self_hosted" + else + PROFILE=${{ matrix.profile }} + fi + + NFT_WORKDIR=~ \ + nf-test test \ + --profile=${{ matrix.profile }} \ + --tap=test.tap \ + --verbose \ + ${{ matrix.path }} - uses: pcolby/tap-summary@v1 with: @@ -130,6 +145,11 @@ jobs: with: report_paths: test.xml + - name: Clean up + if: always() + run: | + sudo rm -rf /home/ubuntu/tests/ + confirm-pass: runs-on: ubuntu-latest needs: From bf86cce3a04bfb265f0ee0d1b8da75dc3526a169 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Tue, 16 Apr 2024 18:58:37 +0000 Subject: [PATCH 06/14] Trying to run CI workflow --- .github/workflows/ci.yml | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0cb57e9..51daf48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,25 +59,21 @@ jobs: nf-test: name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }} + runs-on: ubuntu-latest needs: [nf-test-changes] if: ( needs.nf-test-changes.outputs.paths != '[]' ) - runs-on: ubuntu-latest strategy: fail-fast: false matrix: - NXF_VER: - - "latest-everything" - - "23.04" - nf_test_files: ["${{ fromJson(needs.changes.outputs.nf_test_files) }}"] - profile: - - "docker" + path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] + profile: [docker] steps: - name: Check out pipeline code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: "${{ matrix.NXF_VER }}" @@ -114,13 +110,6 @@ jobs: NFT_DIFF: "pdiff" NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" run: | - # use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker - if [ "${{ matrix.profile }}" == "docker" ]; then - PROFILE="docker_self_hosted" - else - PROFILE=${{ matrix.profile }} - fi - NFT_WORKDIR=~ \ nf-test test \ --profile=${{ matrix.profile }} \ From 475325c140864b1675d86bd2b2986003d0e4b4b4 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Tue, 16 Apr 2024 19:08:46 +0000 Subject: [PATCH 07/14] Tried getting CI workflow to run again --- .github/workflows/ci.yml | 44 ++++++++++------------------------------ 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 51daf48..6a49d0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,20 +10,18 @@ on: branches: - master - dev - env: NXF_ANSI_LOG: false NFT_VER: "0.8.4" NFT_WORKDIR: "~" NFT_DIFF: "pdiff" NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2" - concurrency: group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" cancel-in-progress: true jobs: - nf-test-changes: + changes: name: nf-test-changes runs-on: ubuntu-latest outputs: @@ -31,8 +29,6 @@ jobs: paths: ${{ steps.list.outputs.components }} modules: ${{ steps.outputs.outputs.modules }} subworkflows: ${{ steps.outputs.outputs.subworkflows}} - # Prod for version bumping - steps: - uses: actions/checkout@v4 with: @@ -57,36 +53,37 @@ jobs: echo ${{ steps.outputs.outputs.modules }} echo ${{ steps.outputs.outputs.subworkflows }} - nf-test: + test: name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }} - runs-on: ubuntu-latest needs: [nf-test-changes] if: ( needs.nf-test-changes.outputs.paths != '[]' ) + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] - profile: [docker] + NXF_VER: + - "latest-everything" + - "23.04" + nf_test_files: ["${{ fromJson(needs.changes.outputs.nf_test_files) }}"] + profile: + - "docker" steps: - name: Check out pipeline code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v2 + uses: nf-core/setup-nextflow@v1 with: version: "${{ matrix.NXF_VER }}" - - uses: actions/setup-python@v4 with: python-version: "3.11" architecture: "x64" - - name: Install pdiff to see diff between nf-test snapshots run: | python -m pip install --upgrade pip pip install pdiff - - name: Cache nf-test installation id: cache-software uses: actions/cache@v3 @@ -95,50 +92,32 @@ jobs: /usr/local/bin/nf-test /home/runner/.nf-test/nf-test.jar key: ${{ runner.os }}-${{ env.NFT_VER }}-nftest - - name: Install nf-test if: steps.cache-software.outputs.cache-hit != 'true' run: | wget -qO- https://code.askimed.com/install/nf-test | bash sudo mv nf-test /usr/local/bin/ - - name: Disk space cleanup uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - name: Run nf-test - env: - NFT_DIFF: "pdiff" - NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" run: | - NFT_WORKDIR=~ \ - nf-test test \ - --profile=${{ matrix.profile }} \ - --tap=test.tap \ - --verbose \ - ${{ matrix.path }} - + nf-test test ${{ matrix.nf_test_files }} --verbose --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: path: >- test.tap - - name: Output log on failure if: failure() run: | sudo apt install bat > /dev/null batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log - - name: Publish Test Report uses: mikepenz/action-junit-report@v3 if: always() # always run even if the previous step fails with: report_paths: test.xml - - name: Clean up - if: always() - run: | - sudo rm -rf /home/ubuntu/tests/ - confirm-pass: runs-on: ubuntu-latest needs: @@ -152,7 +131,6 @@ jobs: - name: One or more tests failed if: ${{ contains(needs.*.result, 'failure') }} run: exit 1 - - name: debug-print if: always() run: | From 05e6e8937df5c772a9d61e4fbcbedc1fce1ca5ac Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Tue, 16 Apr 2024 19:30:55 +0000 Subject: [PATCH 08/14] Changed step names to try and run CI action --- .github/python/find_changed_files.py | 377 --------------------------- .github/python/include.yaml | 10 - .github/workflows/ci.yml | 16 +- 3 files changed, 9 insertions(+), 394 deletions(-) delete mode 100644 .github/python/find_changed_files.py delete mode 100644 .github/python/include.yaml diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py deleted file mode 100644 index 4e91751..0000000 --- a/.github/python/find_changed_files.py +++ /dev/null @@ -1,377 +0,0 @@ -#!/usr/bin/env python - -# This script is used to identify *.nf.test files for changed functions/processs/workflows/pipelines and *.nf-test files -# with changed dependencies, then return as a JSON list - -import argparse -import json -import logging -import re -import yaml - -from itertools import chain -from pathlib import Path -from git import Repo - - -def parse_args() -> argparse.Namespace: - """ - Parse command line arguments and return an ArgumentParser object. - - Returns: - argparse.ArgumentParser: The ArgumentParser object with the parsed arguments. - """ - parser = argparse.ArgumentParser( - description="Scan *.nf.test files for function/process/workflow name and return as a JSON list" - ) - parser.add_argument( - "-r", - "--head_ref", - required=True, - help="Head reference branch (Source branch for a PR).", - ) - parser.add_argument( - "-b", - "--base_ref", - required=True, - help="Base reference branch (Target branch for a PR).", - ) - parser.add_argument( - "-x", - "--ignored_files", - nargs="+", - default=[ - ".git/*", - ".gitpod.yml", - ".prettierignore", - ".prettierrc.yml", - "*.md", - "*.png", - "modules.json", - "pyproject.toml", - "tower.yml", - ], - help="List of files or file substrings to ignore.", - ) - parser.add_argument( - "-i", - "--include", - type=Path, - default=".github/python/include.yaml", - help="Path to an include file containing a YAML of key value pairs to include in changed files. I.e., return the current directory if an important file is changed.", - ) - parser.add_argument( - "-l", - "--log-level", - choices=["DEBUG", "INFO", "WARNING", "ERROR"], - default="INFO", - help="Logging level", - ) - parser.add_argument( - "-t", - "--types", - nargs="+", - choices=["function", "process", "workflow", "pipeline"], - default=["function", "process", "workflow", "pipeline"], - help="Types of tests to include.", - ) - return parser.parse_args() - - -def read_yaml_inverted(file_path: str) -> dict: - """ - Read a YAML file and return its contents as a dictionary but reversed, i.e. the values become the keys and the keys become the values. - - Args: - file_path (str): The path to the YAML file. - - Returns: - dict: The contents of the YAML file as a dictionary inverted. - """ - with open(file_path, "r") as f: - data = yaml.safe_load(f) - - # Invert dictionary of lists into contents of lists are keys, values are the original keys - # { "key": ["item1", "item2] } --> { "item1": "key", "item2": "key" } - return {value: key for key, values in data.items() for value in values} - - -def find_changed_files( - branch1: str, - branch2: str, - ignore: list[str], -) -> list[Path]: - """ - Find all *.nf.tests that are associated with files that have been changed between two specified branches. - - Args: - branch1 (str) : The first branch being compared - branch2 (str) : The second branch being compared - ignore (list) : List of files or file substrings to ignore. - - Returns: - list: List of files matching the pattern *.nf.test that have changed between branch2 and branch1. - """ - # create repo - repo = Repo(".") - # identify commit on branch1 - branch1_commit = repo.commit(branch1) - # identify commit on branch2 - branch2_commit = repo.commit(branch2) - # compare two branches - diff_index = branch1_commit.diff(branch2_commit) - - # Start empty list of changed files - changed_files = [] - - # For every file that has changed between commits - for file in diff_index: - # Get pathlib.Path object - filepath = Path(file.a_path) - # If file does not match any in the ignore list, add containing directory to changed_files - if not any(filepath.match(ignored_path) for ignored_path in ignore): - changed_files.append(filepath) - - # Uniqueify the results before returning for efficiency - return list(set(changed_files)) - - -def detect_include_files( - changed_files: list[Path], include_files: dict[str, str] -) -> list[Path]: - """ - Detects the include files based on the changed files. - - Args: - changed_files (list[Path]): List of paths to the changed files. - include_files (dict[str, str]): Key-value pairs to return if a certain file has changed. If a file in a directory has changed, it points to a different directory. - - Returns: - list[Path]: List of paths to representing the keys of the include_files dictionary, where a value matched a path in changed_files. - """ - new_changed_files = [] - for filepath in changed_files: - # If file is in the include_files, we return the key instead of the value - for include_path, include_key in include_files.items(): - if filepath.match(include_path): - new_changed_files.append(Path(include_key)) - return new_changed_files - - -def detect_files(changed_files: list[Path], suffix: str) -> list[Path]: - """ - Detects and returns a list of nf-test files from the given list of changed files. - - Args: - changed_files (list[Path]): A list of file paths. - suffix (str): File suffix to detect - - Returns: - list[Path]: A list of nf-test file paths. - """ - result: list[Path] = [] - for path in changed_files: - # If Path is the exact nf-test file add to list: - if path.match(suffix) and path.exists(): - result.append(path) - # Else recursively search for nf-test files: - else: - # Get the enclosing dir so files in the same dir can be found. - # e.g. - # dir/ - # ├─ main.nf - # ├─ main.nf.test - for file in path.parent.rglob(suffix): - result.append(file) - return result - - -def process_nf_test_files(files: list[Path]) -> list[str]: - """ - Process the files and return lines that begin with 'workflow', 'process', or 'function' and have a single string afterwards. - - Args: - files (list): List of files to process. - - Returns: - list: List of lines that match the criteria. - """ - result = [] - for file in files: - with open(file, "r") as f: - is_pipeline_test = True - lines = f.readlines() - for line in lines: - line = line.strip() - if line.startswith(("workflow", "process", "function")): - words = line.split() - if len(words) == 2 and re.match(r'^".*"$', words[1]): - result.append(line) - is_pipeline_test = False - - # If no results included workflow, process or function - # Add a dummy result to fill the 'pipeline' category - if is_pipeline_test: - result.append("pipeline 'PIPELINE'") - - return result - - -def convert_nf_test_files_to_test_types( - lines: list[str], types: list[str] = ["function", "process", "workflow", "pipeline"] -) -> dict[str, list[str]]: - """ - Generate a dictionary of function, process and workflow lists from the lines. - - Args: - lines (list): List of lines to process. - types (list): List of types to include. - - Returns: - dict: Dictionary with function, process and workflow lists. - """ - # Populate empty dict from types - result: dict[str, list[str]] = {key: [] for key in types} - - for line in lines: - words = line.split() - if len(words) == 2 and re.match(r'^".*"$', words[1]): - keyword = words[0] - name = words[1].strip("'\"") # Strip both single and double quotes - if keyword in types: - result[keyword].append(name) - elif "run" in line: - run_words = words[0].strip(')').split('(') - if len(run_words) == 2 and re.match(r'^".*"$', run_words[1]): - keyword = run_words[0] - name = run_words[1].strip("'\"") # Strip both single and double quotes - if keyword in types: - result[keyword].append(name) - elif "include {" in line and "include" in types: - keyword = words[0] - name = words[2].strip("'\"") # Strip both single and double quotes - if keyword in types: - result[keyword].append(name) - - return result - - -def find_nf_tests_with_changed_dependencies(paths: list[Path], tags: list[str]) -> list[Path]: - """ - Find all *.nf.test files with changed dependencies - (identified as modules loaded in the via setup { run("") } from a list of paths. - - Args: - paths (list): List of directories or files to scan. - tags (list): List of tags identified as having changes. - - Returns: - list: List of *.nf.test files with changed dependencies. - """ - - result: list[Path] = [] - - nf_test_files = detect_files(paths, "*.nf.test") - - # find nf-test files with changed dependencies - for nf_test_file in nf_test_files: - with open(nf_test_file, "r") as f: - lines = f.readlines() - # Get all tags from nf-test file - # Make case insensitive with .casefold() - tags_in_nf_test_file = [ - tag.casefold().replace("/", "_") - for tag in convert_nf_test_files_to_test_types(lines, types=["run"])[ - "run" - ] - ] - # Check if tag in nf-test file appears in a tag. - # Use .casefold() to be case insensitive - if any( - tag.casefold().replace("/", "_") in tags_in_nf_test_file for tag in tags - ): - result.append(nf_test_file) - - return result - - -def find_nf_files_with_changed_dependencies(paths: list[Path], tags: list[str]) -> list[Path]: - """ - Find all *.nf.test files with where the *.nf file uses changed dependencies - (identified via include { }) in *.nf files from a list of paths. - - Args: - paths (list): List of directories or files to scan. - tags (list): List of tags identified as having changes. - - Returns: - list: List of *.nf.test files from *.nf files with changed dependencies. - """ - - nf_files_w_changed_dependencies: list[Path] = [] - - nf_files = detect_files(paths, "*.nf") - - # find nf files with changed dependencies - for nf_file in nf_files: - with open(nf_file, "r") as f: - lines = f.readlines() - # Get all include statements from nf file - # Make case insensitive with .casefold() - includes_in_nf_file = [ - tag.casefold().replace("/", "_") - for tag in convert_nf_test_files_to_test_types(lines, types=["include"])[ - "include" - ] - ] - # Check if include in nf file appears in a tag. - # Use .casefold() to be case insensitive - if any( - tag.casefold().replace("/", "_") in includes_in_nf_file for tag in tags - ): - nf_files_w_changed_dependencies.append(nf_file) - - # find nf-test for nf files with changed dependencies - nf_test_files_for_changed_dependencies = detect_files(nf_files_w_changed_dependencies, "*.nf.test") - - return nf_test_files_for_changed_dependencies - - -if __name__ == "__main__": - - # Utility stuff - args = parse_args() - logging.basicConfig(level=args.log_level) - - # Parse nf-test files for target test tags - changed_files = find_changed_files(args.head_ref, args.base_ref, args.ignored_files) - - # If an additional include YAML is added, we detect additional changed dirs to include - if args.include: - include_files = read_yaml_inverted(args.include) - changed_files = changed_files + detect_include_files( - changed_files, include_files - ) - nf_test_files = detect_files(changed_files, "*.nf.test") - lines = process_nf_test_files(nf_test_files) - result = convert_nf_test_files_to_test_types(lines) - - # Get only relevant results (specified by -t) - # Unique using a set - target_results = list( - {item for sublist in map(result.get, args.types) for item in sublist} - ) - - # Parse nf-test files to identify nf-tests containing "setup" with changed module/subworkflow/workflow - nf_test_changed_setup = find_nf_tests_with_changed_dependencies([Path(".")], target_results) - - # Parse *.nf files to identify nf-files containing include with changed module/subworkflow/workflow - nf_files_changed_include = find_nf_files_with_changed_dependencies([Path(".")], target_results) - - # Combine target nf-test files and nf-test files with changed dependencies - all_nf_tests = [ - str(test_path) for test_path in set(nf_test_files + nf_test_changed_setup + nf_files_changed_include) - ] - - # Print to stdout - print(json.dumps(all_nf_tests)) diff --git a/.github/python/include.yaml b/.github/python/include.yaml deleted file mode 100644 index 6bee890..0000000 --- a/.github/python/include.yaml +++ /dev/null @@ -1,10 +0,0 @@ -".": - - .github/workflows/** - - nf-test.config - - /./nextflow.config -tests: - - assets/* - - bin/* - - conf/* - - /./main.nf - - nextflow_schema.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a49d0a..8f29934 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ concurrency: cancel-in-progress: true jobs: - changes: + nf-test-changes: name: nf-test-changes runs-on: ubuntu-latest outputs: @@ -53,8 +53,8 @@ jobs: echo ${{ steps.outputs.outputs.modules }} echo ${{ steps.outputs.outputs.subworkflows }} - test: - name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }} + nf-test: + name: nf-test needs: [nf-test-changes] if: ( needs.nf-test-changes.outputs.paths != '[]' ) runs-on: ubuntu-latest @@ -102,7 +102,11 @@ jobs: - name: Run nf-test run: | - nf-test test ${{ matrix.nf_test_files }} --verbose --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + nf-test test ${{ matrix.nf_test_files }} \ + --verbose \ + --profile "+${{ matrix.profile }}" -\ + -junitxml=test.xml \ + --tap=test.tap - uses: pcolby/tap-summary@v1 with: path: >- @@ -120,9 +124,7 @@ jobs: confirm-pass: runs-on: ubuntu-latest - needs: - - changes - - test + needs: [nf-test-changes, nf-test] if: always() steps: - name: All tests ok From 9682de7a9c08afb8ac409383d4e237d1881f3809 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Tue, 16 Apr 2024 19:38:41 +0000 Subject: [PATCH 09/14] Changed source of nf-test files --- .github/workflows/ci.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8f29934..0830d53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,10 +61,10 @@ jobs: strategy: fail-fast: false matrix: + path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] NXF_VER: - "latest-everything" - "23.04" - nf_test_files: ["${{ fromJson(needs.changes.outputs.nf_test_files) }}"] profile: - "docker" @@ -101,12 +101,16 @@ jobs: uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - name: Run nf-test + env: + NFT_DIFF: "pdiff" + NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" run: | - nf-test test ${{ matrix.nf_test_files }} \ - --verbose \ - --profile "+${{ matrix.profile }}" -\ - -junitxml=test.xml \ - --tap=test.tap + NFT_WORKDIR=~ \ + nf-test test \ + --profile=${{ matrix.profile }} \ + --tap=test.tap \ + --verbose \ + ${{ matrix.path }} - uses: pcolby/tap-summary@v1 with: path: >- From 75eae223f4a9a0e6c5e469fd28d10712693542e1 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Tue, 23 Apr 2024 16:08:08 +0000 Subject: [PATCH 10/14] Updated CI to match nf-core/fetchngs --- .github/workflows/ci.yml | 66 ++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0830d53..4c535bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,80 +10,75 @@ on: branches: - master - dev + env: NXF_ANSI_LOG: false NFT_VER: "0.8.4" NFT_WORKDIR: "~" NFT_DIFF: "pdiff" NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2" + concurrency: group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" cancel-in-progress: true jobs: - nf-test-changes: - name: nf-test-changes + changes: + name: Check for changes runs-on: ubuntu-latest outputs: - # Expose detected tags as 'modules' and 'workflows' output variables - paths: ${{ steps.list.outputs.components }} - modules: ${{ steps.outputs.outputs.modules }} - subworkflows: ${{ steps.outputs.outputs.subworkflows}} + nf_test_files: ${{ steps.list.outputs.components }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: List nf-test files id: list - uses: adamrtalbot/detect-nf-test-changes@main + uses: adamrtalbot/detect-nf-test-changes@v0.0.1 with: head: ${{ github.sha }} base: origin/${{ github.base_ref }} - n_parents: 2 - - - name: Separate modules and subworkflows - id: outputs - run: | - echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT - echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT + include: .github/include.yaml - - name: debug + - name: print list of nf-test files run: | - echo ${{ steps.outputs.outputs.modules }} - echo ${{ steps.outputs.outputs.subworkflows }} + echo ${{ steps.list.outputs.components }} - nf-test: - name: nf-test - needs: [nf-test-changes] - if: ( needs.nf-test-changes.outputs.paths != '[]' ) + test: + name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }} + needs: [changes] + if: needs.changes.outputs.nf_test_files != '[]' runs-on: ubuntu-latest strategy: fail-fast: false matrix: - path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"] NXF_VER: - "latest-everything" - "23.04" + nf_test_files: ["${{ fromJson(needs.changes.outputs.nf_test_files) }}"] profile: - "docker" steps: - name: Check out pipeline code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@v2 with: version: "${{ matrix.NXF_VER }}" + - uses: actions/setup-python@v4 with: python-version: "3.11" architecture: "x64" + - name: Install pdiff to see diff between nf-test snapshots run: | python -m pip install --upgrade pip pip install pdiff + - name: Cache nf-test installation id: cache-software uses: actions/cache@v3 @@ -92,34 +87,28 @@ jobs: /usr/local/bin/nf-test /home/runner/.nf-test/nf-test.jar key: ${{ runner.os }}-${{ env.NFT_VER }}-nftest + - name: Install nf-test if: steps.cache-software.outputs.cache-hit != 'true' run: | wget -qO- https://code.askimed.com/install/nf-test | bash sudo mv nf-test /usr/local/bin/ - - name: Disk space cleanup - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - name: Run nf-test - env: - NFT_DIFF: "pdiff" - NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2" run: | - NFT_WORKDIR=~ \ - nf-test test \ - --profile=${{ matrix.profile }} \ - --tap=test.tap \ - --verbose \ - ${{ matrix.path }} + nf-test test --verbose ${{ matrix.nf_test_files }} --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + - uses: pcolby/tap-summary@v1 with: path: >- test.tap + - name: Output log on failure if: failure() run: | sudo apt install bat > /dev/null batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log + - name: Publish Test Report uses: mikepenz/action-junit-report@v3 if: always() # always run even if the previous step fails @@ -128,7 +117,9 @@ jobs: confirm-pass: runs-on: ubuntu-latest - needs: [nf-test-changes, nf-test] + needs: + - changes + - test if: always() steps: - name: All tests ok @@ -137,6 +128,7 @@ jobs: - name: One or more tests failed if: ${{ contains(needs.*.result, 'failure') }} run: exit 1 + - name: debug-print if: always() run: | From b20f4c9262d3ce4f54fa89bd5066385e80f460d8 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Tue, 23 Apr 2024 16:09:59 +0000 Subject: [PATCH 11/14] Added include.yaml --- .github/include.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/include.yaml diff --git a/.github/include.yaml b/.github/include.yaml new file mode 100644 index 0000000..a3629f4 --- /dev/null +++ b/.github/include.yaml @@ -0,0 +1,10 @@ +".": + - ./.github/workflows/** + - ./nf-test.config + - ./nextflow.config +tests: + - ./assets/* + - ./bin/* + - ./conf/* + - ./main.nf + - ./nextflow_schema.json From 84ef43025eb9ab78fd478680d0ad820665ba5a8c Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Tue, 23 Apr 2024 16:20:52 +0000 Subject: [PATCH 12/14] Updated genomad/endtoend and removed test changes from mash/sketch and viromeqc/install --- modules/local/viromeqc/install/main.nf | 1 - .../nf-core/genomad/endtoend/environment.yml | 7 + .../genomad/endtoend/genomad-endtoend.diff | 89 ------ modules/nf-core/genomad/endtoend/main.nf | 6 +- modules/nf-core/genomad/endtoend/meta.yml | 9 +- .../nf-core/genomad/endtoend/nextflow.config | 2 +- .../genomad/endtoend/tests/main.nf.test | 83 ++++++ .../genomad/endtoend/tests/main.nf.test.snap | 281 ++++++++++++++++++ .../genomad/endtoend/tests/nextflow.config | 5 + .../nf-core/genomad/endtoend/tests/tags.yml | 2 + modules/nf-core/mash/sketch/main.nf | 1 - 11 files changed, 385 insertions(+), 101 deletions(-) create mode 100644 modules/nf-core/genomad/endtoend/environment.yml delete mode 100644 modules/nf-core/genomad/endtoend/genomad-endtoend.diff create mode 100644 modules/nf-core/genomad/endtoend/tests/main.nf.test create mode 100644 modules/nf-core/genomad/endtoend/tests/main.nf.test.snap create mode 100644 modules/nf-core/genomad/endtoend/tests/nextflow.config create mode 100644 modules/nf-core/genomad/endtoend/tests/tags.yml diff --git a/modules/local/viromeqc/install/main.nf b/modules/local/viromeqc/install/main.nf index a49dadb..7e682a2 100644 --- a/modules/local/viromeqc/install/main.nf +++ b/modules/local/viromeqc/install/main.nf @@ -1,7 +1,6 @@ process VIROMEQC_INSTALL { label 'process_low' - conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/mulled-v2-b28a1a551d380ce8d57f9d83894ccb9559b44404:08a4cf815fcef0080ede5b3633202cbda8edf59b-0': diff --git a/modules/nf-core/genomad/endtoend/environment.yml b/modules/nf-core/genomad/endtoend/environment.yml new file mode 100644 index 0000000..6ec4ebe --- /dev/null +++ b/modules/nf-core/genomad/endtoend/environment.yml @@ -0,0 +1,7 @@ +name: genomad_endtoend +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::genomad=1.7.4 diff --git a/modules/nf-core/genomad/endtoend/genomad-endtoend.diff b/modules/nf-core/genomad/endtoend/genomad-endtoend.diff deleted file mode 100644 index a8b48af..0000000 --- a/modules/nf-core/genomad/endtoend/genomad-endtoend.diff +++ /dev/null @@ -1,89 +0,0 @@ -Changes in module 'nf-core/genomad/endtoend' ---- /dev/null -+++ modules/nf-core/genomad/endtoend/nextflow.config -@@ -0,0 +1,23 @@ -+process { -+ withName: GENOMAD_ENDTOEND { -+ ext.args = [ -+ params.genomad_min_score ? "--min-score ${params.genomad_min_score}" : "", -+ params.genomad_max_fdr ? "--enable-score-calibration --max-fdr ${params.genomad_max_fdr}" : "", -+ params.genomad_splits ? "--splits ${params.genomad_splits}" : "", -+ params.genomad_disable_nn ? "--disable-nn-classification" : "", -+ params.genomad_sensitivity ? "--sensitivity ${params.genomad_sensitivity}" : "" -+ ].join(' ').trim() -+ publishDir = [ -+ [ -+ path: { "${params.outdir}/VirusClassification/genomad/endtoend" }, -+ mode: params.publish_dir_mode, -+ pattern: '*_summary/*_virus.fna.gz' -+ ], -+ [ -+ path: { "${params.outdir}/VirusClassification/genomad/endtoend" }, -+ mode: params.publish_dir_mode, -+ pattern: '*_summary/*_virus_summary.tsv' -+ ] -+ ] -+ } -+} ---- modules/nf-core/genomad/endtoend/main.nf -+++ modules/nf-core/genomad/endtoend/main.nf -@@ -8,22 +8,22 @@ - 'biocontainers/genomad:1.5.2--pyhdfd78af_0' }" - - input: -- tuple val(meta) , path(fasta) -- path genomad_db -+ tuple val(meta), path(fasta) -+ tuple path(genomad_db) - - output: -- tuple val(meta), path("*_aggregated_classification/*_aggregated_classification.tsv") , emit: aggregated_classification -+ tuple val(meta), path("*_aggregated_classification/*_aggregated_classification.tsv") , emit: aggregated_classification , optional: true - tuple val(meta), path("*_annotate/*_taxonomy.tsv") , emit: taxonomy - tuple val(meta), path("*_find_proviruses/*_provirus.tsv") , emit: provirus - tuple val(meta), path("*_score_calibration/*_compositions.tsv") , emit: compositions , optional: true - tuple val(meta), path("*_score_calibration/*_calibrated_aggregated_classification.tsv") , emit: calibrated_classification , optional: true -- tuple val(meta), path("*_summary/*_plasmid.fna") , emit: plasmid_fasta -+ tuple val(meta), path("*_summary/*_plasmid.fna.gz") , emit: plasmid_fasta - tuple val(meta), path("*_summary/*_plasmid_genes.tsv") , emit: plasmid_genes -- tuple val(meta), path("*_summary/*_plasmid_proteins.faa") , emit: plasmid_proteins -+ tuple val(meta), path("*_summary/*_plasmid_proteins.faa.gz") , emit: plasmid_proteins - tuple val(meta), path("*_summary/*_plasmid_summary.tsv") , emit: plasmid_summary -- tuple val(meta), path("*_summary/*_virus.fna") , emit: virus_fasta -+ tuple val(meta), path("*_summary/*_virus.fna.gz") , emit: virus_fasta - tuple val(meta), path("*_summary/*_virus_genes.tsv") , emit: virus_genes -- tuple val(meta), path("*_summary/*_virus_proteins.faa") , emit: virus_proteins -+ tuple val(meta), path("*_summary/*_virus_proteins.faa.gz") , emit: virus_proteins - tuple val(meta), path("*_summary/*_virus_summary.tsv") , emit: virus_summary - path "versions.yml" , emit: versions - -@@ -41,6 +41,9 @@ - $genomad_db \\ - --threads $task.cpus \\ - $args -+ -+ gzip ./**/*.fna -+ gzip ./**/*.faa - - cat <<-END_VERSIONS > versions.yml - "${task.process}": -@@ -65,13 +68,13 @@ - touch ${filename}_score_calibration/${filename}_calibrated_aggregated_classification.tsv - touch ${filename}_score_calibration/${filename}_compositions.tsv - mkdir ${filename}_summary -- touch ${filename}_summary/${filename}_plasmid.fna -+ touch ${filename}_summary/${filename}_plasmid.fna.gz - touch ${filename}_summary/${filename}_plasmid_genes.tsv -- touch ${filename}_summary/${filename}_plasmid_proteins.faa -+ touch ${filename}_summary/${filename}_plasmid_proteins.faa.gz - touch ${filename}_summary/${filename}_plasmid_summary.tsv -- touch ${filename}_summary/${filename}_virus.fna -+ touch ${filename}_summary/${filename}_virus.fna.gz - touch ${filename}_summary/${filename}_virus_genes.tsv -- touch ${filename}_summary/${filename}_virus_proteins.faa -+ touch ${filename}_summary/${filename}_virus_proteins.faa.gz - touch ${filename}_summary/${filename}_virus_summary.tsv - - cat <<-END_VERSIONS > versions.yml - -************************************************************ diff --git a/modules/nf-core/genomad/endtoend/main.nf b/modules/nf-core/genomad/endtoend/main.nf index 5a1937a..0630208 100644 --- a/modules/nf-core/genomad/endtoend/main.nf +++ b/modules/nf-core/genomad/endtoend/main.nf @@ -41,8 +41,8 @@ process GENOMAD_ENDTOEND { $genomad_db \\ --threads $task.cpus \\ $args - - gzip ./**/*.fna + + gzip ./**/*.fna gzip ./**/*.faa cat <<-END_VERSIONS > versions.yml @@ -82,4 +82,4 @@ process GENOMAD_ENDTOEND { genomad: \$(echo \$(genomad --version 2>&1) | sed 's/^.*geNomad, version //; s/ .*\$//') END_VERSIONS """ -} +} \ No newline at end of file diff --git a/modules/nf-core/genomad/endtoend/meta.yml b/modules/nf-core/genomad/endtoend/meta.yml index b5a6f61..044496c 100644 --- a/modules/nf-core/genomad/endtoend/meta.yml +++ b/modules/nf-core/genomad/endtoend/meta.yml @@ -1,5 +1,4 @@ name: "genomad_endtoend" - description: Identify mobile genetic elements present in genomic assemblies keywords: - metagenomics @@ -9,7 +8,6 @@ keywords: - phage - virus - plasmid - tools: - "genomad": description: "Identification of mobile genetic elements" @@ -17,8 +15,7 @@ tools: documentation: https://portal.nersc.gov/genomad/ tool_dev_url: https://github.com/apcamargo/genomad/ doi: 10.1101/2023.03.05.531206 - licence: "['Lawrence Berkeley National Labs BSD variant license']" - + licence: ["Lawrence Berkeley National Labs BSD variant license"] input: - meta: type: map @@ -35,7 +32,6 @@ input: - score_calibration: type: boolean description: true/false value to indicate if score calibration should be enabled - output: - meta: type: map @@ -98,6 +94,7 @@ output: type: file description: File containing software versions pattern: "versions.yml" - authors: - "@CarsonJM" +maintainers: + - "@CarsonJM" diff --git a/modules/nf-core/genomad/endtoend/nextflow.config b/modules/nf-core/genomad/endtoend/nextflow.config index d108d71..570802a 100644 --- a/modules/nf-core/genomad/endtoend/nextflow.config +++ b/modules/nf-core/genomad/endtoend/nextflow.config @@ -20,4 +20,4 @@ process { ] ] } -} \ No newline at end of file +} diff --git a/modules/nf-core/genomad/endtoend/tests/main.nf.test b/modules/nf-core/genomad/endtoend/tests/main.nf.test new file mode 100644 index 0000000..55c6771 --- /dev/null +++ b/modules/nf-core/genomad/endtoend/tests/main.nf.test @@ -0,0 +1,83 @@ +nextflow_process { + + name "Test Process GENOMAD_ENDTOEND" + script "../main.nf" + process "GENOMAD_ENDTOEND" + config "./nextflow.config" + + tag "modules" + tag "modules_nfcore" + tag "genomad" + tag "genomad/download" + tag "genomad/endtoend" + + test("sarscov2 - genome - fasta") { + setup { + run("GENOMAD_DOWNLOAD") { + script "../../download/main.nf" + } + } + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + ] + input[1] = GENOMAD_DOWNLOAD.out.genomad_db + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.aggregated_classification, + process.out.taxonomy, + process.out.provirus, + process.out.compositions, + process.out.calibrated_classification, + process.out.plasmid_genes, + process.out.virus_fasta, + process.out.virus_genes, + process.out.virus_proteins, + process.out.virus_summary, + process.out.versions, + ).match() + }, + { assert file(process.out.plasmid_fasta.get(0).get(1)).exists() }, + { assert file(process.out.plasmid_proteins.get(0).get(1)).exists() } + ) + } + } + + test("sarscov2 - genome - fasta - stub") { + + setup { + run("GENOMAD_DOWNLOAD") { + script "../../download/main.nf" + } + } + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['sarscov2']['genome']['genome_fasta'], checkIfExists: true) + ] + input[1] = GENOMAD_DOWNLOAD.out.genomad_db + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/nf-core/genomad/endtoend/tests/main.nf.test.snap b/modules/nf-core/genomad/endtoend/tests/main.nf.test.snap new file mode 100644 index 0000000..00c02fa --- /dev/null +++ b/modules/nf-core/genomad/endtoend/tests/main.nf.test.snap @@ -0,0 +1,281 @@ +{ + "sarscov2 - genome - fasta": { + "content": [ + [ + + ], + [ + [ + { + "id": "test" + }, + "genome_taxonomy.tsv:md5,44c6cf5ff4c836f66648457ae8deb0d5" + ] + ], + [ + [ + { + "id": "test" + }, + "genome_provirus.tsv:md5,93a6bca59b0bf7f57c0b9b60d2e57082" + ] + ], + [ + [ + { + "id": "test" + }, + "genome_compositions.tsv:md5,9168bea9fe82a979a698d259e1bb906a" + ] + ], + [ + + ], + [ + [ + { + "id": "test" + }, + "genome_plasmid_genes.tsv:md5,55818cae5a57381b77778076e99605e6" + ] + ], + [ + [ + { + "id": "test" + }, + "genome_virus.fna.gz:md5,483f4a5dfe60171c86ee9b7e6dff908b" + ] + ], + [ + [ + { + "id": "test" + }, + "genome_virus_genes.tsv:md5,f84d8a9fce3c92c49e78672bea16561f" + ] + ], + [ + [ + { + "id": "test" + }, + "genome_virus_proteins.faa.gz:md5,6850bd46dcb921e5cfb23bfbb8fcc12b" + ] + ], + [ + [ + { + "id": "test" + }, + "genome_virus_summary.tsv:md5,8cbcf95c2ab397da26b6dca8014e86f4" + ] + ], + [ + "versions.yml:md5,cebe18512f616530fff490a64e595cb6" + ] + ], + "timestamp": "2024-01-03T00:05:48.225084591" + }, + "sarscov2 - genome - fasta - stub": { + "content": [ + { + "0": [ + + ], + "1": [ + [ + { + "id": "test" + }, + "genome_taxonomy.tsv:md5,44c6cf5ff4c836f66648457ae8deb0d5" + ] + ], + "10": [ + [ + { + "id": "test" + }, + "genome_virus_genes.tsv:md5,f84d8a9fce3c92c49e78672bea16561f" + ] + ], + "11": [ + [ + { + "id": "test" + }, + "genome_virus_proteins.faa.gz:md5,6850bd46dcb921e5cfb23bfbb8fcc12b" + ] + ], + "12": [ + [ + { + "id": "test" + }, + "genome_virus_summary.tsv:md5,8cbcf95c2ab397da26b6dca8014e86f4" + ] + ], + "13": [ + "versions.yml:md5,cebe18512f616530fff490a64e595cb6" + ], + "2": [ + [ + { + "id": "test" + }, + "genome_provirus.tsv:md5,93a6bca59b0bf7f57c0b9b60d2e57082" + ] + ], + "3": [ + [ + { + "id": "test" + }, + "genome_compositions.tsv:md5,9168bea9fe82a979a698d259e1bb906a" + ] + ], + "4": [ + + ], + "5": [ + [ + { + "id": "test" + }, + "genome_plasmid.fna.gz:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "6": [ + [ + { + "id": "test" + }, + "genome_plasmid_genes.tsv:md5,55818cae5a57381b77778076e99605e6" + ] + ], + "7": [ + [ + { + "id": "test" + }, + "genome_plasmid_proteins.faa.gz:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "8": [ + [ + { + "id": "test" + }, + "genome_plasmid_summary.tsv:md5,8c4ddaa8a90da779c11537be0fddb799" + ] + ], + "9": [ + [ + { + "id": "test" + }, + "genome_virus.fna.gz:md5,483f4a5dfe60171c86ee9b7e6dff908b" + ] + ], + "aggregated_classification": [ + + ], + "calibrated_classification": [ + + ], + "compositions": [ + [ + { + "id": "test" + }, + "genome_compositions.tsv:md5,9168bea9fe82a979a698d259e1bb906a" + ] + ], + "plasmid_fasta": [ + [ + { + "id": "test" + }, + "genome_plasmid.fna.gz:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "plasmid_genes": [ + [ + { + "id": "test" + }, + "genome_plasmid_genes.tsv:md5,55818cae5a57381b77778076e99605e6" + ] + ], + "plasmid_proteins": [ + [ + { + "id": "test" + }, + "genome_plasmid_proteins.faa.gz:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "plasmid_summary": [ + [ + { + "id": "test" + }, + "genome_plasmid_summary.tsv:md5,8c4ddaa8a90da779c11537be0fddb799" + ] + ], + "provirus": [ + [ + { + "id": "test" + }, + "genome_provirus.tsv:md5,93a6bca59b0bf7f57c0b9b60d2e57082" + ] + ], + "taxonomy": [ + [ + { + "id": "test" + }, + "genome_taxonomy.tsv:md5,44c6cf5ff4c836f66648457ae8deb0d5" + ] + ], + "versions": [ + "versions.yml:md5,cebe18512f616530fff490a64e595cb6" + ], + "virus_fasta": [ + [ + { + "id": "test" + }, + "genome_virus.fna.gz:md5,483f4a5dfe60171c86ee9b7e6dff908b" + ] + ], + "virus_genes": [ + [ + { + "id": "test" + }, + "genome_virus_genes.tsv:md5,f84d8a9fce3c92c49e78672bea16561f" + ] + ], + "virus_proteins": [ + [ + { + "id": "test" + }, + "genome_virus_proteins.faa.gz:md5,6850bd46dcb921e5cfb23bfbb8fcc12b" + ] + ], + "virus_summary": [ + [ + { + "id": "test" + }, + "genome_virus_summary.tsv:md5,8cbcf95c2ab397da26b6dca8014e86f4" + ] + ] + } + ], + "timestamp": "2024-01-03T00:07:52.60021527" + } +} \ No newline at end of file diff --git a/modules/nf-core/genomad/endtoend/tests/nextflow.config b/modules/nf-core/genomad/endtoend/tests/nextflow.config new file mode 100644 index 0000000..149f4a4 --- /dev/null +++ b/modules/nf-core/genomad/endtoend/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: GENOMAD_ENDTOEND { + ext.args = '--splits 6 --enable-score-calibration --sensitivity 0.1 --disable-nn-classification' + } +} diff --git a/modules/nf-core/genomad/endtoend/tests/tags.yml b/modules/nf-core/genomad/endtoend/tests/tags.yml new file mode 100644 index 0000000..497ed72 --- /dev/null +++ b/modules/nf-core/genomad/endtoend/tests/tags.yml @@ -0,0 +1,2 @@ +genomad/endtoend: + - "modules/nf-core/genomad/endtoend/**" diff --git a/modules/nf-core/mash/sketch/main.nf b/modules/nf-core/mash/sketch/main.nf index aee3af0..77e3d4b 100644 --- a/modules/nf-core/mash/sketch/main.nf +++ b/modules/nf-core/mash/sketch/main.nf @@ -6,7 +6,6 @@ process MASH_SKETCH { 'https://depot.galaxyproject.org/singularity/mash:2.3--he348c14_1' : 'biocontainers/mash:2.3--he348c14_1' }" - input: tuple val(meta), path(reads) From 66d082ced68853afa8c0cb1a774b77bde2f34957 Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Tue, 23 Apr 2024 16:24:13 +0000 Subject: [PATCH 13/14] Added patch to genomad --- modules.json | 3 +- .../genomad/endtoend/genomad-endtoend.diff | 57 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 modules/nf-core/genomad/endtoend/genomad-endtoend.diff diff --git a/modules.json b/modules.json index 0fd93f3..99fdeb6 100644 --- a/modules.json +++ b/modules.json @@ -59,7 +59,8 @@ "genomad/endtoend": { "branch": "master", "git_sha": "9b68cbd01a704aaf2c34f6d6bc7c25e9b4670147", - "installed_by": ["modules"] + "installed_by": ["modules"], + "patch": "modules/nf-core/genomad/endtoend/genomad-endtoend.diff" }, "gunzip": { "branch": "master", diff --git a/modules/nf-core/genomad/endtoend/genomad-endtoend.diff b/modules/nf-core/genomad/endtoend/genomad-endtoend.diff new file mode 100644 index 0000000..4cf7685 --- /dev/null +++ b/modules/nf-core/genomad/endtoend/genomad-endtoend.diff @@ -0,0 +1,57 @@ +Changes in module 'nf-core/genomad/endtoend' +--- modules/nf-core/genomad/endtoend/main.nf ++++ modules/nf-core/genomad/endtoend/main.nf +@@ -2,14 +2,14 @@ + tag "$meta.id" + label 'process_high' + +- conda "${moduleDir}/environment.yml" ++ conda "bioconda::genomad=1.5.2" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? +- 'https://depot.galaxyproject.org/singularity/genomad:1.7.4--pyhdfd78af_0': +- 'biocontainers/genomad:1.7.4--pyhdfd78af_0' }" ++ 'https://depot.galaxyproject.org/singularity/genomad:1.5.2--pyhdfd78af_0': ++ 'biocontainers/genomad:1.5.2--pyhdfd78af_0' }" + + input: +- tuple val(meta) , path(fasta) +- path genomad_db ++ tuple val(meta), path(fasta) ++ tuple path(genomad_db) + + output: + tuple val(meta), path("*_aggregated_classification/*_aggregated_classification.tsv") , emit: aggregated_classification , optional: true +@@ -82,4 +82,4 @@ + genomad: \$(echo \$(genomad --version 2>&1) | sed 's/^.*geNomad, version //; s/ .*\$//') + END_VERSIONS + """ +-} ++} +--- /dev/null ++++ modules/nf-core/genomad/endtoend/nextflow.config +@@ -0,0 +1,23 @@ ++process { ++ withName: GENOMAD_ENDTOEND { ++ ext.args = [ ++ params.genomad_min_score ? "--min-score ${params.genomad_min_score}" : "", ++ params.genomad_max_fdr ? "--enable-score-calibration --max-fdr ${params.genomad_max_fdr}" : "", ++ params.genomad_splits ? "--splits ${params.genomad_splits}" : "", ++ params.genomad_disable_nn ? "--disable-nn-classification" : "", ++ params.genomad_sensitivity ? "--sensitivity ${params.genomad_sensitivity}" : "" ++ ].join(' ').trim() ++ publishDir = [ ++ [ ++ path: { "${params.outdir}/VirusClassification/genomad/endtoend" }, ++ mode: params.publish_dir_mode, ++ pattern: '*_summary/*_virus.fna.gz' ++ ], ++ [ ++ path: { "${params.outdir}/VirusClassification/genomad/endtoend" }, ++ mode: params.publish_dir_mode, ++ pattern: '*_summary/*_virus_summary.tsv' ++ ] ++ ] ++ } ++} + +************************************************************ From f9da24f4beb5775735a28d98327bd92755517a8a Mon Sep 17 00:00:00 2001 From: Carson J Miller Date: Tue, 23 Apr 2024 17:09:51 +0000 Subject: [PATCH 14/14] Updated adam's GH action version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c535bd..b102cc2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: - name: List nf-test files id: list - uses: adamrtalbot/detect-nf-test-changes@v0.0.1 + uses: adamrtalbot/detect-nf-test-changes@v0.0.2 with: head: ${{ github.sha }} base: origin/${{ github.base_ref }}