diff --git a/.editorconfig b/.editorconfig index b6b31907..dd9ffa53 100644 --- a/.editorconfig +++ b/.editorconfig @@ -18,7 +18,20 @@ end_of_line = unset insert_final_newline = unset trim_trailing_whitespace = unset indent_style = unset -indent_size = unset +[/subworkflows/nf-core/**] +charset = unset +end_of_line = unset +insert_final_newline = unset +trim_trailing_whitespace = unset +indent_style = unset [/assets/email*] indent_size = unset + +# ignore Readme +[README.md] +indent_style = unset + +# ignore python +[*.{py,md}] +indent_style = unset diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 39045ad2..759496ac 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -27,6 +27,9 @@ If you're not used to this workflow with git, you can start with some [docs from ## Tests +You can optionally test your changes by running the pipeline locally. Then it is recommended to use the `debug` profile to +receive warnings about process selectors and other debug info. Example: `nextflow run . -profile debug,test,docker --outdir `. + When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a8f29dc0..72c2b4ae 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -19,6 +19,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/fetc - [ ] If necessary, also make a PR on the nf-core/fetchngs _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). +- [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. - [ ] `CHANGELOG.md` is updated. diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py new file mode 100644 index 00000000..6dc73449 --- /dev/null +++ b/.github/python/find_changed_files.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python + +## This script is used to generate scan *.nf.test files for function/process/workflow name and return as a JSON list +# It is functionally similar to nf-test list but fills a gap until feature https://github.com/askimed/nf-test/issues/196 is added + +import argparse +import json +import logging +import re + +from itertools import chain +from pathlib import Path + + +def parse_args() -> argparse.Namespace: + """ + Parse command line arguments and return an ArgumentParser object. + + Returns: + argparse.ArgumentParser: The ArgumentParser object with the parsed arguments. + """ + parser = argparse.ArgumentParser( + description="Scan *.nf.test files for function/process/workflow name and return as a JSON list" + ) + parser.add_argument( + "-p", + "--paths", + nargs="+", + default=["."], + help="List of directories or files to scan", + ) + parser.add_argument( + "-l", + "--log-level", + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + default="INFO", + help="Logging level", + ) + parser.add_argument( + "-t", + "--types", + nargs="+", + choices=["function", "process", "workflow", "pipeline"], + default=["function", "process", "workflow", "pipeline"], + help="Types of tests to include.", + ) + return parser.parse_args() + + +def find_files(paths: list[str]) -> list[Path]: + """ + Find all files matching pattern *.nf.test recursively from a list of paths. + + Args: + paths (list): List of directories or files to scan. + + Returns: + list: List of files matching the pattern *.nf.test. + """ + # this is a bit clunky + result = [] + for path in paths: + path_obj = Path(path) + # If Path is the exact nf-test file add to list: + if path_obj.match("*.nf.test"): + result.append(path_obj) + # Else recursively search for nf-test files: + else: + for file in path_obj.rglob("*.nf.test"): + result.append(file) + return result + + +def process_files(files: list[Path]) -> list[str]: + """ + Process the files and return lines that begin with 'workflow', 'process', or 'function' and have a single string afterwards. + + Args: + files (list): List of files to process. + + Returns: + list: List of lines that match the criteria. + """ + result = [] + for file in files: + with open(file, "r") as f: + is_pipeline_test = True + lines = f.readlines() + for line in lines: + line = line.strip() + if line.startswith(("workflow", "process", "function")): + words = line.split() + if len(words) == 2 and re.match(r'^".*"$', words[1]): + result.append(line) + is_pipeline_test = False + + # If no results included workflow, process or function + # Add a dummy result to fill the 'pipeline' category + if is_pipeline_test: + result.append("pipeline 'PIPELINE'") + + return result + + +def generate( + lines: list[str], types: list[str] = ["function", "process", "workflow", "pipeline"] +) -> dict[str, list[str]]: + """ + Generate a dictionary of function, process and workflow lists from the lines. + + Args: + lines (list): List of lines to process. + types (list): List of types to include. + + Returns: + dict: Dictionary with function, process and workflow lists. + """ + result: dict[str, list[str]] = { + "function": [], + "process": [], + "workflow": [], + "pipeline": [], + } + for line in lines: + words = line.split() + if len(words) == 2: + keyword = words[0] + name = words[1].strip("'\"") # Strip both single and double quotes + if keyword in types: + result[keyword].append(name) + return result + + +if __name__ == "__main__": + + # Utility stuff + args = parse_args() + logging.basicConfig(level=args.log_level) + + # Parse nf-test files for targets of tests + files = find_files(args.paths) + lines = process_files(files) + result = generate(lines) + + # Get only relevant results (specified by -t) + # Unique using a set + target_results = list( + {item for sublist in map(result.get, args.types) for item in sublist} + ) + + # Print to stdout + print(json.dumps(target_results)) diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 3747c012..51b2b6dc 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -19,7 +19,7 @@ jobs: # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets - name: Post PR comment if: failure() - uses: mshick/add-pr-comment@v1 + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2 with: message: | ## This PR is against the `master` branch :x: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53480a37..fa75579f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,10 @@ on: env: NXF_ANSI_LOG: false - NFTEST_VER: "0.8.1" + NFT_VER: "0.8.4" + NFT_WORKDIR: "~" + NFT_DIFF: "pdiff" + NFT_DIFF_ARGS: "--line-numbers --expand-tabs=2" concurrency: group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}" @@ -24,56 +27,102 @@ jobs: name: Check for changes runs-on: ubuntu-latest outputs: - # Expose matched filters as job 'tags' output variable - tags: ${{ steps.filter.outputs.changes }} + changes: ${{ steps.changed_files.outputs.any_modified }} + tags: ${{ steps.list.outputs.tags }} steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + architecture: "x64" + - uses: actions/checkout@v3 - - name: Combine all tags.yml files - id: get_username - run: find . -name "tags.yml" -not -path "./.github/*" -exec cat {} + > .github/tags.yml - - name: debug - run: cat .github/tags.yml - - uses: dorny/paths-filter@v2 - id: filter with: - filters: ".github/tags.yml" + fetch-depth: 0 - define_nxf_versions: - name: Choose nextflow versions to test against depending on target branch - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.nxf_versions.outputs.matrix }} - steps: - - id: nxf_versions + - uses: tj-actions/changed-files@v42 + id: changed_files + with: + dir_names: "true" + output_renamed_files_as_deleted_and_added: "true" + # Define list of additional rules for testing paths + # Mostly, we define additional 'pipeline' or 'all' tests here + files_yaml: | + ".": + - .github/workflows/** + - nf-test.config + - nextflow.config + tests: + - assets/* + - bin/* + - conf/* + - main.nf + - nextflow_schema.json + + files_ignore: | + .git* + .gitpod.yml + .prettierignore + .prettierrc.yml + **.md + **.png + modules.json + pyproject.toml + tower.yml + + - name: debug run: | - if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.base_ref }}" == "dev" && "${{ matrix.NXF_VER }}" != "latest-everything" ]]; then - echo matrix='["latest-everything"]' | tee -a $GITHUB_OUTPUT - else - echo matrix='["latest-everything", "23.04.0"]' | tee -a $GITHUB_OUTPUT - fi + echo ${{ steps.changed_files.outputs.any_modified }} + echo ${{ steps.changed_files.outputs.all_changed_files }} + echo ${{ steps.changed_files.outputs.changed_keys }} + + - name: nf-test list tags + id: list + if: ${{ steps.changed_files.outputs.any_modified }} + run: | + echo tags=$(python \ + .github/python/find_changed_files.py \ + -t pipeline workflow process \ + -p ${{ steps.changed_files.outputs.all_changed_files }} ${{ steps.changed_files.outputs.changed_keys }} \ + ) >> $GITHUB_OUTPUT + + - name: debug2 + run: | + echo ${{ steps.list.outputs.tags }} test: - name: ${{ matrix.tags }} ${{ matrix.profile }} NF ${{ matrix.NXF_VER }} - needs: [changes, define_nxf_versions] - if: needs.changes.outputs.tags != '[]' + name: ${{ matrix.tags }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }} + needs: [changes] + if: needs.changes.outputs.changes runs-on: ubuntu-latest strategy: fail-fast: false matrix: - NXF_VER: ${{ fromJson(needs.define_nxf_versions.outputs.matrix) }} + NXF_VER: + - "latest-everything" + - "23.04" tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] profile: - "docker" steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 with: version: "${{ matrix.NXF_VER }}" + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + architecture: "x64" + + - name: Install pdiff to see diff between nf-test snapshots + run: | + python -m pip install --upgrade pip + pip install pdiff + - name: Cache nf-test installation id: cache-software uses: actions/cache@v3 @@ -81,7 +130,7 @@ jobs: path: | /usr/local/bin/nf-test /home/runner/.nf-test/nf-test.jar - key: ${{ runner.os }}-${{ env.NFTEST_VER }}-nftest + key: ${{ runner.os }}-${{ env.NFT_VER }}-nftest - name: Install nf-test if: steps.cache-software.outputs.cache-hit != 'true' @@ -91,7 +140,12 @@ jobs: - name: Run nf-test run: | - nf-test test --tag ${{ matrix.tags }} --profile "test,${{ matrix.profile }}" --junitxml=test.xml + nf-test test --verbose --tag ${{ matrix.tags }} --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + + - uses: pcolby/tap-summary@v1 + with: + path: >- + test.tap - name: Output log on failure if: failure() @@ -104,3 +158,23 @@ jobs: if: always() # always run even if the previous step fails with: report_paths: test.xml + + confirm-pass: + runs-on: ubuntu-latest + needs: + - changes + - test + if: always() + steps: + - name: All tests ok + if: ${{ !contains(needs.*.result, 'failure') }} + run: exit 0 + - name: One or more tests failed + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + + - name: debug-print + if: always() + run: | + echo "toJSON(needs) = ${{ toJSON(needs) }}" + echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}" diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml index 694e90ec..0b6b1f27 100644 --- a/.github/workflows/clean-up.yml +++ b/.github/workflows/clean-up.yml @@ -10,7 +10,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v7 + - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9 with: stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful." diff --git a/.github/workflows/cloud_tests_full.yml b/.github/workflows/cloud_tests_full.yml index d0ed0552..81cabbfd 100644 --- a/.github/workflows/cloud_tests_full.yml +++ b/.github/workflows/cloud_tests_full.yml @@ -19,61 +19,78 @@ jobs: run-full-tests-on-aws: if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'aws' || !github.event.inputs }} runs-on: ubuntu-latest + strategy: + matrix: + download_method: ["aspera", "ftp", "sratools"] steps: - - uses: seqeralabs/action-tower-launch@v1 + - uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_CE_AWS_CPU }} workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/fetchngs/work-${{ github.sha }}" - run_name: "aws_fetchngs_full" + run_name: "aws_fetchngs_full_${{ matrix.download_method }}" + revision: ${{ github.sha }} profiles: test_full parameters: | { "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", - "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/fetchngs/results-${{ github.sha }}" + "download_method": "${{ matrix.download_method }}", + "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/fetchngs/results-${{ github.sha }}/download_method_${{ matrix.download_method }}/" } - uses: actions/upload-artifact@v3 with: name: Tower debug log file path: tower_action_*.log - run-full-tests-on-gcp: - if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'gcp' || !github.event.inputs }} + + run-full-tests-on-azure: + if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'azure' || !github.event.inputs }} runs-on: ubuntu-latest + strategy: + matrix: + download_method: ["aspera", "ftp", "sratools"] steps: - - uses: seqeralabs/action-tower-launch@v1 + - uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} - compute_env: ${{ secrets.TOWER_CE_GCP_CPU }} - workdir: "${{ secrets.TOWER_BUCKET_GCP }}/work/fetchngs/work-${{ github.sha }}" - run_name: "gcp_fetchngs_full" + compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }} + workdir: "${{ secrets.TOWER_BUCKET_AZURE }}/work/fetchngs/work-${{ github.sha }}" + run_name: "azure_fetchngs_full_${{ matrix.download_method }}" + revision: ${{ github.sha }} profiles: test_full parameters: | { "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", - "outdir": "${{ secrets.TOWER_BUCKET_GCP }}/fetchngs/results-${{ github.sha }}" + "download_method": "${{ matrix.download_method }}", + "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/fetchngs/results-${{ github.sha }}/download_method_${{ matrix.download_method }}/" } - uses: actions/upload-artifact@v3 with: name: Tower debug log file path: tower_action_*.log - run-full-tests-on-azure: - if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'azure' || !github.event.inputs }} + + run-full-tests-on-gcp: + if: ${{ github.event.inputs.platform == 'gcp' || !github.event.inputs }} runs-on: ubuntu-latest + strategy: + matrix: + download_method: ["aspera", "ftp", "sratools"] steps: - - uses: seqeralabs/action-tower-launch@v1 + - uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} - compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }} - workdir: "${{ secrets.TOWER_BUCKET_AZURE }}/work/fetchngs/work-${{ github.sha }}" - run_name: "azure_fetchngs_full" + compute_env: ${{ secrets.TOWER_CE_GCP_CPU }} + workdir: "${{ secrets.TOWER_BUCKET_GCP }}/work/fetchngs/work-${{ github.sha }}" + run_name: "gcp_fetchngs_full_${{ matrix.download_method }}" + revision: ${{ github.sha }} profiles: test_full parameters: | { "hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}", - "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/fetchngs/results-${{ github.sha }}" + "download_method": "${{ matrix.download_method }}", + "outdir": "${{ secrets.TOWER_BUCKET_GCP }}/fetchngs/results-${{ github.sha }}/download_method_${{ matrix.download_method }}/" } - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/cloud_tests_small.yml b/.github/workflows/cloud_tests_small.yml index 12b8daab..195eb9d4 100644 --- a/.github/workflows/cloud_tests_small.yml +++ b/.github/workflows/cloud_tests_small.yml @@ -18,57 +18,62 @@ jobs: if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'aws' }} runs-on: ubuntu-latest steps: - - uses: seqeralabs/action-tower-launch@v1 + - uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} compute_env: ${{ secrets.TOWER_CE_AWS_CPU }} workdir: "${{ secrets.TOWER_BUCKET_AWS }}/work/fetchngs/work-${{ github.sha }}" run_name: "aws_fetchngs_small" + revision: ${{ github.sha }} profiles: test parameters: | { - "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/fetchngs/results-test-${{ github.sha }}" + "outdir": "${{ secrets.TOWER_BUCKET_AWS }}/fetchngs/results-test-${{ github.sha }}/" } - uses: actions/upload-artifact@v3 with: name: Tower debug log file path: tower_action_*.log - run-small-tests-on-gcp: - if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'gcp' }} + + run-small-tests-on-azure: + if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'azure' }} runs-on: ubuntu-latest steps: - - uses: seqeralabs/action-tower-launch@v1 + - uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} - compute_env: ${{ secrets.TOWER_CE_GCP_CPU }} - workdir: "${{ secrets.TOWER_BUCKET_GCP }}/work/fetchngs/work-${{ github.sha }}" - run_name: "gcp_fetchngs_small" + compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }} + workdir: "${{ secrets.TOWER_BUCKET_AZURE }}/work/fetchngs/work-${{ github.sha }}" + run_name: "azure_fetchngs_small" + revision: ${{ github.sha }} profiles: test parameters: | { - "outdir": "${{ secrets.TOWER_BUCKET_GCP }}/fetchngs/results-test-${{ github.sha }}" + "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/fetchngs/results-test-${{ github.sha }}/" } - uses: actions/upload-artifact@v3 with: name: Tower debug log file path: tower_action_*.log - run-small-tests-on-azure: - if: ${{ github.event.inputs.platform == 'all' || github.event.inputs.platform == 'azure' }} + + run-small-tests-on-gcp: + if: ${{ github.event.inputs.platform == 'gcp' }} runs-on: ubuntu-latest steps: - - uses: seqeralabs/action-tower-launch@v1 + - uses: seqeralabs/action-tower-launch@v2 with: workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }} access_token: ${{ secrets.TOWER_ACCESS_TOKEN }} - compute_env: ${{ secrets.TOWER_CE_AZURE_CPU }} - workdir: "${{ secrets.TOWER_BUCKET_AZURE }}/work/fetchngs/work-${{ github.sha }}" - run_name: "azure_fetchngs_small" + compute_env: ${{ secrets.TOWER_CE_GCP_CPU }} + workdir: "${{ secrets.TOWER_BUCKET_GCP }}/work/fetchngs/work-${{ github.sha }}" + run_name: "gcp_fetchngs_small" + revision: ${{ github.sha }} profiles: test parameters: | { - "outdir": "${{ secrets.TOWER_BUCKET_AZURE }}/fetchngs/results-test-${{ github.sha }}" + "outdir": "${{ secrets.TOWER_BUCKET_GCP }}/fetchngs/results-test-${{ github.sha }}/" } - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml new file mode 100644 index 00000000..f823210d --- /dev/null +++ b/.github/workflows/download_pipeline.yml @@ -0,0 +1,72 @@ +name: Test successful pipeline download with 'nf-core download' + +# Run the workflow when: +# - dispatched manually +# - when a PR is opened or reopened to master branch +# - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev. +on: + workflow_dispatch: + inputs: + testbranch: + description: "The specific branch you wish to utilize for the test execution of nf-core download." + required: true + default: "dev" + pull_request: + types: + - opened + branches: + - master + pull_request_target: + branches: + - master + +env: + NXF_ANSI_LOG: false + +jobs: + download: + runs-on: ubuntu-latest + steps: + - name: Install Nextflow + uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 + + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + with: + python-version: "3.11" + architecture: "x64" + - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 + with: + singularity-version: 3.8.3 + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install git+https://github.com/nf-core/tools.git@dev + + - name: Get the repository name and current branch set as environment variable + run: | + echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV} + echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} + echo "REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV} + + - name: Download the pipeline + env: + NXF_SINGULARITY_CACHEDIR: ./ + run: | + nf-core download ${{ env.REPO_LOWERCASE }} \ + --revision ${{ env.REPO_BRANCH }} \ + --outdir ./${{ env.REPOTITLE_LOWERCASE }} \ + --compress "none" \ + --container-system 'singularity' \ + --container-library "quay.io" -l "docker.io" -l "ghcr.io" \ + --container-cache-utilisation 'amend' \ + --download-configuration + + - name: Inspect download + run: tree ./${{ env.REPOTITLE_LOWERCASE }} + + - name: Run the downloaded pipeline + env: + NXF_SINGULARITY_CACHEDIR: ./ + NXF_SINGULARITY_HOME_MOUNT: true + run: nextflow run ./${{ env.REPOTITLE_LOWERCASE }}/$( sed 's/\W/_/g' <<< ${{ env.REPO_BRANCH }}) -stub -profile test,singularity --outdir ./results diff --git a/.github/workflows/fix-linting.yml b/.github/workflows/fix-linting.yml index 6e772fce..18efcb70 100644 --- a/.github/workflows/fix-linting.yml +++ b/.github/workflows/fix-linting.yml @@ -4,7 +4,7 @@ on: types: [created] jobs: - deploy: + fix-linting: # Only run if comment is on a PR with the main repo, and if it contains the magic keywords if: > contains(github.event.comment.html_url, '/pull/') && @@ -13,10 +13,17 @@ jobs: runs-on: ubuntu-latest steps: # Use the @nf-core-bot token to check out so we can push later - - uses: actions/checkout@v3 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 with: token: ${{ secrets.nf_core_bot_auth_token }} + # indication that the linting is being fixed + - name: React on comment + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: eyes + # Action runs on the issue comment, so we don't get the PR by default # Use the gh cli to check out the PR - name: Checkout Pull Request @@ -24,32 +31,59 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }} - - uses: actions/setup-node@v3 + # Install and run pre-commit + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + with: + python-version: 3.11 - - name: Install Prettier - run: npm install -g prettier @prettier/plugin-php + - name: Install pre-commit + run: pip install pre-commit - # Check that we actually need to fix something - - name: Run 'prettier --check' - id: prettier_status - run: | - if prettier --check ${GITHUB_WORKSPACE}; then - echo "result=pass" >> $GITHUB_OUTPUT - else - echo "result=fail" >> $GITHUB_OUTPUT - fi + - name: Run pre-commit + id: pre-commit + run: pre-commit run --all-files + continue-on-error: true - - name: Run 'prettier --write' - if: steps.prettier_status.outputs.result == 'fail' - run: prettier --write ${GITHUB_WORKSPACE} + # indication that the linting has finished + - name: react if linting finished succesfully + if: steps.pre-commit.outcome == 'success' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: "+1" - name: Commit & push changes - if: steps.prettier_status.outputs.result == 'fail' + id: commit-and-push + if: steps.pre-commit.outcome == 'failure' run: | git config user.email "core@nf-co.re" git config user.name "nf-core-bot" git config push.default upstream git add . git status - git commit -m "[automated] Fix linting with Prettier" + git commit -m "[automated] Fix code linting" git push + + - name: react if linting errors were fixed + id: react-if-fixed + if: steps.commit-and-push.outcome == 'success' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: hooray + + - name: react if linting errors were not fixed + if: steps.commit-and-push.outcome == 'failure' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + comment-id: ${{ github.event.comment.id }} + reactions: confused + + - name: react if linting errors were not fixed + if: steps.commit-and-push.outcome == 'failure' + uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043 # v4 + with: + issue-number: ${{ github.event.issue.number }} + body: | + @${{ github.actor }} I tried to fix the linting errors, but it didn't work. Please fix them manually. + See [CI log](https://github.com/nf-core/fetchngs/actions/runs/${{ github.run_id }}) for more details. diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index b8bdd214..748b4311 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -11,72 +11,33 @@ on: types: [published] jobs: - EditorConfig: + pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - - uses: actions/setup-node@v3 - - - name: Install editorconfig-checker - run: npm install -g editorconfig-checker - - - name: Run ECLint check - run: editorconfig-checker -exclude README.md $(find .* -type f | grep -v '.git\|.py\|.md\|json\|yml\|yaml\|html\|css\|work\|.nextflow\|build\|nf_core.egg-info\|log.txt\|Makefile') - - Prettier: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-node@v3 - - - name: Install Prettier - run: npm install -g prettier - - - name: Run Prettier --check - run: prettier --check ${GITHUB_WORKSPACE} - - PythonBlack: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Check code lints with Black - uses: psf/black@stable - - # If the above check failed, post a comment on the PR explaining the failure - - name: Post PR comment - if: failure() - uses: mshick/add-pr-comment@v1 + - name: Set up Python 3.11 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: - message: | - ## Python linting (`black`) is failing - - To keep the code consistent with lots of contributors, we run automated code consistency checks. - To fix this CI test, please run: - - * Install [`black`](https://black.readthedocs.io/en/stable/): `pip install black` - * Fix formatting errors in your pipeline: `black .` - - Once you push these changes the test should pass, and you can hide this comment :+1: + python-version: 3.11 + cache: "pip" - We highly recommend setting up Black in your code editor so that this formatting is done automatically on save. Ask about it on Slack for help! + - name: Install pre-commit + run: pip install pre-commit - Thanks again for your contribution! - repo-token: ${{ secrets.GITHUB_TOKEN }} - allow-repeats: false + - name: Run pre-commit + run: pre-commit run --all-files nf-core: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@v1 + uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.11" architecture: "x64" @@ -99,7 +60,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 with: name: linting-logs path: | diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 0bbcd30f..b706875f 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@v2 + uses: dawidd6/action-download-artifact@f6b0bace624032e30a85a8fd9c1a7f8f611f5737 # v3 with: workflow: linting.yml workflow_conclusion: completed @@ -21,7 +21,7 @@ jobs: run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment - uses: marocchino/sticky-pull-request-comment@v2 + uses: marocchino/sticky-pull-request-comment@331f8f5b4215f0445d3c07b4967662a32a2d3e31 # v2 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} number: ${{ steps.pr_number.outputs.pr_number }} diff --git a/.github/workflows/release-announcments.yml b/.github/workflows/release-announcements.yml similarity index 81% rename from .github/workflows/release-announcments.yml rename to .github/workflows/release-announcements.yml index 6ad33927..c3674af2 100644 --- a/.github/workflows/release-announcments.yml +++ b/.github/workflows/release-announcements.yml @@ -9,6 +9,11 @@ jobs: toot: runs-on: ubuntu-latest steps: + - name: get topics and convert to hashtags + id: get_topics + run: | + curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ' > $GITHUB_OUTPUT + - uses: rzr/fediverse-action@master with: access-token: ${{ secrets.MASTODON_ACCESS_TOKEN }} @@ -20,11 +25,13 @@ jobs: Please see the changelog: ${{ github.event.release.html_url }} + ${{ steps.get_topics.outputs.GITHUB_OUTPUT }} #nfcore #openscience #nextflow #bioinformatics + send-tweet: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@v4 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.10" - name: Install dependencies @@ -56,7 +63,7 @@ jobs: bsky-post: runs-on: ubuntu-latest steps: - - uses: zentered/bluesky-post-action@v0.0.2 + - uses: zentered/bluesky-post-action@80dbe0a7697de18c15ad22f4619919ceb5ccf597 # v0.1.0 with: post: | Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}! diff --git a/.gitignore b/.gitignore index ef809d7c..3e70ad89 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,12 @@ -*.pyc -.DS_Store .nextflow* -.nf-test.log +work/ data/ +results/ +.DS_Store +testing/ +testing* +*.pyc +.nf-test.log nf-test .nf-test* -results/ test.xml -testing* -testing/ -work/ diff --git a/.gitpod.yml b/.gitpod.yml index 25488dcc..363d5b1d 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -4,6 +4,9 @@ tasks: command: | pre-commit install --install-hooks nextflow self-update + - name: unset JAVA_TOOL_OPTIONS + command: | + unset JAVA_TOOL_OPTIONS vscode: extensions: # based on nf-core.nf-core-extensionpack diff --git a/.nf-core.yml b/.nf-core.yml index 0e89cae9..3a4ae4a7 100644 --- a/.nf-core.yml +++ b/.nf-core.yml @@ -2,18 +2,6 @@ repository_type: pipeline lint: actions_ci: false files_exist: - - .github/workflows/awsfulltest.yml - - .github/workflows/awstest.yml - - assets/multiqc_config.yml - - conf/igenomes.config - conf/modules.config - - lib/NfcoreTemplate.groovy - - lib/Utils.groovy - - lib/WorkflowFetchngs.groovy - - lib/WorkflowMain.groovy - - lib/nfcore_external_java_deps.jar files_unchanged: - - .gitattributes - - .gitignore - assets/sendmail_template.txt - - lib/NfcoreTemplate.groovy diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0c31cdb9..af57081f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,10 @@ repos: - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v2.7.1" + rev: "v3.1.0" hooks: - id: prettier + - repo: https://github.com/editorconfig-checker/editorconfig-checker.python + rev: "2.7.3" + hooks: + - id: editorconfig-checker + alias: ec diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e484a7a..e0261675 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,85 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [[1.12.0](https://github.com/nf-core/fetchngs/releases/tag/1.12.0)] - 2024-02-29 + +### :warning: Major enhancements + +- The Aspera CLI was recently added to [Bioconda](https://anaconda.org/bioconda/aspera-cli) and we have added it as another way of downloading FastQ files in addition to the existing FTP and sra-tools support. In our limited benchmarks on all public Clouds we found ~50% speed-up in download times compared to FTP! FTP downloads will still be the default download method (i.e. `--download_method ftp`) but you can choose to use sra-tools or Aspera using `--download_method sratools` or `--download_method aspera`, respectively. We would love to have your feedback! +- The `--force_sratools_download` parameter has been deprecated in favour of using `--download_method ` to explicitly specify the download method; available options are `ftp`, `sratools` or `aspera`. +- Support for Synapse ids has been dropped in this release. We haven't had any feedback from users whether it is being used or not. Users can run earlier versions of the pipeline if required. +- We have significantly refactored and standardised the way we are using nf-test within this pipeline. This pipeline is now the current, best-practice implementation for nf-test usage on nf-core. We required a number of features to be added to nf-test and a huge shoutout to [Lukas Forer](https://github.com/lukfor) for entertaining our requests and implementing them within upstream :heart:! + +### Credits + +Special thanks to the following for their contributions to the release: + +- [Adam Talbot](https://github.com/adamrtalbot) +- [Alexandru Mizeranschi](https://github.com/nicolae06) +- [Alexander Blaessle](https://github.com/alexblaessle) +- [Lukas Forer](https://github.com/lukfor) +- [Matt Niederhuber](https://github.com/mniederhuber) +- [Maxime Garcia](https://github.com/maxulysse) +- [Sateesh Peri](https://github.com/sateeshperi) +- [Sebastian Uhrig](https://github.com/suhrig) + +Thank you to everyone else that has contributed by reporting bugs, enhancements or in any other way, shape or form. + +### Enhancements & fixes + +- [PR #238](https://github.com/nf-core/fetchngs/pull/238) - Resolved bug when prefetching large studies ([#236](https://github.com/nf-core/fetchngs/issues/236)) +- [PR #241](https://github.com/nf-core/fetchngs/pull/241) - Use wget instead of curl to download files from FTP ([#169](https://github.com/nf-core/fetchngs/issues/169), [#194](https://github.com/nf-core/fetchngs/issues/194)) +- [PR #242](https://github.com/nf-core/fetchngs/pull/242) - Template update for nf-core/tools v2.11 +- [PR #243](https://github.com/nf-core/fetchngs/pull/243) - Fixes for [PR #238](https://github.com/nf-core/fetchngs/pull/238) +- [PR #245](https://github.com/nf-core/fetchngs/pull/246) - Refactor nf-test CI and test and other pre-release fixes ([#233](https://github.com/nf-core/fetchngs/issues/233)) +- [PR #246](https://github.com/nf-core/fetchngs/pull/246) - Handle dark/light mode for logo in GitHub README properly +- [PR #248](https://github.com/nf-core/fetchngs/pull/248) - Update pipeline level test data path to use mirror on s3 +- [PR #249](https://github.com/nf-core/fetchngs/pull/249) - Update modules which includes absolute paths for test data, making module level test compatible within the pipeline. +- [PR #253](https://github.com/nf-core/fetchngs/pull/253) - Add implicit tags in nf-test files for simpler testing strategy +- [PR #257](https://github.com/nf-core/fetchngs/pull/257) - Template update for nf-core/tools v2.12 +- [PR #258](https://github.com/nf-core/fetchngs/pull/258) - Fixes for [PR #253](https://github.com/nf-core/fetchngs/pull/253) +- [PR #259](https://github.com/nf-core/fetchngs/pull/259) - Add Aspera CLI download support to pipeline ([#68](https://github.com/nf-core/fetchngs/issues/68)) +- [PR #261](https://github.com/nf-core/fetchngs/pull/261) - Revert sratools fasterqdump version ([#221](https://github.com/nf-core/fetchngs/issues/221)) +- [PR #262](https://github.com/nf-core/fetchngs/pull/262) - Use nf-test version v0.8.4 and remove implicit tags +- [PR #263](https://github.com/nf-core/fetchngs/pull/263) - Refine tags used for workflows +- [PR #264](https://github.com/nf-core/fetchngs/pull/264) - Remove synapse workflow from pipeline +- [PR #265](https://github.com/nf-core/fetchngs/pull/265) - Use "+" syntax for profiles to accumulate profiles in nf-test +- [PR #266](https://github.com/nf-core/fetchngs/pull/266) - Make .gitignore match template +- [PR #268](https://github.com/nf-core/fetchngs/pull/268) - Add mermaid diagram +- [PR #273](https://github.com/nf-core/fetchngs/pull/273) - Update utility subworkflows +- [PR #283](https://github.com/nf-core/fetchngs/pull/283) - Template update for nf-core/tools v2.13 +- [PR #288](https://github.com/nf-core/fetchngs/pull/288) - Update Github Action to run full-sized test for all 3 download methods +- [PR #290](https://github.com/nf-core/fetchngs/pull/290) - Remove mentions of deprecated Synapse functionality in pipeline +- [PR #294](https://github.com/nf-core/fetchngs/pull/294) - Replace mermaid diagram with subway map +- [PR #295](https://github.com/nf-core/fetchngs/pull/295) - Be less stringent with test expectations for CI +- [PR #296](https://github.com/nf-core/fetchngs/pull/296) - Remove params.outdir from tests where required and update snapshots +- [PR #298](https://github.com/nf-core/fetchngs/pull/298) - `export CONDA_PREFIX` into container when using Singularity and Apptainer + +### Software dependencies + +| Dependency | Old version | New version | +| ---------- | ----------- | ----------- | +| `wget` | | 1.20.1 | + +> **NB:** Dependency has been **updated** if both old and new version information is present. +> +> **NB:** Dependency has been **added** if just the new version information is present. +> +> **NB:** Dependency has been **removed** if new version information isn't present. + +### Parameters + +| Old parameter | New parameter | +| --------------------------- | ------------------- | +| | `--download_method` | +| `--input_type` | | +| `--force_sratools_download` | | +| `--synapse_config` | | + +> **NB:** Parameter has been **updated** if both old and new parameter information is present. +> **NB:** Parameter has been **added** if just the new parameter information is present. +> **NB:** Parameter has been **removed** if new parameter information isn't present. + ## [[1.11.0](https://github.com/nf-core/fetchngs/releases/tag/1.11.0)] - 2023-10-18 ### Credits @@ -10,7 +89,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Special thanks to the following for their contributions to the release: - [Adam Talbot](https://github.com/adamrtalbot) -- [Edmund Miller](https://github.com/Emiller88) +- [Edmund Miller](https://github.com/edmundmiller) - [Esha Joshi](https://github.com/ejseqera) - [Harshil Patel](https://github.com/drpatelh) - [Lukas Forer](https://github.com/lukfor) @@ -26,8 +105,6 @@ Thank you to everyone else that has contributed by reporting bugs, enhancements - [PR #188](https://github.com/nf-core/fetchngs/pull/188) - Use nf-test for all pipeline testing -### Enhancements & fixes - ## [[1.10.1](https://github.com/nf-core/fetchngs/releases/tag/1.10.1)] - 2023-10-08 ### Credits diff --git a/CITATIONS.md b/CITATIONS.md index 482d5600..62235e72 100644 --- a/CITATIONS.md +++ b/CITATIONS.md @@ -10,6 +10,8 @@ ## Pipeline tools +- [Aspera CLI](https://github.com/IBM/aspera-cli) + - [Python](http://www.python.org) - [Requests](https://docs.python-requests.org/) @@ -34,9 +36,6 @@ > Barrett T, Wilhite SE, Ledoux P, Evangelista C, Kim IF, Tomashevsky M, Marshall KA, Phillippy KH, Sherman PM, Holko M, Yefanov A, Lee H, Zhang N, Robertson CL, Serova N, Davis S, Soboleva A. NCBI GEO: archive for functional genomics data sets--update. Nucleic Acids Res. 2013 Jan;41(Database issue):D991-5. doi: 10.1093/nar/gks1193. Epub 2012 Nov 27. PubMed PMID: 23193258; PubMed Central PMCID: PMC3531084. -- [Synapse](https://pubmed.ncbi.nlm.nih.gov/24071850/) - > Omberg L, Ellrott K, Yuan Y, Kandoth C, Wong C, Kellen MR, Friend SH, Stuart J, Liang H, Margolin AA. Enabling transparent and collaborative computational analysis of 12 tumor types within The Cancer Genome Atlas. Nat Genet. 2013 Oct;45(10):1121-6. doi: 10.1038/ng.2761. PMID: 24071850; PMCID: PMC3950337. - ## Software packaging/containerisation/testing tools - [Anaconda](https://anaconda.com) diff --git a/README.md b/README.md index 02be6056..d6c5c597 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,31 @@ -# ![nf-core/fetchngs](docs/images/nf-core-fetchngs_logo_light.png#gh-light-mode-only) ![nf-core/fetchngs](docs/images/nf-core-fetchngs_logo_dark.png#gh-dark-mode-only) +

+ + + nf-core/fetchngs + +

-[![GitHub Actions CI Status](https://github.com/nf-core/fetchngs/workflows/nf-core%20CI/badge.svg)](https://github.com/nf-core/fetchngs/actions?query=workflow%3A%22nf-core+CI%22) -[![GitHub Actions Linting Status](https://github.com/nf-core/fetchngs/workflows/nf-core%20linting/badge.svg)](https://github.com/nf-core/fetchngs/actions?query=workflow%3A%22nf-core+linting%22) -[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/fetchngs/results) -[![nf-test](https://img.shields.io/badge/tested_with-nf--test-337ab7.svg)](https://github.com/askimed/nf-test) -[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.5070524-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.5070524) +[![GitHub Actions CI Status](https://github.com/nf-core/fetchngs/actions/workflows/ci.yml/badge.svg)](https://github.com/nf-core/fetchngs/actions/workflows/ci.yml) +[![GitHub Actions Linting Status](https://github.com/nf-core/fetchngs/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/fetchngs/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/fetchngs/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.5070524-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.5070524)[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) -[![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/fetchngs) +[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/fetchngs) [![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23fetchngs-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/fetchngs)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Follow on Mastodon](https://img.shields.io/badge/mastodon-nf__core-6364ff?labelColor=FFFFFF&logo=mastodon)](https://mstdn.science/@nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) ## Introduction -**nf-core/fetchngs** is a bioinformatics pipeline to fetch metadata and raw FastQ files from both public and private databases. At present, the pipeline supports SRA / ENA / DDBJ / GEO / Synapse ids (see [usage docs](https://nf-co.re/fetchngs/usage#introduction)). +**nf-core/fetchngs** is a bioinformatics pipeline to fetch metadata and raw FastQ files from both public databases. At present, the pipeline supports SRA / ENA / DDBJ / GEO ids (see [usage docs](https://nf-co.re/fetchngs/usage#introduction)). + +![nf-core/fetchngs metro map](docs/images/nf-core-fetchngs_metro_map_grey.png) ## Usage -:::note -If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how -to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) -with `-profile test` before running the workflow on actual data. -::: +> [!NOTE] +> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data. First, prepare a samplesheet with your input data that looks as follows: @@ -50,11 +51,9 @@ nextflow run nf-core/fetchngs \ --outdir ``` -:::warning -Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those -provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; -see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). -::: +> [!WARNING] +> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_; +> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files). For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/fetchngs/usage) and the [parameter documentation](https://nf-co.re/fetchngs/parameters). @@ -67,17 +66,12 @@ Via a single file of ids, provided one-per-line (see [example input file](https: 1. Resolve database ids back to appropriate experiment-level ids and to be compatible with the [ENA API](https://ena-docs.readthedocs.io/en/latest/retrieval/programmatic-access.html) 2. Fetch extensive id metadata via ENA API 3. Download FastQ files: - - If direct download links are available from the ENA API, fetch in parallel via `curl` and perform `md5sum` check - - Otherwise use [`sra-tools`](https://github.com/ncbi/sra-tools) to download `.sra` files and convert them to FastQ + - If direct download links are available from the ENA API: + - Fetch in parallel via `wget` and perform `md5sum` check (`--download_method ftp`; default). + - Fetch in parallel via `aspera-cli` and perform `md5sum` check. Use `--download_method aspera` to force this behaviour. + - Otherwise use [`sra-tools`](https://github.com/ncbi/sra-tools) to download `.sra` files and convert them to FastQ. Use `--download_method sratools` to force this behaviour. 4. Collate id metadata and paths to FastQ files in a single samplesheet -### Synapse ids - -1. Resolve Synapse directory ids to their corresponding FastQ files ids via the `synapse list` command. -2. Retrieve FastQ file metadata including FastQ file names, md5sums, etags, annotations and other data provenance via the `synapse show` command. -3. Download FastQ files in parallel via `synapse get` -4. Collate paths to FastQ files in a single samplesheet - ## Pipeline output The columns in the output samplesheet can be tailored to be accepted out-of-the-box by selected nf-core pipelines (see [usage docs](https://nf-co.re/fetchngs/usage#samplesheet-format)), these currently include: @@ -93,7 +87,7 @@ For more details about the output files and reports, please refer to the ## Credits -nf-core/fetchngs was originally written by Harshil Patel ([@drpatelh](https://github.com/drpatelh)) from [Seqera Labs, Spain](https://seqera.io/) and Jose Espinosa-Carrasco ([@JoseEspinosa](https://github.com/JoseEspinosa)) from [The Comparative Bioinformatics Group](https://www.crg.eu/en/cedric_notredame) at [The Centre for Genomic Regulation, Spain](https://www.crg.eu/). Support for download of sequencing reads without FTP links via sra-tools was added by Moritz E. Beber ([@Midnighter](https://github.com/Midnighter)) from [Unseen Bio ApS, Denmark](https://unseenbio.com). The Synapse workflow was added by Daisy Han [@daisyhan97](https://github.com/daisyhan97) and Bruno Grande [@BrunoGrandePhD](https://github.com/BrunoGrandePhD) from [Sage Bionetworks, Seattle](https://sagebionetworks.org/). +nf-core/fetchngs was originally written by Harshil Patel ([@drpatelh](https://github.com/drpatelh)) from [Seqera Labs, Spain](https://seqera.io/) and Jose Espinosa-Carrasco ([@JoseEspinosa](https://github.com/JoseEspinosa)) from [The Comparative Bioinformatics Group](https://www.crg.eu/en/cedric_notredame) at [The Centre for Genomic Regulation, Spain](https://www.crg.eu/). Support for download of sequencing reads without FTP links via sra-tools was added by Moritz E. Beber ([@Midnighter](https://github.com/Midnighter)) from [Unseen Bio ApS, Denmark](https://unseenbio.com). ## Contributions and Support diff --git a/assets/email_template.html b/assets/email_template.html index 1848021d..6e593a53 100644 --- a/assets/email_template.html +++ b/assets/email_template.html @@ -12,7 +12,7 @@ -

nf-core/fetchngs v${version}

+

nf-core/fetchngs ${version}

Run Name: $runName

<% if (!success){ diff --git a/assets/email_template.txt b/assets/email_template.txt index f9393aa8..e9f8bcc6 100644 --- a/assets/email_template.txt +++ b/assets/email_template.txt @@ -4,7 +4,7 @@ |\\ | |__ __ / ` / \\ |__) |__ } { | \\| | \\__, \\__/ | \\ |___ \\`-._,-`-, `._,._,' - nf-core/fetchngs v${version} + nf-core/fetchngs ${version} ---------------------------------------------------- Run Name: $runName diff --git a/assets/methods_description_template.yml b/assets/methods_description_template.yml index 49a0a9fd..469f4a46 100644 --- a/assets/methods_description_template.yml +++ b/assets/methods_description_template.yml @@ -3,8 +3,6 @@ description: "Suggested text and references to use when describing pipeline usag section_name: "nf-core/fetchngs Methods Description" section_href: "https://github.com/nf-core/fetchngs" plot_type: "html" -## TODO nf-core: Update the HTML below to your preferred methods description, e.g. add publication citation for this pipeline -## You inject any metadata in the Nextflow '${workflow}' object data: |

Methods

Data was processed using nf-core/fetchngs v${workflow.manifest.version} ${doi_text} of the nf-core collection of workflows (Ewels et al., 2020), utilising reproducible software environments from the Bioconda (Grüning et al., 2018) and Biocontainers (da Veiga Leprevost et al., 2017) projects.

diff --git a/assets/nf-core-fetchngs_logo_light.png b/assets/nf-core-fetchngs_logo_light.png index c201c150..919b9063 100644 Binary files a/assets/nf-core-fetchngs_logo_light.png and b/assets/nf-core-fetchngs_logo_light.png differ diff --git a/assets/schema_input.json b/assets/schema_input.json index d451f9e9..db9ffc00 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -7,9 +7,9 @@ "items": { "type": "object", "properties": { - "id": { + "": { "type": "string", - "pattern": "^(((SR|ER|DR)[APRSX])|(SAM(N|EA|D))|(PRJ(NA|EB|DB))|(GS[EM])|(syn))(\\d+)$", + "pattern": "^(((SR|ER|DR)[APRSX])|(SAM(N|EA|D))|(PRJ(NA|EB|DB))|(GS[EM]))(\\d+)$", "errorMessage": "Please provide a valid SRA, ENA, DDBJ or GEO identifier" } } diff --git a/assets/slackreport.json b/assets/slackreport.json index aa5db2a1..b39981cb 100644 --- a/assets/slackreport.json +++ b/assets/slackreport.json @@ -3,7 +3,7 @@ { "fallback": "Plain-text summary of the attachment.", "color": "<% if (success) { %>good<% } else { %>danger<%} %>", - "author_name": "nf-core/fetchngs v${version} - ${runName}", + "author_name": "nf-core/fetchngs ${version} - ${runName}", "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico", "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>", "fields": [ diff --git a/bin/multiqc_mappings_config.py b/bin/multiqc_mappings_config.py index 2fcdb49a..3ffe35ec 100755 --- a/bin/multiqc_mappings_config.py +++ b/bin/multiqc_mappings_config.py @@ -7,6 +7,7 @@ config = "sample_names_rename_buttons:\n" config += "\n".join([" - " + x.strip('"') for x in header]) config += "sample_names_rename:\n" + rename = [] for line in fin: - config += f" - [{', '.join(line.strip().split(','))}]\n" - fout.write(config) + rename.append(f" - [{', '.join(line.strip().split(','))}]") + fout.write(config + "\n".join(sorted(rename)) + "\n") diff --git a/conf/test.config b/conf/test.config index c09a571c..42b1c34f 100644 --- a/conf/test.config +++ b/conf/test.config @@ -20,7 +20,5 @@ params { max_time = '6.h' // Input data - input = '${projectDir}/tests/sra_ids_test.csv' - - validationSchemaIgnoreParams = 'test_data_base,merge_samplesheet_ids,fastq_ftp_ids,test_data' + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/2732b911c57e607fa7aea5ba0c3d91b25bafb662/testdata/v1.12.0/sra_ids_test.csv' } diff --git a/conf/test_data.config b/conf/test_data.config deleted file mode 100644 index 670115a8..00000000 --- a/conf/test_data.config +++ /dev/null @@ -1,47 +0,0 @@ -params { - // Base directory for test data - test_data_base = "https://raw.githubusercontent.com/nf-core/test-datasets/fetchngs" - - merge_samplesheet_ids = [ "DRX024467_DRR026872", "SRX11047067_SRR14709033", "SRX9504942_SRR13055517", "DRX026011_DRR028935", "SRX17709227_SRR21711856", "SRX9504942_SRR13055518", "ERX1188904_ERR1109373", "SRX17709228_SRR21711855", "SRX9504942_SRR13055519", "ERX1234253_ERR1160846", "SRX6725035_SRR9984183", "SRX9504942_SRR13055520", "SRX10940790_SRR14593545", "SRX9315476_SRR12848126", "SRX9626017_SRR13191702" ] - - def merge_samplesheet_url = "${params.test_data_base}/modules/local/sra_merge_samplesheet/samplesheets/" - def merge_mappings_url = "${params.test_data_base}/modules/local/sra_merge_samplesheet/mappings/" - def merge_samplesheet_urls = [] - def merge_mappings_urls = [] - - merge_samplesheet_ids.each { id -> - merge_samplesheet_urls += "${merge_samplesheet_url}${id}.samplesheet.csv" - merge_mappings_urls += "${merge_mappings_url}/${id}.mappings.csv" - } - - fastq_ftp_ids = ["SRR13191702"] - def fastq_ftp_url = "ftp.sra.ebi.ac.uk/vol1/fastq/SRR131/002/SRR13191702/" - def fastq_ftp_urls = [] - - fastq_ftp_ids.each { id -> - fastq_ftp_urls += "${fastq_ftp_url}${id}_1.fastq.gz" - fastq_ftp_urls += "${fastq_ftp_url}${id}_2.fastq.gz" - } - - test_data { - 'sarscov2'{ - 'illumina' { - SRR11140744_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/sra/SRR11140744.tar.gz" - SRR13255544_tar_gz = "${params.test_data_base}/data/genomics/sarscov2/illumina/sra/SRR13255544.tar.gz" - } - } - 'generic' { - 'config' { - ncbi_user_settings = "${params.test_data_base}/data/generic/config/ncbi_user_settings.mkfg" - } - } - 'modules_local' { - multiqc_mappings_config = "${params.test_data_base}/modules/local/multiqc_mappings_config/SRX9626017_SRR13191702.mappings.csv" - sra_merge_samplesheet_samplesheets = merge_samplesheet_urls - sra_merge_samplesheet_mappings = merge_mappings_urls - sra_to_samplesheet = "${params.test_data_base}/modules/local/sra_to_samplesheet/SRX9626017_SRR13191702.mappings.csv" - sra_fastq_ftp = fastq_ftp_urls - sra_runinfo_to_ftp = "${params.test_data_base}/modules/local/sra_runinfo_to_ftp/SRR13191702.runinfo.tsv" - } - } -} diff --git a/conf/test_full.config b/conf/test_full.config index c16fda7f..84595dfa 100644 --- a/conf/test_full.config +++ b/conf/test_full.config @@ -14,6 +14,6 @@ params { config_profile_name = 'Full test profile' config_profile_description = 'Full test dataset to check pipeline function' - // Input data for full size test - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/fetchngs/sra_ids_test_full.csv' + // File containing SRA ids from nf-core/rnaseq -profile test_full for full-sized test + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/100736c99d87667fb7c247c267bc8acfac647bed/testdata/v1.12.0/sra_ids_rnaseq_test_full.csv' } diff --git a/conf/test_synapse.config b/conf/test_synapse.config deleted file mode 100644 index 1ac1388a..00000000 --- a/conf/test_synapse.config +++ /dev/null @@ -1,25 +0,0 @@ -/* -======================================================================================== - Nextflow config file for running minimal tests -======================================================================================== - Defines input files and everything required to run a fast and simple pipeline test. - - Use as follows: - nextflow run nf-core/fetchngs -profile test_synapse, - ----------------------------------------------------------------------------------------- -*/ - -params { - config_profile_name = 'Test profile using Synapse ids' - config_profile_description = 'Minimal test dataset to check pipeline function' - - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - - // Input data - input = 'https://raw.githubusercontent.com/nf-core/test-datasets/fetchngs/synapse_ids_test.csv' - input_type = 'synapse' -} diff --git a/docs/images/nf-core-fetchngs_logo_dark.png b/docs/images/nf-core-fetchngs_logo_dark.png index e3452bf1..63ff1472 100644 Binary files a/docs/images/nf-core-fetchngs_logo_dark.png and b/docs/images/nf-core-fetchngs_logo_dark.png differ diff --git a/docs/images/nf-core-fetchngs_logo_light.png b/docs/images/nf-core-fetchngs_logo_light.png index c201c150..ffc6d41a 100644 Binary files a/docs/images/nf-core-fetchngs_logo_light.png and b/docs/images/nf-core-fetchngs_logo_light.png differ diff --git a/docs/images/nf-core-fetchngs_metro_map_grey.png b/docs/images/nf-core-fetchngs_metro_map_grey.png new file mode 100644 index 00000000..eb4b49bc Binary files /dev/null and b/docs/images/nf-core-fetchngs_metro_map_grey.png differ diff --git a/docs/images/nf-core-fetchngs_metro_map_grey.svg b/docs/images/nf-core-fetchngs_metro_map_grey.svg new file mode 100644 index 00000000..9857bc12 --- /dev/null +++ b/docs/images/nf-core-fetchngs_metro_map_grey.svg @@ -0,0 +1,12733 @@ + + + +21License:3nf-core/fetchngsENASRADDBJGEODATABASE IDsSTAGE2. Download FASTQ files1. Download metadata3. Downstream pipelinesAsperaFTPsra-toolsFASTQCSVFetchmetadataGetdownloadlinksnf-core/rnaseqnf-core/atacseqnf-core/viralreconnf-core/taxprofilerCSVCSVCSVCSVCSV diff --git a/docs/output.md b/docs/output.md index aad48fe8..5a27bfca 100644 --- a/docs/output.md +++ b/docs/output.md @@ -8,9 +8,7 @@ This document describes the output produced by the pipeline. The directories lis The pipeline is built using [Nextflow](https://www.nextflow.io/) and processes data depending on the type of ids provided: -- Download FastQ files and create samplesheet from: - 1. [SRA / ENA / DDBJ / GEO ids](#sra--ena--ddbj--geo-ids) - 2. [Synapse ids](#synapse-ids) +- Download FastQ files and create samplesheet from [SRA / ENA / DDBJ / GEO ids](#sra--ena--ddbj--geo-ids) - [Pipeline information](#pipeline-information) - Report metrics generated during the workflow execution Please see the [usage documentation](https://nf-co.re/fetchngs/usage#introduction) for a list of supported public repository identifiers and how to provide them to the pipeline. @@ -36,27 +34,6 @@ Please see the [usage documentation](https://nf-co.re/fetchngs/usage#introductio The final sample information for all identifiers is obtained from the ENA which provides direct download links for FastQ files as well as their associated md5 sums. If download links exist, the files will be downloaded in parallel by FTP. Otherwise they are downloaded using sra-tools. -### Synapse ids - -
-Output files - -- `fastq/` - - `*.fastq.gz`: Paired-end/single-end reads downloaded from Synapse. -- `fastq/md5/` - - `*.md5`: Files containing `md5` sum for FastQ files downloaded from the Synapse platform. -- `samplesheet/` - - `samplesheet.csv`: Auto-created samplesheet with collated metadata and paths to downloaded FastQ files. -- `metadata/` - - `*.metadata.txt`: Original metadata file generated using the `synapse show` command. - - `*.list.txt`: Original output of the `synapse list` command, containing the Synapse ids, file version numbers, file names, and other file-specific data for the Synapse directory ID provided. - -
- -FastQ files and corresponding sample information for `Synapse` identifiers are downloaded in parallel directly from the [Synapse](https://www.synapse.org/#) platform. A [configuration file](http://python-docs.synapse.org/build/html/Credentials.html#use-synapseconfig) containing valid login credentials is required for Synapse downloads. - -The final sample information for the FastQ files downloaded from `Synapse` is obtained from the file name itself. The file names are parsed according to the glob pattern `*{1,2}*`. This returns the sample name, presumed to be the longest possible string matching the glob pattern, with the fewest number of wildcard insertions. Further information on sample name parsing can be found in the [usage documentation](https://nf-co.re/fetchngs/usage#introduction). - ### Pipeline information
diff --git a/docs/usage.md b/docs/usage.md index 0cfb95d8..8f27c3d5 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -6,17 +6,17 @@ ## Introduction -The pipeline has been set-up to automatically download and process the raw FastQ files from both public and private repositories. Identifiers can be provided in a file, one-per-line via the `--input` parameter. Currently, the following types of example identifiers are supported: - -| `SRA` | `ENA` | `DDBJ` | `GEO` | `Synapse` | -| ------------ | ------------ | ------------ | ---------- | ----------- | -| SRR11605097 | ERR4007730 | DRR171822 | GSM4432381 | syn26240435 | -| SRX8171613 | ERX4009132 | DRX162434 | GSE147507 | | -| SRS6531847 | ERS4399630 | DRS090921 | | | -| SAMN14689442 | SAMEA6638373 | SAMD00114846 | | | -| SRP256957 | ERP120836 | DRP004793 | | | -| SRA1068758 | ERA2420837 | DRA008156 | | | -| PRJNA625551 | PRJEB37513 | PRJDB4176 | | | +The pipeline has been set-up to automatically download and process the raw FastQ files from public repositories. Identifiers can be provided in a file, one-per-line via the `--input` parameter. Currently, the following types of example identifiers are supported: + +| `SRA` | `ENA` | `DDBJ` | `GEO` | +| ------------ | ------------ | ------------ | ---------- | +| SRR11605097 | ERR4007730 | DRR171822 | GSM4432381 | +| SRX8171613 | ERX4009132 | DRX162434 | GSE147507 | +| SRS6531847 | ERS4399630 | DRS090921 | | +| SAMN14689442 | SAMEA6638373 | SAMD00114846 | | +| SRP256957 | ERP120836 | DRP004793 | | +| SRA1068758 | ERA2420837 | DRA008156 | | +| PRJNA625551 | PRJEB37513 | PRJDB4176 | | ### SRR / ERR / DRR ids @@ -34,25 +34,6 @@ If you have a GEO accession (found in the data availability section of published This downloads a text file called `SRR_Acc_List.txt` that can be directly provided to the pipeline once renamed with a .csv extension e.g. `--input SRR_Acc_List.csv`. -### Synapse ids - -[Synapse](https://www.synapse.org/#) is a collaborative research platform created by [Sage Bionetworks](https://sagebionetworks.org/). Its aim is to promote reproducible research and responsible data sharing throughout the biomedical community. To download data from `Synapse`, the Synapse id of the _directory_ containing all files to be downloaded should be provided. The Synapse id should be an eleven-characters beginning with `syn`. - -This Synapse id will then be resolved to the Synapse id of the corresponding FastQ files contained within the directory. The individual FastQ files are then downloaded in parellel using the `synapse get` command. All Synapse metadata, annotations and data provenance are also downloaded using the `synapse show` command, and are outputted to a separate metadata file. By default, only the md5sums, file sizes, etags, Synapse ids, file names, and file versions are shown. - -In order to download data from Synapse, an account must be created and a user configuration file provided via the parameter `--synapse_config`. For more information about Synapse configuration, please see the [Synapse client configuration](https://help.synapse.org/docs/Client-Configuration.1985446156.html) documentation. - -The final sample information for the FastQ files used for samplesheet generation is obtained from the file name itself. The file names are parsed according to the glob pattern `*{1,2}*`, which returns the sample name, presumed to be the longest possible string matching the glob pattern, with the fewest number of wildcard insertions. - -
-Supported File Names - -- Files named `SRR493366_1.fastq` and `SRR493366_2.fastq` will have a sample name of `SRR493366` -- Files named `SRR_493_367_1.fastq` and `SRR_493_367_2.fastq` will have a sample name of `SRR_493_367` -- Files named `filename12_1.fastq` and `filename12_2.fastq` will have a sample name of `filename12` - -
- ### Samplesheet format As a bonus, the columns in the auto-created samplesheet can be tailored to be accepted out-of-the-box by selected nf-core pipelines, these currently include: @@ -66,9 +47,32 @@ You can use the `--nf_core_pipeline` parameter to customise this behaviour e.g. From v1.9 of this pipeline the default `strandedness` in the output samplesheet will be set to `auto` when using `--nf_core_pipeline rnaseq`. This will only work with v3.10 onwards of nf-core/rnaseq which permits the auto-detection of strandedness during the pipeline execution. You can change this behaviour with the `--nf_core_rnaseq_strandedness` parameter which is set to `auto` by default. -### Bypass `FTP` data download +### Accessions with more than 2 FastQ files + +Using `SRR9320616` as an example, if we run the pipeline with default options to download via Aspera/FTP the ENA API indicates that this sample is associated with a single FastQ file: + +``` +run_accession experiment_accession sample_accession secondary_sample_accession study_accession secondary_study_accession submission_accession run_alias experiment_alias sample_alias study_alias library_layout library_selection library_source library_strategy library_name instrument_model instrument_platform base_count read_count tax_id scientific_name sample_title experiment_title study_title sample_description fastq_md5 fastq_bytes fastq_ftp fastq_galaxy fastq_aspera +SRR9320616 SRX6088086 SAMN12086751 SRS4989433 PRJNA549480 SRP201778 SRA900583 GSM3895942_r1 GSM3895942 GSM3895942 GSE132901 PAIRED cDNA TRANSCRIPTOMIC RNA-Seq Illumina HiSeq 2500 ILLUMINA 11857688850 120996825 10090 Mus musculus Old 3 Kidney Illumina HiSeq 2500 sequencing: GSM3895942: Old 3 Kidney Mus musculus RNA-Seq A murine aging cell atlas reveals cell identity and tissue-specific trajectories of aging Old 3 Kidney 98c939bbae1a1fcf9624905516485b67 7763114613 ftp.sra.ebi.ac.uk/vol1/fastq/SRR932/006/SRR9320616/SRR9320616.fastq.gz ftp.sra.ebi.ac.uk/vol1/fastq/SRR932/006/SRR9320616/SRR9320616.fastq.gz fasp.sra.ebi.ac.uk:/vol1/fastq/SRR932/006/SRR9320616/SRR9320616.fastq.gz +``` + +However, this sample actually has 2 additional FastQ files that are flagged as technical and can only be obtained by running sra-tools. This is particularly important for certain preps like 10x and others using UMI barcodes. + +``` +$ fasterq-dump --threads 6 --split-files --include-technical SRR9320616 --outfile SRR9320616.fastq --progress + +SRR9320616_1.fastq +SRR9320616_2.fastq +SRR9320616_3.fastq +``` + +This highlights that there is a discrepancy between the read data hosted on the ENA API and what can actually be fetched from sra-tools, where the latter seems to be the source of truth. If you anticipate that you may have more than 2 FastQ files per sample, it is recommended to use this pipeline with the `--download_method sratools` parameter. + +See [issue #260](https://github.com/nf-core/fetchngs/issues/260) for more details. + +### Primary options for downloading data -If FTP connections are blocked on your network use the [`--force_sratools_download`](https://nf-co.re/fetchngs/parameters#force_sratools_download) parameter to force the pipeline to download data using sra-tools instead of the ENA FTP. +If the appropriate download links are available, the pipeline uses FTP by default to download FastQ files by setting the `--download_method ftp` parameter. If you are having issues and prefer to use sra-tools or Aspera instead, you can set the [`--download_method`](https://nf-co.re/fetchngs/parameters#download_method) parameter to `--download_method sratools` or `--download_method aspera`, respectively. ### Downloading dbGAP data with JWT diff --git a/main.nf b/main.nf index bf39d7ea..52539e40 100644 --- a/main.nf +++ b/main.nf @@ -13,12 +13,19 @@ nextflow.enable.dsl = 2 /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - NAMED WORKFLOWS FOR PIPELINE + IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -if (params.input_type == 'sra') include { SRA } from './workflows/sra' -if (params.input_type == 'synapse') include { SYNAPSE } from './workflows/synapse' +include { SRA } from './workflows/sra' +include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_fetchngs_pipeline' +include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_fetchngs_pipeline' + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + NAMED WORKFLOWS FOR PIPELINE +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ // // WORKFLOW: Run main nf-core/fetchngs analysis pipeline depending on type of identifier provided @@ -30,45 +37,34 @@ workflow NFCORE_FETCHNGS { main: - ch_versions = Channel.empty() - // // WORKFLOW: Download FastQ files for SRA / ENA / GEO / DDBJ ids // - if (params.input_type == 'sra') { - SRA ( ids ) - ch_versions = SRA.out.versions + SRA ( ids ) - // - // WORKFLOW: Download FastQ files for Synapse ids - // - } else if (params.input_type == 'synapse') { - SYNAPSE ( ids ) - ch_versions = SYNAPSE.out.versions - } - - emit: - versions = ch_versions } /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - RUN ALL WORKFLOWS + RUN MAIN WORKFLOW ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { PIPELINE_INITIALISATION } from './subworkflows/local/nf_core_fetchngs_utils' -include { PIPELINE_COMPLETION } from './subworkflows/local/nf_core_fetchngs_utils' - -// -// WORKFLOW: Execute a single named workflow for the pipeline -// workflow { // // SUBWORKFLOW: Run initialisation tasks // - PIPELINE_INITIALISATION () + PIPELINE_INITIALISATION ( + params.version, + params.help, + params.validate_params, + params.monochrome_logs, + args, + params.outdir, + params.input, + params.ena_metadata_fields + ) // // WORKFLOW: Run primary workflows for the pipeline @@ -81,12 +77,12 @@ workflow { // SUBWORKFLOW: Run completion tasks // PIPELINE_COMPLETION ( - NFCORE_FETCHNGS.out.versions, - params.input_type, params.email, params.email_on_fail, - params.hook_url, - PIPELINE_INITIALISATION.out.summary_params + params.plaintext_email, + params.outdir, + params.monochrome_logs, + params.hook_url ) } diff --git a/modules.json b/modules.json index 81312b77..93f4553b 100644 --- a/modules.json +++ b/modules.json @@ -7,23 +7,23 @@ "nf-core": { "custom/sratoolsncbisettings": { "branch": "master", - "git_sha": "4125fb0c2152efbcec8d9ed71a756c9274b2f7f5", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["fastq_download_prefetch_fasterqdump_sratools"] }, "sratools/fasterqdump": { "branch": "master", - "git_sha": "4125fb0c2152efbcec8d9ed71a756c9274b2f7f5", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["fastq_download_prefetch_fasterqdump_sratools"], "patch": "modules/nf-core/sratools/fasterqdump/sratools-fasterqdump.diff" }, "sratools/prefetch": { "branch": "master", - "git_sha": "4125fb0c2152efbcec8d9ed71a756c9274b2f7f5", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["fastq_download_prefetch_fasterqdump_sratools"] }, "untar": { "branch": "master", - "git_sha": "d0b4fc03af52a1cc8c6fb4493b921b57352b1dd8", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["modules"] } } @@ -32,22 +32,22 @@ "nf-core": { "fastq_download_prefetch_fasterqdump_sratools": { "branch": "master", - "git_sha": "4125fb0c2152efbcec8d9ed71a756c9274b2f7f5", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["subworkflows"] }, - "nextflowpipelineutils": { + "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "1405b3d5078f108f86a2c122846ed2797b4ec8bf", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["subworkflows"] }, - "nfcore_pipeline_utils": { + "utils_nfcore_pipeline": { "branch": "master", - "git_sha": "b1f3d31bbf14fe83959cc03f98db96c6b4f73b37", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["subworkflows"] }, - "nfvalidation_plugin_utils": { + "utils_nfvalidation_plugin": { "branch": "master", - "git_sha": "a8223a72def6cc46f4652a348feba21845ce68c6", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["subworkflows"] } } diff --git a/modules/local/aspera_cli/environment.yml b/modules/local/aspera_cli/environment.yml new file mode 100644 index 00000000..9fbc162f --- /dev/null +++ b/modules/local/aspera_cli/environment.yml @@ -0,0 +1,7 @@ +name: aspera_cli +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::aspera-cli=4.14.0 diff --git a/modules/local/aspera_cli/main.nf b/modules/local/aspera_cli/main.nf new file mode 100644 index 00000000..b38d17c0 --- /dev/null +++ b/modules/local/aspera_cli/main.nf @@ -0,0 +1,68 @@ +process ASPERA_CLI { + tag "$meta.id" + label 'process_medium' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/aspera-cli:4.14.0--hdfd78af_1' : + 'biocontainers/aspera-cli:4.14.0--hdfd78af_1' }" + + input: + tuple val(meta), val(fastq) + val user + + output: + tuple val(meta), path("*fastq.gz"), emit: fastq + tuple val(meta), path("*md5") , emit: md5 + path "versions.yml" , emit: versions + + script: + def args = task.ext.args ?: '' + def conda_prefix = ['singularity', 'apptainer'].contains(workflow.containerEngine) ? "export CONDA_PREFIX=/usr/local" : "" + if (meta.single_end) { + """ + $conda_prefix + + ascp \\ + $args \\ + -i \$CONDA_PREFIX/etc/aspera/aspera_bypass_dsa.pem \\ + ${user}@${fastq[0]} \\ + ${meta.id}.fastq.gz + + echo "${meta.md5_1} ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5 + md5sum -c ${meta.id}.fastq.gz.md5 + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + aspera_cli: \$(ascli --version) + END_VERSIONS + """ + } else { + """ + $conda_prefix + + ascp \\ + $args \\ + -i \$CONDA_PREFIX/etc/aspera/aspera_bypass_dsa.pem \\ + ${user}@${fastq[0]} \\ + ${meta.id}_1.fastq.gz + + echo "${meta.md5_1} ${meta.id}_1.fastq.gz" > ${meta.id}_1.fastq.gz.md5 + md5sum -c ${meta.id}_1.fastq.gz.md5 + + ascp \\ + $args \\ + -i \$CONDA_PREFIX/etc/aspera/aspera_bypass_dsa.pem \\ + ${user}@${fastq[1]} \\ + ${meta.id}_2.fastq.gz + + echo "${meta.md5_2} ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5 + md5sum -c ${meta.id}_2.fastq.gz.md5 + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + aspera_cli: \$(ascli --version) + END_VERSIONS + """ + } +} diff --git a/modules/local/synapse_get/nextflow.config b/modules/local/aspera_cli/nextflow.config similarity index 84% rename from modules/local/synapse_get/nextflow.config rename to modules/local/aspera_cli/nextflow.config index 869f80aa..fa2dbd90 100644 --- a/modules/local/synapse_get/nextflow.config +++ b/modules/local/aspera_cli/nextflow.config @@ -1,5 +1,6 @@ process { - withName: 'SYNAPSE_GET' { + withName: 'ASPERA_CLI' { + ext.args = '-QT -l 300m -P33001' publishDir = [ [ path: { "${params.outdir}/fastq" }, @@ -13,4 +14,4 @@ process { ] ] } -} \ No newline at end of file +} diff --git a/modules/local/aspera_cli/tests/main.nf.test b/modules/local/aspera_cli/tests/main.nf.test new file mode 100644 index 00000000..63347000 --- /dev/null +++ b/modules/local/aspera_cli/tests/main.nf.test @@ -0,0 +1,31 @@ +nextflow_process { + + name "Test process: ASPERA_CLI" + script "../main.nf" + process "ASPERA_CLI" + + test("Should run without failures") { + + when { + process { + """ + input[0] = [ + [ id:'SRX9626017_SRR13191702', single_end:false, md5_1: '89c5be920021a035084d8aeb74f32df7', md5_2: '56271be38a80db78ef3bdfc5d9909b98' ], // meta map + [ + 'fasp.sra.ebi.ac.uk:/vol1/fastq/SRR131/002/SRR13191702/SRR13191702_1.fastq.gz', + 'fasp.sra.ebi.ac.uk:/vol1/fastq/SRR131/002/SRR13191702/SRR13191702_2.fastq.gz' + ] + ] + input[1] = 'era-fasp' + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/local/aspera_cli/tests/main.nf.test.snap b/modules/local/aspera_cli/tests/main.nf.test.snap new file mode 100644 index 00000000..0ba6a643 --- /dev/null +++ b/modules/local/aspera_cli/tests/main.nf.test.snap @@ -0,0 +1,75 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + [ + { + "id": "SRX9626017_SRR13191702", + "single_end": false, + "md5_1": "89c5be920021a035084d8aeb74f32df7", + "md5_2": "56271be38a80db78ef3bdfc5d9909b98" + }, + [ + "SRX9626017_SRR13191702_1.fastq.gz:md5,baaaea61cba4294ec696fdfea1610848", + "SRX9626017_SRR13191702_2.fastq.gz:md5,8e43ad99049fabb6526a4b846da01c32" + ] + ] + ], + "1": [ + [ + { + "id": "SRX9626017_SRR13191702", + "single_end": false, + "md5_1": "89c5be920021a035084d8aeb74f32df7", + "md5_2": "56271be38a80db78ef3bdfc5d9909b98" + }, + [ + "SRX9626017_SRR13191702_1.fastq.gz.md5:md5,055a6916ec9ee478e453d50651f87997", + "SRX9626017_SRR13191702_2.fastq.gz.md5:md5,c30ac785f8d80ec563fabf604d8bf945" + ] + ] + ], + "2": [ + "versions.yml:md5,a51a1dfc6308d71058ddc12c46101dd3" + ], + "fastq": [ + [ + { + "id": "SRX9626017_SRR13191702", + "single_end": false, + "md5_1": "89c5be920021a035084d8aeb74f32df7", + "md5_2": "56271be38a80db78ef3bdfc5d9909b98" + }, + [ + "SRX9626017_SRR13191702_1.fastq.gz:md5,baaaea61cba4294ec696fdfea1610848", + "SRX9626017_SRR13191702_2.fastq.gz:md5,8e43ad99049fabb6526a4b846da01c32" + ] + ] + ], + "md5": [ + [ + { + "id": "SRX9626017_SRR13191702", + "single_end": false, + "md5_1": "89c5be920021a035084d8aeb74f32df7", + "md5_2": "56271be38a80db78ef3bdfc5d9909b98" + }, + [ + "SRX9626017_SRR13191702_1.fastq.gz.md5:md5,055a6916ec9ee478e453d50651f87997", + "SRX9626017_SRR13191702_2.fastq.gz.md5:md5,c30ac785f8d80ec563fabf604d8bf945" + ] + ] + ], + "versions": [ + "versions.yml:md5,a51a1dfc6308d71058ddc12c46101dd3" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:52:00.601018" + } +} \ No newline at end of file diff --git a/modules/local/multiqc_mappings_config/nextflow.config b/modules/local/multiqc_mappings_config/nextflow.config index e2382930..11c58341 100644 --- a/modules/local/multiqc_mappings_config/nextflow.config +++ b/modules/local/multiqc_mappings_config/nextflow.config @@ -6,4 +6,4 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } -} \ No newline at end of file +} diff --git a/modules/local/multiqc_mappings_config/tests/main.nf.test b/modules/local/multiqc_mappings_config/tests/main.nf.test index 75356197..dbb4d74f 100644 --- a/modules/local/multiqc_mappings_config/tests/main.nf.test +++ b/modules/local/multiqc_mappings_config/tests/main.nf.test @@ -3,20 +3,13 @@ nextflow_process { name "Test process: MULTIQC_MAPPINGS_CONFIG" script "../main.nf" process "MULTIQC_MAPPINGS_CONFIG" - tag "modules" - tag "modules_local" - tag "multiqc_mappings_config" test("Should run without failures") { when { - params { - outdir = "$outputDir" - } - process { """ - input[0] = file(params.test_data['modules_local']['multiqc_mappings_config'], checkIfExists: true) + input[0] = file(params.pipelines_testdata_base_path + 'csv/SRX9626017_SRR13191702.mappings.csv', checkIfExists: true) """ } } diff --git a/modules/local/multiqc_mappings_config/tests/main.nf.test.snap b/modules/local/multiqc_mappings_config/tests/main.nf.test.snap index 1a4d16c4..43e46f61 100644 --- a/modules/local/multiqc_mappings_config/tests/main.nf.test.snap +++ b/modules/local/multiqc_mappings_config/tests/main.nf.test.snap @@ -22,6 +22,10 @@ ] } ], - "timestamp": "2023-09-22T10:58:18.132284" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:52:12.65888" } } \ No newline at end of file diff --git a/modules/local/multiqc_mappings_config/tests/tags.yml b/modules/local/multiqc_mappings_config/tests/tags.yml deleted file mode 100644 index 595e2e31..00000000 --- a/modules/local/multiqc_mappings_config/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -multiqc_mappings_config: - - modules/local/multiqc_mappings_config/** diff --git a/modules/local/sra_fastq_ftp/main.nf b/modules/local/sra_fastq_ftp/main.nf index 2b7769ff..e2274d46 100644 --- a/modules/local/sra_fastq_ftp/main.nf +++ b/modules/local/sra_fastq_ftp/main.nf @@ -4,10 +4,10 @@ process SRA_FASTQ_FTP { label 'process_low' label 'error_retry' - conda "bioconda::sra-tools=2.11.0" + conda "conda-forge::wget=1.20.1" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/sra-tools:2.11.0--pl5321ha49a11a_3' : - 'biocontainers/sra-tools:2.11.0--pl5321ha49a11a_3' }" + 'https://depot.galaxyproject.org/singularity/wget:1.20.1' : + 'biocontainers/wget:1.20.1' }" input: tuple val(meta), val(fastq) @@ -21,40 +21,40 @@ process SRA_FASTQ_FTP { def args = task.ext.args ?: '' if (meta.single_end) { """ - curl \\ + wget \\ $args \\ - -L ${fastq[0]} \\ - -o ${meta.id}.fastq.gz + -O ${meta.id}.fastq.gz \\ + ${fastq[0]} echo "${meta.md5_1} ${meta.id}.fastq.gz" > ${meta.id}.fastq.gz.md5 md5sum -c ${meta.id}.fastq.gz.md5 cat <<-END_VERSIONS > versions.yml "${task.process}": - curl: \$(echo \$(curl --version | head -n 1 | sed 's/^curl //; s/ .*\$//')) + wget: \$(echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//')) END_VERSIONS """ } else { """ - curl \\ + wget \\ $args \\ - -L ${fastq[0]} \\ - -o ${meta.id}_1.fastq.gz + -O ${meta.id}_1.fastq.gz \\ + ${fastq[0]} echo "${meta.md5_1} ${meta.id}_1.fastq.gz" > ${meta.id}_1.fastq.gz.md5 md5sum -c ${meta.id}_1.fastq.gz.md5 - curl \\ + wget \\ $args \\ - -L ${fastq[1]} \\ - -o ${meta.id}_2.fastq.gz + -O ${meta.id}_2.fastq.gz \\ + ${fastq[1]} echo "${meta.md5_2} ${meta.id}_2.fastq.gz" > ${meta.id}_2.fastq.gz.md5 md5sum -c ${meta.id}_2.fastq.gz.md5 cat <<-END_VERSIONS > versions.yml "${task.process}": - curl: \$(echo \$(curl --version | head -n 1 | sed 's/^curl //; s/ .*\$//')) + wget: \$(echo \$(wget --version | head -n 1 | sed 's/^GNU Wget //; s/ .*\$//')) END_VERSIONS """ } diff --git a/modules/local/sra_fastq_ftp/nextflow.config b/modules/local/sra_fastq_ftp/nextflow.config index 7b5ebf70..56e43959 100644 --- a/modules/local/sra_fastq_ftp/nextflow.config +++ b/modules/local/sra_fastq_ftp/nextflow.config @@ -1,6 +1,6 @@ process { withName: 'SRA_FASTQ_FTP' { - ext.args = '--retry 5 --continue-at - --max-time 1200' + ext.args = '-t 5 -nv -c -T 60' publishDir = [ [ path: { "${params.outdir}/fastq" }, diff --git a/modules/local/sra_fastq_ftp/tests/main.nf.test b/modules/local/sra_fastq_ftp/tests/main.nf.test index ab846e5d..bf005290 100644 --- a/modules/local/sra_fastq_ftp/tests/main.nf.test +++ b/modules/local/sra_fastq_ftp/tests/main.nf.test @@ -3,22 +3,18 @@ nextflow_process { name "Test process: SRA_FASTQ_FTP" script "../main.nf" process "SRA_FASTQ_FTP" - tag "modules" - tag "modules_local" - tag "sra_fastq_ftp" test("Should run without failures") { when { - params { - outdir = "$outputDir" - } - process { """ input[0] = [ [ id:'SRX9626017_SRR13191702', single_end:false, md5_1: '89c5be920021a035084d8aeb74f32df7', md5_2: '56271be38a80db78ef3bdfc5d9909b98' ], // meta map - params.test_data['modules_local']['sra_fastq_ftp'] + [ + 'ftp.sra.ebi.ac.uk/vol1/fastq/SRR131/002/SRR13191702/SRR13191702_1.fastq.gz', + 'ftp.sra.ebi.ac.uk/vol1/fastq/SRR131/002/SRR13191702/SRR13191702_2.fastq.gz' + ] ] """ } diff --git a/modules/local/sra_fastq_ftp/tests/main.nf.test.snap b/modules/local/sra_fastq_ftp/tests/main.nf.test.snap index fb815455..5b300f8c 100644 --- a/modules/local/sra_fastq_ftp/tests/main.nf.test.snap +++ b/modules/local/sra_fastq_ftp/tests/main.nf.test.snap @@ -11,8 +11,8 @@ "md5_2": "56271be38a80db78ef3bdfc5d9909b98" }, [ - "SRX9626017_SRR13191702_1.fastq.gz:md5,89c5be920021a035084d8aeb74f32df7", - "SRX9626017_SRR13191702_2.fastq.gz:md5,56271be38a80db78ef3bdfc5d9909b98" + "SRX9626017_SRR13191702_1.fastq.gz:md5,baaaea61cba4294ec696fdfea1610848", + "SRX9626017_SRR13191702_2.fastq.gz:md5,8e43ad99049fabb6526a4b846da01c32" ] ] ], @@ -31,7 +31,7 @@ ] ], "2": [ - "versions.yml:md5,6b9d69dea1c1305f74a65197ee871f1b" + "versions.yml:md5,6b60ed6d5805271a1b97798e29c0635c" ], "fastq": [ [ @@ -42,8 +42,8 @@ "md5_2": "56271be38a80db78ef3bdfc5d9909b98" }, [ - "SRX9626017_SRR13191702_1.fastq.gz:md5,89c5be920021a035084d8aeb74f32df7", - "SRX9626017_SRR13191702_2.fastq.gz:md5,56271be38a80db78ef3bdfc5d9909b98" + "SRX9626017_SRR13191702_1.fastq.gz:md5,baaaea61cba4294ec696fdfea1610848", + "SRX9626017_SRR13191702_2.fastq.gz:md5,8e43ad99049fabb6526a4b846da01c32" ] ] ], @@ -62,10 +62,14 @@ ] ], "versions": [ - "versions.yml:md5,6b9d69dea1c1305f74a65197ee871f1b" + "versions.yml:md5,6b60ed6d5805271a1b97798e29c0635c" ] } ], - "timestamp": "2023-09-22T10:58:46.998421" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:51:51.301654" } } \ No newline at end of file diff --git a/modules/local/sra_fastq_ftp/tests/tags.yml b/modules/local/sra_fastq_ftp/tests/tags.yml deleted file mode 100644 index e7474bef..00000000 --- a/modules/local/sra_fastq_ftp/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -sra_fastq_ftp: - - modules/local/sra_fastq_ftp/** diff --git a/modules/local/sra_ids_to_runinfo/nextflow.config b/modules/local/sra_ids_to_runinfo/nextflow.config index 73da969a..9b9d0b16 100644 --- a/modules/local/sra_ids_to_runinfo/nextflow.config +++ b/modules/local/sra_ids_to_runinfo/nextflow.config @@ -5,4 +5,4 @@ process { enabled: false ] } -} \ No newline at end of file +} diff --git a/modules/local/sra_ids_to_runinfo/tests/main.nf.test b/modules/local/sra_ids_to_runinfo/tests/main.nf.test index 087f9069..48797a1a 100644 --- a/modules/local/sra_ids_to_runinfo/tests/main.nf.test +++ b/modules/local/sra_ids_to_runinfo/tests/main.nf.test @@ -3,17 +3,10 @@ nextflow_process { name "Test process: SRA_IDS_TO_RUNINFO" script "../main.nf" process "SRA_IDS_TO_RUNINFO" - tag "modules" - tag "modules_local" - tag "sra_ids_to_runinfo" test("Should run without failures") { when { - params { - outdir = "$outputDir" - } - process { """ input[0] = 'SRR13191702' diff --git a/modules/local/sra_ids_to_runinfo/tests/main.nf.test.snap b/modules/local/sra_ids_to_runinfo/tests/main.nf.test.snap index 241efe2a..f7b6cee5 100644 --- a/modules/local/sra_ids_to_runinfo/tests/main.nf.test.snap +++ b/modules/local/sra_ids_to_runinfo/tests/main.nf.test.snap @@ -16,6 +16,10 @@ ] } ], - "timestamp": "2023-09-22T10:58:56.721948" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:52:05.345153" } } \ No newline at end of file diff --git a/modules/local/sra_ids_to_runinfo/tests/tags.yml b/modules/local/sra_ids_to_runinfo/tests/tags.yml deleted file mode 100644 index 63ea7db3..00000000 --- a/modules/local/sra_ids_to_runinfo/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -sra_ids_to_runinfo: - - modules/local/sra_ids_to_runinfo/** diff --git a/modules/local/sra_merge_samplesheet/main.nf b/modules/local/sra_merge_samplesheet/main.nf deleted file mode 100644 index 1c2ee7df..00000000 --- a/modules/local/sra_merge_samplesheet/main.nf +++ /dev/null @@ -1,34 +0,0 @@ -process SRA_MERGE_SAMPLESHEET { - - conda "conda-forge::sed=4.7" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'nf-core/ubuntu:20.04' }" - - input: - path ('samplesheets/*') - path ('mappings/*') - - output: - path "samplesheet.csv", emit: samplesheet - path "id_mappings.csv", emit: mappings - path "versions.yml" , emit: versions - - script: - """ - head -n 1 `ls ./samplesheets/* | head -n 1` > samplesheet.csv - for fileid in `ls ./samplesheets/*`; do - awk 'NR>1' \$fileid >> samplesheet.csv - done - - head -n 1 `ls ./mappings/* | head -n 1` > id_mappings.csv - for fileid in `ls ./mappings/*`; do - awk 'NR>1' \$fileid >> id_mappings.csv - done - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - sed: \$(echo \$(sed --version 2>&1) | sed 's/^.*GNU sed) //; s/ .*\$//') - END_VERSIONS - """ -} diff --git a/modules/local/sra_merge_samplesheet/nextflow.config b/modules/local/sra_merge_samplesheet/nextflow.config deleted file mode 100644 index f2926529..00000000 --- a/modules/local/sra_merge_samplesheet/nextflow.config +++ /dev/null @@ -1,9 +0,0 @@ -process { - withName: 'SRA_MERGE_SAMPLESHEET' { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } -} \ No newline at end of file diff --git a/modules/local/sra_merge_samplesheet/tests/main.nf.test b/modules/local/sra_merge_samplesheet/tests/main.nf.test deleted file mode 100644 index b7c98622..00000000 --- a/modules/local/sra_merge_samplesheet/tests/main.nf.test +++ /dev/null @@ -1,32 +0,0 @@ -nextflow_process { - - name "Test process: SRA_MERGE_SAMPLESHEET" - script "../main.nf" - process "SRA_MERGE_SAMPLESHEET" - tag "modules" - tag "modules_local" - tag "sra_merge_samplesheet" - - test("Should run without failures") { - - when { - params { - outdir = "$outputDir" - } - - process { - """ - input[0] = params.test_data['modules_local']['sra_merge_samplesheet_samplesheets'].collect { file(it, checkIfExists: true) } - input[1] = params.test_data['modules_local']['sra_merge_samplesheet_mappings'].collect { file(it, checkIfExists: true) } - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot(process.out).match() } - ) - } - } -} diff --git a/modules/local/sra_merge_samplesheet/tests/main.nf.test.snap b/modules/local/sra_merge_samplesheet/tests/main.nf.test.snap deleted file mode 100644 index 203395ae..00000000 --- a/modules/local/sra_merge_samplesheet/tests/main.nf.test.snap +++ /dev/null @@ -1,27 +0,0 @@ -{ - "Should run without failures": { - "content": [ - { - "0": [ - "samplesheet.csv:md5,ef756557b0735becd2574c2a3f5840b2" - ], - "1": [ - "id_mappings.csv:md5,07262807636ce4b50102308eabdcf253" - ], - "2": [ - "versions.yml:md5,410006679d5e496d8c55e58e78ca6b34" - ], - "mappings": [ - "id_mappings.csv:md5,07262807636ce4b50102308eabdcf253" - ], - "samplesheet": [ - "samplesheet.csv:md5,ef756557b0735becd2574c2a3f5840b2" - ], - "versions": [ - "versions.yml:md5,410006679d5e496d8c55e58e78ca6b34" - ] - } - ], - "timestamp": "2023-09-22T10:59:14.022116" - } -} \ No newline at end of file diff --git a/modules/local/sra_merge_samplesheet/tests/tags.yml b/modules/local/sra_merge_samplesheet/tests/tags.yml deleted file mode 100644 index a4292b70..00000000 --- a/modules/local/sra_merge_samplesheet/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -sra_merge_samplesheet: - - modules/local/sra_merge_samplesheet/** diff --git a/modules/local/sra_runinfo_to_ftp/nextflow.config b/modules/local/sra_runinfo_to_ftp/nextflow.config index aee1848c..43263648 100644 --- a/modules/local/sra_runinfo_to_ftp/nextflow.config +++ b/modules/local/sra_runinfo_to_ftp/nextflow.config @@ -6,4 +6,4 @@ process { saveAs: { filename -> filename.equals('versions.yml') ? null : filename } ] } -} \ No newline at end of file +} diff --git a/modules/local/sra_runinfo_to_ftp/tests/main.nf.test b/modules/local/sra_runinfo_to_ftp/tests/main.nf.test index 3b1405f9..39db814a 100644 --- a/modules/local/sra_runinfo_to_ftp/tests/main.nf.test +++ b/modules/local/sra_runinfo_to_ftp/tests/main.nf.test @@ -3,20 +3,13 @@ nextflow_process { name "Test process: SRA_RUNINFO_TO_FTP" script "../main.nf" process "SRA_RUNINFO_TO_FTP" - tag "modules" - tag "modules_local" - tag "sra_runinfo_to_ftp" test("Should run without failures") { when { - params { - outdir = "$outputDir" - } - process { """ - input[0] = file(params.test_data['modules_local']['sra_runinfo_to_ftp'], checkIfExists: true) + input[0] = file(params.pipelines_testdata_base_path + 'tsv/SRR13191702.runinfo.tsv', checkIfExists: true) """ } } diff --git a/modules/local/sra_runinfo_to_ftp/tests/main.nf.test.snap b/modules/local/sra_runinfo_to_ftp/tests/main.nf.test.snap index cd246303..be190f5a 100644 --- a/modules/local/sra_runinfo_to_ftp/tests/main.nf.test.snap +++ b/modules/local/sra_runinfo_to_ftp/tests/main.nf.test.snap @@ -16,6 +16,10 @@ ] } ], - "timestamp": "2023-09-22T10:58:36.16611" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:51:45.748227" } } \ No newline at end of file diff --git a/modules/local/sra_runinfo_to_ftp/tests/tags.yml b/modules/local/sra_runinfo_to_ftp/tests/tags.yml deleted file mode 100644 index 0987a1e6..00000000 --- a/modules/local/sra_runinfo_to_ftp/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -sra_runinfo_to_ftp: - - modules/local/sra_runinfo_to_ftp/** diff --git a/modules/local/sra_to_samplesheet/tests/main.nf.test b/modules/local/sra_to_samplesheet/tests/main.nf.test index 6094717b..ed765158 100644 --- a/modules/local/sra_to_samplesheet/tests/main.nf.test +++ b/modules/local/sra_to_samplesheet/tests/main.nf.test @@ -3,17 +3,10 @@ nextflow_process { name "Test process: SRA_TO_SAMPLESHEET" script "../main.nf" process "SRA_TO_SAMPLESHEET" - tag "modules" - tag "modules_local" - tag "sra_to_samplesheet" test("Should run without failures") { when { - params { - outdir = "$outputDir" - } - process { """ input[0] = [id:'ERX1188904_ERR1109373', run_accession:'ERR1109373', experiment_accession:'ERX1188904', sample_accession:'SAMEA3643867', experiment_alias:'ena-EXPERIMENT-CAM-03-11-2015-17:01:52:847-7', run_alias:'ena-RUN-CAM-03-11-2015-17:01:52:847-7', sample_alias:'sample_56', study_alias:'ena-STUDY-CAM-02-11-2015-17:42:24:189-13', library_layout:'PAIRED', experiment_title:'Illumina HiSeq 2500 paired end sequencing', sample_title:'RNA-Seq reads mapped onto L. Boulardi Toti-like virus genome', sample_description:'RNA-Seq reads mapped onto L. Boulardi Toti-like virus genome', fastq_md5:'8d7d7b854d0207d1226477a30103fade;9fd57225d6c07a31843276d6df9b15c0;5a62e8f785687dce890cfb4fe3e607f9', fastq_ftp:'ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373_1.fastq.gz;ftp.sra.ebi.ac.uk/vol1/fastq/ERR110/003/ERR1109373/ERR1109373_2.fastq.gz', fastq_1:'./results/fastq/ERX1188904_ERR1109373_1.fastq.gz', fastq_2:'./results/fastq/ERX1188904_ERR1109373_2.fastq.gz', md5_1:'9fd57225d6c07a31843276d6df9b15c0', md5_2:'5a62e8f785687dce890cfb4fe3e607f9', single_end:false] diff --git a/modules/local/sra_to_samplesheet/tests/main.nf.test.snap b/modules/local/sra_to_samplesheet/tests/main.nf.test.snap index 8cc8686a..568f3ea7 100644 --- a/modules/local/sra_to_samplesheet/tests/main.nf.test.snap +++ b/modules/local/sra_to_samplesheet/tests/main.nf.test.snap @@ -108,6 +108,10 @@ ] } ], - "timestamp": "2023-09-22T10:58:27.169349" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:51:38.244046" } } \ No newline at end of file diff --git a/modules/local/sra_to_samplesheet/tests/tags.yml b/modules/local/sra_to_samplesheet/tests/tags.yml deleted file mode 100644 index 2f2d527b..00000000 --- a/modules/local/sra_to_samplesheet/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -sra_to_samplesheet: - - modules/local/sra_to_samplesheet/** diff --git a/modules/local/synapse_get/main.nf b/modules/local/synapse_get/main.nf deleted file mode 100644 index c8a6d7a4..00000000 --- a/modules/local/synapse_get/main.nf +++ /dev/null @@ -1,37 +0,0 @@ - -process SYNAPSE_GET { - tag "$meta.id" - label 'process_low' - label 'error_retry' - - conda "bioconda::synapseclient=2.7.1" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/synapseclient:2.7.1--pyh7cba7a3_0' : - 'biocontainers/synapseclient:2.7.1--pyh7cba7a3_0' }" - - input: - val meta - path config - - output: - tuple val(meta), path("*.fastq.gz"), emit: fastq - tuple val(meta), path("*md5") , emit: md5 - path "versions.yml" , emit: versions - - script: - def args = task.ext.args ?: '' - """ - synapse \\ - -c $config \\ - get \\ - $args \\ - $meta.id - - echo "${meta.md5} \t ${meta.name}" > ${meta.id}.md5 - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - synapse: \$(synapse --version | sed -e "s/Synapse Client //g") - END_VERSIONS - """ -} diff --git a/modules/local/synapse_list/main.nf b/modules/local/synapse_list/main.nf deleted file mode 100644 index 0c03f8b2..00000000 --- a/modules/local/synapse_list/main.nf +++ /dev/null @@ -1,36 +0,0 @@ - -process SYNAPSE_LIST { - tag "$id" - label 'process_low' - - conda "bioconda::synapseclient=2.7.1" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/synapseclient:2.7.1--pyh7cba7a3_0' : - 'biocontainers/synapseclient:2.7.1--pyh7cba7a3_0' }" - - input: - val id - path config - - output: - path "*.txt" , emit: txt - path "versions.yml", emit: versions - - script: - def args = task.ext.args ?: '' - def args2 = task.ext.args2 ?: '' - """ - synapse \\ - -c $config \\ - list \\ - $args \\ - $id \\ - $args2 \\ - > ${id}.list.txt - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - syanpse: \$(synapse --version | sed -e "s/Synapse Client //g") - END_VERSIONS - """ -} diff --git a/modules/local/synapse_list/nextflow.config b/modules/local/synapse_list/nextflow.config deleted file mode 100644 index 15124234..00000000 --- a/modules/local/synapse_list/nextflow.config +++ /dev/null @@ -1,10 +0,0 @@ -process { - withName: SYNAPSE_LIST { - ext.args = '--long' - publishDir = [ - path: { "${params.outdir}/metadata" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } -} diff --git a/modules/local/synapse_merge_samplesheet/main.nf b/modules/local/synapse_merge_samplesheet/main.nf deleted file mode 100644 index 4cb2abc3..00000000 --- a/modules/local/synapse_merge_samplesheet/main.nf +++ /dev/null @@ -1,28 +0,0 @@ - -process SYNAPSE_MERGE_SAMPLESHEET { - - conda "conda-forge::sed=4.7" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : - 'nf-core/ubuntu:20.04' }" - - input: - path ('samplesheets/*') - - output: - path "samplesheet.csv", emit: samplesheet - path "versions.yml" , emit: versions - - script: - """ - head -n 1 `ls ./samplesheets/* | head -n 1` > samplesheet.csv - for fileid in `ls ./samplesheets/*`; do - awk 'NR>1' \$fileid >> samplesheet.csv - done - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - sed: \$(echo \$(sed --version 2>&1) | sed 's/^.*GNU sed) //; s/ .*\$//') - END_VERSIONS - """ -} diff --git a/modules/local/synapse_merge_samplesheet/nextflow.config b/modules/local/synapse_merge_samplesheet/nextflow.config deleted file mode 100644 index 9befef9c..00000000 --- a/modules/local/synapse_merge_samplesheet/nextflow.config +++ /dev/null @@ -1,9 +0,0 @@ -process { - withName: SYNAPSE_MERGE_SAMPLESHEET { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } -} \ No newline at end of file diff --git a/modules/local/synapse_show/main.nf b/modules/local/synapse_show/main.nf deleted file mode 100644 index e1f756a5..00000000 --- a/modules/local/synapse_show/main.nf +++ /dev/null @@ -1,36 +0,0 @@ - -process SYNAPSE_SHOW { - tag "$id" - label 'process_low' - - conda "bioconda::synapseclient=2.7.1" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/synapseclient:2.7.1--pyh7cba7a3_0' : - 'biocontainers/synapseclient:2.7.1--pyh7cba7a3_0' }" - - input: - val id - path config - - output: - path "*.txt" , emit: metadata - path "versions.yml", emit: versions - - script: - def args = task.ext.args ?: '' - def args2 = task.ext.args2 ?: '' - """ - synapse \\ - -c $config \\ - show \\ - $args \\ - $id \\ - $args2 \\ - > ${id}.metadata.txt - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - synapse: \$(synapse --version | sed -e "s/Synapse Client //g") - END_VERSIONS - """ -} diff --git a/modules/local/synapse_show/nextflow.config b/modules/local/synapse_show/nextflow.config deleted file mode 100644 index a1864dfe..00000000 --- a/modules/local/synapse_show/nextflow.config +++ /dev/null @@ -1,9 +0,0 @@ -process { - withName: 'SYNAPSE_SHOW' { - publishDir = [ - path: { "${params.outdir}/metadata" }, - mode: params.publish_dir_mode, - saveAs: { filename -> filename.equals('versions.yml') ? null : filename } - ] - } -} \ No newline at end of file diff --git a/modules/local/synapse_to_samplesheet/main.nf b/modules/local/synapse_to_samplesheet/main.nf deleted file mode 100644 index 393203de..00000000 --- a/modules/local/synapse_to_samplesheet/main.nf +++ /dev/null @@ -1,55 +0,0 @@ - -process SYNAPSE_TO_SAMPLESHEET { - tag "$meta.id" - - executor 'local' - memory 100.MB - - input: - tuple val(meta), path(fastq) - val pipeline - val strandedness - - output: - tuple val(meta), path("*.csv"), emit: samplesheet - - exec: - - // Remove custom keys - def meta_map = meta.clone() - meta_map.remove("id") - - def fastq_1 = "${params.outdir}/fastq/${fastq}" - def fastq_2 = '' - if (fastq instanceof List && fastq.size() == 2) { - fastq_1 = "${params.outdir}/fastq/${fastq[0]}" - fastq_2 = "${params.outdir}/fastq/${fastq[1]}" - } - - // Add relevant fields to the beginning of the map - pipeline_map = [ - sample : "${meta.id}", - fastq_1 : fastq_1, - fastq_2 : fastq_2 - ] - - // Add nf-core pipeline specific entries - if (pipeline) { - if (pipeline == 'rnaseq') { - pipeline_map << [ strandedness: strandedness ] - } else if (pipeline == 'atacseq') { - pipeline_map << [ replicate: 1 ] - } else if (pipeline == 'taxprofiler') { - pipeline_map << [ fasta: '' ] - } - } - pipeline_map << meta_map - - // Create a samplesheet - samplesheet = pipeline_map.keySet().collect{ '"' + it + '"'}.join(",") + '\n' - samplesheet += pipeline_map.values().collect{ '"' + it + '"'}.join(",") - - // Write samplesheet to file - def samplesheet_file = task.workDir.resolve("${meta.id}.samplesheet.csv") - samplesheet_file.text = samplesheet -} diff --git a/modules/local/synapse_to_samplesheet/nextflow.config b/modules/local/synapse_to_samplesheet/nextflow.config deleted file mode 100644 index b286c12c..00000000 --- a/modules/local/synapse_to_samplesheet/nextflow.config +++ /dev/null @@ -1,8 +0,0 @@ -process { - withName: SYNAPSE_TO_SAMPLESHEET { - publishDir = [ - path: { "${params.outdir}/samplesheet" }, - enabled: false - ] - } -} \ No newline at end of file diff --git a/modules/nf-core/custom/sratoolsncbisettings/environment.yml b/modules/nf-core/custom/sratoolsncbisettings/environment.yml new file mode 100644 index 00000000..44a1b008 --- /dev/null +++ b/modules/nf-core/custom/sratoolsncbisettings/environment.yml @@ -0,0 +1,7 @@ +name: custom_sratoolsncbisettings +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::sra-tools=3.0.8 diff --git a/modules/nf-core/custom/sratoolsncbisettings/main.nf b/modules/nf-core/custom/sratoolsncbisettings/main.nf index 5deb8892..577117ed 100644 --- a/modules/nf-core/custom/sratoolsncbisettings/main.nf +++ b/modules/nf-core/custom/sratoolsncbisettings/main.nf @@ -2,10 +2,13 @@ process CUSTOM_SRATOOLSNCBISETTINGS { tag 'ncbi-settings' label 'process_low' - conda "bioconda::sra-tools=2.11.0" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/sra-tools:2.11.0--pl5321ha49a11a_3' : - 'biocontainers/sra-tools:2.11.0--pl5321ha49a11a_3' }" + 'https://depot.galaxyproject.org/singularity/sra-tools:3.0.8--h9f5acd7_0' : + 'biocontainers/sra-tools:3.0.8--h9f5acd7_0' }" + + input: + val ids output: path('*.mkfg') , emit: ncbi_settings diff --git a/modules/nf-core/custom/sratoolsncbisettings/meta.yml b/modules/nf-core/custom/sratoolsncbisettings/meta.yml index 01e98856..46a6cd32 100644 --- a/modules/nf-core/custom/sratoolsncbisettings/meta.yml +++ b/modules/nf-core/custom/sratoolsncbisettings/meta.yml @@ -1,4 +1,4 @@ -name: "sratoolsncbisettings" +name: "custom_sratoolsncbisettings" description: Test for the presence of suitable NCBI settings or create them on the fly. keywords: - NCBI @@ -12,8 +12,7 @@ tools: homepage: https://github.com/ncbi/sra-tools documentation: https://github.com/ncbi/sra-tools/wiki tool_dev_url: https://github.com/ncbi/sra-tools - licence: "['Public Domain']" - + licence: ["Public Domain"] output: - versions: type: file @@ -23,6 +22,7 @@ output: type: file description: An NCBI user settings file. pattern: "*.mkfg" - authors: - "@Midnighter" +maintainers: + - "@Midnighter" diff --git a/modules/nf-core/custom/sratoolsncbisettings/nextflow.config b/modules/nf-core/custom/sratoolsncbisettings/nextflow.config deleted file mode 100644 index e69de29b..00000000 diff --git a/modules/nf-core/custom/sratoolsncbisettings/tests/main.nf.test b/modules/nf-core/custom/sratoolsncbisettings/tests/main.nf.test index ec4a7797..e9ea68dc 100644 --- a/modules/nf-core/custom/sratoolsncbisettings/tests/main.nf.test +++ b/modules/nf-core/custom/sratoolsncbisettings/tests/main.nf.test @@ -4,23 +4,20 @@ nextflow_process { script "../main.nf" process "CUSTOM_SRATOOLSNCBISETTINGS" config "modules/nf-core/custom/sratoolsncbisettings/tests/nextflow.config" - tag "modules" - tag "modules_nfcore" - tag "custom" - tag "custom/sratoolsncbisettings" test("Should run without failures") { when { params { settings_path = '/tmp/.ncbi' - settings_file = "${params.settings_path}/user-settings.mkfg" + settings_file = "${params.settings_path}/user-settings.mkfg" } process { """ + input[0] = ["SRX6725035"] file(params.settings_path).mkdirs() - def settings = file(params.test_data['generic']['config']['ncbi_user_settings'], checkIfExists: true) + def settings = file(params.modules_testdata_base_path + 'generic/config/ncbi_user_settings.mkfg', checkIfExists: true) settings.copyTo(params.settings_file) """ } @@ -37,78 +34,5 @@ nextflow_process { assert path(get(0)).readLines().any { it.contains('/libs/cloud/report_instance_identity') } } } - - } - - test("Should fail") { - - when { - params { - settings_path = '/tmp/.ncbi' - settings_file = "${params.settings_path}/user-settings.mkfg" - } - - process { - """ - file(params.settings_path).mkdirs() - def settings = file(params.settings_file) - settings.text = ''' - ## auto-generated configuration file - DO NOT EDIT ## - config/default = "false" - /repository/remote/main/CGI/resolver-cgi = "https://trace.ncbi.nlm.nih.gov/Traces/names/names.fcgi" - /repository/remote/protected/CGI/resolver-cgi = "https://trace.ncbi.nlm.nih.gov/Traces/names/names.fcgi" - /repository/user/ad/public/apps/file/volumes/flatAd = "." - /repository/user/ad/public/apps/refseq/volumes/refseqAd = "." - /repository/user/ad/public/apps/sra/volumes/sraAd = "." - /repository/user/ad/public/apps/sraPileup/volumes/ad = "." - /repository/user/ad/public/apps/sraRealign/volumes/ad = "." - /repository/user/ad/public/apps/wgs/volumes/wgsAd = "." - /repository/user/ad/public/root = "." - /repository/user/default-path = "/root/ncbi" - '''.stripIndent() - """ - } - } - - then { - assert process.failed - assert snapshot( - process.out.versions - ).match() - assert process.stdout.any { it.contains('Command error:') } - assert process.stdout.any { it.contains('missing the required entries') } - assert process.stdout.any { it.contains('/LIBS/GUID') } - assert process.stdout.any { it.contains('/libs/cloud/report_instance_identity') } - } - } - - test("Should run with nonexisting") { - - when { - params { - settings_path = '/tmp/.ncbi' - settings_file = "${params.settings_path}/user-settings.mkfg" - } - - process { - """ - def settings = file(params.settings_file) - settings.delete() - """ - } - } - - then { - assert process.success - assert snapshot(process.out.versions).match() - - with(process.out.ncbi_settings) { - { assert path(get(0)).readLines().any { it.contains('/LIBS/GUID') } } - { assert path(get(0)).readLines().any { it.contains('/libs/cloud/report_instance_identity') } } - } - } - - } - } diff --git a/modules/nf-core/custom/sratoolsncbisettings/tests/main.nf.test.snap b/modules/nf-core/custom/sratoolsncbisettings/tests/main.nf.test.snap index edb3b351..5e314f0b 100644 --- a/modules/nf-core/custom/sratoolsncbisettings/tests/main.nf.test.snap +++ b/modules/nf-core/custom/sratoolsncbisettings/tests/main.nf.test.snap @@ -1,18 +1,14 @@ { - "Should run with nonexisting": { - "content": [ - [ - "versions.yml:md5,fec13b593c3b42ddd38f2fc77df25b70" - ] - ], - "timestamp": "2023-10-12T12:24:24.023849" - }, "Should run without failures": { "content": [ [ - "versions.yml:md5,fec13b593c3b42ddd38f2fc77df25b70" + "versions.yml:md5,3d6ee88cce1ee517e198633f062589a8" ] ], - "timestamp": "2023-10-12T10:40:51.717351" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:47:15.824443" } } \ No newline at end of file diff --git a/modules/nf-core/custom/sratoolsncbisettings/tests/nextflow.config b/modules/nf-core/custom/sratoolsncbisettings/tests/nextflow.config index c4a96e94..df5def04 100644 --- a/modules/nf-core/custom/sratoolsncbisettings/tests/nextflow.config +++ b/modules/nf-core/custom/sratoolsncbisettings/tests/nextflow.config @@ -4,8 +4,6 @@ params.settings_file = "${params.settings_path}/user-settings.mkfg" env.NCBI_SETTINGS = params.settings_file process { - - publishDir = { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" } withName: CUSTOM_SRATOOLSNCBISETTINGS { containerOptions = { (workflow.containerEngine == 'singularity') ? diff --git a/modules/nf-core/custom/sratoolsncbisettings/tests/tags.yml b/modules/nf-core/custom/sratoolsncbisettings/tests/tags.yml deleted file mode 100644 index fb4a08a7..00000000 --- a/modules/nf-core/custom/sratoolsncbisettings/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -custom/sratoolsncbisettings: - - modules/nf-core/custom/sratoolsncbisettings/** diff --git a/modules/nf-core/sratools/fasterqdump/environment.yml b/modules/nf-core/sratools/fasterqdump/environment.yml new file mode 100644 index 00000000..dd0faa56 --- /dev/null +++ b/modules/nf-core/sratools/fasterqdump/environment.yml @@ -0,0 +1,8 @@ +name: sratools_fasterqdump +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::sra-tools=2.11.0 + - conda-forge::pigz=2.6 diff --git a/modules/nf-core/sratools/fasterqdump/main.nf b/modules/nf-core/sratools/fasterqdump/main.nf index 2c5cc8f6..e7cf157a 100644 --- a/modules/nf-core/sratools/fasterqdump/main.nf +++ b/modules/nf-core/sratools/fasterqdump/main.nf @@ -2,10 +2,10 @@ process SRATOOLS_FASTERQDUMP { tag "$meta.id" label 'process_medium' - conda "bioconda::sra-tools=3.0.8 conda-forge::pigz=2.6" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:2f4a4c900edd6801ff0068c2b3048b4459d119eb-0' : - 'biocontainers/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:2f4a4c900edd6801ff0068c2b3048b4459d119eb-0' }" + 'https://depot.galaxyproject.org/singularity/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:6a9ff0e76ec016c3d0d27e0c0d362339f2d787e6-0' : + 'quay.io/biocontainers/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:6a9ff0e76ec016c3d0d27e0c0d362339f2d787e6-0' }" input: tuple val(meta), path(sra) @@ -24,7 +24,12 @@ process SRATOOLS_FASTERQDUMP { def args2 = task.ext.args2 ?: '' def prefix = task.ext.prefix ?: "${meta.id}" def outfile = meta.single_end ? "${prefix}.fastq" : prefix - def key_file = certificate ? "--perm ${certificate}" : '' + def key_file = '' + if (certificate.toString().endsWith('.jwt')) { + key_file += " --perm ${certificate}" + } else if (certificate.toString().endsWith('.ngc')) { + key_file += " --ngc ${certificate}" + } """ export NCBI_SETTINGS="\$PWD/${ncbi_settings}" diff --git a/modules/nf-core/sratools/fasterqdump/meta.yml b/modules/nf-core/sratools/fasterqdump/meta.yml index 629bdca5..b5e0175a 100644 --- a/modules/nf-core/sratools/fasterqdump/meta.yml +++ b/modules/nf-core/sratools/fasterqdump/meta.yml @@ -11,13 +11,12 @@ tools: documentation: https://github.com/ncbi/sra-tools/wiki tool_dev_url: https://github.com/ncbi/sra-tools licence: ["Public Domain"] - input: - meta: type: map description: > - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] + Groovy Map containing sample information e.g. [ id:'test', single_end:false ] + - sra: type: directory description: Directory containing ETL data for the given SRA. @@ -26,18 +25,20 @@ input: type: file description: > An NCBI user settings file. + pattern: "*.mkfg" - certificate: type: file description: > Path to a JWT cart file used to access protected dbGAP data on SRA using the sra-toolkit + pattern: "*.cart" output: - meta: type: map description: > - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] + Groovy Map containing sample information e.g. [ id:'test', single_end:false ] + - versions: type: file description: File containing software versions @@ -46,6 +47,7 @@ output: type: file description: Extracted FASTQ file or files if the sequencing reads are paired-end. pattern: "*.fastq.gz" - authors: - "@Midnighter" +maintainers: + - "@Midnighter" diff --git a/modules/nf-core/sratools/fasterqdump/sratools-fasterqdump.diff b/modules/nf-core/sratools/fasterqdump/sratools-fasterqdump.diff index 5275d3e4..089862bf 100644 --- a/modules/nf-core/sratools/fasterqdump/sratools-fasterqdump.diff +++ b/modules/nf-core/sratools/fasterqdump/sratools-fasterqdump.diff @@ -1,4 +1,18 @@ Changes in module 'nf-core/sratools/fasterqdump' +--- modules/nf-core/sratools/fasterqdump/main.nf ++++ modules/nf-core/sratools/fasterqdump/main.nf +@@ -4,8 +4,8 @@ + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? +- 'https://depot.galaxyproject.org/singularity/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:2f4a4c900edd6801ff0068c2b3048b4459d119eb-0' : +- 'biocontainers/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:2f4a4c900edd6801ff0068c2b3048b4459d119eb-0' }" ++ 'https://depot.galaxyproject.org/singularity/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:6a9ff0e76ec016c3d0d27e0c0d362339f2d787e6-0' : ++ 'quay.io/biocontainers/mulled-v2-5f89fe0cd045cb1d615630b9261a1d17943a9b6a:6a9ff0e76ec016c3d0d27e0c0d362339f2d787e6-0' }" + + input: + tuple val(meta), path(sra) + --- /dev/null +++ modules/nf-core/sratools/fasterqdump/nextflow.config @@ -0,0 +1,10 @@ @@ -12,25 +26,37 @@ Changes in module 'nf-core/sratools/fasterqdump' + ] + } +} ---- modules/nf-core/sratools/fasterqdump/tests/main.nf.test.snap -+++ modules/nf-core/sratools/fasterqdump/tests/main.nf.test.snap -@@ -8,7 +8,7 @@ - "id": "test_single_end", - "single_end": true - }, -- "test_single_end.fastq.gz:md5,1054c7b71884acdb5eed8a378f18be82" -+ "test_single_end_1.fastq.gz:md5,1054c7b71884acdb5eed8a378f18be82" - ] - ], - "1": [ -@@ -20,7 +20,7 @@ - "id": "test_single_end", - "single_end": true - }, -- "test_single_end.fastq.gz:md5,1054c7b71884acdb5eed8a378f18be82" -+ "test_single_end_1.fastq.gz:md5,1054c7b71884acdb5eed8a378f18be82" - ] - ], - "versions": [ +--- modules/nf-core/sratools/fasterqdump/environment.yml ++++ modules/nf-core/sratools/fasterqdump/environment.yml +@@ -4,5 +4,5 @@ + - bioconda + - defaults + dependencies: +- - bioconda::sra-tools=3.0.8 ++ - bioconda::sra-tools=2.11.0 + - conda-forge::pigz=2.6 + +--- modules/nf-core/sratools/fasterqdump/tests/main.nf.test ++++ modules/nf-core/sratools/fasterqdump/tests/main.nf.test +@@ -3,11 +3,8 @@ + script "../main.nf" + config "./nextflow.config" + process "SRATOOLS_FASTERQDUMP" +- tag "modules" +- tag "modules_nfcore" +- tag "untar" +- tag "sratools" +- tag "sratools/fasterqdump" ++ ++ tag "UNTAR" + + test("Single-end") { + + +--- modules/nf-core/sratools/fasterqdump/tests/tags.yml ++++ /dev/null +@@ -1,2 +0,0 @@ +-sratools/fasterqdump: +- - modules/nf-core/sratools/fasterqdump/** ************************************************************ diff --git a/modules/nf-core/sratools/fasterqdump/tests/main.nf.test b/modules/nf-core/sratools/fasterqdump/tests/main.nf.test index ee4c5844..695394d4 100644 --- a/modules/nf-core/sratools/fasterqdump/tests/main.nf.test +++ b/modules/nf-core/sratools/fasterqdump/tests/main.nf.test @@ -1,10 +1,10 @@ nextflow_process { name "Test Process SRATOOLS_FASTERQDUMP" script "../main.nf" + config "./nextflow.config" process "SRATOOLS_FASTERQDUMP" - tag "modules" - tag "modules_nfcore" - tag "sratools/fasterqdump" + + tag "UNTAR" test("Single-end") { @@ -13,7 +13,7 @@ nextflow_process { script "modules/nf-core/untar/main.nf" process { """ - input[0] = Channel.of([ [], file(params.test_data['sarscov2']['illumina']['SRR13255544_tar_gz'], checkIfExists: true) ]) + input[0] = Channel.of([ [], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/sra/SRR13255544.tar.gz', checkIfExists: true) ]) """ } } @@ -22,8 +22,8 @@ nextflow_process { when { process { """ - input[0] = UNTAR.out.untar.collect{ meta, files -> files }.map{ files -> [ [ id:'test_single_end', single_end:true ], files]} - input[1] = file(params.test_data['generic']['config']['ncbi_user_settings'], checkIfExists: true) + input[0] = UNTAR.out.untar.map{ meta, files -> [ [ id:'test_single_end', single_end:true ], files]} + input[1] = file(params.modules_testdata_base_path + 'generic/config/ncbi_user_settings.mkfg', checkIfExists: true) input[2] = [] """ } @@ -44,7 +44,7 @@ nextflow_process { script "modules/nf-core/untar/main.nf" process { """ - input[0] = Channel.of([ [], file(params.test_data['sarscov2']['illumina']['SRR11140744_tar_gz'], checkIfExists: true) ]) + input[0] = Channel.of([ [], file(params.modules_testdata_base_path + 'genomics/sarscov2/illumina/sra/SRR11140744.tar.gz', checkIfExists: true) ]) """ } } @@ -53,8 +53,8 @@ nextflow_process { when { process { """ - input[0] = UNTAR.out.untar.collect{ meta, files -> files }.map{ files -> [ [ id:'test_paired_end', single_end:false ], files]} - input[1] = file(params.test_data['generic']['config']['ncbi_user_settings'], checkIfExists: true) + input[0] = UNTAR.out.untar.map{ meta, files -> [ [ id:'test_paired_end', single_end:false ], files]} + input[1] = file(params.modules_testdata_base_path + 'generic/config/ncbi_user_settings.mkfg', checkIfExists: true) input[2] = [] """ } diff --git a/modules/nf-core/sratools/fasterqdump/tests/main.nf.test.snap b/modules/nf-core/sratools/fasterqdump/tests/main.nf.test.snap index 77fd6c79..ce0f9800 100644 --- a/modules/nf-core/sratools/fasterqdump/tests/main.nf.test.snap +++ b/modules/nf-core/sratools/fasterqdump/tests/main.nf.test.snap @@ -8,11 +8,11 @@ "id": "test_single_end", "single_end": true }, - "test_single_end_1.fastq.gz:md5,1054c7b71884acdb5eed8a378f18be82" + "test_single_end.fastq.gz:md5,674d78c1cc3c1308d6d39d6369a42887" ] ], "1": [ - "versions.yml:md5,a3d61a9761e1606ef8459f0b68821d7a" + "versions.yml:md5,6ff2d50b15c3f0eb9c72cd13a4a20295" ], "reads": [ [ @@ -20,15 +20,19 @@ "id": "test_single_end", "single_end": true }, - "test_single_end_1.fastq.gz:md5,1054c7b71884acdb5eed8a378f18be82" + "test_single_end.fastq.gz:md5,674d78c1cc3c1308d6d39d6369a42887" ] ], "versions": [ - "versions.yml:md5,a3d61a9761e1606ef8459f0b68821d7a" + "versions.yml:md5,6ff2d50b15c3f0eb9c72cd13a4a20295" ] } ], - "timestamp": "2023-10-10T18:11:12.87053" + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-28T15:25:52.837288" }, "Paired-end": { "content": [ @@ -40,13 +44,13 @@ "single_end": false }, [ - "test_paired_end_1.fastq.gz:md5,193809c784a4ea132ab2a253fa4f55b6", - "test_paired_end_2.fastq.gz:md5,3e3b3af3413f50a1685fd7b3f1456d4e" + "test_paired_end_1.fastq.gz:md5,8573015c91d099b6e30789f8bab2f43c", + "test_paired_end_2.fastq.gz:md5,37e6f719a022dc3c9994c80fbc20c311" ] ] ], "1": [ - "versions.yml:md5,a3d61a9761e1606ef8459f0b68821d7a" + "versions.yml:md5,6ff2d50b15c3f0eb9c72cd13a4a20295" ], "reads": [ [ @@ -55,16 +59,20 @@ "single_end": false }, [ - "test_paired_end_1.fastq.gz:md5,193809c784a4ea132ab2a253fa4f55b6", - "test_paired_end_2.fastq.gz:md5,3e3b3af3413f50a1685fd7b3f1456d4e" + "test_paired_end_1.fastq.gz:md5,8573015c91d099b6e30789f8bab2f43c", + "test_paired_end_2.fastq.gz:md5,37e6f719a022dc3c9994c80fbc20c311" ] ] ], "versions": [ - "versions.yml:md5,a3d61a9761e1606ef8459f0b68821d7a" + "versions.yml:md5,6ff2d50b15c3f0eb9c72cd13a4a20295" ] } ], - "timestamp": "2023-10-10T18:11:50.928006" + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-28T15:26:42.466223" } } \ No newline at end of file diff --git a/modules/nf-core/sratools/fasterqdump/tests/nextflow.config b/modules/nf-core/sratools/fasterqdump/tests/nextflow.config new file mode 100644 index 00000000..23e4100b --- /dev/null +++ b/modules/nf-core/sratools/fasterqdump/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: SRATOOLS_FASTERQDUMP { + ext.args = '' + } +} \ No newline at end of file diff --git a/modules/nf-core/sratools/fasterqdump/tests/tags.yml b/modules/nf-core/sratools/fasterqdump/tests/tags.yml deleted file mode 100644 index 5d1ddcb3..00000000 --- a/modules/nf-core/sratools/fasterqdump/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -sratools/fasterqdump: - - modules/nf-core/sratools/fasterqdump/** diff --git a/modules/nf-core/sratools/prefetch/environment.yml b/modules/nf-core/sratools/prefetch/environment.yml new file mode 100644 index 00000000..cfc7d9a8 --- /dev/null +++ b/modules/nf-core/sratools/prefetch/environment.yml @@ -0,0 +1,7 @@ +name: sratools_prefetch +channels: + - conda-forge + - bioconda + - defaults +dependencies: + - bioconda::sra-tools=3.0.8 diff --git a/modules/nf-core/sratools/prefetch/main.nf b/modules/nf-core/sratools/prefetch/main.nf index fb226962..3c30739a 100644 --- a/modules/nf-core/sratools/prefetch/main.nf +++ b/modules/nf-core/sratools/prefetch/main.nf @@ -2,7 +2,7 @@ process SRATOOLS_PREFETCH { tag "$id" label 'process_low' - conda "bioconda::sra-tools=3.0.8" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/sra-tools:3.0.8--h9f5acd7_0' : 'biocontainers/sra-tools:3.0.8--h9f5acd7_0' }" @@ -21,7 +21,15 @@ process SRATOOLS_PREFETCH { shell: args = task.ext.args ?: '' - args += certificate ? " --perm ${certificate}" : '' args2 = task.ext.args2 ?: '5 1 100' // + if (certificate) { + if (certificate.toString().endsWith('.jwt')) { + args += " --perm ${certificate}" + } + else if (certificate.toString().endsWith('.ngc')) { + args += " --ngc ${certificate}" + } + } + template 'retry_with_backoff.sh' } diff --git a/modules/nf-core/sratools/prefetch/meta.yml b/modules/nf-core/sratools/prefetch/meta.yml index 9817b0b2..ff54229f 100644 --- a/modules/nf-core/sratools/prefetch/meta.yml +++ b/modules/nf-core/sratools/prefetch/meta.yml @@ -11,42 +11,46 @@ tools: documentation: https://github.com/ncbi/sra-tools/wiki tool_dev_url: https://github.com/ncbi/sra-tools licence: ["Public Domain"] - input: - meta: type: map description: > - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] + Groovy Map containing sample information e.g. [ id:'test', single_end:false ] + - id: type: string description: > A string denoting an SRA id. + - ncbi_settings: type: file description: > An NCBI user settings file. + pattern: "*.mkfg" - certificate: type: file description: > Path to a JWT cart file used to access protected dbGAP data on SRA using the sra-toolkit + pattern: "*.cart" output: - meta: type: map description: > - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] + Groovy Map containing sample information e.g. [ id:'test', single_end:false ] + - sra: type: directory description: > Directory containing the ETL data for the given SRA id. + pattern: "*/*.sra" - versions: type: file description: File containing software versions pattern: "versions.yml" - authors: - "@Midnighter" +maintainers: + - "@Midnighter" diff --git a/modules/nf-core/sratools/prefetch/tests/main.nf.test b/modules/nf-core/sratools/prefetch/tests/main.nf.test index fcf4036d..c3ebd91e 100644 --- a/modules/nf-core/sratools/prefetch/tests/main.nf.test +++ b/modules/nf-core/sratools/prefetch/tests/main.nf.test @@ -2,20 +2,14 @@ nextflow_process { name "Test Process SRATOOLS_PREFETCH" script "../main.nf" process "SRATOOLS_PREFETCH" - tag "modules" - tag "modules_nfcore" - tag "sratools/prefetch" test("sratools/prefetch") { when { - params { - outdir = "output" - } process { """ input[0] = Channel.of([ [ id:'test', single_end:false ], 'DRR000774' ]) - input[1] = file(params.test_data['generic']['config']['ncbi_user_settings'], checkIfExists: true) + input[1] = file(params.modules_testdata_base_path + 'generic/config/ncbi_user_settings.mkfg', checkIfExists: true) input[2] = [] """ } @@ -32,13 +26,10 @@ nextflow_process { test("sratools/prefetch with sralite") { when { - params { - outdir = "output" - } process { """ input[0] = Channel.of([ [ id:'test', single_end:false ], 'SRR1170046' ]) - input[1] = file(params.test_data['generic']['config']['ncbi_user_settings'], checkIfExists: true) + input[1] = file(params.modules_testdata_base_path + 'generic/config/ncbi_user_settings.mkfg', checkIfExists: true) input[2] = [] """ } diff --git a/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap b/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap index ab1d2088..946fc869 100644 --- a/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap +++ b/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap @@ -32,7 +32,11 @@ ] } ], - "timestamp": "2023-10-13T12:11:24.563510389" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:49:02.309737" }, "sratools/prefetch": { "content": [ @@ -67,6 +71,10 @@ ] } ], - "timestamp": "2023-10-13T12:11:02.75256571" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:48:37.428307" } } \ No newline at end of file diff --git a/modules/nf-core/sratools/prefetch/tests/tags.yml b/modules/nf-core/sratools/prefetch/tests/tags.yml deleted file mode 100644 index 52110bfd..00000000 --- a/modules/nf-core/sratools/prefetch/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -sratools/prefetch: - - modules/nf-core/sratools/prefetch/** diff --git a/modules/nf-core/untar/environment.yml b/modules/nf-core/untar/environment.yml new file mode 100644 index 00000000..0c9cbb10 --- /dev/null +++ b/modules/nf-core/untar/environment.yml @@ -0,0 +1,11 @@ +name: untar + +channels: + - conda-forge + - bioconda + - defaults + +dependencies: + - conda-forge::grep=3.11 + - conda-forge::sed=4.7 + - conda-forge::tar=1.34 diff --git a/modules/nf-core/untar/main.nf b/modules/nf-core/untar/main.nf index 61461c39..8a75bb95 100644 --- a/modules/nf-core/untar/main.nf +++ b/modules/nf-core/untar/main.nf @@ -2,7 +2,7 @@ process UNTAR { tag "$archive" label 'process_single' - conda "conda-forge::sed=4.7 conda-forge::grep=3.11 conda-forge::tar=1.34" + conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? 'https://depot.galaxyproject.org/singularity/ubuntu:20.04' : 'nf-core/ubuntu:20.04' }" diff --git a/modules/nf-core/untar/meta.yml b/modules/nf-core/untar/meta.yml index db241a6e..a9a2110f 100644 --- a/modules/nf-core/untar/meta.yml +++ b/modules/nf-core/untar/meta.yml @@ -39,3 +39,8 @@ authors: - "@drpatelh" - "@matthdsm" - "@jfy133" +maintainers: + - "@joseespinosa" + - "@drpatelh" + - "@matthdsm" + - "@jfy133" diff --git a/modules/nf-core/untar/tests/main.nf.test b/modules/nf-core/untar/tests/main.nf.test new file mode 100644 index 00000000..98b769ad --- /dev/null +++ b/modules/nf-core/untar/tests/main.nf.test @@ -0,0 +1,45 @@ +nextflow_process { + + name "Test Process UNTAR" + script "../main.nf" + process "UNTAR" + + test("test_untar") { + + when { + process { + """ + input[0] = [ [], file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kraken2.tar.gz', checkIfExists: true) ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out.untar).match("test_untar") }, + ) + } + + } + + test("test_untar_onlyfiles") { + + when { + process { + """ + input[0] = [ [], file(params.modules_testdata_base_path + 'generic/tar/hello.tar.gz', checkIfExists: true) ] + """ + } + } + + then { + assertAll ( + { assert process.success }, + { assert snapshot(process.out.untar).match("test_untar_onlyfiles") }, + ) + } + + } + +} diff --git a/modules/nf-core/untar/tests/main.nf.test.snap b/modules/nf-core/untar/tests/main.nf.test.snap new file mode 100644 index 00000000..64550292 --- /dev/null +++ b/modules/nf-core/untar/tests/main.nf.test.snap @@ -0,0 +1,42 @@ +{ + "test_untar_onlyfiles": { + "content": [ + [ + [ + [ + + ], + [ + "hello.txt:md5,e59ff97941044f85df5297e1c302d260" + ] + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:49:41.320643" + }, + "test_untar": { + "content": [ + [ + [ + [ + + ], + [ + "hash.k2d:md5,8b8598468f54a7087c203ad0190555d9", + "opts.k2d:md5,a033d00cf6759407010b21700938f543", + "taxo.k2d:md5,094d5891cdccf2f1468088855c214b2c" + ] + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:49:33.795172" + } +} \ No newline at end of file diff --git a/nextflow.config b/nextflow.config index f9f7962a..bb6b6b79 100644 --- a/nextflow.config +++ b/nextflow.config @@ -11,13 +11,11 @@ params { // Input options input = null - input_type = 'sra' nf_core_pipeline = null nf_core_rnaseq_strandedness = 'auto' ena_metadata_fields = null sample_mapping_fields = 'experiment_accession,run_accession,sample_accession,experiment_alias,run_alias,sample_alias,experiment_title,sample_title,sample_description' - synapse_config = null - force_sratools_download = false + download_method = 'ftp' skip_fastq_download = false dbgap_key = null @@ -53,6 +51,10 @@ params { validationSchemaIgnoreParams = '' validate_params = true + // Deprecated options + // See: https://github.com/nf-core/fetchngs/pull/279/files#r1494459480 + force_sratools_download = false + } // Load base.config by default for all pipelines @@ -66,25 +68,14 @@ try { } // Workflow specific configs -if (params.input_type == 'sra') { - includeConfig './workflows/sra/nextflow.config' -} else if (params.input_type == 'synapse') { - includeConfig './workflows/synapse/nextflow.config' -} - -// Load nf-core/fetchngs custom profiles from different institutions. -// Warning: Uncomment only if a pipeline-specific instititutional config already exists on nf-core/configs! -// try { -// includeConfig "${params.custom_config_base}/pipeline/fetchngs.config" -// } catch (Exception e) { -// System.err.println("WARNING: Could not load nf-core/config/fetchngs profiles: ${params.custom_config_base}/pipeline/fetchngs.config") -// } +includeConfig './workflows/sra/nextflow.config' profiles { debug { dumpHashes = true process.beforeScript = 'echo $HOSTNAME' cleanup = false + nextflow.enable.configProcessNamesValidation = true } conda { conda.enabled = true @@ -93,6 +84,7 @@ profiles { podman.enabled = false shifter.enabled = false charliecloud.enabled = false + channels = ['conda-forge', 'bioconda', 'defaults'] apptainer.enabled = false } mamba { @@ -107,16 +99,16 @@ profiles { } docker { docker.enabled = true - docker.userEmulation = true conda.enabled = false singularity.enabled = false podman.enabled = false shifter.enabled = false charliecloud.enabled = false apptainer.enabled = false + docker.runOptions = '-u $(id -u):$(id -g)' } arm { - docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' + docker.runOptions = '-u $(id -u):$(id -g) --platform=linux/amd64' } singularity { singularity.enabled = true @@ -170,12 +162,8 @@ profiles { executor.cpus = 4 executor.memory = 8.GB } - test { - includeConfig 'conf/test.config' - includeConfig 'conf/test_data.config' - } - test_synapse { includeConfig 'conf/test_synapse.config' } - test_full { includeConfig 'conf/test_full.config' } + test { includeConfig 'conf/test.config' } + test_full { includeConfig 'conf/test_full.config' } } // Set default registry for Apptainer, Docker, Podman and Singularity independent of -profile @@ -188,7 +176,7 @@ singularity.registry = 'quay.io' // Nextflow plugins plugins { - id 'nf-validation' // Validation of pipeline parameters and creation of an input channel from a sample sheet + id 'nf-validation@1.1.3' // Validation of pipeline parameters and creation of an input channel from a sample sheet } // Export these variables to prevent local Python/R libraries from conflicting with those in the container @@ -205,6 +193,9 @@ env { // Capture exit codes from upstream processes when piping process.shell = ['/bin/bash', '-euo', 'pipefail'] +// Disable process selector warnings by default. Use debug profile to enable warnings. +nextflow.enable.configProcessNamesValidation = false + def trace_timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') timeline { enabled = true @@ -230,7 +221,7 @@ manifest { description = """Pipeline to fetch metadata and raw FastQ files from public databases""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = '1.11.0' + version = '1.12.0' doi = 'https://doi.org/10.5281/zenodo.5070524' } diff --git a/nextflow_schema.json b/nextflow_schema.json index 31d9f17a..29f7b710 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -16,19 +16,12 @@ "type": "string", "format": "file-path", "exists": true, + "schema": "assets/schema_input.json", "mimetype": "text/csv", "pattern": "^\\S+\\.(csv|tsv|txt)$", - "schema": "assets/schema_input.json", "fa_icon": "fas fa-file-excel", "description": "File containing SRA/ENA/GEO/DDBJ identifiers one per line to download their associated metadata and FastQ files." }, - "input_type": { - "type": "string", - "default": "sra", - "description": "Specifies the type of identifier provided via `--input` - available options are 'sra' and 'synapse'.", - "fa_icon": "fas fa-keyboard", - "enum": ["sra", "synapse"] - }, "ena_metadata_fields": { "type": "string", "fa_icon": "fas fa-columns", @@ -37,7 +30,7 @@ }, "sample_mapping_fields": { "type": "string", - "fa_icon": "fas fa-globe-americas", + "fa_icon": "fas fa-columns", "description": "Comma-separated list of ENA metadata fields used to create a separate 'id_mappings.csv' and 'multiqc_config.yml' with selected fields that can be used to rename samples in general and in MultiQC.", "default": "experiment_accession,run_accession,sample_accession,experiment_alias,run_alias,sample_alias,experiment_title,sample_title,sample_description" }, @@ -49,16 +42,18 @@ }, "nf_core_rnaseq_strandedness": { "type": "string", - "fa_icon": "fas fa-car", + "fa_icon": "fas fa-dna", "description": "Value for 'strandedness' entry added to samplesheet created when using '--nf_core_pipeline rnaseq'.", "help_text": "The default is 'auto' which can be used with nf-core/rnaseq v3.10 onwards to auto-detect strandedness during the pipeline execution.", "default": "auto" }, - "force_sratools_download": { - "type": "boolean", - "fa_icon": "fas fa-tools", - "description": "Force download FASTQ files via sra-tools instead of via the ENA FTP.", - "help_text": "If FTP connections are blocked on your network use this flag to force the pipeline to download data using sra-tools instead of the ENA FTP." + "download_method": { + "type": "string", + "default": "ftp", + "fa_icon": "fas fa-download", + "enum": ["aspera", "ftp", "sratools"], + "description": "Method to download FastQ files. Available options are 'aspera', 'ftp' or 'sratools'. Default is 'ftp'.", + "help_text": "FTP and Aspera CLI download FastQ files directly from the ENA FTP whereas sratools uses sra-tools to download *.sra files and convert to FastQ." }, "skip_fastq_download": { "type": "boolean", @@ -84,12 +79,6 @@ "fa_icon": "fas fa-envelope", "help_text": "Set this parameter to your e-mail address to get a summary e-mail with details of the run sent to you when the workflow exits. If set in your user config file (`~/.nextflow/config`) then you don't need to specify this on the command line for every run.", "pattern": "^([a-zA-Z0-9_\\-\\.]+)@([a-zA-Z0-9_\\-\\.]+)\\.([a-zA-Z]{2,5})$" - }, - "synapse_config": { - "type": "string", - "description": "Path to Synapse configuration file", - "fa_icon": "fas fa-users-cog", - "hidden": true } } }, @@ -260,6 +249,22 @@ "help_text": "Allows string values that are parseable as numbers or booleans. For further information see [JSONSchema docs](https://github.com/everit-org/json-schema#lenient-mode)." } } + }, + "deprecated_options": { + "title": "Deprecated options", + "type": "object", + "description": "List of parameters that have been deprecated.", + "default": "", + "fa_icon": "fas fa-calendar-times", + "properties": { + "force_sratools_download": { + "type": "boolean", + "fa_icon": "fas fa-times-circle", + "description": "This parameter has been deprecated. Please use '--download_method sratools' instead.", + "enum": [false], + "hidden": true + } + } } }, "allOf": [ @@ -274,6 +279,9 @@ }, { "$ref": "#/definitions/generic_options" + }, + { + "$ref": "#/definitions/deprecated_options" } ] } diff --git a/nf-test.config b/nf-test.config index cb656b94..6f5e2c47 100644 --- a/nf-test.config +++ b/nf-test.config @@ -1,16 +1,10 @@ config { - // location for all nf-tests + // Location of nf-tests testsDir "." - // nf-test directory including temporary files for each test - workDir "/tmp" + // nf-test directory used to create temporary files for each test + workDir System.getenv("NFT_WORKDIR") ?: ".nf-test" - // location of library folder that is added automatically to the classpath - libDir "lib/" - - // location of an optional nextflow.config file specific for executing tests - configFile "nextflow.config" - - // run all test with the defined docker profile from the main nextflow.config - profile "" + // Location of an optional nextflow.config file specific for executing pipeline tests + configFile "tests/nextflow.config" } diff --git a/pyproject.toml b/pyproject.toml index 0d62beb6..56110621 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,15 @@ -# Config file for Python. Mostly used to configure linting of bin/check_samplesheet.py with Black. +# Config file for Python. Mostly used to configure linting of bin/*.py with Ruff. # Should be kept the same as nf-core/tools to avoid fighting with template synchronisation. -[tool.black] +[tool.ruff] line-length = 120 -target_version = ["py37", "py38", "py39", "py310"] +target-version = "py38" +cache-dir = "~/.cache/ruff" -[tool.isort] -profile = "black" -known_first_party = ["nf_core"] -multi_line_output = 3 +[tool.ruff.lint] +select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] + +[tool.ruff.lint.isort] +known-first-party = ["nf_core"] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["E402", "F401"] diff --git a/subworkflows/local/nf_core_fetchngs_utils/main.nf b/subworkflows/local/nf_core_fetchngs_utils/main.nf deleted file mode 100644 index 509a4473..00000000 --- a/subworkflows/local/nf_core_fetchngs_utils/main.nf +++ /dev/null @@ -1,313 +0,0 @@ -// -// Subworkflow with functionality specific to the nf-core/fetchngs pipeline -// - -/* -======================================================================================== - IMPORT MODULES/SUBWORKFLOWS -======================================================================================== -*/ - -include { NEXTFLOW_PIPELINE_UTILS; getWorkflowVersion } from '../../nf-core/nextflowpipelineutils/main' -include { NF_VALIDATION_PLUGIN_UTILS } from '../../nf-core/nfvalidation_plugin_utils/main.nf' -include { - NFCORE_PIPELINE_UTILS; - workflowCitation; - nfCoreLogo; - dashedLine; - completionEmail; - completionSummary; - imNotification -} from '../../nf-core/nfcore_pipeline_utils' - -/* -======================================================================================== - SUBWORKFLOW TO INITIALISE PIPELINE -======================================================================================== -*/ - -workflow PIPELINE_INITIALISATION { - - main: - - // - // Print version and exit if required and dump pipeline parameters to JSON file - // - NEXTFLOW_PIPELINE_UTILS ( - params.version, - true, - params.outdir, - workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1 - ) - - // - // Validate parameters and generate parameter summary to stdout - // - def pre_help_text = nfCoreLogo(getWorkflowVersion()) - def post_help_text = '\n' + workflowCitation() + '\n' + dashedLine() - def String workflow_command = "nextflow run ${workflow.manifest.name} -profile --input ids.csv --outdir " - NF_VALIDATION_PLUGIN_UTILS ( - params.help, - workflow_command, - pre_help_text, - post_help_text, - params.validate_params, - "nextflow_schema.json" - ) - - // - // Check config provided to the pipeline - // - NFCORE_PIPELINE_UTILS () - - // - // Auto-detect input id type - // - ch_input = file(params.input) - def input_type = '' - if (isSraId(ch_input)) { - input_type = 'sra' - sraCheckENAMetadataFields() - } else if (isSynapseId(ch_input)) { - input_type = 'synapse' - } else { - error('Ids provided via --input not recognised please make sure they are either SRA / ENA / GEO / DDBJ or Synapse ids!') - } - - if (params.input_type != input_type) { - error("Ids auto-detected as ${input_type}. Please provide '--input_type ${input_type}' as a parameter to the pipeline!") - } - - // Read in ids from --input file - Channel - .from(ch_input) - .splitCsv(header:false, sep:'', strip:true) - .map { it[0] } - .unique() - .set { ch_ids } - - emit: - ids = ch_ids - summary_params = NF_VALIDATION_PLUGIN_UTILS.out.summary_params -} - -/* -======================================================================================== - SUBWORKFLOW FOR PIPELINE COMPLETION -======================================================================================== -*/ - -workflow PIPELINE_COMPLETION { - - take: - versions // channel: software tools versions - input_type // string: 'sra' or 'synapse' - email // string: email address - email_on_fail // string: email address sent on pipeline failure - hook_url // string: hook URL for notifications - summary_params // map: Groovy map of the parameters used in the pipeline - - main: - - // - // MODULE: Dump software versions for all tools used in the workflow - // - pipeline_version_info = Channel.of("""\"workflow\": - nextflow: ${workflow.nextflow.version} - ${workflow.manifest.name}: ${workflow.manifest.version} - """.stripIndent()) - - versions = versions.mix(pipeline_version_info) - versions.collectFile(name: 'fetchngs_mqc_versions.yml', storeDir: "${params.outdir}/pipeline_info") - - // - // Completion email and summary - // - workflow.onComplete { - if (email || email_on_fail) { - completionEmail(summary_params) - } - - completionSummary() - - if (hook_url) { - imNotification(summary_params) - } - - if (input_type == 'sra') { - sraCurateSamplesheetWarn() - } else if (input_type == 'synapse') { - synapseCurateSamplesheetWarn() - } - } -} - -/* -======================================================================================== - FUNCTIONS -======================================================================================== -*/ - -// -// Check if input ids are from the SRA -// -def isSraId(input) { - def is_sra = false - def total_ids = 0 - def no_match_ids = [] - def pattern = /^(((SR|ER|DR)[APRSX])|(SAM(N|EA|D))|(PRJ(NA|EB|DB))|(GS[EM]))(\d+)$/ - input.eachLine { line -> - total_ids += 1 - if (!(line =~ pattern)) { - no_match_ids << line - } - } - - def num_match = total_ids - no_match_ids.size() - if (num_match > 0) { - if (num_match == total_ids) { - is_sra = true - } else { - error("Mixture of ids provided via --input: ${no_match_ids.join(', ')}\nPlease provide either SRA / ENA / GEO / DDBJ or Synapse ids!") - } - } - return is_sra -} - -// -// Check if input ids are from the Synapse platform -// -def isSynapseId(input) { - def is_synapse = false - def total_ids = 0 - def no_match_ids = [] - def pattern = /^syn\d{8}$/ - input.eachLine { line -> - total_ids += 1 - if (!(line =~ pattern)) { - no_match_ids << line - } - } - - def num_match = total_ids - no_match_ids.size() - if (num_match > 0) { - if (num_match == total_ids) { - is_synapse = true - } else { - error("Mixture of ids provided via --input: ${no_match_ids.join(', ')}\nPlease provide either SRA / ENA / GEO / DDBJ or Synapse ids!") - } - } - return is_synapse -} - -// -// Check and validate parameters -// -def sraCheckENAMetadataFields() { - // Check minimal ENA fields are provided to download FastQ files - def valid_ena_metadata_fields = ['run_accession', 'experiment_accession', 'library_layout', 'fastq_ftp', 'fastq_md5'] - def ena_metadata_fields = params.ena_metadata_fields ? params.ena_metadata_fields.split(',').collect{ it.trim().toLowerCase() } : valid_ena_metadata_fields - if (!ena_metadata_fields.containsAll(valid_ena_metadata_fields)) { - error("Invalid option: '${params.ena_metadata_fields}'. Minimally required fields for '--ena_metadata_fields': '${valid_ena_metadata_fields.join(',')}'") - } -} - -// -// Print a warning after pipeline has completed -// -def sraCurateSamplesheetWarn() { - log.warn "=============================================================================\n" + - " Please double-check the samplesheet that has been auto-created by the pipeline.\n\n" + - " Public databases don't reliably hold information such as strandedness\n" + - " information, controls etc\n\n" + - " All of the sample metadata obtained from the ENA has been appended\n" + - " as additional columns to help you manually curate the samplesheet before\n" + - " running nf-core/other pipelines.\n" + - "===================================================================================" -} - -// -// Convert metadata obtained from the 'synapse show' command to a Groovy map -// -def synapseShowToMap(synapse_file) { - def meta = [:] - def category = '' - synapse_file.eachLine { line -> - def entries = [null, null] - if (!line.startsWith(' ') && !line.trim().isEmpty()) { - category = line.tokenize(':')[0] - } else { - entries = line.trim().tokenize('=') - } - meta["${category}|${entries[0]}"] = entries[1] - } - meta.id = meta['properties|id'] - meta.name = meta['properties|name'] - meta.md5 = meta['File|md5'] - return meta.findAll{ it.value != null } -} - -// -// Print a warning after pipeline has completed -// -def synapseCurateSamplesheetWarn() { - log.warn "=============================================================================\n" + - " Please double-check the samplesheet that has been auto-created by the pipeline.\n\n" + - " Where applicable, default values will be used for sample-specific metadata\n" + - " such as strandedness, controls etc as this information is not provided\n" + - " in a standardised manner when uploading data to Synapse.\n" + - "===================================================================================" -} - -// -// Obtain Sample ID from File Name -// -def synapseSampleNameFromFastQ(input_file, pattern) { - - def sampleids = "" - - def filePattern = pattern.toString() - int p = filePattern.lastIndexOf('/') - if( p != -1 ) - filePattern = filePattern.substring(p+1) - - input_file.each { - String fileName = input_file.getFileName().toString() - - String indexOfWildcards = filePattern.findIndexOf { it=='*' || it=='?' } - String indexOfBrackets = filePattern.findIndexOf { it=='{' || it=='[' } - if( indexOfWildcards==-1 && indexOfBrackets==-1 ) { - if( fileName == filePattern ) - return actual.getSimpleName() - throw new IllegalArgumentException("Not a valid file pair globbing pattern: pattern=$filePattern file=$fileName") - } - - int groupCount = 0 - for( int i=0; i= 1 + ) + + // + // Validate parameters and generate parameter summary to stdout + // + pre_help_text = nfCoreLogo(monochrome_logs) + post_help_text = '\n' + workflowCitation() + '\n' + dashedLine(monochrome_logs) + def String workflow_command = "nextflow run ${workflow.manifest.name} -profile --input ids.csv --outdir " + UTILS_NFVALIDATION_PLUGIN ( + help, + workflow_command, + pre_help_text, + post_help_text, + validate_params, + "nextflow_schema.json" + ) + + // + // Check config provided to the pipeline + // + UTILS_NFCORE_PIPELINE ( + nextflow_cli_args + ) + + // + // Auto-detect input id type + // + ch_input = file(input) + if (isSraId(ch_input)) { + sraCheckENAMetadataFields(ena_metadata_fields) + } else { + error('Ids provided via --input not recognised please make sure they are either SRA / ENA / GEO / DDBJ ids!') + } + + // Read in ids from --input file + Channel + .from(ch_input) + .splitCsv(header:false, sep:'', strip:true) + .map { it[0] } + .unique() + .set { ch_ids } + + emit: + ids = ch_ids +} + +/* +======================================================================================== + SUBWORKFLOW FOR PIPELINE COMPLETION +======================================================================================== +*/ + +workflow PIPELINE_COMPLETION { + + take: + email // string: email address + email_on_fail // string: email address sent on pipeline failure + plaintext_email // boolean: Send plain-text email instead of HTML + outdir // path: Path to output directory where results will be published + monochrome_logs // boolean: Disable ANSI colour codes in log output + hook_url // string: hook URL for notifications + + main: + + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + + // + // Completion email and summary + // + workflow.onComplete { + if (email || email_on_fail) { + completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs) + } + + completionSummary(monochrome_logs) + + if (hook_url) { + imNotification(summary_params, hook_url) + } + + sraCurateSamplesheetWarn() + } +} + +/* +======================================================================================== + FUNCTIONS +======================================================================================== +*/ + +// +// Check if input ids are from the SRA +// +def isSraId(input) { + def is_sra = false + def total_ids = 0 + def no_match_ids = [] + def pattern = /^(((SR|ER|DR)[APRSX])|(SAM(N|EA|D))|(PRJ(NA|EB|DB))|(GS[EM]))(\d+)$/ + input.eachLine { line -> + total_ids += 1 + if (!(line =~ pattern)) { + no_match_ids << line + } + } + + def num_match = total_ids - no_match_ids.size() + if (num_match > 0) { + if (num_match == total_ids) { + is_sra = true + } else { + error("Mixture of ids provided via --input: ${no_match_ids.join(', ')}\nPlease provide either SRA / ENA / GEO / DDBJ ids!") + } + } + return is_sra +} + +// +// Check and validate parameters +// +def sraCheckENAMetadataFields(ena_metadata_fields) { + // Check minimal ENA fields are provided to download FastQ files + def valid_ena_metadata_fields = ['run_accession', 'experiment_accession', 'library_layout', 'fastq_ftp', 'fastq_md5'] + def actual_ena_metadata_fields = ena_metadata_fields ? ena_metadata_fields.split(',').collect{ it.trim().toLowerCase() } : valid_ena_metadata_fields + if (!actual_ena_metadata_fields.containsAll(valid_ena_metadata_fields)) { + error("Invalid option: '${ena_metadata_fields}'. Minimally required fields for '--ena_metadata_fields': '${valid_ena_metadata_fields.join(',')}'") + } +} + +// +// Print a warning after pipeline has completed +// +def sraCurateSamplesheetWarn() { + log.warn "=============================================================================\n" + + " Please double-check the samplesheet that has been auto-created by the pipeline.\n\n" + + " Public databases don't reliably hold information such as strandedness\n" + + " information, controls etc\n\n" + + " All of the sample metadata obtained from the ENA has been appended\n" + + " as additional columns to help you manually curate the samplesheet before\n" + + " running nf-core/other pipelines.\n" + + "===================================================================================" +} diff --git a/subworkflows/local/utils_nfcore_fetchngs_pipeline/tests/main.function.nf.test b/subworkflows/local/utils_nfcore_fetchngs_pipeline/tests/main.function.nf.test new file mode 100644 index 00000000..f2e3f12a --- /dev/null +++ b/subworkflows/local/utils_nfcore_fetchngs_pipeline/tests/main.function.nf.test @@ -0,0 +1,79 @@ + +nextflow_function { + + name "Test Functions" + script "subworkflows/local/utils_nfcore_fetchngs_pipeline/main.nf" + tag "UTILS_NFCORE_FETCHNGS_PIPELINE" + + test("Function isSraId") { + + function "isSraId" + + when { + function { + """ + input[0] = 'DRR000774' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Function sraCheckENAMetadataFields [success]") { + + function "sraCheckENAMetadataFields" + + when { + function { + """ + input[0] = 'run_accession,experiment_accession,library_layout,fastq_ftp,fastq_md5' + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Function sraCheckENAMetadataFields [failure]") { + + function "sraCheckENAMetadataFields" + + when { + function { + """ + input[0] = 'run_accession,experiment_accession,library_layout,fastq_ftp' + """ + } + } + + then { + assertAll( + { assert !function.success } + ) + } + } + + test("Function sraCurateSamplesheetWarn") { + + function "sraCurateSamplesheetWarn" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + +} diff --git a/subworkflows/local/utils_nfcore_fetchngs_pipeline/tests/main.function.nf.test.snap b/subworkflows/local/utils_nfcore_fetchngs_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 00000000..99fb20e7 --- /dev/null +++ b/subworkflows/local/utils_nfcore_fetchngs_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,28 @@ +{ + "Function sraCurateSamplesheetWarn": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:55:41.001798" + }, + "Function sraCheckENAMetadataFields [success]": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:55:33.679255" + }, + "Function isSraId": { + "content": [ + true + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:55:29.999289" + } +} \ No newline at end of file diff --git a/subworkflows/local/utils_nfcore_fetchngs_pipeline/tests/main.workflow_pipeline_completion.nf.test b/subworkflows/local/utils_nfcore_fetchngs_pipeline/tests/main.workflow_pipeline_completion.nf.test new file mode 100644 index 00000000..b94f72cf --- /dev/null +++ b/subworkflows/local/utils_nfcore_fetchngs_pipeline/tests/main.workflow_pipeline_completion.nf.test @@ -0,0 +1,36 @@ +nextflow_workflow { + + name "Test Workflow PIPELINE_COMPLETION" + script "subworkflows/local/utils_nfcore_fetchngs_pipeline/main.nf" + workflow "PIPELINE_COMPLETION" + tag "UTILS_NFCORE_FETCHNGS_PIPELINE" + + test("Should run") { + + when { + workflow { + """ + email = null + email_on_fail = null + plaintext_email = false + outdir = 'results' + monochrome_logs = false + hook_url = null + + input[0] = email + input[1] = email_on_fail + input[2] = plaintext_email + input[3] = outdir + input[4] = monochrome_logs + input[5] = hook_url + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } +} diff --git a/subworkflows/local/utils_nfcore_fetchngs_pipeline/tests/main.workflow_pipeline_initialisation.nf.test b/subworkflows/local/utils_nfcore_fetchngs_pipeline/tests/main.workflow_pipeline_initialisation.nf.test new file mode 100644 index 00000000..e04aa726 --- /dev/null +++ b/subworkflows/local/utils_nfcore_fetchngs_pipeline/tests/main.workflow_pipeline_initialisation.nf.test @@ -0,0 +1,39 @@ +nextflow_workflow { + + name "Test Workflow PIPELINE_INITIALISATION" + script "subworkflows/local/utils_nfcore_fetchngs_pipeline/main.nf" + workflow "PIPELINE_INITIALISATION" + tag "UTILS_NFCORE_FETCHNGS_PIPELINE" + + test("Should run") { + + when { + workflow { + """ + version = false + help = false + validate_params = false + monochrome_logs = false + nextflow_cli_args = [] + outdir = 'results' + ena_metadata_fields = null + + input[0] = version + input[1] = help + input[2] = validate_params + input[3] = monochrome_logs + input[4] = nextflow_cli_args + input[5] = outdir + input[6] = 'https://raw.githubusercontent.com/nf-core/test-datasets/2732b911c57e607fa7aea5ba0c3d91b25bafb662/testdata/v1.12.0/sra_ids_test.csv' + input[7] = ena_metadata_fields + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } +} diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/main.nf b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/main.nf index de31637e..fbeacf4a 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/main.nf +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/main.nf @@ -17,7 +17,7 @@ workflow FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS { // // Detect existing NCBI user settings or create new ones. // - CUSTOM_SRATOOLSNCBISETTINGS() + CUSTOM_SRATOOLSNCBISETTINGS ( ch_sra_ids.collect() ) ch_ncbi_settings = CUSTOM_SRATOOLSNCBISETTINGS.out.ncbi_settings ch_versions = ch_versions.mix(CUSTOM_SRATOOLSNCBISETTINGS.out.versions) diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/meta.yml b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/meta.yml index 6a36b225..1b968acc 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/meta.yml +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/meta.yml @@ -16,24 +16,26 @@ input: - meta: type: map description: > - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] + Groovy Map containing sample information e.g. [ id:'test', single_end:false ] + - id: type: string description: > SRA run identifier. + - certificate: type: file description: > Path to a JWT cart file used to access protected dbGAP data on SRA using the sra-toolkit + pattern: "*.cart" # TODO Update when we decide on a standard for subworkflow docs output: - meta: type: map description: > - Groovy Map containing sample information - e.g. [ id:'test', single_end:false ] + Groovy Map containing sample information e.g. [ id:'test', single_end:false ] + - reads: type: file description: Extracted FASTQ file or files if the sequencing reads are paired-end. @@ -45,3 +47,6 @@ output: authors: - "@Midnighter" - "@drpatelh" +maintainers: + - "@Midnighter" + - "@drpatelh" diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config index 996c1888..de803a38 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/nextflow.config @@ -1,3 +1,2 @@ includeConfig '../../../modules/nf-core/sratools/prefetch/nextflow.config' includeConfig '../../../modules/nf-core/sratools/fasterqdump/nextflow.config' -includeConfig '../../../modules/nf-core/custom/sratoolsncbisettings/nextflow.config' \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test index be18fc3b..7b41ed7e 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test @@ -3,12 +3,10 @@ nextflow_workflow { name "Test workflow: fastq_download_prefetch_fasterqdump_sratools/main.nf" script "../main.nf" workflow "FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS" - tag "subworkflow" - tag "subworkflow_nfcore" - tag "custom/sratoolsncbisettings" - tag "sratools/prefetch" - tag "sratools/fasterqdump" - tag "fastq_download_prefetch_fasterqdump_sratools" + + tag "CUSTOM_SRATOOLSNCBISETTINGS" + tag "SRATOOLS_PREFETCH" + tag "SRATOOLS_FASTERQDUMP" test("Parameters: default") { @@ -22,15 +20,21 @@ nextflow_workflow { input[1] = [] """ } - params { - outdir = "output" - } } then { + def pelines1 = path(workflow.out.reads[0][1][0]).linesGzip + def pelines2 = path(workflow.out.reads[0][1][1]).linesGzip + def selines = path(workflow.out.reads[1][1]).linesGzip assertAll( { assert workflow.success }, - { assert snapshot(workflow.out).match() } + { assert snapshot(pelines1[0..5]).match("test_pe_reads_1_lines") }, + { assert snapshot(pelines1.size()).match("test_pe_reads_1_size") }, + { assert snapshot(pelines2[0..5]).match("test_pe_reads_2_lines") }, + { assert snapshot(pelines2.size()).match("test_pe_reads_2_size") }, + { assert snapshot(selines[0..5]).match("test_se_reads_lines") }, + { assert snapshot(selines.size()).match("test_se_reads_size") }, + { assert snapshot(workflow.out.versions).match("versions") } ) } } diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap index a73679e0..485411da 100644 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap +++ b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/main.nf.test.snap @@ -1,57 +1,97 @@ { - "Parameters: default": { + "test_se_reads_size": { "content": [ - { - "0": [ - [ - { - "id": "test_paired_end", - "single_end": false - }, - [ - "test_paired_end_1.fastq.gz:md5,193809c784a4ea132ab2a253fa4f55b6", - "test_paired_end_2.fastq.gz:md5,3e3b3af3413f50a1685fd7b3f1456d4e" - ] - ], - [ - { - "id": "test_single_end", - "single_end": true - }, - "test_single_end.fastq.gz:md5,19029a1132115b55277a0d79ee089b49" - ] - ], - "1": [ - "versions.yml:md5,98d78bba9f3da39a0b7db6e9c7dcc224", - "versions.yml:md5,9c558ff624585a6eee82a19c8c0136db", - "versions.yml:md5,db0e1352064b53c651661f43f2620765" - ], - "reads": [ - [ - { - "id": "test_paired_end", - "single_end": false - }, - [ - "test_paired_end_1.fastq.gz:md5,193809c784a4ea132ab2a253fa4f55b6", - "test_paired_end_2.fastq.gz:md5,3e3b3af3413f50a1685fd7b3f1456d4e" - ] - ], - [ - { - "id": "test_single_end", - "single_end": true - }, - "test_single_end.fastq.gz:md5,19029a1132115b55277a0d79ee089b49" - ] - ], - "versions": [ - "versions.yml:md5,98d78bba9f3da39a0b7db6e9c7dcc224", - "versions.yml:md5,9c558ff624585a6eee82a19c8c0136db", - "versions.yml:md5,db0e1352064b53c651661f43f2620765" - ] - } + 19996 ], - "timestamp": "2023-10-12T20:09:55.255971" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:56.176292" + }, + "test_pe_reads_2_lines": { + "content": [ + [ + "@SRR11140744.1 M01472:285:000000000-CYHNP:1:1101:12117:3295 length=251", + "ACAGGACACGAGTAACTCGTCTATCTTCTGCTGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTTCGTCCGGGTGTGACCGAAAGGTAAGATGGAGAGCCTTGTCCCTGGTTTCAACGAGAAAACACACGTCCAACTCAGTTTGCCTGTTTTACAGGTTCGCGACGTGCTCGTACGTGGCTTTGGAGACTCCGTGGAGGAGGTCTTATCAGAGGCACGTCAACATCTTAAA", + "+SRR11140744.1 M01472:285:000000000-CYHNP:1:1101:12117:3295 length=251", + "ABAAAFBFFBDBGGGGGGGGGGHHHHHHHHHHCHGHGGGHHHGGHGGHGHGGGHFHHHHHHHHGGGGGHHHHHHHHHFHHHHGHHHGHGGGGGEFGDGHHGFGGGHHHHHGHHGGHHFHHHHGHHHHHHHHHHHHHHGFFGGHHHHHHGGHHGGHHHHHEGHHHHHHHGHHGHHFHHHHHGGGGGGGGGGGGAGGG9BEFFFFFFFFFFFFFFEEFFFFFFFA.FFFFFFFEFEFFFFFFF.BFFFFFFFB", + "@SRR11140744.2 M01472:285:000000000-CYHNP:1:1101:20752:3564 length=238", + "GTGCACTCACGCAGTATAATTAATAACTAATTACTGTCGTTGACAGGACACGAGTAACTCGTCTATCTTCTGCAGGCTGCTTACGGTTTCGTCCGTGTTGCAGCCGATCATCAGCACATCTAGGTTTCGTCCGGGTGTGACCGAAAGGTAAGATGGAGAGCCTTGTCCCTGGTTTCAACGAGAAAACACACGTCCAACTCAGTTTGCCTGTTTTACAGGTTCGCGACGTGCTCGTACG" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:56.166207" + }, + "test_pe_reads_2_size": { + "content": [ + 2011460 + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:56.168869" + }, + "versions": { + "content": [ + [ + "versions.yml:md5,1a2218ff913fc33408bffccb081b5048", + "versions.yml:md5,2f3b3a13b36dabf13f09327613d5558d", + "versions.yml:md5,98d78bba9f3da39a0b7db6e9c7dcc224" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-28T15:19:18.755939" + }, + "test_pe_reads_1_size": { + "content": [ + 2013376 + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.01.0" + }, + "timestamp": "2024-02-28T15:19:18.677234" + }, + "test_se_reads_lines": { + "content": [ + [ + "@DRR000774.1 1 length=421", + "ACGCAGGTGCCAGCAGCCGCGGTAATACGTAGGATCCGAGCGTTGTCCGGATTTATTGGGCGTAAAGGGTGCGTAGGCGGCTTGTCAAGTCTCATGTGAAATCTCCCGGCTCAACTGGGAGGGTCATGGGAAACTGATGAGCTCGAGGGCAGTAGAGGGAAGCGGAATTCCGAGAGTAGTGGTGAAATGCGTAGATACTCGGAGGAACACCAGTGGCGAAAGCGGCTTCCTGGACTGTACCTGACGCTGAGGCACGAAAGCGTGGGGAGCAAACCGGATTAGATACCCGGGTAGTCCACGCCCTAAACGATGGATACTAGATATAGGGGGTATCGACCCTCTGTGTCGAAGCTAACGCATTAAGTATCCCGCCTGAGGAGTACGGCCGCAAGGCTAAAACTTAAGGAATTGACGGCTGCGT", + "+DRR000774.1 1 length=421", + "FFFFFFFFFFFIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIIHHFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF:88FFF888???DBBBBB666F222ADDDFFF::;FFFFFFFFFFFFFFFFFFFFFFFFFFFF9:::FFFFCCCFFFFDDDFFFFF<<<<<8888886623//38><83238@B@@<;855557,,,,,,,0/0;;8:==DDDDDDDDD9:", + "@DRR000774.2 2 length=126", + "ACGCAGGTGCCAGCAGCCGCGGTAATACGGAGGGAGCTAGCGTTGTTCGGAATTACTGGGCGTAAAGCGCACGTAGGCGGCTTTTCAAGTCAGGGGTGGAAATACCCGGGGCCGTCAACCCGACCG" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:56.171227" + }, + "test_pe_reads_1_lines": { + "content": [ + [ + "@SRR11140744.1 M01472:285:000000000-CYHNP:1:1101:12117:3295 length=251", + "ACATAGGGCTGTTCAAGTTGAGGCAAAACGCCTTTTTCAACTTCTACTAAGCCACAAGTGCCATCTTTAAGATGTTGACGTGCCTCTGATAAGACCTCCTCCACGGAGTCTCCAAAGCCACGTACGAGCACGTCGCGAACCTGTAAAACAGGCAAACTGAGTTGGACGTGTGTTTTCTCGTTGAAACCAGGGACAAGGCTCTCCATCTTACCTTTCGGTCACACCCGGACGAAACCTAGATGTGCTGATGA", + "+SRR11140744.1 M01472:285:000000000-CYHNP:1:1101:12117:3295 length=251", + "BCCCCFFFFFCFGGGGGGGGGGHGGHHHHGGGHGHHHHHHHHHHHHHHHHHHHHHHHGHHHHHHHHHHHHHHHHHHHHHGGGHHHHHGHHGHHHHHHHHHHHHHGGGGGHHHHHHHHHHHHGHHHGGGGGHGHHGGGGGGGHHHHHHHHHHHGGHHHHHFHHHHHHHGGGHHHHHHHHHGGGHHHHHHHHGGGGGGGFGGGGGGGGGGGGGGGGGGGGGGFFFFFFFFFDFFFFFFFFFFFFFFFFFFFFB", + "@SRR11140744.2 M01472:285:000000000-CYHNP:1:1101:20752:3564 length=238", + "CGTACGAGCACGTCGCGAACCTGTAAAACAGGCAAACTGAGTTGGACGTGTGTTTTCTCGTTGAAACCAGGGACAAGGCTCTCCATCTTACCTTTCGGTCACACCCGGACGAAACCTAGATGTGCTGATGATCGGCTGCAACACGGACGAAACCGTAAGCAGCCTGCAGAAGATAGACGAGTTACTCGTGTCCTGTCAACGACAGTAATTAGTTATTAATTATACTGCGTGAGTGCAC" + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:56.161354" } } \ No newline at end of file diff --git a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/tags.yml b/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/tags.yml deleted file mode 100644 index ab064503..00000000 --- a/subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -fastq_download_prefetch_fasterqdump_sratools: - - subworkflows/nf-core/fastq_download_prefetch_fasterqdump_sratools/** diff --git a/subworkflows/nf-core/nextflowpipelineutils/tests/main.functions.nf.test b/subworkflows/nf-core/nextflowpipelineutils/tests/main.functions.nf.test deleted file mode 100644 index 1a1577eb..00000000 --- a/subworkflows/nf-core/nextflowpipelineutils/tests/main.functions.nf.test +++ /dev/null @@ -1,71 +0,0 @@ - -nextflow_function { - - name "Test Functions" - script "subworkflows/nf-core/nextflowpipelineutils/main.nf" - config "subworkflows/nf-core/nextflowpipelineutils/tests/nextflow.config" - tag 'subworkflows' - tag 'nextflowpipelineutils' - tag 'subworkflows/nextflowpipelineutils' - - test("Test Function getWorkflowVersion") { - - function "getWorkflowVersion" - - when { - function { - """ - // no inputs - """ - } - } - - then { - assert function.success - assert snapshot(function.result).match() - } - - } - - - test("Test Function dumpParametersToJSON") { - - function "dumpParametersToJSON" - - when { - function { - """ - // define inputs of the function here. Example: - input[0] = "$outputDir" - """.stripIndent() - } - } - - then { - assert function.success - assert function.result =~ /publish_dir_mode/ - assert function.result =~ /copy/ - } - - } - - test("Test Function checkCondaChannels") { - - function "checkCondaChannels" - - when { - function { - """ - // no inputs - """ - } - } - - then { - assert function.success - assert snapshot(function.result).match() - } - - } - -} \ No newline at end of file diff --git a/subworkflows/nf-core/nextflowpipelineutils/tests/main.functions.nf.test.snap b/subworkflows/nf-core/nextflowpipelineutils/tests/main.functions.nf.test.snap deleted file mode 100644 index 939dbbad..00000000 --- a/subworkflows/nf-core/nextflowpipelineutils/tests/main.functions.nf.test.snap +++ /dev/null @@ -1,12 +0,0 @@ -{ - "Test Function getWorkflowVersion": { - "content": [ - "v9.9.9" - ], - "timestamp": "2023-10-15T13:43:55.128868" - }, - "Test Function checkCondaChannels": { - "content": null, - "timestamp": "2023-10-12T17:09:24.933468" - } -} \ No newline at end of file diff --git a/subworkflows/nf-core/nextflowpipelineutils/tests/main.workflow.nf.test b/subworkflows/nf-core/nextflowpipelineutils/tests/main.workflow.nf.test deleted file mode 100644 index 3cf7f034..00000000 --- a/subworkflows/nf-core/nextflowpipelineutils/tests/main.workflow.nf.test +++ /dev/null @@ -1,104 +0,0 @@ -nextflow_workflow { - - name "Test Workflow NEXTFLOW_PIPELINE_UTILS" - script "../main.nf" - config "subworkflows/nf-core/nextflowpipelineutils/tests/nextflow.config" - workflow "NEXTFLOW_PIPELINE_UTILS" - tag 'subworkflows' - tag 'nextflowpipelineutils' - tag 'subworkflows/nextflowpipelineutils' - - test("Should run no inputs") { - - when { - params { - outdir = "tests/results" - } - workflow { - """ - input[0] = false - input[1] = false - input[2] = null - input[3] = false - """ - } - } - - then { - assert workflow.success - assert snapshot(workflow.out).match() - } - - } - - test("Should print version") { - - when { - params { - outdir = "tests/results" - } - workflow { - """ - input[0] = true - input[1] = false - input[2] = null - input[3] = false - """ - } - } - - then { - assert workflow.success - assert workflow.stdout.contains("nextflow_workflow v9.9.9") - } - - } - - - test("Should dump params") { - - when { - params { - outdir = "$outputDir" - } - workflow { - """ - input[0] = false - input[1] = true - input[2] = params.outdir - input[3] = false - """ - } - } - - then { - assert workflow.success - assert snapshot(workflow.out).match() - } - - } - - test("Should not create params JSON if no output directory") { - - when { - params { - outdir = "$outputDir" - } - workflow { - """ - input[0] = false - input[1] = true - input[2] = null - input[3] = false - """ - } - } - - then { - assert workflow.success - assert snapshot(workflow.out, path(params.outdir).list()).match() - } - - } - -} diff --git a/subworkflows/nf-core/nextflowpipelineutils/tests/main.workflow.nf.test.snap b/subworkflows/nf-core/nextflowpipelineutils/tests/main.workflow.nf.test.snap deleted file mode 100644 index 08040fd4..00000000 --- a/subworkflows/nf-core/nextflowpipelineutils/tests/main.workflow.nf.test.snap +++ /dev/null @@ -1,44 +0,0 @@ -{ - "Should not create params JSON if no output directory": { - "content": [ - { - "0": [ - true - ], - "out": [ - true - ] - }, - [ - - ] - ], - "timestamp": "2023-10-13T11:55:54.576748" - }, - "Should dump params": { - "content": [ - { - "0": [ - true - ], - "out": [ - true - ] - } - ], - "timestamp": "2023-10-13T11:55:52.711329" - }, - "Should run no inputs": { - "content": [ - { - "0": [ - true - ], - "out": [ - true - ] - } - ], - "timestamp": "2023-10-13T11:55:49.048315" - } -} \ No newline at end of file diff --git a/subworkflows/nf-core/nfcore_pipeline_utils/tests/main.workflow.nf.test b/subworkflows/nf-core/nfcore_pipeline_utils/tests/main.workflow.nf.test deleted file mode 100644 index ad7c73a8..00000000 --- a/subworkflows/nf-core/nfcore_pipeline_utils/tests/main.workflow.nf.test +++ /dev/null @@ -1,33 +0,0 @@ -nextflow_workflow { - - name "Test Workflow NFCORE_PIPELINE_UTILS" - script "../main.nf" - config "subworkflows/nf-core/nfcore_pipeline_utils/tests/nextflow.config" - workflow "NFCORE_PIPELINE_UTILS" - tag "subworkflows" - tag "subworkflows_nfcore" - tag "nfcorepipelineutils" - tag "subworkflows/nfcorepipelineutils" - - test("Should run without failures") { - - when { - params { - // define parameters here. Example: - // outdir = "tests/results" - } - workflow { - """ - // define inputs of the workflow here. Example: - // input[0] = file("test-file.txt") - """ - } - } - - then { - assert workflow.success - } - - } - -} diff --git a/subworkflows/nf-core/nfvalidation_plugin_utils/main.nf b/subworkflows/nf-core/nfvalidation_plugin_utils/main.nf deleted file mode 100644 index 22531719..00000000 --- a/subworkflows/nf-core/nfvalidation_plugin_utils/main.nf +++ /dev/null @@ -1,62 +0,0 @@ -// -// Subworkflow that uses the nf-validation plugin to render help text and parameter summary -// - -/* -======================================================================================== - IMPORT NF-VALIDATION PLUGIN -======================================================================================== -*/ - -include { paramsHelp; paramsSummaryLog; paramsSummaryMap; validateParameters } from 'plugin/nf-validation' - -/* -======================================================================================== - SUBWORKFLOW DEFINITION -======================================================================================== -*/ - -workflow NF_VALIDATION_PLUGIN_UTILS { - - take: - print_help // bool - workflow_command // string: default commmand used to run pipeline - pre_help_text // string: string to be printed before help text and summary log - post_help_text // string: string to be printed after help text and summary log - validate_params // bool: Validate parameters - schema_filename // path: JSON schema file, null to use default value - - main: - - log.debug "Using schema file: ${schema_filename}" - - // Default values for strings - pre_help_text = pre_help_text ?: '' - post_help_text = post_help_text ?: '' - workflow_command = workflow_command ?: '' - - // - // Print help message if needed - // - if (print_help) { - log.info pre_help_text + paramsHelp(workflow_command, parameters_schema: schema_filename) + post_help_text - System.exit(0) - } - - // - // Print parameter summary to stdout - // - log.info pre_help_text + paramsSummaryLog(workflow, parameters_schema: schema_filename) + post_help_text - - // - // Validate parameters relative to the parameter JSON schema - // - if (validate_params){ - validateParameters(parameters_schema: schema_filename) - } - - summary_params = paramsSummaryMap(workflow, parameters_schema: schema_filename) - - emit: - summary_params = summary_params -} diff --git a/subworkflows/nf-core/nfvalidation_plugin_utils/meta.yml b/subworkflows/nf-core/nfvalidation_plugin_utils/meta.yml deleted file mode 100644 index 6117b455..00000000 --- a/subworkflows/nf-core/nfvalidation_plugin_utils/meta.yml +++ /dev/null @@ -1,49 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: "nfvalidationpluginutils" -## TODO nf-core: Add a description of the subworkflow and list keywords -description: Sort SAM/BAM/CRAM file -keywords: - - sort - - bam - - sam - - cram -## TODO nf-core: Add a list of the modules and/or subworkflows used in the subworkflow -components: - - samtools/sort - - samtools/index -## TODO nf-core: List all of the channels used as input with a description and their structure -input: - - ch_bam: - type: file - description: | - The input channel containing the BAM/CRAM/SAM files - Structure: [ val(meta), path(bam) ] - pattern: "*.{bam/cram/sam}" -## TODO nf-core: List all of the channels used as output with a descriptions and their structure -output: - - bam: - type: file - description: | - Channel containing BAM files - Structure: [ val(meta), path(bam) ] - pattern: "*.bam" - - bai: - type: file - description: | - Channel containing indexed BAM (BAI) files - Structure: [ val(meta), path(bai) ] - pattern: "*.bai" - - csi: - type: file - description: | - Channel containing CSI files - Structure: [ val(meta), path(csi) ] - pattern: "*.csi" - - versions: - type: file - description: | - File containing software versions - Structure: [ path(versions.yml) ] - pattern: "versions.yml" -authors: - - "@adamrtalbot" diff --git a/subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test.snap b/subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test.snap deleted file mode 100644 index ae33a930..00000000 --- a/subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test.snap +++ /dev/null @@ -1,26 +0,0 @@ -{ - "Should run help": { - "content": [ - { - "stderr": [ - - ], - "errorReport": "", - "exitStatus": 0, - "failed": false, - "stdout": [ - - ], - "errorMessage": "", - "trace": { - "tasksFailed": 0, - "tasksCount": 0, - "tasksSucceeded": 0 - }, - "name": "workflow", - "success": true - } - ], - "timestamp": "2023-10-13T13:18:16.933251413" - } -} diff --git a/subworkflows/nf-core/nextflowpipelineutils/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf similarity index 82% rename from subworkflows/nf-core/nextflowpipelineutils/main.nf rename to subworkflows/nf-core/utils_nextflow_pipeline/main.nf index 5cb9af8c..ac31f28f 100644 --- a/subworkflows/nf-core/nextflowpipelineutils/main.nf +++ b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -4,6 +4,7 @@ import org.yaml.snakeyaml.Yaml import groovy.json.JsonOutput +import nextflow.extension.FilesEx /* ======================================================================================== @@ -11,13 +12,13 @@ import groovy.json.JsonOutput ======================================================================================== */ -workflow NEXTFLOW_PIPELINE_UTILS { +workflow UTILS_NEXTFLOW_PIPELINE { take: - print_version // bool - dump_parameters // bool - output_directory // path: base directory used to publish pipeline results - check_conda_channels // bool + print_version // boolean: print version + dump_parameters // boolean: dump parameters + outdir // path: base directory used to publish pipeline results + check_conda_channels // boolean: check conda channels main: @@ -32,8 +33,8 @@ workflow NEXTFLOW_PIPELINE_UTILS { // // Dump pipeline parameters to a JSON file // - if (dump_parameters && output_directory) { - dumpParametersToJSON(output_directory) + if (dump_parameters && outdir) { + dumpParametersToJSON(outdir) } // @@ -44,7 +45,7 @@ workflow NEXTFLOW_PIPELINE_UTILS { } emit: - out = true + dummy_emit = true } /* @@ -74,16 +75,15 @@ def getWorkflowVersion() { // // Dump pipeline parameters to a JSON file // -def dumpParametersToJSON(output_directory) { - def output_d = new File("${output_directory}/pipeline_info/") - if (!output_d.exists()) { - output_d.mkdirs() - } - +def dumpParametersToJSON(outdir) { def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') - def output_pf = new File(output_d, "params_${timestamp}.json") + def filename = "params_${timestamp}.json" + def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") def jsonStr = JsonOutput.toJson(params) - output_pf.text = JsonOutput.prettyPrint(jsonStr) + temp_pf.text = JsonOutput.prettyPrint(jsonStr) + + FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") + temp_pf.delete() } // diff --git a/subworkflows/nf-core/nextflowpipelineutils/meta.yml b/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml similarity index 83% rename from subworkflows/nf-core/nextflowpipelineutils/meta.yml rename to subworkflows/nf-core/utils_nextflow_pipeline/meta.yml index f56b827b..e5c3a0a8 100644 --- a/subworkflows/nf-core/nextflowpipelineutils/meta.yml +++ b/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml @@ -1,5 +1,5 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: "nextflow_pipeline_utils" +name: "UTILS_NEXTFLOW_PIPELINE" description: Subworkflow with functionality that may be useful for any Nextflow pipeline keywords: - utility @@ -25,10 +25,14 @@ input: description: | Check if the conda channel priority is correct. output: - - out: + - dummy_emit: type: boolean description: | - Dummy output for making using in process chaining. + Dummy emit to make nf-core subworkflows lint happy authors: - "@adamrtalbot" - "@drpatelh" +maintainers: + - "@adamrtalbot" + - "@drpatelh" + - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test new file mode 100644 index 00000000..e49d617f --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test @@ -0,0 +1,52 @@ + +nextflow_function { + + name "Test Functions" + script "subworkflows/nf-core/utils_nextflow_pipeline/main.nf" + config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" + tag "UTILS_NEXTFLOW_PIPELINE" + + test("Test Function getWorkflowVersion") { + + function "getWorkflowVersion" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function dumpParametersToJSON") { + + function "dumpParametersToJSON" + + when { + function { + """ + // define inputs of the function here. Example: + input[0] = "$outputDir" + """.stripIndent() + } + } + + then { + assertAll( + { assert function.success } + ) + } + } + + test("Test Function checkCondaChannels") { + + function "checkCondaChannels" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 00000000..e3f0baf4 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,20 @@ +{ + "Test Function getWorkflowVersion": { + "content": [ + "v9.9.9" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:05.308243" + }, + "Test Function checkCondaChannels": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:12.425833" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test new file mode 100644 index 00000000..13782037 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,108 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NEXTFLOW_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" + workflow "UTILS_NEXTFLOW_PIPELINE" + + test("Should run no inputs") { + + when { + workflow { + """ + print_version = false + dump_parameters = false + outdir = null + check_conda_channels = false + + input[0] = print_version + input[1] = dump_parameters + input[2] = outdir + input[3] = check_conda_channels + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should print version") { + + when { + workflow { + """ + print_version = true + dump_parameters = false + outdir = null + check_conda_channels = false + + input[0] = print_version + input[1] = dump_parameters + input[2] = outdir + input[3] = check_conda_channels + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.stdout.contains("nextflow_workflow v9.9.9") } + ) + } + } + + test("Should dump params") { + + when { + workflow { + """ + print_version = false + dump_parameters = true + outdir = 'results' + check_conda_channels = false + + input[0] = false + input[1] = true + input[2] = outdir + input[3] = false + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should not create params JSON if no output directory") { + + when { + workflow { + """ + print_version = false + dump_parameters = true + outdir = null + check_conda_channels = false + + input[0] = false + input[1] = true + input[2] = outdir + input[3] = false + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } +} diff --git a/subworkflows/nf-core/nextflowpipelineutils/tests/nextflow.config b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config similarity index 99% rename from subworkflows/nf-core/nextflowpipelineutils/tests/nextflow.config rename to subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config index 53574ffe..d0a926bf 100644 --- a/subworkflows/nf-core/nextflowpipelineutils/tests/nextflow.config +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -6,4 +6,4 @@ manifest { nextflowVersion = '!>=23.04.0' version = '9.9.9' doi = 'https://doi.org/10.5281/zenodo.5070524' -} \ No newline at end of file +} diff --git a/subworkflows/nf-core/nfcore_pipeline_utils/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf similarity index 50% rename from subworkflows/nf-core/nfcore_pipeline_utils/main.nf rename to subworkflows/nf-core/utils_nfcore_pipeline/main.nf index b3b136dc..a8b55d6f 100644 --- a/subworkflows/nf-core/nfcore_pipeline_utils/main.nf +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -2,20 +2,26 @@ // Subworkflow with utility functions specific to the nf-core pipeline template // +import org.yaml.snakeyaml.Yaml +import nextflow.extension.FilesEx + /* ======================================================================================== SUBWORKFLOW DEFINITION ======================================================================================== */ -workflow NFCORE_PIPELINE_UTILS { +workflow UTILS_NFCORE_PIPELINE { + + take: + nextflow_cli_args main: - checkConfigProvided() + valid_config = checkConfigProvided() + checkProfileProvided(nextflow_cli_args) emit: - success = true - + valid_config } /* @@ -28,6 +34,7 @@ workflow NFCORE_PIPELINE_UTILS { // Warn if a -profile or Nextflow config has not been provided to run the pipeline // def checkConfigProvided() { + valid_config = true if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { log.warn "[$workflow.manifest.name] You are attempting to run the pipeline without any custom configuration!\n\n" + "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + @@ -35,6 +42,22 @@ def checkConfigProvided() { " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + "Please refer to the quick start section and usage docs for the pipeline.\n " + valid_config = false + } + return valid_config +} + +// +// Exit pipeline if --profile contains spaces +// +def checkProfileProvided(nextflow_cli_args) { + if (workflow.profile.endsWith(',')) { + error "The `-profile` option cannot end with a trailing comma, please remove it and re-run the pipeline!\n" + + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + } + if (nextflow_cli_args[0]) { + log.warn "nf-core pipelines do not accept positional arguments. The positional argument `${nextflow_cli_args[0]}` has been detected.\n" + + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" } } @@ -51,21 +74,98 @@ def workflowCitation() { " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" } +// +// Generate workflow version string +// +def getWorkflowVersion() { + String version_string = "" + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string +} + +// +// Get software versions for pipeline +// +def processVersionsFromYAML(yaml_file) { + Yaml yaml = new Yaml() + versions = yaml.load(yaml_file).collectEntries { k, v -> [ k.tokenize(':')[-1], v ] } + return yaml.dumpAsMap(versions).trim() +} + +// +// Get workflow version for pipeline +// +def workflowVersionToYAML() { + return """ + Workflow: + $workflow.manifest.name: ${getWorkflowVersion()} + Nextflow: $workflow.nextflow.version + """.stripIndent().trim() +} + +// +// Get channel of software versions used in pipeline in YAML format +// +def softwareVersionsToYAML(ch_versions) { + return ch_versions + .unique() + .map { processVersionsFromYAML(it) } + .unique() + .mix(Channel.of(workflowVersionToYAML())) +} + +// +// Get workflow summary for MultiQC +// +def paramsSummaryMultiqc(summary_params) { + def summary_section = '' + for (group in summary_params.keySet()) { + def group_params = summary_params.get(group) // This gets the parameters of that particular group + if (group_params) { + summary_section += "

$group

\n" + summary_section += "
\n" + for (param in group_params.keySet()) { + summary_section += "
$param
${group_params.get(param) ?: 'N/A'}
\n" + } + summary_section += "
\n" + } + } + + String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" + yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" + yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" + yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" + yaml_file_text += "plot_type: 'html'\n" + yaml_file_text += "data: |\n" + yaml_file_text += "${summary_section}" + + return yaml_file_text +} + // // nf-core logo // -def nfCoreLogo(workflow_version) { - Map colors = logColours() +def nfCoreLogo(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) String.format( """\n - ${dashedLine()} + ${dashedLine(monochrome_logs)} ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} ${colors.green}`._,._,\'${colors.reset} - ${colors.purple} ${workflow.manifest.name} ${workflow_version}${colors.reset} - ${dashedLine()} + ${colors.purple} ${workflow.manifest.name} ${getWorkflowVersion()}${colors.reset} + ${dashedLine(monochrome_logs)} """.stripIndent() ) } @@ -73,83 +173,106 @@ def nfCoreLogo(workflow_version) { // // Return dashed line // -def dashedLine() { - Map colors = logColours() +def dashedLine(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) return "-${colors.dim}----------------------------------------------------${colors.reset}-" } // // ANSII colours used for terminal logging // -def logColours() { +def logColours(monochrome_logs=true) { Map colorcodes = [:] // Reset / Meta - colorcodes['reset'] = params.monochrome_logs ? '' : "\033[0m" - colorcodes['bold'] = params.monochrome_logs ? '' : "\033[1m" - colorcodes['dim'] = params.monochrome_logs ? '' : "\033[2m" - colorcodes['underlined'] = params.monochrome_logs ? '' : "\033[4m" - colorcodes['blink'] = params.monochrome_logs ? '' : "\033[5m" - colorcodes['reverse'] = params.monochrome_logs ? '' : "\033[7m" - colorcodes['hidden'] = params.monochrome_logs ? '' : "\033[8m" + colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" + colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" + colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" + colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" + colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" + colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" + colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" // Regular Colors - colorcodes['black'] = params.monochrome_logs ? '' : "\033[0;30m" - colorcodes['red'] = params.monochrome_logs ? '' : "\033[0;31m" - colorcodes['green'] = params.monochrome_logs ? '' : "\033[0;32m" - colorcodes['yellow'] = params.monochrome_logs ? '' : "\033[0;33m" - colorcodes['blue'] = params.monochrome_logs ? '' : "\033[0;34m" - colorcodes['purple'] = params.monochrome_logs ? '' : "\033[0;35m" - colorcodes['cyan'] = params.monochrome_logs ? '' : "\033[0;36m" - colorcodes['white'] = params.monochrome_logs ? '' : "\033[0;37m" + colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" + colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" + colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" + colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" + colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" + colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" + colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" + colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" // Bold - colorcodes['bblack'] = params.monochrome_logs ? '' : "\033[1;30m" - colorcodes['bred'] = params.monochrome_logs ? '' : "\033[1;31m" - colorcodes['bgreen'] = params.monochrome_logs ? '' : "\033[1;32m" - colorcodes['byellow'] = params.monochrome_logs ? '' : "\033[1;33m" - colorcodes['bblue'] = params.monochrome_logs ? '' : "\033[1;34m" - colorcodes['bpurple'] = params.monochrome_logs ? '' : "\033[1;35m" - colorcodes['bcyan'] = params.monochrome_logs ? '' : "\033[1;36m" - colorcodes['bwhite'] = params.monochrome_logs ? '' : "\033[1;37m" + colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" + colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" + colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" + colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" + colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" + colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" + colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" + colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" // Underline - colorcodes['ublack'] = params.monochrome_logs ? '' : "\033[4;30m" - colorcodes['ured'] = params.monochrome_logs ? '' : "\033[4;31m" - colorcodes['ugreen'] = params.monochrome_logs ? '' : "\033[4;32m" - colorcodes['uyellow'] = params.monochrome_logs ? '' : "\033[4;33m" - colorcodes['ublue'] = params.monochrome_logs ? '' : "\033[4;34m" - colorcodes['upurple'] = params.monochrome_logs ? '' : "\033[4;35m" - colorcodes['ucyan'] = params.monochrome_logs ? '' : "\033[4;36m" - colorcodes['uwhite'] = params.monochrome_logs ? '' : "\033[4;37m" + colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" + colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" + colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" + colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" + colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" + colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" + colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" + colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" // High Intensity - colorcodes['iblack'] = params.monochrome_logs ? '' : "\033[0;90m" - colorcodes['ired'] = params.monochrome_logs ? '' : "\033[0;91m" - colorcodes['igreen'] = params.monochrome_logs ? '' : "\033[0;92m" - colorcodes['iyellow'] = params.monochrome_logs ? '' : "\033[0;93m" - colorcodes['iblue'] = params.monochrome_logs ? '' : "\033[0;94m" - colorcodes['ipurple'] = params.monochrome_logs ? '' : "\033[0;95m" - colorcodes['icyan'] = params.monochrome_logs ? '' : "\033[0;96m" - colorcodes['iwhite'] = params.monochrome_logs ? '' : "\033[0;97m" + colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" + colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" + colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" + colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" + colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" + colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" + colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" + colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" // Bold High Intensity - colorcodes['biblack'] = params.monochrome_logs ? '' : "\033[1;90m" - colorcodes['bired'] = params.monochrome_logs ? '' : "\033[1;91m" - colorcodes['bigreen'] = params.monochrome_logs ? '' : "\033[1;92m" - colorcodes['biyellow'] = params.monochrome_logs ? '' : "\033[1;93m" - colorcodes['biblue'] = params.monochrome_logs ? '' : "\033[1;94m" - colorcodes['bipurple'] = params.monochrome_logs ? '' : "\033[1;95m" - colorcodes['bicyan'] = params.monochrome_logs ? '' : "\033[1;96m" - colorcodes['biwhite'] = params.monochrome_logs ? '' : "\033[1;97m" + colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" + colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" + colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" + colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" + colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" + colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" + colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" + colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" return colorcodes } +// +// Attach the multiqc report to email +// +def attachMultiqcReport(multiqc_report) { + def mqc_report = null + try { + if (workflow.success) { + mqc_report = multiqc_report.getVal() + if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { + if (mqc_report.size() > 1) { + log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" + } + mqc_report = mqc_report[0] + } + } + } catch (all) { + if (multiqc_report) { + log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" + } + } + return mqc_report +} + // // Construct and send completion email // -def completionEmail(summary_params) { +def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true, multiqc_report=null) { // Set up the e-mail variables def subject = "[$workflow.manifest.name] Successful: $workflow.runName" @@ -175,7 +298,7 @@ def completionEmail(summary_params) { misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp def email_fields = [:] - email_fields['version'] = NfcoreTemplate.version(workflow) + email_fields['version'] = getWorkflowVersion() email_fields['runName'] = workflow.runName email_fields['success'] = workflow.success email_fields['dateComplete'] = workflow.complete @@ -187,10 +310,13 @@ def completionEmail(summary_params) { email_fields['projectDir'] = workflow.projectDir email_fields['summary'] = summary << misc_fields + // On success try attach the multiqc report + def mqc_report = attachMultiqcReport(multiqc_report) + // Check if we are only sending emails on failure - def email_address = params.email - if (!params.email && params.email_on_fail && !workflow.success) { - email_address = params.email_on_fail + def email_address = email + if (!email && email_on_fail && !workflow.success) { + email_address = email_on_fail } // Render the TXT template @@ -205,17 +331,20 @@ def completionEmail(summary_params) { def email_html = html_template.toString() // Render the sendmail template - def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}" ] + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit + def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") def sendmail_template = engine.createTemplate(sf).make(smail_fields) def sendmail_html = sendmail_template.toString() // Send the HTML e-mail - Map colors = logColours() + Map colors = logColours(monochrome_logs) if (email_address) { try { - if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } + if (plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } // Try to send HTML e-mail using sendmail + def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") + sendmail_tf.withWriter { w -> w << sendmail_html } [ 'sendmail', '-t' ].execute() << sendmail_html log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" } catch (all) { @@ -227,21 +356,23 @@ def completionEmail(summary_params) { } // Write summary e-mail HTML to a file - def output_d = new File("${params.outdir}/pipeline_info/") - if (!output_d.exists()) { - output_d.mkdirs() - } - def output_hf = new File(output_d, "pipeline_report.html") + def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") output_hf.withWriter { w -> w << email_html } - def output_tf = new File(output_d, "pipeline_report.txt") + FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html"); + output_hf.delete() + + // Write summary e-mail TXT to a file + def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") output_tf.withWriter { w -> w << email_txt } + FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt"); + output_tf.delete() } // // Print pipeline summary on completion // -def completionSummary() { - Map colors = logColours() +def completionSummary(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) if (workflow.success) { if (workflow.stats.ignoredCount == 0) { log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" @@ -256,9 +387,7 @@ def completionSummary() { // // Construct and send a notification to a web server as JSON e.g. Microsoft Teams and Slack // -def imNotification(summary_params) { - def hook_url = params.hook_url - +def imNotification(summary_params, hook_url) { def summary = [:] for (group in summary_params.keySet()) { summary << summary_params[group] @@ -277,7 +406,7 @@ def imNotification(summary_params) { misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp def msg_fields = [:] - msg_fields['version'] = NfcoreTemplate.version(workflow) + msg_fields['version'] = getWorkflowVersion() msg_fields['runName'] = workflow.runName msg_fields['success'] = workflow.success msg_fields['dateComplete'] = workflow.complete diff --git a/subworkflows/nf-core/nfcore_pipeline_utils/meta.yml b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml similarity index 68% rename from subworkflows/nf-core/nfcore_pipeline_utils/meta.yml rename to subworkflows/nf-core/utils_nfcore_pipeline/meta.yml index 5721b7e3..d08d2434 100644 --- a/subworkflows/nf-core/nfcore_pipeline_utils/meta.yml +++ b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml @@ -1,5 +1,5 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json -name: "NFCORE_PIPELINE_UTILS" +name: "UTILS_NFCORE_PIPELINE" description: Subworkflow with utility functions specific to the nf-core pipeline template keywords: - utility @@ -7,7 +7,11 @@ keywords: - initialise - version components: [] -input: [] +input: + - nextflow_cli_args: + type: list + description: | + Nextflow CLI positional arguments output: - success: type: boolean @@ -15,3 +19,6 @@ output: Dummy output to indicate success authors: - "@adamrtalbot" +maintainers: + - "@adamrtalbot" + - "@maxulysse" diff --git a/subworkflows/nf-core/nfcore_pipeline_utils/tests/main.function.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test similarity index 54% rename from subworkflows/nf-core/nfcore_pipeline_utils/tests/main.function.nf.test rename to subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test index 265ece39..1dc317f8 100644 --- a/subworkflows/nf-core/nfcore_pipeline_utils/tests/main.function.nf.test +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test @@ -3,118 +3,132 @@ nextflow_function { name "Test Functions" script "../main.nf" - config "subworkflows/nf-core/nfcore_pipeline_utils/tests/nextflow.config" - + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + test("Test Function checkConfigProvided") { function "checkConfigProvided" + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function checkProfileProvided") { + + function "checkProfileProvided" + when { function { """ - // define inputs of the function here. Example: - // input[0] = 1 + input[0] = [] """ } } then { - assertAll ( - { assert function.success }, + assertAll( + { assert function.success }, { assert snapshot(function.result).match() } ) } - } test("Test Function workflowCitation") { function "workflowCitation" + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function nfCoreLogo") { + + function "nfCoreLogo" + when { function { """ - // define inputs of the function here. Example: - // input[0] = 1 + input[0] = false """ } } then { - assertAll ( - { assert function.success }, + assertAll( + { assert function.success }, { assert snapshot(function.result).match() } ) } - } - test("Test Function nfCoreLogo") { + test("Test Function dashedLine") { - function "nfCoreLogo" + function "dashedLine" when { function { """ - input[0] = "9.9.9" + input[0] = false """ } } then { - assertAll ( - { assert function.success }, + assertAll( + { assert function.success }, { assert snapshot(function.result).match() } ) } - } - + test("Test Function without logColours") { - test("Test Function dashedLine") { - - function "dashedLine" + function "logColours" when { function { """ - // define inputs of the function here. Example: - // input[0] = 1 + input[0] = true """ } } then { - assertAll ( - { assert function.success }, + assertAll( + { assert function.success }, { assert snapshot(function.result).match() } ) } - } - - - test("Test Function logColours") { - + test("Test Function with logColours") { function "logColours" when { function { """ - // define inputs of the function here. Example: - // input[0] = 1 + input[0] = false """ } } then { - assertAll ( - { assert function.success }, + assertAll( + { assert function.success }, { assert snapshot(function.result).match() } ) } - } -} \ No newline at end of file +} diff --git a/subworkflows/nf-core/nfcore_pipeline_utils/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap similarity index 53% rename from subworkflows/nf-core/nfcore_pipeline_utils/tests/main.function.nf.test.snap rename to subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap index 3ea51f68..1037232c 100644 --- a/subworkflows/nf-core/nfcore_pipeline_utils/tests/main.function.nf.test.snap +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap @@ -1,5 +1,111 @@ { - "Test Function logColours": { + "Test Function checkProfileProvided": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:03.360873" + }, + "Test Function checkConfigProvided": { + "content": [ + true + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:59.729647" + }, + "Test Function nfCoreLogo": { + "content": [ + "\n\n-\u001b[2m----------------------------------------------------\u001b[0m-\n \u001b[0;32m,--.\u001b[0;30m/\u001b[0;32m,-.\u001b[0m\n\u001b[0;34m ___ __ __ __ ___ \u001b[0;32m/,-._.--~'\u001b[0m\n\u001b[0;34m |\\ | |__ __ / ` / \\ |__) |__ \u001b[0;33m} {\u001b[0m\n\u001b[0;34m | \\| | \\__, \\__/ | \\ |___ \u001b[0;32m\\`-._,-`-,\u001b[0m\n \u001b[0;32m`._,._,'\u001b[0m\n\u001b[0;35m nextflow_workflow v9.9.9\u001b[0m\n-\u001b[2m----------------------------------------------------\u001b[0m-\n" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:10.562934" + }, + "Test Function workflowCitation": { + "content": [ + "If you use nextflow_workflow for your analysis please cite:\n\n* The pipeline\n https://doi.org/10.5281/zenodo.5070524\n\n* The nf-core framework\n https://doi.org/10.1038/s41587-020-0439-x\n\n* Software dependencies\n https://github.com/nextflow_workflow/blob/master/CITATIONS.md" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:07.019761" + }, + "Test Function without logColours": { + "content": [ + { + "reset": "", + "bold": "", + "dim": "", + "underlined": "", + "blink": "", + "reverse": "", + "hidden": "", + "black": "", + "red": "", + "green": "", + "yellow": "", + "blue": "", + "purple": "", + "cyan": "", + "white": "", + "bblack": "", + "bred": "", + "bgreen": "", + "byellow": "", + "bblue": "", + "bpurple": "", + "bcyan": "", + "bwhite": "", + "ublack": "", + "ured": "", + "ugreen": "", + "uyellow": "", + "ublue": "", + "upurple": "", + "ucyan": "", + "uwhite": "", + "iblack": "", + "ired": "", + "igreen": "", + "iyellow": "", + "iblue": "", + "ipurple": "", + "icyan": "", + "iwhite": "", + "biblack": "", + "bired": "", + "bigreen": "", + "biyellow": "", + "biblue": "", + "bipurple": "", + "bicyan": "", + "biwhite": "" + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:17.969323" + }, + "Test Function dashedLine": { + "content": [ + "-\u001b[2m----------------------------------------------------\u001b[0m-" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:14.366181" + }, + "Test Function with logColours": { "content": [ { "reset": "\u001b[0m", @@ -51,28 +157,10 @@ "biwhite": "\u001b[1;97m" } ], - "timestamp": "2023-10-16T14:27:09.843064" - }, - "Test Function checkConfigProvided": { - "content": null, - "timestamp": "2023-10-16T14:27:01.415849" - }, - "Test Function nfCoreLogo": { - "content": [ - "\n\n-\u001b[2m----------------------------------------------------\u001b[0m-\n \u001b[0;32m,--.\u001b[0;30m/\u001b[0;32m,-.\u001b[0m\n\u001b[0;34m ___ __ __ __ ___ \u001b[0;32m/,-._.--~'\u001b[0m\n\u001b[0;34m |\\ | |__ __ / ` / \\ |__) |__ \u001b[0;33m} {\u001b[0m\n\u001b[0;34m | \\| | \\__, \\__/ | \\ |___ \u001b[0;32m\\`-._,-`-,\u001b[0m\n \u001b[0;32m`._,._,'\u001b[0m\n\u001b[0;35m nextflow_workflow 9.9.9\u001b[0m\n-\u001b[2m----------------------------------------------------\u001b[0m-\n" - ], - "timestamp": "2023-10-16T14:27:51.566211" - }, - "Test Function workflowCitation": { - "content": [ - "If you use nextflow_workflow for your analysis please cite:\n\n* The pipeline\n https://doi.org/10.5281/zenodo.5070524\n\n* The nf-core framework\n https://doi.org/10.1038/s41587-020-0439-x\n\n* Software dependencies\n https://github.com/nextflow_workflow/blob/master/CITATIONS.md" - ], - "timestamp": "2023-10-16T14:27:03.505737" - }, - "Test Function dashedLine": { - "content": [ - "-\u001b[2m----------------------------------------------------\u001b[0m-" - ], - "timestamp": "2023-10-16T14:27:07.721916" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:21.714424" } } \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test new file mode 100644 index 00000000..8940d32d --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + when { + workflow { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap new file mode 100644 index 00000000..859d1030 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap @@ -0,0 +1,19 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/nfcore_pipeline_utils/tests/nextflow.config b/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config similarity index 99% rename from subworkflows/nf-core/nfcore_pipeline_utils/tests/nextflow.config rename to subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config index 53574ffe..d0a926bf 100644 --- a/subworkflows/nf-core/nfcore_pipeline_utils/tests/nextflow.config +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config @@ -6,4 +6,4 @@ manifest { nextflowVersion = '!>=23.04.0' version = '9.9.9' doi = 'https://doi.org/10.5281/zenodo.5070524' -} \ No newline at end of file +} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf b/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf new file mode 100644 index 00000000..2585b65d --- /dev/null +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf @@ -0,0 +1,62 @@ +// +// Subworkflow that uses the nf-validation plugin to render help text and parameter summary +// + +/* +======================================================================================== + IMPORT NF-VALIDATION PLUGIN +======================================================================================== +*/ + +include { paramsHelp } from 'plugin/nf-validation' +include { paramsSummaryLog } from 'plugin/nf-validation' +include { validateParameters } from 'plugin/nf-validation' + +/* +======================================================================================== + SUBWORKFLOW DEFINITION +======================================================================================== +*/ + +workflow UTILS_NFVALIDATION_PLUGIN { + + take: + print_help // boolean: print help + workflow_command // string: default commmand used to run pipeline + pre_help_text // string: string to be printed before help text and summary log + post_help_text // string: string to be printed after help text and summary log + validate_params // boolean: validate parameters + schema_filename // path: JSON schema file, null to use default value + + main: + + log.debug "Using schema file: ${schema_filename}" + + // Default values for strings + pre_help_text = pre_help_text ?: '' + post_help_text = post_help_text ?: '' + workflow_command = workflow_command ?: '' + + // + // Print help message if needed + // + if (print_help) { + log.info pre_help_text + paramsHelp(workflow_command, parameters_schema: schema_filename) + post_help_text + System.exit(0) + } + + // + // Print parameter summary to stdout + // + log.info pre_help_text + paramsSummaryLog(workflow, parameters_schema: schema_filename) + post_help_text + + // + // Validate parameters relative to the parameter JSON schema + // + if (validate_params){ + validateParameters(parameters_schema: schema_filename) + } + + emit: + dummy_emit = true +} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml b/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml new file mode 100644 index 00000000..3d4a6b04 --- /dev/null +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml @@ -0,0 +1,44 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NFVALIDATION_PLUGIN" +description: Use nf-validation to initiate and validate a pipeline +keywords: + - utility + - pipeline + - initialise + - validation +components: [] +input: + - print_help: + type: boolean + description: | + Print help message and exit + - workflow_command: + type: string + description: | + The command to run the workflow e.g. "nextflow run main.nf" + - pre_help_text: + type: string + description: | + Text to print before the help message + - post_help_text: + type: string + description: | + Text to print after the help message + - validate_params: + type: boolean + description: | + Validate the parameters and error if invalid. + - schema_filename: + type: string + description: | + The filename of the schema to validate against. +output: + - dummy_emit: + type: boolean + description: | + Dummy emit to make nf-core subworkflows lint happy +authors: + - "@adamrtalbot" +maintainers: + - "@adamrtalbot" + - "@maxulysse" diff --git a/subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test similarity index 64% rename from subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test rename to subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test index 6a0efd1e..5649654e 100644 --- a/subworkflows/nf-core/nfvalidation_plugin_utils/tests/main.nf.test +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test @@ -1,12 +1,8 @@ nextflow_workflow { - name "Test Workflow NF_VALIDATION_PLUGIN_UTILS" + name "Test Workflow UTILS_NFVALIDATION_PLUGIN" script "../main.nf" - workflow "NF_VALIDATION_PLUGIN_UTILS" - tag "subworkflow" - tag "subworkflow_nfcore" - tag "nfvalidationpluginutils" - tag "subworkflows/nfvalidationpluginutils" + workflow "UTILS_NFVALIDATION_PLUGIN" test("Should run nothing") { @@ -19,11 +15,11 @@ nextflow_workflow { workflow { """ - help = false + help = false workflow_command = null - pre_help_text = null - post_help_text = null - validate_params = false + pre_help_text = null + post_help_text = null + validate_params = false schema_filename = "$moduleTestDir/nextflow_schema.json" input[0] = help @@ -37,9 +33,10 @@ nextflow_workflow { } then { - assert workflow.success + assertAll( + { assert workflow.success } + ) } - } test("Should run help") { @@ -71,18 +68,17 @@ nextflow_workflow { } then { - assert workflow.success - assert workflow.exitStatus == 0 - assert workflow.stdout.any { it.contains('Input/output options') } - assert workflow.stdout.any { it.contains('--outdir') } - assert workflow.stdout.any { it.contains('--outdir') } + assertAll( + { assert workflow.success }, + { assert workflow.exitStatus == 0 }, + { assert workflow.stdout.any { it.contains('Input/output options') } }, + { assert workflow.stdout.any { it.contains('--outdir') } } + ) } - } test("Should run help with command") { - when { params { @@ -92,7 +88,7 @@ nextflow_workflow { workflow { """ help = true - workflow_command = "nextflow run noorg/doesntexist" + workflow_command = "nextflow run noorg/doesntexist" pre_help_text = null post_help_text = null validate_params = false @@ -109,14 +105,14 @@ nextflow_workflow { } then { - assert workflow.success - assert workflow.exitStatus == 0 - assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } - assert workflow.stdout.any { it.contains('Input/output options') } - assert workflow.stdout.any { it.contains('--outdir') } - assert workflow.stdout.any { it.contains('--outdir') } + assertAll( + { assert workflow.success }, + { assert workflow.exitStatus == 0 }, + { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, + { assert workflow.stdout.any { it.contains('Input/output options') } }, + { assert workflow.stdout.any { it.contains('--outdir') } } + ) } - } test("Should run help with extra text") { @@ -131,7 +127,7 @@ nextflow_workflow { workflow { """ help = true - workflow_command = "nextflow run noorg/doesntexist" + workflow_command = "nextflow run noorg/doesntexist" pre_help_text = "pre-help-text" post_help_text = "post-help-text" validate_params = false @@ -148,31 +144,31 @@ nextflow_workflow { } then { - assert workflow.success - assert workflow.exitStatus == 0 - assert workflow.stdout.any { it.contains('pre-help-text') } - assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } - assert workflow.stdout.any { it.contains('Input/output options') } - assert workflow.stdout.any { it.contains('--outdir') } - assert workflow.stdout.any { it.contains('--outdir') } - assert workflow.stdout.any { it.contains('post-help-text') } + assertAll( + { assert workflow.success }, + { assert workflow.exitStatus == 0 }, + { assert workflow.stdout.any { it.contains('pre-help-text') } }, + { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, + { assert workflow.stdout.any { it.contains('Input/output options') } }, + { assert workflow.stdout.any { it.contains('--outdir') } }, + { assert workflow.stdout.any { it.contains('post-help-text') } } + ) } - } test("Should validate params") { - when { params { monochrome_logs = true test_data = '' + outdir = 1 } workflow { """ help = false - workflow_command = null + workflow_command = null pre_help_text = null post_help_text = null validate_params = true @@ -189,11 +185,10 @@ nextflow_workflow { } then { - assert workflow.failed - assert workflow.stdout.any { it.contains('ERROR ~ ERROR: Validation of pipeline parameters failed!') } - assert workflow.stdout.any { it.contains('The following invalid input values have been detected:') } + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ ERROR: Validation of pipeline parameters failed!') } } + ) } - } - } diff --git a/subworkflows/nf-core/nfvalidation_plugin_utils/tests/nextflow_schema.json b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json similarity index 100% rename from subworkflows/nf-core/nfvalidation_plugin_utils/tests/nextflow_schema.json rename to subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json diff --git a/tests/main.nf.test b/tests/main.nf.test index ac05aea9..dc3662cd 100644 --- a/tests/main.nf.test +++ b/tests/main.nf.test @@ -2,27 +2,80 @@ nextflow_pipeline { name "Test pipeline" script "../main.nf" - tag "pipeline" - tag "pipeline_fetchngs" + tag "PIPELINE" test("Run with profile test") { when { params { - outdir = "results" - max_cpus = 2 - max_memory = '6.GB' - max_time = '6.h' - input = "$projectDir/tests/sra_ids_test.csv" - validationSchemaIgnoreParams = 'test_data_base,merge_samplesheet_ids,fastq_ftp_ids,test_data' + outdir = "$outputDir" + input = 'https://raw.githubusercontent.com/nf-core/test-datasets/2732b911c57e607fa7aea5ba0c3d91b25bafb662/testdata/v1.12.0/sra_ids_test.csv' } } then { + assert workflow.success + assertAll( - { assert workflow.success } + { assert new File("$outputDir/samplesheet/samplesheet.csv").readLines().size() == 15 }, + { assert new File("$outputDir/samplesheet/samplesheet.csv").readLines()*.split(',')[0].take(4) == ['"sample"', '"fastq_1"', '"fastq_2"', '"run_accession"'] }, + { assert new File("$outputDir/samplesheet/samplesheet.csv").readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX024467"', '"DRX026011"', '"ERX1234253"', '"SRX10940790"', '"SRX11047067"', '"SRX17709227"', '"SRX17709228"', '"SRX6725035"', '"SRX9315476"', '"SRX9504942"', '"SRX9504942"', '"SRX9504942"', '"SRX9504942"', '"SRX9626017"'] }, + { assert new File("$outputDir/samplesheet/samplesheet.csv").text.contains('Illumina HiSeq 2500') }, + { assert new File("$outputDir/custom/user-settings.mkfg").exists() }, + { assert new File("$outputDir/fastq/md5/DRX024467_DRR026872.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/DRX026011_DRR028935_1.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/DRX026011_DRR028935_2.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/ERX1234253_ERR1160846.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX17709227_SRR21711856.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX17709228_SRR21711855.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX6725035_SRR9984183.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055517_1.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055517_2.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055518_1.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055518_2.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055519_1.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055519_2.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055520_1.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX9504942_SRR13055520_2.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX9626017_SRR13191702_1.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/md5/SRX9626017_SRR13191702_2.fastq.gz.md5").exists() }, + { assert new File("$outputDir/fastq/DRX024467_DRR026872.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/DRX026011_DRR028935_1.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/DRX026011_DRR028935_2.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/ERX1234253_ERR1160846.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX10940790_SRR14593545_1.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX10940790_SRR14593545_2.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX11047067_SRR14709033.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX17709227_SRR21711856.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX17709228_SRR21711855.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX6725035_SRR9984183.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX9315476_SRR12848126_1.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX9315476_SRR12848126_2.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX9504942_SRR13055517_1.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX9504942_SRR13055517_2.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX9504942_SRR13055518_1.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX9504942_SRR13055518_2.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX9504942_SRR13055519_1.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX9504942_SRR13055519_2.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX9504942_SRR13055520_1.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX9504942_SRR13055520_2.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX9626017_SRR13191702_1.fastq.gz").exists() }, + { assert new File("$outputDir/fastq/SRX9626017_SRR13191702_2.fastq.gz").exists() }, + { assert new File("$outputDir/metadata/DRR026872.runinfo_ftp.tsv").exists() }, + { assert new File("$outputDir/metadata/DRR028935.runinfo_ftp.tsv").exists() }, + { assert new File("$outputDir/metadata/ERR1160846.runinfo_ftp.tsv").exists() }, + { assert new File("$outputDir/metadata/GSE214215.runinfo_ftp.tsv").exists() }, + { assert new File("$outputDir/metadata/GSM4907283.runinfo_ftp.tsv").exists() }, + { assert new File("$outputDir/metadata/SRR12848126.runinfo_ftp.tsv").exists() }, + { assert new File("$outputDir/metadata/SRR13191702.runinfo_ftp.tsv").exists() }, + { assert new File("$outputDir/metadata/SRR14593545.runinfo_ftp.tsv").exists() }, + { assert new File("$outputDir/metadata/SRR14709033.runinfo_ftp.tsv").exists() }, + { assert new File("$outputDir/metadata/SRR9984183.runinfo_ftp.tsv").exists() }, + { assert new File("$outputDir/pipeline_info/nf_core_fetchngs_software_mqc_versions.yml").exists() }, + { assert new File("$outputDir/samplesheet/id_mappings.csv").exists() }, + { assert new File("$outputDir/samplesheet/multiqc_config.yml").exists() }, + { assert new File("$outputDir/samplesheet/samplesheet.csv").exists() } ) } } - } diff --git a/tests/nextflow.config b/tests/nextflow.config new file mode 100644 index 00000000..d75ad44b --- /dev/null +++ b/tests/nextflow.config @@ -0,0 +1,27 @@ +params { + // Base directory for nf-core/modules test data + modules_testdata_base_path = 's3://ngi-igenomes/testdata/nf-core/modules/' + + // Base directory for nf-core/fetchngs test data + pipelines_testdata_base_path = 's3://ngi-igenomes/testdata/nf-core/pipelines/fetchngs/1.15.0/' +} + +// Impose sensible resource limits for testing +process { + withName: '.*' { + cpus = 2 + memory = 3.GB + time = 2.h + } +} + +// Impose same minimum Nextflow version as the pipeline for testing +manifest { + nextflowVersion = '!>=23.04.0' +} + +// Disable all Nextflow reporting options +timeline { enabled = false } +report { enabled = false } +trace { enabled = false } +dag { enabled = false } diff --git a/tests/sra_ids_test.csv b/tests/sra_ids_test.csv deleted file mode 100644 index d9a20c22..00000000 --- a/tests/sra_ids_test.csv +++ /dev/null @@ -1,3 +0,0 @@ -ERR1160846 -GSE214215 -SRR12848126 diff --git a/tests/tags.yml b/tests/tags.yml deleted file mode 100644 index 80e5fd75..00000000 --- a/tests/tags.yml +++ /dev/null @@ -1,5 +0,0 @@ -pipeline_fetchngs: - - "**.nf" - - "**.config" - - "**.nf.test" - - "**.json" diff --git a/workflows/sra/main.nf b/workflows/sra/main.nf index ddac096d..0c8cac0c 100644 --- a/workflows/sra/main.nf +++ b/workflows/sra/main.nf @@ -7,9 +7,10 @@ include { MULTIQC_MAPPINGS_CONFIG } from '../../modules/local/multiqc_mappings_config' include { SRA_FASTQ_FTP } from '../../modules/local/sra_fastq_ftp' include { SRA_IDS_TO_RUNINFO } from '../../modules/local/sra_ids_to_runinfo' -include { SRA_MERGE_SAMPLESHEET } from '../../modules/local/sra_merge_samplesheet' include { SRA_RUNINFO_TO_FTP } from '../../modules/local/sra_runinfo_to_ftp' +include { ASPERA_CLI } from '../../modules/local/aspera_cli' include { SRA_TO_SAMPLESHEET } from '../../modules/local/sra_to_samplesheet' +include { softwareVersionsToYAML } from '../../subworkflows/nf-core/utils_nfcore_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -54,25 +55,37 @@ workflow SRA { .out .tsv .splitCsv(header:true, sep:'\t') - .map{ meta -> - def meta_clone = meta.clone() - meta_clone.single_end = meta_clone.single_end.toBoolean() - return meta_clone + .map { + meta -> + def meta_clone = meta.clone() + meta_clone.single_end = meta_clone.single_end.toBoolean() + return meta_clone } .unique() .set { ch_sra_metadata } - fastq_files = Channel.empty() if (!params.skip_fastq_download) { ch_sra_metadata - .map { - meta -> - [ meta, [ meta.fastq_1, meta.fastq_2 ] ] - } .branch { - ftp: it[0].fastq_1 && !params.force_sratools_download - sra: !it[0].fastq_1 || params.force_sratools_download + meta -> + def download_method = 'ftp' + // meta.fastq_aspera is a metadata string with ENA fasp links supported by Aspera + // For single-end: 'fasp.sra.ebi.ac.uk:/vol1/fastq/ERR116/006/ERR1160846/ERR1160846.fastq.gz' + // For paired-end: 'fasp.sra.ebi.ac.uk:/vol1/fastq/SRR130/020/SRR13055520/SRR13055520_1.fastq.gz;fasp.sra.ebi.ac.uk:/vol1/fastq/SRR130/020/SRR13055520/SRR13055520_2.fastq.gz' + if (meta.fastq_aspera && params.download_method == 'aspera') { + download_method = 'aspera' + } + if ((!meta.fastq_aspera && !meta.fastq_1) || params.download_method == 'sratools') { + download_method = 'sratools' + } + + aspera: download_method == 'aspera' + return [ meta, meta.fastq_aspera.tokenize(';').take(2) ] + ftp: download_method == 'ftp' + return [ meta, [ meta.fastq_1, meta.fastq_2 ] ] + sratools: download_method == 'sratools' + return [ meta, meta.run_accession ] } .set { ch_sra_reads } @@ -88,14 +101,26 @@ workflow SRA { // SUBWORKFLOW: Download sequencing reads without FTP links using sra-tools. // FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS ( - ch_sra_reads.sra.map { meta, reads -> [ meta, meta.run_accession ] }, + ch_sra_reads.sratools, params.dbgap_key ? file(params.dbgap_key, checkIfExists: true) : [] ) ch_versions = ch_versions.mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.versions.first()) + // + // MODULE: If Aspera link is provided in run information then download FastQ directly via Aspera CLI and validate with md5sums + // + ASPERA_CLI ( + ch_sra_reads.aspera, + 'era-fasp' + ) + ch_versions = ch_versions.mix(ASPERA_CLI.out.versions.first()) + // Isolate FASTQ channel which will be added to emit block - fastq_files - .mix(SRA_FASTQ_FTP.out.fastq, FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.reads) + SRA_FASTQ_FTP + .out + .fastq + .mix(FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS.out.reads) + .mix(ASPERA_CLI.out.fastq) .map { meta, fastq -> def reads = fastq instanceof List ? fastq.flatten() : [ fastq ] @@ -119,14 +144,24 @@ workflow SRA { params.sample_mapping_fields ) - // - // MODULE: Create a merged samplesheet across all samples for the pipeline - // - SRA_MERGE_SAMPLESHEET ( - SRA_TO_SAMPLESHEET.out.samplesheet.collect{it[1]}, - SRA_TO_SAMPLESHEET.out.mappings.collect{it[1]} - ) - ch_versions = ch_versions.mix(SRA_MERGE_SAMPLESHEET.out.versions) + // Merge samplesheets and mapping files across all samples + SRA_TO_SAMPLESHEET + .out + .samplesheet + .map { it[1] } + .collectFile(name:'tmp_samplesheet.csv', newLine: true, keepHeader: true, sort: { it.baseName }) + .map { it.text.tokenize('\n').join('\n') } + .collectFile(name:'samplesheet.csv', storeDir: "${params.outdir}/samplesheet") + .set { ch_samplesheet } + + SRA_TO_SAMPLESHEET + .out + .mappings + .map { it[1] } + .collectFile(name:'tmp_id_mappings.csv', newLine: true, keepHeader: true, sort: { it.baseName }) + .map { it.text.tokenize('\n').join('\n') } + .collectFile(name:'id_mappings.csv', storeDir: "${params.outdir}/samplesheet") + .set { ch_mappings } // // MODULE: Create a MutiQC config file with sample name mappings @@ -134,17 +169,23 @@ workflow SRA { ch_sample_mappings_yml = Channel.empty() if (params.sample_mapping_fields) { MULTIQC_MAPPINGS_CONFIG ( - SRA_MERGE_SAMPLESHEET.out.mappings + ch_mappings ) ch_versions = ch_versions.mix(MULTIQC_MAPPINGS_CONFIG.out.versions) ch_sample_mappings_yml = MULTIQC_MAPPINGS_CONFIG.out.yml } + // + // Collate and save software versions + // + softwareVersionsToYAML(ch_versions) + .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_fetchngs_software_mqc_versions.yml', sort: true, newLine: true) + emit: - fastq = fastq_files - samplesheet = SRA_MERGE_SAMPLESHEET.out.samplesheet - mappings = SRA_MERGE_SAMPLESHEET.out.mappings + samplesheet = ch_samplesheet + mappings = ch_mappings sample_mappings = ch_sample_mappings_yml + sra_metadata = ch_sra_metadata versions = ch_versions.unique() } diff --git a/workflows/sra/nextflow.config b/workflows/sra/nextflow.config index 6b66fcd5..d242c238 100644 --- a/workflows/sra/nextflow.config +++ b/workflows/sra/nextflow.config @@ -1,7 +1,7 @@ includeConfig "../../modules/local/multiqc_mappings_config/nextflow.config" +includeConfig "../../modules/local/aspera_cli/nextflow.config" includeConfig "../../modules/local/sra_fastq_ftp/nextflow.config" includeConfig "../../modules/local/sra_ids_to_runinfo/nextflow.config" -includeConfig "../../modules/local/sra_merge_samplesheet/nextflow.config" includeConfig "../../modules/local/sra_runinfo_to_ftp/nextflow.config" includeConfig "../../modules/local/sra_to_samplesheet/nextflow.config" includeConfig "../../modules/nf-core/sratools/prefetch/nextflow.config" diff --git a/workflows/sra/tests/main.nf.test b/workflows/sra/tests/main.nf.test index ef5b7c9d..1e2856c1 100644 --- a/workflows/sra/tests/main.nf.test +++ b/workflows/sra/tests/main.nf.test @@ -3,33 +3,49 @@ nextflow_workflow { name "Test workflow: sra/main.nf" script "../main.nf" workflow "SRA" - tag "workflows" - tag "workflows_sra" - tag "multiqc_mappings_config" - tag "sra_fastq_ftp" - tag "sra_ids_to_runinfo" - tag "sra_merge_samplesheet" - tag "sra_runinfo_to_ftp" - tag "sra_to_samplesheet" - tag "sra_default_parameters" + tag "SRA_DEFAULT" + + // Dependencies + tag "SRA_IDS_TO_RUNINFO" + tag "SRA_RUNINFO_TO_FTP" + tag "SRA_FASTQ_FTP" + tag "SRA_TO_SAMPLESHEET" + tag "MULTIQC_MAPPINGS_CONFIG" test("Parameters: default") { when { workflow { """ - input[0] = Channel.from('ERR1160846', 'GSE214215', 'SRR12848126') + input[0] = Channel.from("DRX026011", "ERX1234253", "SRX6725035") """ } - params { - outdir = "results" - } } then { + assert workflow.success + assertAll( - { assert workflow.success }, - { assert snapshot(workflow.out).match() } + { + with(workflow.out.samplesheet) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',')[0].take(4) == ['"sample"', '"fastq_1"', '"fastq_2"', '"run_accession"'] + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.mappings) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.sample_mappings) { + assert path(get(0)[0]).md5 == "1ac06bb95b503703430e74660bbdd768" + } + } ) } } diff --git a/workflows/sra/tests/main.nf.test.snap b/workflows/sra/tests/main.nf.test.snap deleted file mode 100644 index 885b7d6a..00000000 --- a/workflows/sra/tests/main.nf.test.snap +++ /dev/null @@ -1,55 +0,0 @@ -{ - "Parameters: default": { - "content": [ - { - "0": [ - - ], - "1": [ - "samplesheet.csv:md5,eda595eca3cb5ed450641565c38390c2" - ], - "2": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "3": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "4": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47", - "versions.yml:md5,e977678bd13a54fc9c31addf135be15e" - ], - "fastq": [ - - ], - "mappings": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "sample_mappings": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "samplesheet": [ - "samplesheet.csv:md5,eda595eca3cb5ed450641565c38390c2" - ], - "versions": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47", - "versions.yml:md5,e977678bd13a54fc9c31addf135be15e" - ] - } - ], - "timestamp": "2023-10-18T09:49:19.020482551" - } -} \ No newline at end of file diff --git a/workflows/sra/tests/sra_custom_ena_metadata_fields.nf.test b/workflows/sra/tests/sra_custom_ena_metadata_fields.nf.test index 62cabb7b..c8d17876 100644 --- a/workflows/sra/tests/sra_custom_ena_metadata_fields.nf.test +++ b/workflows/sra/tests/sra_custom_ena_metadata_fields.nf.test @@ -3,26 +3,24 @@ nextflow_workflow { name "Test workflow: sra/main.nf" script "../main.nf" workflow "SRA" - tag "workflows" - tag "workflows_sra" - tag "multiqc_mappings_config" - tag "sra_fastq_ftp" - tag "sra_ids_to_runinfo" - tag "sra_merge_samplesheet" - tag "sra_runinfo_to_ftp" - tag "sra_to_samplesheet" - tag "sra_custom_ena_metadata_fields" + tag "SRA_CUSTOM_ENA_METADATA_FIELDS" + + // Dependencies + tag "SRA_IDS_TO_RUNINFO" + tag "SRA_RUNINFO_TO_FTP" + tag "SRA_FASTQ_FTP" + tag "SRA_TO_SAMPLESHEET" + tag "MULTIQC_MAPPINGS_CONFIG" test("Parameters: --nf_core_pipeline rnaseq --ena_metadata_fields ... --sample_mapping_fields ...") { when { workflow { """ - input[0] = Channel.from('ERR1160846', 'GSE214215', 'SRR12848126') + input[0] = Channel.from("DRX026011", "ERX1234253", "SRX6725035") """ } params { - outdir = "results" nf_core_pipeline = "rnaseq" ena_metadata_fields = "run_accession,experiment_accession,library_layout,fastq_ftp,fastq_md5" sample_mapping_fields = "run_accession,library_layout" @@ -30,9 +28,29 @@ nextflow_workflow { } then { + assert workflow.success + assertAll( - { assert workflow.success }, - { assert snapshot(workflow.out).match() } + { + with(workflow.out.samplesheet) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',')[0].take(5) == ['"sample"', '"fastq_1"', '"fastq_2"', '"strandedness"' , '"run_accession"'] + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('SINGLE') + } + }, + { + with(workflow.out.mappings) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('SINGLE') + } + }, + { + with(workflow.out.sample_mappings) { + assert path(get(0)[0]).md5 == "3b70bc9658eab4ba2f4ec98cb749ac9d" + } + } ) } } diff --git a/workflows/sra/tests/sra_custom_ena_metadata_fields.nf.test.snap b/workflows/sra/tests/sra_custom_ena_metadata_fields.nf.test.snap deleted file mode 100644 index 45e007d0..00000000 --- a/workflows/sra/tests/sra_custom_ena_metadata_fields.nf.test.snap +++ /dev/null @@ -1,55 +0,0 @@ -{ - "Parameters: --nf_core_pipeline rnaseq --ena_metadata_fields ... --sample_mapping_fields ...": { - "content": [ - { - "0": [ - - ], - "1": [ - "samplesheet.csv:md5,a9e9da506288c364af14d46b86dafeb1" - ], - "2": [ - "id_mappings.csv:md5,8ed2bd72d432eff4a8ed7d909af6f60c" - ], - "3": [ - [ - "multiqc_config.yml:md5,3bc981f0de28023083cdf13691d249d5", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "4": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47", - "versions.yml:md5,e977678bd13a54fc9c31addf135be15e" - ], - "fastq": [ - - ], - "mappings": [ - "id_mappings.csv:md5,8ed2bd72d432eff4a8ed7d909af6f60c" - ], - "sample_mappings": [ - [ - "multiqc_config.yml:md5,3bc981f0de28023083cdf13691d249d5", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "samplesheet": [ - "samplesheet.csv:md5,a9e9da506288c364af14d46b86dafeb1" - ], - "versions": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47", - "versions.yml:md5,e977678bd13a54fc9c31addf135be15e" - ] - } - ], - "timestamp": "2023-10-18T09:36:57.154883885" - } -} \ No newline at end of file diff --git a/workflows/sra/tests/sra_download_method_aspera.nf.test b/workflows/sra/tests/sra_download_method_aspera.nf.test new file mode 100644 index 00000000..d431a571 --- /dev/null +++ b/workflows/sra/tests/sra_download_method_aspera.nf.test @@ -0,0 +1,55 @@ +nextflow_workflow { + + name "Test workflow: sra/main.nf" + script "../main.nf" + workflow "SRA" + tag "SRA_DOWNLOAD_METHOD_ASPERA" + + // Dependencies + tag "SRA_IDS_TO_RUNINFO" + tag "SRA_RUNINFO_TO_FTP" + tag "ASPERA_CLI" + tag "SRA_TO_SAMPLESHEET" + tag "MULTIQC_MAPPINGS_CONFIG" + + test("Parameters: --download_method aspera") { + + when { + workflow { + """ + input[0] = Channel.from("DRX026011", "ERX1234253", "SRX6725035") + """ + } + params { + download_method = 'aspera' + } + } + + then { + assert workflow.success + + assertAll( + { + with(workflow.out.samplesheet) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',')[0].take(4) == ['"sample"', '"fastq_1"', '"fastq_2"', '"run_accession"'] + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.mappings) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.sample_mappings) { + assert path(get(0)[0]).md5 == "1ac06bb95b503703430e74660bbdd768" + } + } + ) + } + } +} diff --git a/workflows/sra/tests/sra_download_method_sratools.nf.test b/workflows/sra/tests/sra_download_method_sratools.nf.test new file mode 100644 index 00000000..e4f8c2d2 --- /dev/null +++ b/workflows/sra/tests/sra_download_method_sratools.nf.test @@ -0,0 +1,55 @@ +nextflow_workflow { + + name "Test workflow: sra/main.nf" + script "../main.nf" + workflow "SRA" + tag "SRA_DOWNLOAD_METHOD_SRATOOLS" + + // Dependencies + tag "FASTQ_DOWNLOAD_PREFETCH_FASTERQDUMP_SRATOOLS" + tag "SRA_IDS_TO_RUNINFO" + tag "SRA_RUNINFO_TO_FTP" + tag "SRA_TO_SAMPLESHEET" + tag "MULTIQC_MAPPINGS_CONFIG" + + test("Parameters: --download_method sratools") { + + when { + workflow { + """ + input[0] = Channel.from("DRX026011", "ERX1234253", "SRX6725035") + """ + } + params { + download_method = 'sratools' + } + } + + then { + assert workflow.success + + assertAll( + { + with(workflow.out.samplesheet) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',')[0].take(4) == ['"sample"', '"fastq_1"', '"fastq_2"', '"run_accession"'] + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.mappings) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.sample_mappings) { + assert path(get(0)[0]).md5 == "1ac06bb95b503703430e74660bbdd768" + } + } + ) + } + } +} diff --git a/workflows/sra/tests/sra_force_sratools_download.nf.test b/workflows/sra/tests/sra_force_sratools_download.nf.test deleted file mode 100644 index df751a6b..00000000 --- a/workflows/sra/tests/sra_force_sratools_download.nf.test +++ /dev/null @@ -1,37 +0,0 @@ -nextflow_workflow { - - name "Test workflow: sra/main.nf" - script "../main.nf" - workflow "SRA" - tag "workflows" - tag "workflows_sra" - tag "multiqc_mappings_config" - tag "sra_fastq_ftp" - tag "sra_ids_to_runinfo" - tag "sra_merge_samplesheet" - tag "sra_runinfo_to_ftp" - tag "sra_to_samplesheet" - tag "sra_force_sratools_download" - - test("Parameters: --force_sratools_download") { - - when { - workflow { - """ - input[0] = Channel.from('ERR1160846', 'GSE214215', 'SRR12848126') - """ - } - params { - outdir = "results" - force_sratools_download = true - } - } - - then { - assertAll( - { assert workflow.success }, - { assert snapshot(workflow.out).match() } - ) - } - } -} diff --git a/workflows/sra/tests/sra_force_sratools_download.nf.test.snap b/workflows/sra/tests/sra_force_sratools_download.nf.test.snap deleted file mode 100644 index f445296a..00000000 --- a/workflows/sra/tests/sra_force_sratools_download.nf.test.snap +++ /dev/null @@ -1,53 +0,0 @@ -{ - "Parameters: --force_sratools_download": { - "content": [ - { - "0": [ - - ], - "1": [ - "samplesheet.csv:md5,7e6a65057fe000f562132ae9e608e87b" - ], - "2": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "3": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "4": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47" - ], - "fastq": [ - - ], - "mappings": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "sample_mappings": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "samplesheet": [ - "samplesheet.csv:md5,7e6a65057fe000f562132ae9e608e87b" - ], - "versions": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47" - ] - } - ], - "timestamp": "2023-10-18T09:43:16.468494798" - } -} \ No newline at end of file diff --git a/workflows/sra/tests/sra_nf_core_pipeline_atacseq.nf.test b/workflows/sra/tests/sra_nf_core_pipeline_atacseq.nf.test index f4837f76..72f8d595 100644 --- a/workflows/sra/tests/sra_nf_core_pipeline_atacseq.nf.test +++ b/workflows/sra/tests/sra_nf_core_pipeline_atacseq.nf.test @@ -3,34 +3,52 @@ nextflow_workflow { name "Test workflow: sra/main.nf" script "../main.nf" workflow "SRA" - tag "workflows" - tag "workflows_sra" - tag "multiqc_mappings_config" - tag "sra_fastq_ftp" - tag "sra_ids_to_runinfo" - tag "sra_merge_samplesheet" - tag "sra_runinfo_to_ftp" - tag "sra_to_samplesheet" - tag "sra_nf_core_pipeline_atacseq" + tag "SRA_NF_CORE_PIPELINE_ATACSEQ" + + // Dependencies + tag "SRA_IDS_TO_RUNINFO" + tag "SRA_RUNINFO_TO_FTP" + tag "SRA_FASTQ_FTP" + tag "SRA_TO_SAMPLESHEET" + tag "MULTIQC_MAPPINGS_CONFIG" test("Parameters: --nf_core_pipeline atacseq") { when { workflow { """ - input[0] = Channel.from('ERR1160846', 'GSE214215', 'SRR12848126') + input[0] = Channel.from("DRX026011", "ERX1234253", "SRX6725035") """ } params { - outdir = "results" nf_core_pipeline = "atacseq" } } then { + assert workflow.success + assertAll( - { assert workflow.success }, - { assert snapshot(workflow.out).match() } + { + with(workflow.out.samplesheet) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',')[0].take(5) == ['"sample"', '"fastq_1"', '"fastq_2"', '"replicate"', '"run_accession"'] + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.mappings) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.sample_mappings) { + assert path(get(0)[0]).md5 == "1ac06bb95b503703430e74660bbdd768" + } + } ) } } diff --git a/workflows/sra/tests/sra_nf_core_pipeline_atacseq.nf.test.snap b/workflows/sra/tests/sra_nf_core_pipeline_atacseq.nf.test.snap deleted file mode 100644 index 3e9e7f31..00000000 --- a/workflows/sra/tests/sra_nf_core_pipeline_atacseq.nf.test.snap +++ /dev/null @@ -1,55 +0,0 @@ -{ - "Parameters: --nf_core_pipeline atacseq": { - "content": [ - { - "0": [ - - ], - "1": [ - "samplesheet.csv:md5,35292d2bea78b087d75c3333f6319b6b" - ], - "2": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "3": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "4": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47", - "versions.yml:md5,e977678bd13a54fc9c31addf135be15e" - ], - "fastq": [ - - ], - "mappings": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "sample_mappings": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "samplesheet": [ - "samplesheet.csv:md5,35292d2bea78b087d75c3333f6319b6b" - ], - "versions": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47", - "versions.yml:md5,e977678bd13a54fc9c31addf135be15e" - ] - } - ], - "timestamp": "2023-10-18T09:47:56.240260295" - } -} \ No newline at end of file diff --git a/workflows/sra/tests/sra_nf_core_pipeline_rnaseq.nf.test b/workflows/sra/tests/sra_nf_core_pipeline_rnaseq.nf.test index 871207ab..2e20902a 100644 --- a/workflows/sra/tests/sra_nf_core_pipeline_rnaseq.nf.test +++ b/workflows/sra/tests/sra_nf_core_pipeline_rnaseq.nf.test @@ -3,34 +3,52 @@ nextflow_workflow { name "Test workflow: sra/main.nf" script "../main.nf" workflow "SRA" - tag "workflows" - tag "workflows_sra" - tag "multiqc_mappings_config" - tag "sra_fastq_ftp" - tag "sra_ids_to_runinfo" - tag "sra_merge_samplesheet" - tag "sra_runinfo_to_ftp" - tag "sra_to_samplesheet" - tag "sra_nf_core_pipeline_rnaseq" + tag "SRA_NF_CORE_PIPELINE_RNASEQ" + + // Dependencies + tag "SRA_IDS_TO_RUNINFO" + tag "SRA_RUNINFO_TO_FTP" + tag "SRA_FASTQ_FTP" + tag "SRA_TO_SAMPLESHEET" + tag "MULTIQC_MAPPINGS_CONFIG" test("Parameters: --nf_core_pipeline rnaseq") { when { workflow { """ - input[0] = Channel.from('ERR1160846', 'GSE214215', 'SRR12848126') + input[0] = Channel.from("DRX026011", "ERX1234253", "SRX6725035") """ } params { - outdir = "results" nf_core_pipeline = "rnaseq" } } then { + assert workflow.success + assertAll( - { assert workflow.success }, - { assert snapshot(workflow.out).match() } + { + with(workflow.out.samplesheet) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',')[0].take(5) == ['"sample"', '"fastq_1"', '"fastq_2"', '"strandedness"' , '"run_accession"'] + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.mappings) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.sample_mappings) { + assert path(get(0)[0]).md5 == "1ac06bb95b503703430e74660bbdd768" + } + } ) } } diff --git a/workflows/sra/tests/sra_nf_core_pipeline_rnaseq.nf.test.snap b/workflows/sra/tests/sra_nf_core_pipeline_rnaseq.nf.test.snap deleted file mode 100644 index 8805d3ac..00000000 --- a/workflows/sra/tests/sra_nf_core_pipeline_rnaseq.nf.test.snap +++ /dev/null @@ -1,55 +0,0 @@ -{ - "Parameters: --nf_core_pipeline rnaseq": { - "content": [ - { - "0": [ - - ], - "1": [ - "samplesheet.csv:md5,b7ac0ff84f5031daaa9405c736a37125" - ], - "2": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "3": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "4": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47", - "versions.yml:md5,e977678bd13a54fc9c31addf135be15e" - ], - "fastq": [ - - ], - "mappings": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "sample_mappings": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "samplesheet": [ - "samplesheet.csv:md5,b7ac0ff84f5031daaa9405c736a37125" - ], - "versions": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47", - "versions.yml:md5,e977678bd13a54fc9c31addf135be15e" - ] - } - ], - "timestamp": "2023-10-18T09:40:15.590249346" - } -} \ No newline at end of file diff --git a/workflows/sra/tests/sra_nf_core_pipeline_taxprofiler.nf.test b/workflows/sra/tests/sra_nf_core_pipeline_taxprofiler.nf.test index 995301dd..50650b79 100644 --- a/workflows/sra/tests/sra_nf_core_pipeline_taxprofiler.nf.test +++ b/workflows/sra/tests/sra_nf_core_pipeline_taxprofiler.nf.test @@ -3,34 +3,52 @@ nextflow_workflow { name "Test workflow: sra/main.nf" script "../main.nf" workflow "SRA" - tag "workflows" - tag "workflows_sra" - tag "multiqc_mappings_config" - tag "sra_fastq_ftp" - tag "sra_ids_to_runinfo" - tag "sra_merge_samplesheet" - tag "sra_runinfo_to_ftp" - tag "sra_to_samplesheet" - tag "sra_nf_core_pipeline_taxprofiler" + tag "SRA_NF_CORE_PIPELINE_TAXPROFILER" + + // Dependencies + tag "SRA_IDS_TO_RUNINFO" + tag "SRA_RUNINFO_TO_FTP" + tag "SRA_FASTQ_FTP" + tag "SRA_TO_SAMPLESHEET" + tag "MULTIQC_MAPPINGS_CONFIG" test("Parameters: --nf_core_pipeline taxprofiler") { when { workflow { """ - input[0] = Channel.from('ERR1160846', 'GSE214215', 'SRR12848126') + input[0] = Channel.from("DRX026011", "ERX1234253", "SRX6725035") """ } params { - outdir = "results" nf_core_pipeline = "taxprofiler" } } then { + assert workflow.success + assertAll( - { assert workflow.success }, - { assert snapshot(workflow.out).match() } + { + with(workflow.out.samplesheet) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',')[0].take(5) == ['"sample"', '"fastq_1"', '"fastq_2"', '"fasta"', '"run_accession"'] + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.mappings) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.sample_mappings) { + assert path(get(0)[0]).md5 == "1ac06bb95b503703430e74660bbdd768" + } + } ) } } diff --git a/workflows/sra/tests/sra_nf_core_pipeline_taxprofiler.nf.test.snap b/workflows/sra/tests/sra_nf_core_pipeline_taxprofiler.nf.test.snap deleted file mode 100644 index 0853eef8..00000000 --- a/workflows/sra/tests/sra_nf_core_pipeline_taxprofiler.nf.test.snap +++ /dev/null @@ -1,55 +0,0 @@ -{ - "Parameters: --nf_core_pipeline taxprofiler": { - "content": [ - { - "0": [ - - ], - "1": [ - "samplesheet.csv:md5,296836917bbdf34bda106a313424fa95" - ], - "2": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "3": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "4": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47", - "versions.yml:md5,e977678bd13a54fc9c31addf135be15e" - ], - "fastq": [ - - ], - "mappings": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "sample_mappings": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "samplesheet": [ - "samplesheet.csv:md5,296836917bbdf34bda106a313424fa95" - ], - "versions": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47", - "versions.yml:md5,e977678bd13a54fc9c31addf135be15e" - ] - } - ], - "timestamp": "2023-10-18T09:49:51.04235046" - } -} \ No newline at end of file diff --git a/workflows/sra/tests/sra_nf_core_pipeline_viralrecon.nf.test b/workflows/sra/tests/sra_nf_core_pipeline_viralrecon.nf.test index 4bf6a3f0..1fe9cc41 100644 --- a/workflows/sra/tests/sra_nf_core_pipeline_viralrecon.nf.test +++ b/workflows/sra/tests/sra_nf_core_pipeline_viralrecon.nf.test @@ -3,34 +3,52 @@ nextflow_workflow { name "Test workflow: sra/main.nf" script "../main.nf" workflow "SRA" - tag "workflows" - tag "workflows_sra" - tag "multiqc_mappings_config" - tag "sra_fastq_ftp" - tag "sra_ids_to_runinfo" - tag "sra_merge_samplesheet" - tag "sra_runinfo_to_ftp" - tag "sra_to_samplesheet" - tag "sra_nf_core_pipeline_viralrecon" + tag "SRA_NF_CORE_PIPELINE_VIRALRECON" + + // Dependencies + tag "SRA_IDS_TO_RUNINFO" + tag "SRA_RUNINFO_TO_FTP" + tag "SRA_FASTQ_FTP" + tag "SRA_TO_SAMPLESHEET" + tag "MULTIQC_MAPPINGS_CONFIG" test("Parameters: --nf_core_pipeline viralrecon") { when { workflow { """ - input[0] = Channel.from('ERR1160846', 'GSE214215', 'SRR12848126') + input[0] = Channel.from("DRX026011", "ERX1234253", "SRX6725035") """ } params { - outdir = "results" nf_core_pipeline = "viralrecon" } } then { + assert workflow.success + assertAll( - { assert workflow.success }, - { assert snapshot(workflow.out).match() } + { + with(workflow.out.samplesheet) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',')[0].take(4) == ['"sample"', '"fastq_1"', '"fastq_2"', '"run_accession"'] + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.mappings) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.sample_mappings) { + assert path(get(0)[0]).md5 == "1ac06bb95b503703430e74660bbdd768" + } + } ) } } diff --git a/workflows/sra/tests/sra_nf_core_pipeline_viralrecon.nf.test.snap b/workflows/sra/tests/sra_nf_core_pipeline_viralrecon.nf.test.snap deleted file mode 100644 index e364b3a9..00000000 --- a/workflows/sra/tests/sra_nf_core_pipeline_viralrecon.nf.test.snap +++ /dev/null @@ -1,55 +0,0 @@ -{ - "Parameters: --nf_core_pipeline viralrecon": { - "content": [ - { - "0": [ - - ], - "1": [ - "samplesheet.csv:md5,eda595eca3cb5ed450641565c38390c2" - ], - "2": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "3": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "4": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47", - "versions.yml:md5,e977678bd13a54fc9c31addf135be15e" - ], - "fastq": [ - - ], - "mappings": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "sample_mappings": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "samplesheet": [ - "samplesheet.csv:md5,eda595eca3cb5ed450641565c38390c2" - ], - "versions": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,45e4a369df2d3af023b8622aa3f6d5a8", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47", - "versions.yml:md5,e977678bd13a54fc9c31addf135be15e" - ] - } - ], - "timestamp": "2023-10-18T09:48:46.587502263" - } -} \ No newline at end of file diff --git a/workflows/sra/tests/sra_skip_fastq_download.nf.test b/workflows/sra/tests/sra_skip_fastq_download.nf.test index 705facbb..8663cf3f 100644 --- a/workflows/sra/tests/sra_skip_fastq_download.nf.test +++ b/workflows/sra/tests/sra_skip_fastq_download.nf.test @@ -3,34 +3,51 @@ nextflow_workflow { name "Test workflow: sra/main.nf" script "../main.nf" workflow "SRA" - tag "workflows" - tag "workflows_sra" - tag "multiqc_mappings_config" - tag "sra_fastq_ftp" - tag "sra_ids_to_runinfo" - tag "sra_merge_samplesheet" - tag "sra_runinfo_to_ftp" - tag "sra_to_samplesheet" - tag "sra_skip_fastq_download" + tag "SRA_SKIP_FASTQ_DOWNLOAD" + + // Dependencies + tag "SRA_IDS_TO_RUNINFO" + tag "SRA_RUNINFO_TO_FTP" + tag "SRA_TO_SAMPLESHEET" + tag "MULTIQC_MAPPINGS_CONFIG" test("Parameters: --skip_fastq_download") { when { workflow { """ - input[0] = Channel.from('ERR1160846', 'GSE214215', 'SRR12848126') + input[0] = Channel.from("DRX026011", "ERX1234253", "SRX6725035") """ } params { - outdir = "results" skip_fastq_download = true } } then { + assert workflow.success + assertAll( - { assert workflow.success }, - { assert snapshot(workflow.out).match() } + { + with(workflow.out.samplesheet) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',')[0].take(4) == ['"sample"', '"fastq_1"', '"fastq_2"', '"run_accession"'] + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.mappings) { + assert path(get(0)).readLines().size() == 4 + assert path(get(0)).readLines()*.split(',').collect { it[0] } == ['"sample"', '"DRX026011"', '"ERX1234253"', '"SRX6725035"'] + assert path(get(0)).text.contains('Illumina HiSeq 2500') + } + }, + { + with(workflow.out.sample_mappings) { + assert path(get(0)[0]).md5 == "1ac06bb95b503703430e74660bbdd768" + } + } ) } } diff --git a/workflows/sra/tests/sra_skip_fastq_download.nf.test.snap b/workflows/sra/tests/sra_skip_fastq_download.nf.test.snap deleted file mode 100644 index cdb0965c..00000000 --- a/workflows/sra/tests/sra_skip_fastq_download.nf.test.snap +++ /dev/null @@ -1,51 +0,0 @@ -{ - "Parameters: --skip_fastq_download": { - "content": [ - { - "0": [ - - ], - "1": [ - "samplesheet.csv:md5,0a9b99509bcc64ca245408b6bb634f15" - ], - "2": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "3": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "4": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47" - ], - "fastq": [ - - ], - "mappings": [ - "id_mappings.csv:md5,3e70c965568c59f8c8fbb8e99e7a8b79" - ], - "sample_mappings": [ - [ - "multiqc_config.yml:md5,6d4c3e5137704358474330207f5f2b5c", - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9" - ] - ], - "samplesheet": [ - "samplesheet.csv:md5,0a9b99509bcc64ca245408b6bb634f15" - ], - "versions": [ - "versions.yml:md5,1496d1cbc9041e07ab8a0c25f0b054d9", - "versions.yml:md5,9b17045ca8bdc272cb3f9d349a81d206", - "versions.yml:md5,b21338b74b9a8fe0c7a114f7686c07cd", - "versions.yml:md5,b52279f7d6b891a6523d9321f3f85b47" - ] - } - ], - "timestamp": "2023-10-18T09:50:04.300075498" - } -} \ No newline at end of file diff --git a/workflows/sra/tests/tags.yml b/workflows/sra/tests/tags.yml deleted file mode 100644 index ae41e37d..00000000 --- a/workflows/sra/tests/tags.yml +++ /dev/null @@ -1,16 +0,0 @@ -sra_custom_ena_metadata_fields: - - workflows/sra/** -sra_default_parameters: - - workflows/sra/** -sra_force_sratools_download: - - workflows/sra/** -sra_nf_core_pipeline_atacseq: - - workflows/sra/** -sra_nf_core_pipeline_rnaseq: - - workflows/sra/** -sra_nf_core_pipeline_taxprofiler: - - workflows/sra/** -sra_nf_core_pipeline_viralrecon: - - workflows/sra/** -sra_skip_fastq_download: - - workflows/sra/** diff --git a/workflows/synapse/main.nf b/workflows/synapse/main.nf deleted file mode 100644 index 6b5ea02c..00000000 --- a/workflows/synapse/main.nf +++ /dev/null @@ -1,118 +0,0 @@ -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - IMPORT LOCAL MODULES/SUBWORKFLOWS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -include { SYNAPSE_LIST } from '../../modules/local/synapse_list' -include { SYNAPSE_SHOW } from '../../modules/local/synapse_show' -include { SYNAPSE_GET } from '../../modules/local/synapse_get' -include { SYNAPSE_TO_SAMPLESHEET } from '../../modules/local/synapse_to_samplesheet' -include { SYNAPSE_MERGE_SAMPLESHEET } from '../../modules/local/synapse_merge_samplesheet' - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - RUN MAIN WORKFLOW -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -workflow SYNAPSE { - - take: - ids // channel: [ ids ] - ch_synapse_config // channel: [ synapse_config ] - - main: - ch_versions = Channel.empty() - - // - // MODULE: Expand synapse ids for individual FastQ files - // - SYNAPSE_LIST ( - ids, - ch_synapse_config - ) - ch_versions = ch_versions.mix(SYNAPSE_LIST.out.versions.first()) - - // Create channel for FastQ synapse ids - SYNAPSE_LIST - .out - .txt - .splitCsv(header:false, sep:' ') - .map { it[0] } - .unique() - .set { ch_samples } - - // - // MODULE: Download metadata for each synapse id - // - SYNAPSE_SHOW ( - ch_samples, - ch_synapse_config - ) - ch_versions = ch_versions.mix(SYNAPSE_SHOW.out.versions.first()) - - // Get metadata into channels - SYNAPSE_SHOW - .out - .metadata - .map { it -> WorkflowMain.synapseShowToMap(it) } - .set { ch_samples_meta } - - // - // MODULE: Download FastQs by synapse id - // - SYNAPSE_GET ( - ch_samples_meta, - ch_synapse_config - ) - ch_versions = ch_versions.mix(SYNAPSE_GET.out.versions.first()) - - // Combine channels for PE/SE FastQs: [ [ id:SRR6357070, synapse_ids:syn26240474;syn26240477 ], [ fastq_1, fastq_2 ] ] - SYNAPSE_GET - .out - .fastq - .map { meta, fastq -> [ WorkflowMain.synapseSampleNameFromFastQ( fastq , "*{1,2}*"), fastq ] } - .groupTuple(sort: { it -> it.baseName }) - .set { ch_fastq } - - SYNAPSE_GET - .out - .fastq - .map { meta, fastq -> [ WorkflowMain.synapseSampleNameFromFastQ( fastq , "*{1,2}*"), meta.id ] } - .groupTuple() - .join(ch_fastq) - .map { id, synids, fastq -> - def meta = [ id:id, synapse_ids:synids.join(';') ] - [ meta, fastq ] - } - .set { ch_fastq } - - // - // MODULE: Create samplesheet per sample - // - SYNAPSE_TO_SAMPLESHEET ( - ch_fastq, - params.nf_core_pipeline ?: '', - params.nf_core_rnaseq_strandedness ?: 'auto' - ) - - // - // MODULE: Merge samplesheets - // - SYNAPSE_MERGE_SAMPLESHEET ( - SYNAPSE_TO_SAMPLESHEET.out.samplesheet.collect{ it[1] } - ) - ch_versions = ch_versions.mix(SYNAPSE_MERGE_SAMPLESHEET.out.versions) - - emit: - fastq = ch_fastq - samplesheet = SYNAPSE_MERGE_SAMPLESHEET.out.samplesheet - versions = ch_versions.unique() -} - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - THE END -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ diff --git a/workflows/synapse/nextflow.config b/workflows/synapse/nextflow.config deleted file mode 100644 index 01b71558..00000000 --- a/workflows/synapse/nextflow.config +++ /dev/null @@ -1,5 +0,0 @@ -includeConfig "../../modules/local/synapse_get/nextflow.config" -includeConfig "../../modules/local/synapse_to_samplesheet/nextflow.config" -includeConfig "../../modules/local/synapse_list/nextflow.config" -includeConfig "../../modules/local/synapse_merge_samplesheet/nextflow.config" -includeConfig "../../modules/local/synapse_show/nextflow.config" \ No newline at end of file