diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 4ecfbfe..b290e09 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -10,15 +10,7 @@ "vscode": { // Set *default* container specific settings.json values on container create. "settings": { - "python.defaultInterpreterPath": "/opt/conda/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/opt/conda/bin/autopep8", - "python.formatting.yapfPath": "/opt/conda/bin/yapf", - "python.linting.flake8Path": "/opt/conda/bin/flake8", - "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", - "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", - "python.linting.pylintPath": "/opt/conda/bin/pylint" + "python.defaultInterpreterPath": "/opt/conda/bin/python" }, // Add the IDs of extensions you want installed when the container is created. diff --git a/.editorconfig b/.editorconfig index 9b99008..dd9ffa5 100644 --- a/.editorconfig +++ b/.editorconfig @@ -18,7 +18,12 @@ end_of_line = unset insert_final_newline = unset trim_trailing_whitespace = unset indent_style = unset -indent_size = unset +[/subworkflows/nf-core/**] +charset = unset +end_of_line = unset +insert_final_newline = unset +trim_trailing_whitespace = unset +indent_style = unset [/assets/email*] indent_size = unset @@ -28,5 +33,5 @@ indent_size = unset indent_style = unset # ignore python -[*.{py}] +[*.{py,md}] indent_style = unset diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 6ef91e2..90cd7e8 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -9,9 +9,8 @@ Please use the pre-filled template to save time. However, don't be put off by this template - other more general issues and suggestions are welcome! Contributions to the code are even more welcome ;) -:::info -If you need help using or modifying nf-core/spatialtranscriptomics then the best place to ask is on the nf-core Slack [#spatialtranscriptomics](https://nfcore.slack.com/channels/spatialtranscriptomics) channel ([join our Slack here](https://nf-co.re/join/slack)). -::: +> [!NOTE] +> If you need help using or modifying nf-core/spatialtranscriptomics then the best place to ask is on the nf-core Slack [#spatialtranscriptomics](https://nfcore.slack.com/channels/spatialtranscriptomics) channel ([join our Slack here](https://nf-co.re/join/slack)). ## Contribution workflow @@ -27,8 +26,11 @@ If you're not used to this workflow with git, you can start with some [docs from ## Tests -You can optionally test your changes by running the pipeline locally. Then it is recommended to use the `debug` profile to -receive warnings about process selectors and other debug info. Example: `nextflow run . -profile debug,test,docker --outdir `. +You have the option to test your changes locally by running the pipeline. For receiving warnings about process selectors and other `debug` information, it is recommended to use the debug profile. Execute all the tests with the following command: + +```bash +nf-test test --profile debug,test,docker --verbose +``` When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests. Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then. @@ -90,7 +92,7 @@ Once there, use `nf-core schema build` to add to `nextflow_schema.json`. Sensible defaults for process resource requirements (CPUs / memory / time) for a process should be defined in `conf/base.config`. These should generally be specified generic with `withLabel:` selectors so they can be shared across multiple processes/steps of the pipeline. A nf-core standard set of labels that should be followed where possible can be seen in the [nf-core pipeline template](https://github.com/nf-core/tools/blob/master/nf_core/pipeline-template/conf/base.config), which has the default process as a single core-process, and then different levels of multi-core configurations for increasingly large memory requirements defined with standardised labels. -The process resources can be passed on to the tool dynamically within the process with the `${task.cpu}` and `${task.memory}` variables in the `script:` block. +The process resources can be passed on to the tool dynamically within the process with the `${task.cpus}` and `${task.memory}` variables in the `script:` block. ### Naming schemes diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1213782..d90b4d8 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -18,7 +18,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/spat - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/spatialtranscriptomics/tree/master/.github/CONTRIBUTING.md) - [ ] If necessary, also make a PR on the nf-core/spatialtranscriptomics _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). +- [ ] Ensure the test suite passes (`nf-test test main.nf.test -profile test,docker`). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. diff --git a/.github/workflows/branch.yml b/.github/workflows/branch.yml index 5f10613..9ab998b 100644 --- a/.github/workflows/branch.yml +++ b/.github/workflows/branch.yml @@ -19,7 +19,7 @@ jobs: # NOTE - this doesn't currently work if the PR is coming from a fork, due to limitations in GitHub actions secrets - name: Post PR comment if: failure() - uses: mshick/add-pr-comment@v2 + uses: mshick/add-pr-comment@b8f338c590a895d50bcbfa6c5859251edc8952fc # v2 with: message: | ## This PR is against the `master` branch :x: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f659755..c696b85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,7 @@ jobs: - tests/pipeline/test_downstream.nf.test steps: - name: Check out pipeline code - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 # Install Nextflow - name: Install Nextflow diff --git a/.github/workflows/clean-up.yml b/.github/workflows/clean-up.yml index e37cfda..0b6b1f2 100644 --- a/.github/workflows/clean-up.yml +++ b/.github/workflows/clean-up.yml @@ -10,7 +10,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v9 + - uses: actions/stale@28ca1036281a5e5922ead5184a1bbf96e5fc984e # v9 with: stale-issue-message: "This issue has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment otherwise this issue will be closed in 20 days." stale-pr-message: "This PR has been tagged as awaiting-changes or awaiting-feedback by an nf-core contributor. Remove stale label or add a comment if it is still useful." diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml index 8a33004..08622fd 100644 --- a/.github/workflows/download_pipeline.yml +++ b/.github/workflows/download_pipeline.yml @@ -6,6 +6,11 @@ name: Test successful pipeline download with 'nf-core download' # - the head branch of the pull request is updated, i.e. if fixes for a release are pushed last minute to dev. on: workflow_dispatch: + inputs: + testbranch: + description: "The specific branch you wish to utilize for the test execution of nf-core download." + required: true + default: "dev" pull_request: types: - opened @@ -25,11 +30,11 @@ jobs: - name: Install Nextflow uses: nf-core/setup-nextflow@v1 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.11" architecture: "x64" - - uses: eWaterCycle/setup-singularity@v7 + - uses: eWaterCycle/setup-singularity@931d4e31109e875b13309ae1d07c70ca8fbc8537 # v7 with: singularity-version: 3.8.3 @@ -42,13 +47,13 @@ jobs: run: | echo "REPO_LOWERCASE=${GITHUB_REPOSITORY,,}" >> ${GITHUB_ENV} echo "REPOTITLE_LOWERCASE=$(basename ${GITHUB_REPOSITORY,,})" >> ${GITHUB_ENV} - echo "REPO_BRANCH=${GITHUB_REF#refs/heads/}" >> ${GITHUB_ENV} + echo "REPO_BRANCH=${{ github.event.inputs.testbranch || 'dev' }}" >> ${GITHUB_ENV} - name: Download the pipeline env: NXF_SINGULARITY_CACHEDIR: ./ run: | - nf-core download ${{ env.REPO_LOWERCASE }} \ + nf-core download ${{ env.REPO_LOWERCASE }} \ --revision ${{ env.REPO_BRANCH }} \ --outdir ./${{ env.REPOTITLE_LOWERCASE }} \ --compress "none" \ diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 81cd098..073e187 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -14,10 +14,10 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Set up Python 3.11 - uses: actions/setup-python@v5 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: 3.11 cache: "pip" @@ -32,12 +32,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out pipeline code - uses: actions/checkout@v4 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 - name: Install Nextflow uses: nf-core/setup-nextflow@v1 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.11" architecture: "x64" @@ -60,7 +60,7 @@ jobs: - name: Upload linting log file artifact if: ${{ always() }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4 with: name: linting-logs path: | diff --git a/.github/workflows/linting_comment.yml b/.github/workflows/linting_comment.yml index 147bcd1..b706875 100644 --- a/.github/workflows/linting_comment.yml +++ b/.github/workflows/linting_comment.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Download lint results - uses: dawidd6/action-download-artifact@v3 + uses: dawidd6/action-download-artifact@f6b0bace624032e30a85a8fd9c1a7f8f611f5737 # v3 with: workflow: linting.yml workflow_conclusion: completed @@ -21,7 +21,7 @@ jobs: run: echo "pr_number=$(cat linting-logs/PR_number.txt)" >> $GITHUB_OUTPUT - name: Post PR comment - uses: marocchino/sticky-pull-request-comment@v2 + uses: marocchino/sticky-pull-request-comment@331f8f5b4215f0445d3c07b4967662a32a2d3e31 # v2 with: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} number: ${{ steps.pr_number.outputs.pr_number }} diff --git a/.github/workflows/release-announcements.yml b/.github/workflows/release-announcements.yml index 21ac3f0..d468aea 100644 --- a/.github/workflows/release-announcements.yml +++ b/.github/workflows/release-announcements.yml @@ -9,6 +9,11 @@ jobs: toot: runs-on: ubuntu-latest steps: + - name: get topics and convert to hashtags + id: get_topics + run: | + curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ' >> $GITHUB_OUTPUT + - uses: rzr/fediverse-action@master with: access-token: ${{ secrets.MASTODON_ACCESS_TOKEN }} @@ -20,11 +25,13 @@ jobs: Please see the changelog: ${{ github.event.release.html_url }} + ${{ steps.get_topics.outputs.GITHUB_OUTPUT }} #nfcore #openscience #nextflow #bioinformatics + send-tweet: runs-on: ubuntu-latest steps: - - uses: actions/setup-python@v5 + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: python-version: "3.10" - name: Install dependencies @@ -56,7 +63,7 @@ jobs: bsky-post: runs-on: ubuntu-latest steps: - - uses: zentered/bluesky-post-action@v0.1.0 + - uses: zentered/bluesky-post-action@80dbe0a7697de18c15ad22f4619919ceb5ccf597 # v0.1.0 with: post: | Pipeline release! ${{ github.repository }} v${{ github.event.release.tag_name }} - ${{ github.event.release.name }}! diff --git a/.gitpod.yml b/.gitpod.yml index 363d5b1..105a182 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -10,13 +10,11 @@ tasks: vscode: extensions: # based on nf-core.nf-core-extensionpack - - codezombiech.gitignore # Language support for .gitignore files - # - cssho.vscode-svgviewer # SVG viewer - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code - - eamodio.gitlens # Quickly glimpse into whom, why, and when a line or code block was changed - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar - mechatroner.rainbow-csv # Highlight columns in csv files in different colors - # - nextflow.nextflow # Nextflow syntax highlighting + # - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - streetsidesoftware.code-spell-checker # Spelling checker for source code + - charliermarsh.ruff # Code linter Ruff diff --git a/CHANGELOG.md b/CHANGELOG.md index ee3eb02..066d74d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ compatible with further downstream analyses and/or exploration in _e.g._ ### `Added` +- Add MultiQC support for Space Ranger outputs [[#70](https://github.com/nf-core/spatialtranscriptomics/pull/70)] +- Use the QUARTONOTEBOOK nf-core module instead of local Quarto-based modules [[#68](https://github.com/nf-core/spatialtranscriptomics/pull/68)] +- Add support for SpatialData [[$67](https://github.com/nf-core/spatialtranscriptomics/pull/67)] - Add a custom nf-core Quarto template for the downstream analysis reports [[#64](https://github.com/nf-core/spatialtranscriptomics/pull/64)] - Allow input directories `fastq_dir` and `spaceranger_dir` to be specified as tar archives (`.tar.gz`) - Add a check to make sure that there are spots left after filtering [[#46](https://github.com/nf-core/spatialtranscriptomics/issues/46)] diff --git a/CITATIONS.md b/CITATIONS.md index b6b79a1..d82d84d 100644 --- a/CITATIONS.md +++ b/CITATIONS.md @@ -10,7 +10,7 @@ ## Pipeline tools -- [AnnData](https://github.com/theislab/anndata) +- [AnnData](https://github.com/scverse/anndata) > Virshup I, Rybakov S, Theis FJ, Angerer P, Wolf FA. bioRxiv 2021.12.16.473007; doi: https://doi.org/10.1101/2021.12.16.473007 @@ -32,11 +32,15 @@ - [Space Ranger](https://www.10xgenomics.com/support/software/space-ranger) - > 10x Genomics Space Ranger 2.1.0 + > 10x Genomics Space Ranger 2.1.0 [Online] -- [SpatialDE](https://github.com/Teichlab/SpatialDE) +- [SpatialData](https://www.biorxiv.org/content/10.1101/2023.05.05.539647v1) - > Svensson V, Teichmann S, Stegle O. SpatialDE: identification of spatially variable genes. Nat Methods 15, 343–346 (2018). doi: https://doi.org/10.1038/nmeth.4636 + > Marconato L, Palla G, Yamauchi K, Virshup I, Heidari E, Treis T, Toth M, Shrestha R, Vöhringer H, Huber W, Gerstung M, Moore J, Theis F, Stegle O. SpatialData: an open and universal data framework for spatial omics. bioRxiv 2023.05.05.539647; doi: https://doi.org/10.1101/2023.05.05.539647 + +- [Squipy](https://www.nature.com/articles/s41592-021-01358-2) + + > Palla G, Spitzer H, Klein M et al. Squidpy: a scalable framework for spatial omics analysis. Nat Methods 19, 171–178 (2022). doi: https://doi.org/10.1038/s41592-021-01358-2 ## Software packaging/containerisation tools diff --git a/README.md b/README.md index b12fcd7..0322b77 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,16 @@ nf-core/spatialtranscriptomics -[![GitHub Actions CI Status](https://github.com/nf-core/spatialtranscriptomics/workflows/nf-core%20CI/badge.svg)](https://github.com/nf-core/spatialtranscriptomics/actions?query=workflow%3A%22nf-core+CI%22) -[![GitHub Actions Linting Status](https://github.com/nf-core/spatialtranscriptomics/workflows/nf-core%20linting/badge.svg)](https://github.com/nf-core/spatialtranscriptomics/actions?query=workflow%3A%22nf-core+linting%22)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/spatialtranscriptomics/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) + +[![GitHub Actions CI Status](https://github.com/nf-core/spatialtranscriptomics/actions/workflows/ci.yml/badge.svg)](https://github.com/nf-core/spatialtranscriptomics/actions/workflows/ci.yml) +[![GitHub Actions Linting Status](https://github.com/nf-core/spatialtranscriptomics/actions/workflows/linting.yml/badge.svg)](https://github.com/nf-core/spatialtranscriptomics/actions/workflows/linting.yml)[![AWS CI](https://img.shields.io/badge/CI%20tests-full%20size-FF9900?labelColor=000000&logo=Amazon%20AWS)](https://nf-co.re/spatialtranscriptomics/results)[![Cite with Zenodo](http://img.shields.io/badge/DOI-10.5281/zenodo.XXXXXXX-1073c8?labelColor=000000)](https://doi.org/10.5281/zenodo.XXXXXXX) +[![nf-test](https://img.shields.io/badge/unit_tests-nf--test-337ab7.svg)](https://www.nf-test.com) [![Nextflow](https://img.shields.io/badge/nextflow%20DSL2-%E2%89%A523.04.0-23aa62.svg)](https://www.nextflow.io/) [![run with conda](http://img.shields.io/badge/run%20with-conda-3EB049?labelColor=000000&logo=anaconda)](https://docs.conda.io/en/latest/) [![run with docker](https://img.shields.io/badge/run%20with-docker-0db7ed?labelColor=000000&logo=docker)](https://www.docker.com/) [![run with singularity](https://img.shields.io/badge/run%20with-singularity-1d355c.svg?labelColor=000000)](https://sylabs.io/docs/) -[![Launch on Nextflow Tower](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Nextflow%20Tower-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/spatialtranscriptomics) +[![Launch on Seqera Platform](https://img.shields.io/badge/Launch%20%F0%9F%9A%80-Seqera%20Platform-%234256e7)](https://tower.nf/launch?pipeline=https://github.com/nf-core/spatialtranscriptomics) [![Get help on Slack](http://img.shields.io/badge/slack-nf--core%20%23spatialtranscriptomics-4A154B?labelColor=000000&logo=slack)](https://nfcore.slack.com/channels/spatialtranscriptomics)[![Follow on Twitter](http://img.shields.io/badge/twitter-%40nf__core-1DA1F2?labelColor=000000&logo=twitter)](https://twitter.com/nf_core)[![Follow on Mastodon](https://img.shields.io/badge/mastodon-nf__core-6364ff?labelColor=FFFFFF&logo=mastodon)](https://mstdn.science/@nf_core)[![Watch on YouTube](http://img.shields.io/badge/youtube-nf--core-FF0000?labelColor=000000&logo=youtube)](https://www.youtube.com/c/nf-core) diff --git a/assets/_extensions/nf-core/_extension.yml b/assets/_extensions/nf-core/_extension.yml index 2089122..69579df 100644 --- a/assets/_extensions/nf-core/_extension.yml +++ b/assets/_extensions/nf-core/_extension.yml @@ -11,11 +11,15 @@ contributes: highlight-style: nf-core.theme smooth-scroll: true theme: [default, nf-core.scss] - toc-location: left toc: true + toc-image: nf-core-spatialtranscriptomics_logo_light.png + toc-location: left + template-partials: + - toc.html revealjs: code-line-numbers: false embed-resources: true + logo: nf-core-spatialtranscriptomics_logo_light.png slide-level: 2 slide-number: false theme: [default, nf-core.scss] diff --git a/assets/_extensions/nf-core/nf-core-spatialtranscriptomics_logo_light.png b/assets/_extensions/nf-core/nf-core-spatialtranscriptomics_logo_light.png new file mode 120000 index 0000000..a64fe77 --- /dev/null +++ b/assets/_extensions/nf-core/nf-core-spatialtranscriptomics_logo_light.png @@ -0,0 +1 @@ +../../nf-core-spatialtranscriptomics_logo_light.png \ No newline at end of file diff --git a/assets/_extensions/nf-core/toc.html b/assets/_extensions/nf-core/toc.html new file mode 100644 index 0000000..b402642 --- /dev/null +++ b/assets/_extensions/nf-core/toc.html @@ -0,0 +1,7 @@ + diff --git a/assets/methods_description_template.yml b/assets/methods_description_template.yml index d8e566d..c65a0c4 100644 --- a/assets/methods_description_template.yml +++ b/assets/methods_description_template.yml @@ -3,7 +3,6 @@ description: "Suggested text and references to use when describing pipeline usag section_name: "nf-core/spatialtranscriptomics Methods Description" section_href: "https://github.com/nf-core/spatialtranscriptomics" plot_type: "html" -## TODO nf-core: Update the HTML below to your preferred methods description, e.g. add publication citation for this pipeline ## You inject any metadata in the Nextflow '${workflow}' object data: |

Methods

@@ -12,13 +11,7 @@ data: | 
${workflow.commandLine}

${tool_citations}

References

-
    -
  • Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. doi: 10.1038/nbt.3820
    • -
  • Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. doi: 10.1038/s41587-020-0439-x
    • -
  • Grüning, B., Dale, R., Sjödin, A., Chapman, B. A., Rowe, J., Tomkins-Tinch, C. H., Valieris, R., Köster, J., & Bioconda Team. (2018). Bioconda: sustainable and comprehensive software distribution for the life sciences. Nature Methods, 15(7), 475–476. doi: 10.1038/s41592-018-0046-7
    • -
  • da Veiga Leprevost, F., Grüning, B. A., Alves Aflitos, S., Röst, H. L., Uszkoreit, J., Barsnes, H., Vaudel, M., Moreno, P., Gatto, L., Weber, J., Bai, M., Jimenez, R. C., Sachsenberg, T., Pfeuffer, J., Vera Alvarez, R., Griss, J., Nesvizhskii, A. I., & Perez-Riverol, Y. (2017). BioContainers: an open-source and community-driven framework for software standardization. Bioinformatics (Oxford, England), 33(16), 2580–2582. doi: 10.1093/bioinformatics/btx192
    • - ${tool_bibliography} -
+
    ${tool_bibliography}
Notes:
    diff --git a/assets/multiqc_config.yml b/assets/multiqc_config.yml index c73cf3b..2ea9059 100644 --- a/assets/multiqc_config.yml +++ b/assets/multiqc_config.yml @@ -11,3 +11,5 @@ report_section_order: order: -1002 export_plots: true + +disable_version_detection: true diff --git a/assets/schema_input.json b/assets/schema_input.json index 4bf3d96..628cbe3 100644 --- a/assets/schema_input.json +++ b/assets/schema_input.json @@ -10,27 +10,10 @@ "sample": { "type": "string", "pattern": "^\\S+$", - "errorMessage": "Sample name must be provided and cannot contain spaces" - }, - "fastq_1": { - "type": "string", - "pattern": "^\\S+\\.f(ast)?q\\.gz$", - "errorMessage": "FastQ file for reads 1 must be provided, cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'" - }, - "fastq_2": { - "errorMessage": "FastQ file for reads 2 cannot contain spaces and must have extension '.fq.gz' or '.fastq.gz'", - "anyOf": [ - { - "type": "string", - "pattern": "^\\S+\\.f(ast)?q\\.gz$" - }, - { - "type": "string", - "maxLength": 0 - } - ] + "errorMessage": "Sample name must be provided and cannot contain spaces", + "meta": ["id"] } }, - "required": ["sample", "fastq_1"] + "required": ["sample"] } } diff --git a/bin/st_clustering.qmd b/bin/clustering.qmd similarity index 53% rename from bin/st_clustering.qmd rename to bin/clustering.qmd index be8e496..3985af8 100644 --- a/bin/st_clustering.qmd +++ b/bin/clustering.qmd @@ -3,39 +3,68 @@ title: "nf-core/spatialtranscriptomics" subtitle: "Dimensionality reduction and clustering" format: nf-core-html: default -execute: - keep-ipynb: true jupyter: python3 --- ```{python} #| tags: [parameters] #| echo: false - -input_adata_filtered = "st_adata_filtered.h5ad" # Name of the input anndata file +input_sdata = "sdata_filtered.zarr" # Input: SpatialData file cluster_resolution = 1 # Resolution for Leiden clustering n_hvgs = 2000 # Number of HVGs to use for analyses -output_adata_processed = "st_adata_processed.h5ad" # Name of the output anndata file +artifact_dir = "artifacts" # Output directory +output_adata = "adata_processed.h5ad" # Output: AnnData file +output_sdata = "sdata_processed.zarr" # Output: SpatialData file ``` The data has already been filtered in the _quality controls_ reports and is -saved in the AnnData format: +saved in the SpatialData format: ```{python} #| warning: false +import spatialdata +import os import scanpy as sc import numpy as np import pandas as pd +from anndata import AnnData from umap import UMAP from matplotlib import pyplot as plt import seaborn as sns +import leidenalg from IPython.display import display, Markdown ``` ```{python} -st_adata = sc.read("./" + input_adata_filtered) -print("Content of the AnnData object:") -print(st_adata) +# Make sure we can use scanpy plots with the AnnData object exported from +# `sdata.tables`. This code is taken from the early version of https://github.com/scverse/spatialdata-io/pull/102/ +# Once that PR is merged into spatialdata-io, we should instead use +# `spatialdata_io.to_legacy_anndata(sdata)`. +def to_legacy_anndata(sdata: spatialdata.SpatialData) -> AnnData: + adata = sdata.tables["table"] + for dataset_id in adata.uns["spatial"]: + adata.uns["spatial"][dataset_id]["images"] = { + "hires": np.array(sdata.images[f"{dataset_id}_hires_image"]).transpose([1, 2, 0]), + "lowres": np.array(sdata.images[f"{dataset_id}_lowres_image"]).transpose([1, 2, 0]), + } + adata.uns["spatial"][dataset_id]["scalefactors"] = { + "tissue_hires_scalef": spatialdata.transformations.get_transformation( + sdata.shapes[dataset_id], to_coordinate_system="downscaled_hires" + ).scale[0], + "tissue_lowres_scalef": spatialdata.transformations.get_transformation( + sdata.shapes[dataset_id], to_coordinate_system="downscaled_lowres" + ).scale[0], + "spot_diameter_fullres": sdata.shapes[dataset_id]["radius"][0] * 2, + } + return adata +``` + +```{python} +sdata = spatialdata.read_zarr(input_sdata, ["images", "tables", "shapes"]) +adata = to_legacy_anndata(sdata) + +print("Content of the SpatialData table object:") +print(adata) ``` # Normalization @@ -45,8 +74,8 @@ use the built-in `normalize_total` method from [Scanpy](https://scanpy.readthedo followed by a log-transformation. ```{python} -sc.pp.normalize_total(st_adata, inplace=True) -sc.pp.log1p(st_adata) +sc.pp.normalize_total(adata, inplace=True) +sc.pp.log1p(adata) ``` # Feature selection @@ -59,13 +88,13 @@ regards to yielding a good separation of clusters. ```{python} # layout-nrow: 1 # Find top HVGs and print results -sc.pp.highly_variable_genes(st_adata, flavor="seurat", n_top_genes=n_hvgs) -var_genes_all = st_adata.var.highly_variable +sc.pp.highly_variable_genes(adata, flavor="seurat", n_top_genes=n_hvgs) +var_genes_all = adata.var.highly_variable print("Extracted highly variable genes: %d"%sum(var_genes_all)) # Plot the HVGs plt.rcParams["figure.figsize"] = (4.5, 4.5) -sc.pl.highly_variable_genes(st_adata) +sc.pl.highly_variable_genes(adata) ``` # Clustering @@ -77,10 +106,10 @@ Manifold Approximation and Projection) is used for visualization. The Leiden algorithm is employed for clustering with a given resolution. ```{python} -sc.pp.pca(st_adata) -sc.pp.neighbors(st_adata) -sc.tl.umap(st_adata) -sc.tl.leiden(st_adata, key_added="clusters", resolution=cluster_resolution) +sc.pp.pca(adata) +sc.pp.neighbors(adata) +sc.tl.umap(adata) +sc.tl.leiden(adata, key_added="clusters", resolution=cluster_resolution) Markdown(f"Resolution for Leiden clustering: `{cluster_resolution}`") ``` @@ -91,7 +120,7 @@ We then generate UMAP plots to visualize the distribution of clusters: ```{python} #| warning: false plt.rcParams["figure.figsize"] = (7, 7) -sc.pl.umap(st_adata, color="clusters") +sc.pl.umap(adata, color="clusters") ``` ## Counts and genes @@ -102,7 +131,7 @@ the UMAP: ```{python} # Make plots of UMAP of ST spots clusters plt.rcParams["figure.figsize"] = (3.5, 3.5) -sc.pl.umap(st_adata, color=["total_counts", "n_genes_by_counts"]) +sc.pl.umap(adata, color=["total_counts", "n_genes_by_counts"]) ``` ## Individual clusters @@ -111,8 +140,8 @@ An additional visualisation is to show where the various spots are in each individual cluster while ignoring all other cluster: ```{python} -sc.tl.embedding_density(st_adata, basis="umap", groupby="clusters") -sc.pl.embedding_density(st_adata, groupby="clusters", ncols=2) +sc.tl.embedding_density(adata, basis="umap", groupby="clusters") +sc.pl.embedding_density(adata, groupby="clusters", ncols=2) ``` # Spatial visualisation @@ -123,8 +152,8 @@ spatial coordinates by overlaying the spots on the tissue image itself. ```{python} #| layout-nrow: 2 plt.rcParams["figure.figsize"] = (8, 8) -sc.pl.spatial(st_adata, img_key="hires", color="total_counts", size=1.25) -sc.pl.spatial(st_adata, img_key="hires", color="n_genes_by_counts", size=1.25) +sc.pl.spatial(adata, img_key="hires", color="total_counts", size=1.25) +sc.pl.spatial(adata, img_key="hires", color="n_genes_by_counts", size=1.25) ``` To gain insights into tissue organization and potential inter-cellular @@ -136,10 +165,13 @@ organization of cells. ```{python} # TODO: Can the colour bar on this figure be fit to the figure? plt.rcParams["figure.figsize"] = (7, 7) -sc.pl.spatial(st_adata, img_key="hires", color="clusters", size=1.25) +sc.pl.spatial(adata, img_key="hires", color="clusters", size=1.25) ``` ```{python} #| echo: false -st_adata.write(output_adata_processed) +del sdata.tables["table"] +sdata.tables["table"] = adata +adata.write(os.path.join(artifact_dir, output_adata)) +sdata.write(os.path.join(artifact_dir, output_sdata)) ``` diff --git a/bin/st_quality_controls.qmd b/bin/quality_controls.qmd similarity index 57% rename from bin/st_quality_controls.qmd rename to bin/quality_controls.qmd index a841d8f..4afabcc 100644 --- a/bin/st_quality_controls.qmd +++ b/bin/quality_controls.qmd @@ -10,55 +10,86 @@ jupyter: python3 Spatial Transcriptomics data analysis involves several steps, including quality controls (QC) and pre-processing, to ensure the reliability of downstream -analyses. This is an essential step in spatial transcriptomics to -identify and filter out spots and genes that may introduce noise and/or bias -into the analysis. +analyses. This is an essential step in spatial transcriptomics to identify and +filter out spots and genes that may introduce noise and/or bias into the +analysis. This report outlines the QC and pre-processing steps for Visium Spatial Transcriptomics data using the [AnnData format](https://anndata.readthedocs.io/en/latest/tutorials/notebooks/getting-started.html) and the [`scanpy` Python package](https://scanpy.readthedocs.io/en/stable/). -The anndata format is utilized to organize and store the Spatial Transcriptomics +The AnnData format is utilized to organize and store the Spatial Transcriptomics data. It includes information about counts, features, observations, and -additional metadata. The anndata format ensures compatibility with various +additional metadata. The AnnData format ensures compatibility with various analysis tools and facilitates seamless integration into existing workflows. +The AnnData object is saved in the `Tables` element of a zarr [SpatialData object](https://spatialdata.scverse.org/en/latest/design_doc.html#table-table-of-annotations-for-regions). ```{python} #| tags: [parameters] #| echo: false -input_adata_raw = "st_adata_raw.h5ad" # Name of the input anndata file +input_sdata = "sdata_raw.zarr" # Input: SpatialData file min_counts = 500 # Min counts per spot min_genes = 250 # Min genes per spot min_spots = 1 # Min spots per gene mito_threshold = 20 # Mitochondrial content threshold (%) ribo_threshold = 0 # Ribosomal content threshold (%) hb_threshold = 100 # content threshold (%) -output_adata_filtered = "st_adata_filtered.h5ad" # Name of the output anndata file +artifact_dir = "artifacts" +output_adata = "adata_filtered.h5ad" # Output: AnnData file +output_sdata = "sdata_filtered.zarr" # Output: SpatialData file ``` ```{python} -import scanpy as sc -import scipy import pandas as pd import matplotlib.pyplot as plt +import numpy as np +import os +import scanpy as sc +import scipy import seaborn as sns +import spatialdata +from anndata import AnnData from IPython.display import display, Markdown from textwrap import dedent plt.rcParams["figure.figsize"] = (6, 6) ``` +```{python} +# Make sure we can use scanpy plots with the AnnData object exported from sdata.tables +# This code is taken from the early version of https://github.com/scverse/spatialdata-io/pull/102/ +# Once the PR will be merged in spatialdata-io, we should use spatialdata_io.to_legacy_anndata(sdata). +def to_legacy_anndata(sdata: spatialdata.SpatialData) -> AnnData: + adata = sdata.tables["table"] + for dataset_id in adata.uns["spatial"]: + adata.uns["spatial"][dataset_id]["images"] = { + "hires": np.array(sdata.images[f"{dataset_id}_hires_image"]).transpose([1, 2, 0]), + "lowres": np.array(sdata.images[f"{dataset_id}_lowres_image"]).transpose([1, 2, 0]), + } + adata.uns["spatial"][dataset_id]["scalefactors"] = { + "tissue_hires_scalef": spatialdata.transformations.get_transformation( + sdata.shapes[dataset_id], to_coordinate_system="downscaled_hires" + ).scale[0], + "tissue_lowres_scalef": spatialdata.transformations.get_transformation( + sdata.shapes[dataset_id], to_coordinate_system="downscaled_lowres" + ).scale[0], + "spot_diameter_fullres": sdata.shapes[dataset_id]["radius"][0] * 2, + } + return adata +``` + ```{python} # Read the data -st_adata = sc.read("./" + input_adata_raw) +sdata = spatialdata.read_zarr(input_sdata, ["images", "tables", "shapes"]) +adata = to_legacy_anndata(sdata) -# Convert X matrix from csr to csc dense matrix for output compatibility: -st_adata.X = scipy.sparse.csc_matrix(st_adata.X) +# Convert X matrix from CSR to CSC dense matrix for output compatibility +adata.X = scipy.sparse.csc_matrix(adata.X) # Store the raw data so that it can be used for analyses from scratch if desired -st_adata.layers['raw'] = st_adata.X.copy() +adata.layers['raw'] = adata.X.copy() # Print the anndata object for inspection print("Content of the AnnData object:") -print(st_adata) +print(adata) ``` # Quality controls @@ -71,14 +102,14 @@ percentage of counts from mitochondrial, ribosomal and haemoglobin genes ```{python} # Calculate mitochondrial, ribosomal and haemoglobin percentages -st_adata.var['mt'] = st_adata.var_names.str.startswith('MT-') -st_adata.var['ribo'] = st_adata.var_names.str.contains(("^RP[LS]")) -st_adata.var['hb'] = st_adata.var_names.str.contains(("^HB[AB]")) -sc.pp.calculate_qc_metrics(st_adata, qc_vars=["mt", "ribo", "hb"], +adata.var['mt'] = adata.var_names.str.startswith('MT-') +adata.var['ribo'] = adata.var_names.str.contains(("^RP[LS]")) +adata.var['hb'] = adata.var_names.str.contains(("^HB[AB]")) +sc.pp.calculate_qc_metrics(adata, qc_vars=["mt", "ribo", "hb"], inplace=True, log1p=False) # Save a copy of data as a restore-point if filtering results in 0 spots left -st_adata_before_filtering = st_adata.copy() +adata_before_filtering = adata.copy() ``` ## Violin plots @@ -89,9 +120,9 @@ mitochondrial, ribosomal and haemoglobin genes: ```{python} #| layout-nrow: 2 -sc.pl.violin(st_adata, ['n_genes_by_counts', 'total_counts'], +sc.pl.violin(adata, ['n_genes_by_counts', 'total_counts'], multi_panel=True, jitter=0.4, rotation= 45) -sc.pl.violin(st_adata, ['pct_counts_mt', 'pct_counts_ribo', 'pct_counts_hb'], +sc.pl.violin(adata, ['pct_counts_mt', 'pct_counts_ribo', 'pct_counts_hb'], multi_panel=True, jitter=0.4, rotation= 45) ``` @@ -102,8 +133,8 @@ spatial patterns may be discerned: ```{python} #| layout-nrow: 2 -sc.pl.spatial(st_adata, color = ["total_counts", "n_genes_by_counts"], size=1.25) -sc.pl.spatial(st_adata, color = ["pct_counts_mt", "pct_counts_ribo", "pct_counts_hb"], size=1.25) +sc.pl.spatial(adata, color = ["total_counts", "n_genes_by_counts"], size=1.25) +sc.pl.spatial(adata, color = ["pct_counts_mt", "pct_counts_ribo", "pct_counts_hb"], size=1.25) ``` ## Scatter plots @@ -114,8 +145,8 @@ counts versus the number of genes: ```{python} #| layout-ncol: 2 -sc.pl.scatter(st_adata, x='pct_counts_ribo', y='pct_counts_mt') -sc.pl.scatter(st_adata, x='total_counts', y='n_genes_by_counts') +sc.pl.scatter(adata, x='pct_counts_ribo', y='pct_counts_mt') +sc.pl.scatter(adata, x='total_counts', y='n_genes_by_counts') ``` ## Top expressed genes @@ -124,7 +155,7 @@ It can also be informative to see which genes are the most expressed in the dataset; the following figure shows the top 20 most expressed genes. ```{python} -sc.pl.highest_expr_genes(st_adata, n_top=20) +sc.pl.highest_expr_genes(adata, n_top=20) ``` # Filtering @@ -136,16 +167,16 @@ are uninformative and are thus removed. ```{python} # Create a string observation "obs/in_tissue_str" with "In tissue" and "Outside tissue": -st_adata.obs["in_tissue_str"] = ["In tissue" if x == 1 else "Outside tissue" for x in st_adata.obs["in_tissue"]] +adata.obs["in_tissue_str"] = ["In tissue" if x == 1 else "Outside tissue" for x in adata.obs["in_tissue"]] # Plot spots inside tissue -sc.pl.spatial(st_adata, color=["in_tissue_str"], title="Spots in tissue", size=1.25) -del st_adata.obs["in_tissue_str"] +sc.pl.spatial(adata, color=["in_tissue_str"], title="Spots in tissue", size=1.25) +del adata.obs["in_tissue_str"] # Remove spots outside tissue and print results -n_spots = st_adata.shape[0] -st_adata = st_adata[st_adata.obs["in_tissue"] == 1] -n_spots_in_tissue = st_adata.shape[0] +n_spots = adata.shape[0] +adata = adata[adata.obs["in_tissue"] == 1] +n_spots_in_tissue = adata.shape[0] Markdown(f"""A total of `{n_spots_in_tissue}` spots are situated inside the tissue, out of `{n_spots}` spots in total.""") ``` @@ -159,18 +190,18 @@ your knowledge of the specific tissue at hand. ```{python} #| warning: false # Filter spots based on counts -n_spots = st_adata.shape[0] -n_genes = st_adata.shape[1] -sc.pp.filter_cells(st_adata, min_counts=min_counts) -n_spots_filtered_min_counts = st_adata.shape[0] +n_spots = adata.shape[0] +n_genes = adata.shape[1] +sc.pp.filter_cells(adata, min_counts=min_counts) +n_spots_filtered_min_counts = adata.shape[0] # Filter spots based on genes -sc.pp.filter_cells(st_adata, min_genes=min_genes) -n_spots_filtered_min_genes = st_adata.shape[0] +sc.pp.filter_cells(adata, min_genes=min_genes) +n_spots_filtered_min_genes = adata.shape[0] # Filter genes based on spots -sc.pp.filter_genes(st_adata, min_cells=min_spots) -n_genes_filtered_min_spots = st_adata.shape[1] +sc.pp.filter_genes(adata, min_cells=min_spots) +n_genes_filtered_min_spots = adata.shape[1] # Print results Markdown(f""" @@ -189,16 +220,16 @@ ribosomal nor haemoglobin content is filtered by default. ```{python} # Filter spots -st_adata = st_adata[st_adata.obs["pct_counts_mt"] <= mito_threshold] -n_spots_filtered_mito = st_adata.shape[0] -st_adata = st_adata[st_adata.obs["pct_counts_ribo"] >= ribo_threshold] -n_spots_filtered_ribo = st_adata.shape[0] -st_adata = st_adata[st_adata.obs["pct_counts_hb"] <= hb_threshold] -n_spots_filtered_hb = st_adata.shape[0] +adata = adata[adata.obs["pct_counts_mt"] <= mito_threshold] +n_spots_filtered_mito = adata.shape[0] +adata = adata[adata.obs["pct_counts_ribo"] >= ribo_threshold] +n_spots_filtered_ribo = adata.shape[0] +adata = adata[adata.obs["pct_counts_hb"] <= hb_threshold] +n_spots_filtered_hb = adata.shape[0] # Print results Markdown(f""" -- Removed `{st_adata.shape[0] - n_spots_filtered_mito}` spots with more than `{mito_threshold}%` mitochondrial content. +- Removed `{adata.shape[0] - n_spots_filtered_mito}` spots with more than `{mito_threshold}%` mitochondrial content. - Removed `{n_spots_filtered_mito - n_spots_filtered_ribo}` spots with less than `{ribo_threshold}%` ribosomal content. - Removed `{n_spots_filtered_ribo - n_spots_filtered_hb}` spots with more than `{hb_threshold}%` haemoglobin content. """) @@ -207,8 +238,8 @@ Markdown(f""" ```{python} #| echo: false # Restore non-filtered data if filtering results in 0 spots left -if (st_adata.shape[0] == 0 or st_adata.shape[1] == 0): - st_adata = st_adata_before_filtering +if (adata.shape[0] == 0 or adata.shape[1] == 0): + adata = adata_before_filtering display( Markdown(dedent( """ @@ -238,21 +269,21 @@ if (st_adata.shape[0] == 0 or st_adata.shape[1] == 0): Markdown(f""" The final results of all the filtering is as follows: -- A total of `{st_adata.shape[0]}` spots out of `{n_spots}` remain after filtering. -- A total of `{st_adata.shape[1]}` genes out of `{n_genes}` remain after filtering. +- A total of `{adata.shape[0]}` spots out of `{n_spots}` remain after filtering. +- A total of `{adata.shape[1]}` genes out of `{n_genes}` remain after filtering. """) ``` ```{python} #| layout-nrow: 2 -sc.pl.violin(st_adata, ['n_genes_by_counts', 'total_counts'], +sc.pl.violin(adata, ['n_genes_by_counts', 'total_counts'], multi_panel=True, jitter=0.4, rotation= 45) -sc.pl.violin(st_adata, ['pct_counts_mt', 'pct_counts_ribo', 'pct_counts_hb'], +sc.pl.violin(adata, ['pct_counts_mt', 'pct_counts_ribo', 'pct_counts_hb'], multi_panel=True, jitter=0.4, rotation= 45) ``` ```{python} -#| echo: false -# Write filtered data to disk -st_adata.write(output_adata_filtered) +del sdata.tables["table"] +sdata.tables["table"] = adata +sdata.write(os.path.join(artifact_dir, output_sdata)) ``` diff --git a/bin/read_data.py b/bin/read_data.py new file mode 100755 index 0000000..fd27753 --- /dev/null +++ b/bin/read_data.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python + +# Load packages +import argparse + +import spatialdata_io + +if __name__ == "__main__": + # Parse command-line arguments + parser = argparse.ArgumentParser( + description="Load spatial transcriptomics data from MTX matrices and aligned images." + ) + parser.add_argument( + "--SRCountDir", + metavar="SRCountDir", + type=str, + default=None, + help="Input directory with Spaceranger data.", + ) + parser.add_argument( + "--output_sdata", + metavar="output_sdata", + type=str, + default=None, + help="Output spatialdata zarr path.", + ) + # TODO Add argument with meta.id for dataset_id + + args = parser.parse_args() + + # Read Visium data + spatialdata = spatialdata_io.visium( + args.SRCountDir, counts_file="raw_feature_bc_matrix.h5", dataset_id="visium" + ) + + # Write raw spatialdata to file + spatialdata.write(args.output_sdata, overwrite=True) diff --git a/bin/read_st_data.py b/bin/read_st_data.py deleted file mode 100755 index 7c542af..0000000 --- a/bin/read_st_data.py +++ /dev/null @@ -1,209 +0,0 @@ -#!/usr/bin/env python - -# Load packages -import argparse -from scanpy import read_10x_h5 -from pathlib import Path -from typing import Union, Optional - -import json -import pandas as pd -from matplotlib.image import imread -from anndata import AnnData - -from scanpy import logging as logg - - -def read_visium( - path: Union[str, Path], - genome: Optional[str] = None, - *, - count_file: str = "filtered_feature_bc_matrix.h5", - library_id: str = None, - load_images: Optional[bool] = True, - source_image_path: Optional[Union[str, Path]] = None, -) -> AnnData: - """\ - Read 10x-Genomics-formatted visum dataset. - - In addition to reading regular 10x output, - this looks for the `spatial` folder and loads images, - coordinates and scale factors. - Based on the `Space Ranger output docs`_. - - See :func:`~scanpy.pl.spatial` for a compatible plotting function. - - .. _Space Ranger output docs: https://support.10xgenomics.com/spatial-gene-expression/software/pipelines/latest/output/overview - - Parameters - ---------- - path - Path to directory for visium datafiles. - genome - Filter expression to genes within this genome. - count_file - Which file in the passed directory to use as the count file. Typically would be one of: - 'filtered_feature_bc_matrix.h5' or 'raw_feature_bc_matrix.h5'. - library_id - Identifier for the visium library. Can be modified when concatenating multiple adata objects. - source_image_path - Path to the high-resolution tissue image. Path will be included in - `.uns["spatial"][library_id]["metadata"]["source_image_path"]`. - - Returns - ------- - Annotated data matrix, where observations/cells are named by their - barcode and variables/genes by gene name. Stores the following information: - - :attr:`~anndata.AnnData.X` - The data matrix is stored - :attr:`~anndata.AnnData.obs_names` - Cell names - :attr:`~anndata.AnnData.var_names` - Gene names for a feature barcode matrix, probe names for a probe bc matrix - :attr:`~anndata.AnnData.var`\\ `['gene_ids']` - Gene IDs - :attr:`~anndata.AnnData.var`\\ `['feature_types']` - Feature types - :attr:`~anndata.AnnData.obs`\\ `[filtered_barcodes]` - filtered barcodes if present in the matrix - :attr:`~anndata.AnnData.var` - Any additional metadata present in /matrix/features is read in. - :attr:`~anndata.AnnData.uns`\\ `['spatial']` - Dict of spaceranger output files with 'library_id' as key - :attr:`~anndata.AnnData.uns`\\ `['spatial'][library_id]['images']` - Dict of images (`'hires'` and `'lowres'`) - :attr:`~anndata.AnnData.uns`\\ `['spatial'][library_id]['scalefactors']` - Scale factors for the spots - :attr:`~anndata.AnnData.uns`\\ `['spatial'][library_id]['metadata']` - Files metadata: 'chemistry_description', 'software_version', 'source_image_path' - :attr:`~anndata.AnnData.obsm`\\ `['spatial']` - Spatial spot coordinates, usable as `basis` by :func:`~scanpy.pl.embedding`. - """ - path = Path(path) - adata = read_10x_h5(path / "raw_feature_bc_matrix.h5") - # use ensemble IDs as index, because they are unique - adata.var["gene_symbol"] = adata.var_names - adata.var.set_index("gene_ids", inplace=True) - - adata.uns["spatial"] = dict() - - from h5py import File - - with File(path / count_file, mode="r") as f: - attrs = dict(f.attrs) - if library_id is None: - library_id = str(attrs.pop("library_ids")[0], "utf-8") - - adata.uns["spatial"][library_id] = dict() - - if load_images: - tissue_positions_file = ( - path / "spatial/tissue_positions.csv" - if (path / "spatial/tissue_positions.csv").exists() - else path / "spatial/tissue_positions_list.csv" - ) - files = dict( - tissue_positions_file=tissue_positions_file, - scalefactors_json_file=path / "spatial/scalefactors_json.json", - hires_image=path / "spatial/tissue_hires_image.png", - lowres_image=path / "spatial/tissue_lowres_image.png", - ) - - # check if files exists, continue if images are missing - for f in files.values(): - if not f.exists(): - if any(x in str(f) for x in ["hires_image", "lowres_image"]): - logg.warning( - f"You seem to be missing an image file.\n" - f"Could not find '{f}'." - ) - else: - raise OSError(f"Could not find '{f}'") - - adata.uns["spatial"][library_id]["images"] = dict() - for res in ["hires", "lowres"]: - try: - adata.uns["spatial"][library_id]["images"][res] = imread( - str(files[f"{res}_image"]) - ) - except Exception: - raise OSError(f"Could not find '{res}_image'") - - # read json scalefactors - adata.uns["spatial"][library_id]["scalefactors"] = json.loads( - files["scalefactors_json_file"].read_bytes() - ) - - adata.uns["spatial"][library_id]["metadata"] = { - k: (str(attrs[k], "utf-8") if isinstance(attrs[k], bytes) else attrs[k]) - for k in ("chemistry_description", "software_version") - if k in attrs - } - - # read coordinates - positions = pd.read_csv( - files["tissue_positions_file"], - header=0 if tissue_positions_file.name == "tissue_positions.csv" else None, - index_col=0, - ) - positions.columns = [ - "in_tissue", - "array_row", - "array_col", - "pxl_col_in_fullres", - "pxl_row_in_fullres", - ] - - adata.obs = adata.obs.join(positions, how="left") - - adata.obsm["spatial"] = adata.obs[ - ["pxl_row_in_fullres", "pxl_col_in_fullres"] - ].to_numpy() - adata.obs.drop( - columns=["pxl_row_in_fullres", "pxl_col_in_fullres"], - inplace=True, - ) - - # put image path in uns - if source_image_path is not None: - # get an absolute path - source_image_path = str(Path(source_image_path).resolve()) - adata.uns["spatial"][library_id]["metadata"]["source_image_path"] = str( - source_image_path - ) - - return adata - - -if __name__ == "__main__": - # Parse command-line arguments - parser = argparse.ArgumentParser( - description="Load spatial transcriptomics data from MTX matrices and aligned images." - ) - parser.add_argument( - "--SRCountDir", - metavar="SRCountDir", - type=str, - default=None, - help="Input directory with Spaceranger data.", - ) - parser.add_argument( - "--outAnnData", - metavar="outAnnData", - type=str, - default=None, - help="Output h5ad file path.", - ) - args = parser.parse_args() - - # Read Visium data - st_adata = read_visium( - args.SRCountDir, - count_file="raw_feature_bc_matrix.h5", - library_id=None, - load_images=True, - ) - - # Write raw anndata to file - st_adata.write(args.outAnnData) diff --git a/bin/spatially_variable_genes.qmd b/bin/spatially_variable_genes.qmd new file mode 100644 index 0000000..e5762ea --- /dev/null +++ b/bin/spatially_variable_genes.qmd @@ -0,0 +1,138 @@ +--- +title: "nf-core/spatialtranscriptomics" +subtitle: "Neighborhood enrichment analysis and Spatially variable genes" +format: + nf-core-html: default +jupyter: python3 +--- + +```{python} +#| tags: [parameters] +#| echo: false +input_sdata = "sdata_processed.zarr" # Input: SpatialData file +n_top_svgs = 14 # Number of spatially variable genes to plot +artifact_dir = "artifacts" # Output directory +output_csv = "spatially_variable_genes.csv" # Output: gene list +output_adata = "adata_spatially_variable_genes.h5ad" # Output: AnnData file +output_sdata = "sdata.zarr" # Output: SpatialData file +``` + +```{python} +import numpy as np +import os +import pandas as pd +import scanpy as sc +import squidpy as sq +import spatialdata +from anndata import AnnData +from matplotlib import pyplot as plt +``` + +```{python} +# Make sure we can use scanpy plots with the AnnData object exported from sdata.tables +# This code is taken from the early version of https://github.com/scverse/spatialdata-io/pull/102/ +# Once the PR will be merged in spatialdata-io, we should use spatialdata_io.to_legacy_anndata(sdata). +def to_legacy_anndata(sdata: spatialdata.SpatialData) -> AnnData: + adata = sdata.tables["table"] + for dataset_id in adata.uns["spatial"]: + adata.uns["spatial"][dataset_id]["images"] = { + "hires": np.array(sdata.images[f"{dataset_id}_hires_image"]).transpose([1, 2, 0]), + "lowres": np.array(sdata.images[f"{dataset_id}_lowres_image"]).transpose([1, 2, 0]), + } + adata.uns["spatial"][dataset_id]["scalefactors"] = { + "tissue_hires_scalef": spatialdata.transformations.get_transformation( + sdata.shapes[dataset_id], to_coordinate_system="downscaled_hires" + ).scale[0], + "tissue_lowres_scalef": spatialdata.transformations.get_transformation( + sdata.shapes[dataset_id], to_coordinate_system="downscaled_lowres" + ).scale[0], + "spot_diameter_fullres": sdata.shapes[dataset_id]["radius"][0] * 2, + } + return adata +``` + +```{python} +# Read data +sdata = spatialdata.read_zarr(input_sdata, ["images", "tables", "shapes"]) + +adata = to_legacy_anndata(sdata) +print("Content of the AnnData object:") +print(adata) + +# Suppress scanpy-specific warnings +sc.settings.verbosity = 0 +``` + +# Differential gene expression + +Before we look for spatially variable genes we first find differentially +expressed genes (DEG) across the different clusters found in the data. We can +visualize the top DEGs in a heatmap: + +```{python} +#| warning: false +sc.tl.rank_genes_groups(adata, 'clusters', method='t-test') +sc.pl.rank_genes_groups_heatmap(adata, n_genes=5, groupby="clusters") +``` + +A different but similar visualization of the DEGs is the dot plot, where we can +also include the gene names: + +```{python} +#| warning: false +sc.pl.rank_genes_groups_dotplot(adata, n_genes=5, groupby="clusters") +``` + +::: {.callout-note} +Please note that you may need to scroll sidewise in these figures, as their +height and width depends on the number of clusters as well as the number and +intersection of the DEGs that are being plotted. +::: + +# Neighborhood enrichment analysis + +We can perform a neighborhood enrichment analysis to find out which +genes are enriched in the neighborhood of each cluster: + +```{python} +#| warning: false +sq.gr.spatial_neighbors(adata, coord_type="generic") +sq.gr.nhood_enrichment(adata, cluster_key="clusters") +sq.pl.nhood_enrichment(adata, cluster_key="clusters", method="ward", vmin=-100, vmax=100) +``` + +We visualize the interaction matrix between the different clusters: + +```{python} +#| warning: false +sq.gr.interaction_matrix(adata, cluster_key="clusters") +sq.pl.interaction_matrix(adata, cluster_key="clusters", method="ward") +``` + +# Spatially variable genes with spatial autocorrelation statistics + +Spatial transcriptomics data can give insight into how genes are expressed in +different areas in a tissue, allowing identification of spatial gene expression +patterns. Here we use [Moran's I](https://en.wikipedia.org/wiki/Moran%27s_I) autocorrelation score to identify such patterns. + +```{python} +adata.var_names_make_unique() +sq.gr.spatial_autocorr(adata, mode="moran") +adata.uns["moranI"].head(n_top_svgs) +#[TODO] add gearyC as optional mode +``` + +```{python} +#| echo: false +# Save the spatially variable genes to a CSV file: +adata.uns["moranI"].to_csv(os.path.join(artifact_dir, output_csv)) +``` + +```{python} +#| echo: false +#| info: false +adata.write(output_adata) +del sdata.tables["table"] +sdata.tables["table"] = adata +sdata.write("./" + output_sdata) +``` diff --git a/bin/st_spatial_de.qmd b/bin/st_spatial_de.qmd deleted file mode 100644 index 0e1e211..0000000 --- a/bin/st_spatial_de.qmd +++ /dev/null @@ -1,99 +0,0 @@ ---- -title: "nf-core/spatialtranscriptomics" -subtitle: "Differential gene expression" -format: - nf-core-html: default -jupyter: python3 ---- - -```{python} -#| tags: [parameters] -#| echo: false -input_adata_processed = "st_adata_processed.h5ad" -output_spatial_degs = "st_spatial_de.csv" -n_top_spatial_degs = 14 -``` - -```{python} -import scanpy as sc -import pandas as pd -import SpatialDE -from matplotlib import pyplot as plt -``` - -```{python} -# Read data -st_adata = sc.read(input_adata_processed) -print("Content of the AnnData object:") -print(st_adata) - -# Fix for scanpy issue https://github.com/scverse/scanpy/issues/2181 -st_adata.uns['log1p']['base'] = None - -# Suppress scanpy-specific warnings -sc.settings.verbosity = 0 -``` - -# Differential gene expression - -Before we look for spatially variable genes we first find differentially -expressed genes across the different clusters found in the data. We can -visualize the top DEGs in a heatmap: - -```{python} -#| warning: false -sc.tl.rank_genes_groups(st_adata, 'clusters', method='t-test') -sc.pl.rank_genes_groups_heatmap(st_adata, n_genes=5, groupby="clusters") -``` - -A different but similar visualization of the DEGs is the dot plot, where we can -also include the gene names: - -```{python} -#| warning: false -sc.pl.rank_genes_groups_dotplot(st_adata, n_genes=5, groupby="clusters") -``` - -::: {.callout-note} -Please note that you may need to scroll sidewise in these figures, as their -height and width depends on the number of clusters as well as the number and -intersection of the DEGs that are being plotted. -::: - -# Spatial gene expression - -Spatial transcriptomics data can give insight into how genes are expressed in -different areas in a tissue, allowing identification of spatial gene expression -patterns. Here we use [SpatialDE](https://www.nature.com/articles/nmeth.4636) to -identify such patterns. - -```{python} -#| output: false -results = SpatialDE.run(st_adata.obsm["spatial"], st_adata.to_df()) -``` - -We can then inspect the spatial DEGs in a table: - -```{python} -results.set_index("g", inplace=True) -# workaround for https://github.com/Teichlab/SpatialDE/issues/36 -results = results.loc[~results.index.duplicated(keep="first")] - -# Add annotations -st_adata.var = pd.concat([st_adata.var, results.loc[st_adata.var.index.values, :]], axis=1) - -# Print results table -results_tab = st_adata.var.sort_values("qval", ascending=True) -results_tab.to_csv(output_spatial_degs) -results_tab.head(n_top_spatial_degs) -``` - -We can also plot the top spatially variable genes on top of the tissue image -itself to visualize the patterns: - -```{python} -symbols = results_tab.iloc[: n_top_spatial_degs]["gene_symbol"] -plt.rcParams["figure.figsize"] = (3.5, 4) -sc.pl.spatial(st_adata, img_key="hires", color=symbols.index, alpha=0.7, - ncols=2, title=symbols, size=1.25) -``` diff --git a/conf/modules.config b/conf/modules.config index 30a52b1..274e825 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -10,12 +10,6 @@ ---------------------------------------------------------------------------------------- */ -// Environment specification needed for Quarto -env { - XDG_CACHE_HOME = "./.xdg_cache_home" - XDG_DATA_HOME = "./.xdg_data_home" -} - process { publishDir = [ @@ -61,6 +55,7 @@ process { } withName: SPACERANGER_COUNT { + ext.args = '--create-bam false' publishDir = [ path: { "${params.outdir}/${meta.id}/spaceranger" }, mode: params.publish_dir_mode, @@ -68,22 +63,37 @@ process { ] } - withName: 'ST_READ_DATA|ST_QUALITY_CONTROLS|ST_CLUSTERING|ST_SPATIAL_DE' { + withName: 'READ_DATA|QUALITY_CONTROLS|CLUSTERING|SPATIALLY_VARIABLE_GENES' { + ext.prefix = { "${notebook.baseName}" } publishDir = [ [ path: { "${params.outdir}/${meta.id}/reports" }, mode: params.publish_dir_mode, - pattern: "*{.html,_files}" + pattern: "*{.html,.qmd,_extensions}" + ], + [ + path: { "${params.outdir}/${meta.id}/reports" }, + mode: params.publish_dir_mode, + pattern: "params.yml", + saveAs: { "${notebook.baseName}.yml" } ], [ path: { "${params.outdir}/${meta.id}/data" }, mode: params.publish_dir_mode, - pattern: "st_adata_processed.h5ad" + pattern: "artifacts/sdata_processed.zarr", + saveAs: { "sdata_processed.zarr" } ], [ - path: { "${params.outdir}/${meta.id}/degs" }, + path: { "${params.outdir}/${meta.id}/data" }, + mode: params.publish_dir_mode, + pattern: "artifacts/adata_processed.h5ad", + saveAs: { "adata_processed.h5ad" } + ], + [ + path: { "${params.outdir}/${meta.id}/data" }, mode: params.publish_dir_mode, - pattern: "*.csv" + pattern: "artifacts/spatially_variable_genes.csv", + saveAs: { "spatially_variable_genes.csv" } ] ] } diff --git a/conf/test.config b/conf/test.config index ce4909c..97b801b 100644 --- a/conf/test.config +++ b/conf/test.config @@ -25,7 +25,7 @@ params { spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" // Parameters - st_qc_min_counts = 5 - st_qc_min_genes = 3 + qc_min_counts = 5 + qc_min_genes = 3 outdir = 'results' } diff --git a/conf/test_downstream.config b/conf/test_downstream.config index 89a56b2..51a6dc9 100644 --- a/conf/test_downstream.config +++ b/conf/test_downstream.config @@ -11,8 +11,8 @@ */ params { - config_profile_name = 'Downstream test profile' - config_profile_description = 'Test pipeline for downstream (post-Space Ranger) functionality' + config_profile_name = 'Downstream test profile' + config_profile_description = 'Test pipeline for downstream (post-Space Ranger) functionality' // Limit resources so that this can run on GitHub Actions max_cpus = 2 @@ -25,7 +25,7 @@ params { spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" // Parameters - st_qc_min_counts = 5 - st_qc_min_genes = 3 + qc_min_counts = 5 + qc_min_genes = 3 outdir = 'results' } diff --git a/conf/test_spaceranger_v1.config b/conf/test_spaceranger_v1.config index 2fca10e..5bce856 100644 --- a/conf/test_spaceranger_v1.config +++ b/conf/test_spaceranger_v1.config @@ -25,7 +25,7 @@ params { spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" // Parameters - st_qc_min_counts = 5 - st_qc_min_genes = 3 + qc_min_counts = 5 + qc_min_genes = 3 outdir = 'results' } diff --git a/docs/output.md b/docs/output.md index 23a88d8..3d6d220 100644 --- a/docs/output.md +++ b/docs/output.md @@ -53,25 +53,38 @@ information about these files at the [10X website](https://support.10xgenomics.c Output files - `/data/` - - `st_adata_processed.h5ad`: Filtered, normalised and clustered adata. + - `sdata_processed.zarr`: Processed data in SpatialData format. + - `adata_processed.h5ad`: Processed data in AnnData format. + - `spatially_variable_genes.csv`: List of spatially variable genes. -Data in `.h5ad` format as processed by the pipeline, which can be used for -further downstream analyses if desired; unprocessed data is also present in this -file. It can also be used by the [TissUUmaps](https://tissuumaps.github.io/) +Data in `.zarr` and `.h5ad` formats as processed by the pipeline, which can be +used for further downstream analyses if desired; unprocessed data is also +present in these files. It can also be used by the [TissUUmaps](https://tissuumaps.github.io/) browser-based tool for visualisation and exploration, allowing you to delve into -the data in an interactive way. +the data in an interactive way. The list of spatially variable genes are added +as a convenience if you want to explore them in _e.g._ Excel. ## Reports +
    +Output files + +- `/reports/` + - `_extensions/`: Quarto nf-core extension, common to all reports. + +
    + ### Quality controls and filtering
    Output files - `/reports/` - - `st_quality_controls.html`: HTML report. + - `quality_controls.html`: Rendered HTML report. + - `quality_controls.yml`: YAML file containing parameters used in the report. + - `quality_controls.qmd`: Quarto document used for rendering the report.
    @@ -85,7 +98,9 @@ well as presence in tissue; you can find more details in the report itself. Output files - `/reports/` - - `st_clustering.html`: HTML report. + - `clustering.html`: Rendered HTML report. + - `clustering.yml`: YAML file containing parameters used in the report. + - `clustering.qmd`: Quarto document used for rendering the report. @@ -93,21 +108,21 @@ Report containing analyses related to normalisation, dimensionality reduction, clustering and spatial visualisation. Leiden clustering is currently the only option; you can find more details in the report itself. -### Differential expression +### Spatially variable genes
    Output files - `/reports/` - - `st_spatial_de.html`: HTML report. -- `/degs/` - - `st_spatial_de.csv`: List of spatially differentially expressed genes. + - `spatially_variable_genes.html`: Rendered HTML report. + - `spatially_variable_genes.yml`: YAML file containing parameters used in the report. + - `spatially_variable_genes.qmd`: Quarto document used for rendering the report.
    Report containing analyses related to differential expression testing and -spatially varying genes. The [SpatialDE](https://github.com/Teichlab/SpatialDE) -package is currently the only option for spatial testing; you can find more +spatially varying genes. The [Moran 1](https://en.wikipedia.org/wiki/Moran%27s_I) +score is currently the only option for spatial testing; you can find more details in the report itself. ## Workflow reporting @@ -122,6 +137,9 @@ details in the report itself. - Reports generated by the pipeline: `pipeline_report.html`, `pipeline_report.txt` and `software_versions.yml`. The `pipeline_report*` files will only be present if the `--email` / `--email_on_fail` parameter's are used when running the pipeline. - Reformatted samplesheet files used as input to the pipeline: `samplesheet.valid.csv`. - Parameters used by the pipeline run: `params.json`. +- `multiqc/` + - Report generated by MultiQC: `multiqc_report.html`. + - Data and plots generated by MultiQC: `multiqc_data/` and `multiqc_plots/`. diff --git a/env/Dockerfile b/env/Dockerfile new file mode 100644 index 0000000..8c44eed --- /dev/null +++ b/env/Dockerfile @@ -0,0 +1,45 @@ +# +# First stage: Quarto installation +# +FROM ubuntu:20.04 as quarto +ARG QUARTO_VERSION=1.3.450 +ARG TARGETARCH +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + && apt-get clean + +RUN mkdir -p /opt/quarto \ + && curl -o quarto.tar.gz -L "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-${TARGETARCH}.tar.gz" \ + && tar -zxvf quarto.tar.gz -C /opt/quarto/ --strip-components=1 \ + && rm quarto.tar.gz + +# +# Second stage: Conda environment +# +FROM condaforge/mambaforge:23.11.0-0 +COPY --from=quarto /opt/quarto /opt/quarto +ENV PATH="${PATH}:/opt/quarto/bin" +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + pkg-config \ + && apt-get clean + +# Install packages using Mamba; also remove static libraries, python bytecode +# files and javascript source maps that are not required for execution +COPY environment.yml ./ +RUN mamba env update --name base --file environment.yml \ + && mamba clean --all --force-pkgs-dirs --yes \ + && find /opt/conda -follow -type f -name '*.a' -delete \ + && find /opt/conda -follow -type f -name '*.pyc' -delete \ + && find /opt/conda -follow -type f -name '*.js.map' -delete + +# Set environment variable for leidenalg-related ARM64 issue +ENV LD_PRELOAD=/opt/conda/lib/libgomp.so.1 + +CMD /bin/bash + +LABEL \ + authors = "Erik Fasterius, Christophe Avenel" \ + description = "Dockerfile for nf-core/spatialtranscriptomics report modules" diff --git a/env/environment.yml b/env/environment.yml new file mode 100644 index 0000000..c376fb2 --- /dev/null +++ b/env/environment.yml @@ -0,0 +1,19 @@ +channels: + - conda-forge + - bioconda +dependencies: + - python=3.10 + - jupyter=1.0.0 + - leidenalg=0.9.1 + - papermill=2.3.4 + - pip=23.0.1 + - gcc=13.2.0 + - libgdal=3.8.3 + - gxx=13.2.0 + - imagecodecs=2024.1.1 + - pip: + - scanpy==1.10.0 + - squidpy==1.4.1 + - spatialdata==0.1.2 + - spatialdata-io==0.1.2 + - spatialdata-plot==0.2.1 diff --git a/env/reports/Dockerfile b/env/reports/Dockerfile deleted file mode 100644 index e1cb40e..0000000 --- a/env/reports/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -# First stage: multi-platform Quarto image -FROM jdutant/quarto-minimal:1.3.313 as quarto - -# Second stage: multi-platform Mamba image -FROM condaforge/mambaforge:23.1.0-1 - -# Copy Quarto installation from first stage and add to PATH -COPY --from=quarto /opt/quarto /opt/quarto -ENV PATH="${PATH}:/opt/quarto/bin" - -# Install packages using Mamba; also remove static libraries, python bytecode -# files and javascript source maps that are not required for execution -COPY environment.yml ./ -RUN mamba env update --name base --file environment.yml \ - && mamba clean --all --force-pkgs-dirs --yes \ - && find /opt/conda -follow -type f -name '*.a' -delete \ - && find /opt/conda -follow -type f -name '*.pyc' -delete \ - && find /opt/conda -follow -type f -name '*.js.map' -delete - -CMD /bin/bash - -LABEL \ - authors = "Erik Fasterius, Christophe Avenel" \ - description = "Dockerfile for nf-core/spatialtranscriptomics report modules" diff --git a/env/reports/environment.yml b/env/reports/environment.yml deleted file mode 100644 index f8f1923..0000000 --- a/env/reports/environment.yml +++ /dev/null @@ -1,11 +0,0 @@ -channels: - - conda-forge - - bioconda -dependencies: - - jupyter=1.0.0 - - leidenalg=0.9.1 - - papermill=2.3.4 - - pip=23.0.1 - - scanpy=1.9.6 - - pip: - - SpatialDE==1.1.3 diff --git a/env/st_spatial_de/environment.yml b/env/st_spatial_de/environment.yml deleted file mode 100644 index 28457d1..0000000 --- a/env/st_spatial_de/environment.yml +++ /dev/null @@ -1,13 +0,0 @@ -channels: - - conda-forge - - bioconda - - defaults -dependencies: - - quarto=1.3.353 - - jupyter=1.0.0 - - leidenalg=0.9.1 - - papermill=2.3.4 - - pip=23.0.1 - - scanpy=1.9.3 - - pip: - - SpatialDE==1.1.3 diff --git a/lib/NfcoreTemplate.groovy b/lib/NfcoreTemplate.groovy deleted file mode 100755 index e248e4c..0000000 --- a/lib/NfcoreTemplate.groovy +++ /dev/null @@ -1,356 +0,0 @@ -// -// This file holds several functions used within the nf-core pipeline template. -// - -import org.yaml.snakeyaml.Yaml -import groovy.json.JsonOutput -import nextflow.extension.FilesEx - -class NfcoreTemplate { - - // - // Check AWS Batch related parameters have been specified correctly - // - public static void awsBatch(workflow, params) { - if (workflow.profile.contains('awsbatch')) { - // Check params.awsqueue and params.awsregion have been set if running on AWSBatch - assert (params.awsqueue && params.awsregion) : "Specify correct --awsqueue and --awsregion parameters on AWSBatch!" - // Check outdir paths to be S3 buckets if running on AWSBatch - assert params.outdir.startsWith('s3:') : "Outdir not on S3 - specify S3 Bucket to run on AWSBatch!" - } - } - - // - // Warn if a -profile or Nextflow config has not been provided to run the pipeline - // - public static void checkConfigProvided(workflow, log) { - if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { - log.warn "[$workflow.manifest.name] You are attempting to run the pipeline without any custom configuration!\n\n" + - "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + - " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + - " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + - " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + - "Please refer to the quick start section and usage docs for the pipeline.\n " - } - } - - // - // Generate version string - // - public static String version(workflow) { - String version_string = "" - - if (workflow.manifest.version) { - def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' - version_string += "${prefix_v}${workflow.manifest.version}" - } - - if (workflow.commitId) { - def git_shortsha = workflow.commitId.substring(0, 7) - version_string += "-g${git_shortsha}" - } - - return version_string - } - - // - // Construct and send completion email - // - public static void email(workflow, params, summary_params, projectDir, log, multiqc_report=[]) { - - // Set up the e-mail variables - def subject = "[$workflow.manifest.name] Successful: $workflow.runName" - if (!workflow.success) { - subject = "[$workflow.manifest.name] FAILED: $workflow.runName" - } - - def summary = [:] - for (group in summary_params.keySet()) { - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['Date Started'] = workflow.start - misc_fields['Date Completed'] = workflow.complete - misc_fields['Pipeline script file path'] = workflow.scriptFile - misc_fields['Pipeline script hash ID'] = workflow.scriptId - if (workflow.repository) misc_fields['Pipeline repository Git URL'] = workflow.repository - if (workflow.commitId) misc_fields['Pipeline repository Git Commit'] = workflow.commitId - if (workflow.revision) misc_fields['Pipeline Git branch/tag'] = workflow.revision - misc_fields['Nextflow Version'] = workflow.nextflow.version - misc_fields['Nextflow Build'] = workflow.nextflow.build - misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp - - def email_fields = [:] - email_fields['version'] = NfcoreTemplate.version(workflow) - email_fields['runName'] = workflow.runName - email_fields['success'] = workflow.success - email_fields['dateComplete'] = workflow.complete - email_fields['duration'] = workflow.duration - email_fields['exitStatus'] = workflow.exitStatus - email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - email_fields['errorReport'] = (workflow.errorReport ?: 'None') - email_fields['commandLine'] = workflow.commandLine - email_fields['projectDir'] = workflow.projectDir - email_fields['summary'] = summary << misc_fields - - // On success try attach the multiqc report - def mqc_report = null - try { - if (workflow.success) { - mqc_report = multiqc_report.getVal() - if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { - if (mqc_report.size() > 1) { - log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" - } - mqc_report = mqc_report[0] - } - } - } catch (all) { - if (multiqc_report) { - log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" - } - } - - // Check if we are only sending emails on failure - def email_address = params.email - if (!params.email && params.email_on_fail && !workflow.success) { - email_address = params.email_on_fail - } - - // Render the TXT template - def engine = new groovy.text.GStringTemplateEngine() - def tf = new File("$projectDir/assets/email_template.txt") - def txt_template = engine.createTemplate(tf).make(email_fields) - def email_txt = txt_template.toString() - - // Render the HTML template - def hf = new File("$projectDir/assets/email_template.html") - def html_template = engine.createTemplate(hf).make(email_fields) - def email_html = html_template.toString() - - // Render the sendmail template - def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit - def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "$projectDir", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] - def sf = new File("$projectDir/assets/sendmail_template.txt") - def sendmail_template = engine.createTemplate(sf).make(smail_fields) - def sendmail_html = sendmail_template.toString() - - // Send the HTML e-mail - Map colors = logColours(params.monochrome_logs) - if (email_address) { - try { - if (params.plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } - // Try to send HTML e-mail using sendmail - def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") - sendmail_tf.withWriter { w -> w << sendmail_html } - [ 'sendmail', '-t' ].execute() << sendmail_html - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" - } catch (all) { - // Catch failures and try with plaintext - def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] - if ( mqc_report != null && mqc_report.size() <= max_multiqc_email_size.toBytes() ) { - mail_cmd += [ '-A', mqc_report ] - } - mail_cmd.execute() << email_html - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (mail)-" - } - } - - // Write summary e-mail HTML to a file - def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") - output_hf.withWriter { w -> w << email_html } - FilesEx.copyTo(output_hf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.html"); - output_hf.delete() - - // Write summary e-mail TXT to a file - def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") - output_tf.withWriter { w -> w << email_txt } - FilesEx.copyTo(output_tf.toPath(), "${params.outdir}/pipeline_info/pipeline_report.txt"); - output_tf.delete() - } - - // - // Construct and send a notification to a web server as JSON - // e.g. Microsoft Teams and Slack - // - public static void IM_notification(workflow, params, summary_params, projectDir, log) { - def hook_url = params.hook_url - - def summary = [:] - for (group in summary_params.keySet()) { - summary << summary_params[group] - } - - def misc_fields = [:] - misc_fields['start'] = workflow.start - misc_fields['complete'] = workflow.complete - misc_fields['scriptfile'] = workflow.scriptFile - misc_fields['scriptid'] = workflow.scriptId - if (workflow.repository) misc_fields['repository'] = workflow.repository - if (workflow.commitId) misc_fields['commitid'] = workflow.commitId - if (workflow.revision) misc_fields['revision'] = workflow.revision - misc_fields['nxf_version'] = workflow.nextflow.version - misc_fields['nxf_build'] = workflow.nextflow.build - misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp - - def msg_fields = [:] - msg_fields['version'] = NfcoreTemplate.version(workflow) - msg_fields['runName'] = workflow.runName - msg_fields['success'] = workflow.success - msg_fields['dateComplete'] = workflow.complete - msg_fields['duration'] = workflow.duration - msg_fields['exitStatus'] = workflow.exitStatus - msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') - msg_fields['errorReport'] = (workflow.errorReport ?: 'None') - msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") - msg_fields['projectDir'] = workflow.projectDir - msg_fields['summary'] = summary << misc_fields - - // Render the JSON template - def engine = new groovy.text.GStringTemplateEngine() - // Different JSON depending on the service provider - // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format - def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" - def hf = new File("$projectDir/assets/${json_path}") - def json_template = engine.createTemplate(hf).make(msg_fields) - def json_message = json_template.toString() - - // POST - def post = new URL(hook_url).openConnection(); - post.setRequestMethod("POST") - post.setDoOutput(true) - post.setRequestProperty("Content-Type", "application/json") - post.getOutputStream().write(json_message.getBytes("UTF-8")); - def postRC = post.getResponseCode(); - if (! postRC.equals(200)) { - log.warn(post.getErrorStream().getText()); - } - } - - // - // Dump pipeline parameters in a json file - // - public static void dump_parameters(workflow, params) { - def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') - def filename = "params_${timestamp}.json" - def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") - def jsonStr = JsonOutput.toJson(params) - temp_pf.text = JsonOutput.prettyPrint(jsonStr) - - FilesEx.copyTo(temp_pf.toPath(), "${params.outdir}/pipeline_info/params_${timestamp}.json") - temp_pf.delete() - } - - // - // Print pipeline summary on completion - // - public static void summary(workflow, params, log) { - Map colors = logColours(params.monochrome_logs) - if (workflow.success) { - if (workflow.stats.ignoredCount == 0) { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" - } else { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-" - } - } else { - log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed with errors${colors.reset}-" - } - } - - // - // ANSII Colours used for terminal logging - // - public static Map logColours(Boolean monochrome_logs) { - Map colorcodes = [:] - - // Reset / Meta - colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" - colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" - colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" - colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" - colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" - colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" - colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" - - // Regular Colors - colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" - colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" - colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" - colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" - colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" - colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" - colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" - colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" - - // Bold - colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" - colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" - colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" - colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" - colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" - colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" - colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" - colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" - - // Underline - colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" - colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" - colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" - colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" - colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" - colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" - colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" - colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" - - // High Intensity - colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" - colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" - colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" - colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" - colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" - colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" - colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" - colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" - - // Bold High Intensity - colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" - colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" - colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" - colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" - colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" - colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" - colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" - colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" - - return colorcodes - } - - // - // Does what is says on the tin - // - public static String dashedLine(monochrome_logs) { - Map colors = logColours(monochrome_logs) - return "-${colors.dim}----------------------------------------------------${colors.reset}-" - } - - // - // nf-core logo - // - public static String logo(workflow, monochrome_logs) { - Map colors = logColours(monochrome_logs) - String workflow_version = NfcoreTemplate.version(workflow) - String.format( - """\n - ${dashedLine(monochrome_logs)} - ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} - ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} - ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} - ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} - ${colors.green}`._,._,\'${colors.reset} - ${colors.purple} ${workflow.manifest.name} ${workflow_version}${colors.reset} - ${dashedLine(monochrome_logs)} - """.stripIndent() - ) - } -} diff --git a/lib/Utils.groovy b/lib/Utils.groovy deleted file mode 100644 index 9d9d635..0000000 --- a/lib/Utils.groovy +++ /dev/null @@ -1,55 +0,0 @@ -// -// This file holds several Groovy functions that could be useful for any Nextflow pipeline -// - -import org.yaml.snakeyaml.Yaml - -class Utils { - - public static List DOWNSTREAM_REQUIRED_SPACERANGER_FILES = [ - "raw_feature_bc_matrix.h5", - "tissue_positions.csv", - "scalefactors_json.json", - "tissue_hires_image.png", - "tissue_lowres_image.png" - ] - - // - // When running with -profile conda, warn if channels have not been set-up appropriately - // - public static void checkCondaChannels(log) { - Yaml parser = new Yaml() - def channels = [] - try { - def config = parser.load("conda config --show channels".execute().text) - channels = config.channels - } catch(NullPointerException | IOException e) { - log.warn "Could not verify conda channel configuration." - return - } - - // Check that all channels are present - // This channel list is ordered by required channel priority. - def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] - def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean - - // Check that they are in the right order - def channel_priority_violation = false - def n = required_channels_in_order.size() - for (int i = 0; i < n - 1; i++) { - channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) - } - - if (channels_missing | channel_priority_violation) { - log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + - " There is a problem with your Conda configuration!\n\n" + - " You will need to set-up the conda-forge and bioconda channels correctly.\n" + - " Please refer to https://bioconda.github.io/\n" + - " The observed channel order is \n" + - " ${channels}\n" + - " but the following channel order is required:\n" + - " ${required_channels_in_order}\n" + - "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - } - } -} diff --git a/lib/WorkflowMain.groovy b/lib/WorkflowMain.groovy deleted file mode 100755 index 85469a7..0000000 --- a/lib/WorkflowMain.groovy +++ /dev/null @@ -1,65 +0,0 @@ -// -// This file holds several functions specific to the main.nf workflow in the nf-core/spatialtranscriptomics pipeline -// - -import nextflow.Nextflow - -class WorkflowMain { - - // - // Citation string for pipeline - // - public static String citation(workflow) { - return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + - // TODO nf-core: Add Zenodo DOI for pipeline after first release - //"* The pipeline\n" + - //" https://doi.org/10.5281/zenodo.XXXXXXX\n\n" + - "* The nf-core framework\n" + - " https://doi.org/10.1038/s41587-020-0439-x\n\n" + - "* Software dependencies\n" + - " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" - } - - // - // Validate parameters and print summary to screen - // - public static void initialise(workflow, params, log, args) { - - // Print workflow version and exit on --version - if (params.version) { - String workflow_version = NfcoreTemplate.version(workflow) - log.info "${workflow.manifest.name} ${workflow_version}" - System.exit(0) - } - - // Check that a -profile or Nextflow config has been provided to run the pipeline - NfcoreTemplate.checkConfigProvided(workflow, log) - // Check that the profile doesn't contain spaces and doesn't end with a trailing comma - checkProfile(workflow.profile, args, log) - - // Check that conda channels are set-up correctly - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - Utils.checkCondaChannels(log) - } - - // Check AWS batch settings - NfcoreTemplate.awsBatch(workflow, params) - - // Check input has been provided - if (!params.input) { - Nextflow.error("Please provide an input samplesheet to the pipeline e.g. '--input samplesheet.csv'") - } - } - - // - // Exit pipeline if --profile contains spaces - // - private static void checkProfile(profile, args, log) { - if (profile.endsWith(',')) { - Nextflow.error "Profile cannot end with a trailing comma. Please remove the comma from the end of the profile string.\nHint: A common mistake is to provide multiple values to `-profile` separated by spaces. Please use commas to separate profiles instead,e.g., `-profile docker,test`." - } - if (args[0]) { - log.warn "nf-core pipelines do not accept positional arguments. The positional argument `${args[0]}` has been detected.\n Hint: A common mistake is to provide multiple values to `-profile` separated by spaces. Please use commas to separate profiles instead,e.g., `-profile docker,test`." - } - } -} diff --git a/lib/WorkflowSpatialtranscriptomics.groovy b/lib/WorkflowSpatialtranscriptomics.groovy deleted file mode 100755 index 99fb1ad..0000000 --- a/lib/WorkflowSpatialtranscriptomics.groovy +++ /dev/null @@ -1,107 +0,0 @@ -// -// This file holds several functions specific to the workflow/spatialtranscriptomics.nf in the nf-core/spatialtranscriptomics pipeline -// - -import nextflow.Nextflow -import groovy.text.SimpleTemplateEngine - -class WorkflowSpatialtranscriptomics { - - // - // Check and validate parameters - // - public static void initialise(params, log) { - - // if (!params.fasta) { - // Nextflow.error "Genome fasta file not specified with e.g. '--fasta genome.fa' or via a detectable config file." - // } - } - - // - // Get workflow summary for MultiQC - // - public static String paramsSummaryMultiqc(workflow, summary) { - String summary_section = '' - for (group in summary.keySet()) { - def group_params = summary.get(group) // This gets the parameters of that particular group - if (group_params) { - summary_section += "

    $group

    \n" - summary_section += "
    \n" - for (param in group_params.keySet()) { - summary_section += "
    $param
    ${group_params.get(param) ?: 'N/A'}
    \n" - } - summary_section += "
    \n" - } - } - - String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" - yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" - yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" - yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" - yaml_file_text += "plot_type: 'html'\n" - yaml_file_text += "data: |\n" - yaml_file_text += "${summary_section}" - return yaml_file_text - } - - // - // Generate methods description for MultiQC - // - - public static String toolCitationText(params) { - - def citation_text = [ - "Tools used in the workflow included:", - "AnnData (Virshup et al. 2021),", - "FastQC (Andrews 2010),", - "MultiQC (Ewels et al. 2016),", - "Quarto (Allaire et al. 2022),", - "Scanpy (Wolf et al. 2018),", - "Space Ranger (10x Genomics) and", - "SpatialDE (Svensson et al. 2018)." - ].join(' ').trim() - - return citation_text - } - - public static String toolBibliographyText(params) { - - def reference_text = [ - "
  • Virshup I, Rybakov S, Theis FJ, Angerer P, Wolf FA. bioRxiv 2021.12.16.473007; doi: 10.1101/2021.12.16.473007
    • ", - "
  • Andrews S, (2010) FastQC, URL: bioinformatics.babraham.ac.uk.
    • ", - "
  • Ewels, P., Magnusson, M., Lundin, S., & Käller, M. (2016). MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics , 32(19), 3047–3048. doi: 10.1093/bioinformatics/btw354
    • ", - "
  • Allaire J, Teague C, Scheidegger C, Xie Y, Dervieux C. Quarto (2022). doi: 10.5281/zenodo.5960048
    • ", - "
  • Wolf F, Angerer P, Theis F. SCANPY: large-scale single-cell gene expression data analysis. Genome Biol 19, 15 (2018). doi: 10.1186/s13059-017-1382-0
    • ", - "
  • 10x Genomics Space Ranger 2.1.0, URL: 10xgenomics.com/support/software/space-ranger
    • ", - "
  • Svensson V, Teichmann S, Stegle O. SpatialDE: identification of spatially variable genes. Nat Methods 15, 343–346 (2018). doi: 10.1038/nmeth.4636
    • ", - ].join(' ').trim() - - return reference_text - } - - public static String methodsDescriptionText(run_workflow, mqc_methods_yaml, params) { - // Convert to a named map so can be used as with familar NXF ${workflow} variable syntax in the MultiQC YML file - def meta = [:] - meta.workflow = run_workflow.toMap() - meta["manifest_map"] = run_workflow.manifest.toMap() - - // Pipeline DOI - meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" - meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " - - // Tool references - meta["tool_citations"] = "" - meta["tool_bibliography"] = "" - meta["tool_citations"] = toolCitationText(params).replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".") - meta["tool_bibliography"] = toolBibliographyText(params) - - - def methods_text = mqc_methods_yaml.text - - def engine = new SimpleTemplateEngine() - def description_html = engine.createTemplate(methods_text).make(meta) - - return description_html - } - -} diff --git a/lib/commons-csv-1.9.0.jar b/lib/commons-csv-1.9.0.jar deleted file mode 100644 index 0e3f678..0000000 Binary files a/lib/commons-csv-1.9.0.jar and /dev/null differ diff --git a/main.nf b/main.nf index ae2df75..8afe21f 100644 --- a/main.nf +++ b/main.nf @@ -11,58 +11,85 @@ nextflow.enable.dsl = 2 - /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - VALIDATE & PRINT PARAMETER SUMMARY + IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS / WORKFLOWS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { validateParameters; paramsHelp } from 'plugin/nf-validation' - -// Print help message if needed -if (params.help) { - def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) - def citation = '\n' + WorkflowMain.citation(workflow) + '\n' - def String command = "nextflow run ${workflow.manifest.name} --input samplesheet.csv --genome GRCh37 -profile docker" - log.info logo + paramsHelp(command) + citation + NfcoreTemplate.dashedLine(params.monochrome_logs) - System.exit(0) -} - -// Validate input parameters -if (params.validate_params) { - validateParameters() -} - -WorkflowMain.initialise(workflow, params, log, args) +include { SPATIALTRANSCRIPTOMICS } from './workflows/spatialtranscriptomics' +include { PIPELINE_INITIALISATION } from './subworkflows/local/utils_nfcore_spatialtranscriptomics_pipeline' +include { PIPELINE_COMPLETION } from './subworkflows/local/utils_nfcore_spatialtranscriptomics_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - NAMED WORKFLOW FOR PIPELINE + NAMED WORKFLOWS FOR PIPELINE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { ST } from './workflows/spatialtranscriptomics' - // -// WORKFLOW: Run main nf-core/spatialtranscriptomics analysis pipeline +// WORKFLOW: Run main analysis pipeline depending on type of input // workflow NFCORE_SPATIALTRANSCRIPTOMICS { - ST () -} + take: + samplesheet // file: samplesheet read in from --input + + main: + + // + // WORKFLOW: Run pipeline + // + SPATIALTRANSCRIPTOMICS ( + samplesheet + ) + + emit: + multiqc_report = SPATIALTRANSCRIPTOMICS.out.multiqc_report // channel: /path/to/multiqc_report.html + +} /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - RUN ALL WORKFLOWS + RUN MAIN WORKFLOW ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -// -// WORKFLOW: Execute a single named workflow for the pipeline -// See: https://github.com/nf-core/rnaseq/issues/619 -// workflow { - NFCORE_SPATIALTRANSCRIPTOMICS () + + main: + + // + // SUBWORKFLOW: Run initialisation tasks + // + PIPELINE_INITIALISATION ( + params.version, + params.help, + params.validate_params, + params.monochrome_logs, + args, + params.outdir, + params.input + ) + + // + // WORKFLOW: Run main workflow + // + NFCORE_SPATIALTRANSCRIPTOMICS ( + params.input + ) + + // + // SUBWORKFLOW: Run completion tasks + // + PIPELINE_COMPLETION ( + params.email, + params.email_on_fail, + params.plaintext_email, + params.outdir, + params.monochrome_logs, + params.hook_url, + NFCORE_SPATIALTRANSCRIPTOMICS.out.multiqc_report + ) } /* diff --git a/modules.json b/modules.json index c114074..e071f8c 100644 --- a/modules.json +++ b/modules.json @@ -5,11 +5,6 @@ "https://github.com/nf-core/modules.git": { "modules": { "nf-core": { - "custom/dumpsoftwareversions": { - "branch": "master", - "git_sha": "8ec825f465b9c17f9d83000022995b4f7de6fe93", - "installed_by": ["modules"] - }, "fastqc": { "branch": "master", "git_sha": "f4ae1d942bd50c5c0b9bd2de1393ce38315ba57c", @@ -17,20 +12,45 @@ }, "multiqc": { "branch": "master", - "git_sha": "9e71d8519dfbfc328c078bba14d4bd4c99e39a94", + "git_sha": "b7ebe95761cd389603f9cc0e0dc384c0f663815a", "installed_by": ["modules"] }, + "quartonotebook": { + "branch": "master", + "git_sha": "93b7e1bf63944488fe77ad490a9de62a73959bed", + "installed_by": ["modules"], + "patch": "modules/nf-core/quartonotebook/quartonotebook.diff" + }, "spaceranger/count": { "branch": "master", - "git_sha": "3bd057bfdfb64578636ff3ae7f7cb8eeab3c0cb6", + "git_sha": "2f0ef0cd414ea43e33625023c72b6af936dce63d", "installed_by": ["modules"] }, "untar": { "branch": "master", - "git_sha": "e719354ba77df0a1bd310836aa2039b45c29d620", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", "installed_by": ["modules"] } } + }, + "subworkflows": { + "nf-core": { + "utils_nextflow_pipeline": { + "branch": "master", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "installed_by": ["subworkflows"] + }, + "utils_nfcore_pipeline": { + "branch": "master", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "installed_by": ["subworkflows"] + }, + "utils_nfvalidation_plugin": { + "branch": "master", + "git_sha": "5caf7640a9ef1d18d765d55339be751bb0969dfa", + "installed_by": ["subworkflows"] + } + } } } } diff --git a/modules/local/read_data.nf b/modules/local/read_data.nf new file mode 100644 index 0000000..613efd9 --- /dev/null +++ b/modules/local/read_data.nf @@ -0,0 +1,48 @@ +// +// Read ST 10x visium and SC 10x data with spatialdata_io and save to `SpatialData` file +// +process READ_DATA { + + tag "${meta.id}" + label 'process_low' + + container "docker.io/erikfas/spatialtranscriptomics" + + input: + tuple val (meta), path("${meta.id}/*") + + output: + tuple val(meta), path("sdata_raw.zarr"), emit: sdata_raw + path("versions.yml") , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + exit 1, "The READ_DATA module does not support Conda/Mamba, please use Docker / Singularity / Podman instead." + } + """ + # Fix required directory structure + mkdir "${meta.id}/spatial" + mv "${meta.id}/scalefactors_json.json" \\ + "${meta.id}/tissue_hires_image.png" \\ + "${meta.id}/tissue_lowres_image.png" \\ + "${meta.id}/tissue_positions.csv" \\ + "${meta.id}/spatial/" + + # Set environment variables + export XDG_CACHE_HOME="./.xdg_cache_home" + export XDG_DATA_HOME="./.xdg_data_home" + + # Execute read data script + read_data.py \\ + --SRCountDir "${meta.id}" \\ + --output_sdata sdata_raw.zarr + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + spatialdata_io: \$(python -c "import spatialdata_io; print(spatialdata_io.__version__)") + END_VERSIONS + """ +} diff --git a/modules/local/st_clustering.nf b/modules/local/st_clustering.nf deleted file mode 100644 index 0819c7a..0000000 --- a/modules/local/st_clustering.nf +++ /dev/null @@ -1,49 +0,0 @@ -// -// Dimensionality reduction and clustering -// -process ST_CLUSTERING { - - // TODO: Update Conda directive when Quarto/Pandoc works on ARM64 - - tag "${meta.id}" - label 'process_low' - - conda "conda-forge::quarto=1.3.353 conda-forge::scanpy=1.9.3 conda-forge::papermill=2.3.4 conda-forge::jupyter=1.0.0 conda-forge::leidenalg=0.9.1" - container "docker.io/erikfas/spatialtranscriptomics" - - // Exit if running this module with -profile conda / -profile mamba on ARM64 - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - architecture = System.getProperty("os.arch") - if (architecture == "arm64" || architecture == "aarch64") { - exit 1, "The ST_CLUSTERING module does not support Conda on ARM64. Please use Docker / Singularity / Podman instead." - } - } - - input: - path(report) - path(report_template) - tuple val(meta), path(st_adata_filtered) - - output: - tuple val(meta), path("st_adata_processed.h5ad"), emit: st_adata_processed - tuple val(meta), path("st_clustering.html") , emit: html - path("versions.yml") , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - """ - quarto render ${report} \ - -P input_adata_filtered:${st_adata_filtered} \ - -P cluster_resolution:${params.st_cluster_resolution} \ - -P n_hvgs:${params.st_cluster_n_hvgs} \ - -P output_adata_processed:st_adata_processed.h5ad - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - quarto: \$(quarto -v) - scanpy: \$(python -c "import scanpy; print(scanpy.__version__)") - END_VERSIONS - """ -} diff --git a/modules/local/st_quality_controls.nf b/modules/local/st_quality_controls.nf deleted file mode 100644 index f52b6bb..0000000 --- a/modules/local/st_quality_controls.nf +++ /dev/null @@ -1,53 +0,0 @@ -// -// Quality controls and filtering -// -process ST_QUALITY_CONTROLS { - - // TODO: Update Conda directive when Quarto/Pandoc works on ARM64 - - tag "${meta.id}" - label 'process_low' - - conda "conda-forge::quarto=1.3.353 conda-forge::scanpy=1.9.3 conda-forge::papermill=2.3.4 conda-forge::jupyter=1.0.0" - container "docker.io/erikfas/spatialtranscriptomics" - - // Exit if running this module with -profile conda / -profile mamba on ARM64 - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - architecture = System.getProperty("os.arch") - if (architecture == "arm64" || architecture == "aarch64") { - exit 1, "The ST_QUALITY_CONTROLS module does not support Conda on ARM64. Please use Docker / Singularity / Podman instead." - } - } - - input: - path(report) - path(report_template) - tuple val(meta), path(st_adata_raw) - - output: - tuple val(meta), path("st_adata_filtered.h5ad") , emit: st_adata_filtered - tuple val(meta), path("st_quality_controls.html"), emit: html - path("versions.yml") , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - """ - quarto render ${report} \ - -P input_adata_raw:${st_adata_raw} \ - -P min_counts:${params.st_qc_min_counts} \ - -P min_genes:${params.st_qc_min_genes} \ - -P min_spots:${params.st_qc_min_spots} \ - -P mito_threshold:${params.st_qc_mito_threshold} \ - -P ribo_threshold:${params.st_qc_ribo_threshold} \ - -P hb_threshold:${params.st_qc_hb_threshold} \ - -P output_adata_filtered:st_adata_filtered.h5ad - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - quarto: \$(quarto -v) - scanpy: \$(python -c "import scanpy; print(scanpy.__version__)") - END_VERSIONS - """ -} diff --git a/modules/local/st_read_data.nf b/modules/local/st_read_data.nf deleted file mode 100644 index a39c9c3..0000000 --- a/modules/local/st_read_data.nf +++ /dev/null @@ -1,42 +0,0 @@ -// -// Read ST 10x visium and SC 10x data with Scanpy and save to `anndata` file -// -process ST_READ_DATA { - - tag "${meta.id}" - label 'process_low' - - conda "conda-forge::scanpy=1.7.2 conda-forge::matplotlib=3.6.3 conda-forge::pandas=1.5.3" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/scanpy:1.7.2--pyhdfd78af_0' : - 'biocontainers/scanpy:1.7.2--pyhdfd78af_0' }" - - input: - tuple val (meta), path("${meta.id}/*") - - output: - tuple val(meta), path("st_adata_raw.h5ad"), emit: st_adata_raw - path("versions.yml") , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - """ - mkdir "${meta.id}/spatial" - mv "${meta.id}/scalefactors_json.json" \\ - "${meta.id}/tissue_hires_image.png" \\ - "${meta.id}/tissue_lowres_image.png" \\ - "${meta.id}/tissue_positions.csv" \\ - "${meta.id}/spatial/" - - read_st_data.py \\ - --SRCountDir "${meta.id}" \\ - --outAnnData st_adata_raw.h5ad - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - scanpy: \$(python -c "import scanpy; print(scanpy.__version__)") - END_VERSIONS - """ -} diff --git a/modules/local/st_spatial_de.nf b/modules/local/st_spatial_de.nf deleted file mode 100644 index 249fda1..0000000 --- a/modules/local/st_spatial_de.nf +++ /dev/null @@ -1,49 +0,0 @@ -// -// Spatial differential expression -// -process ST_SPATIAL_DE { - - // TODO: Update Conda directive when Quarto/Pandoc works on ARM64 - - tag "${meta.id}" - label 'process_medium' - - conda "env/st_spatial_de/environment.yml" - container "docker.io/erikfas/spatialtranscriptomics" - - // Exit if running this module with -profile conda / -profile mamba on ARM64 - if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { - architecture = System.getProperty("os.arch") - if (architecture == "arm64" || architecture == "aarch64") { - exit 1, "The ST_SPATIAL_DE module does not support Conda on ARM64. Please use Docker / Singularity / Podman instead." - } - } - input: - path(report) - path(report_template) - tuple val(meta), path(st_adata_processed) - - output: - tuple val(meta), path("*.csv") , emit: degs - tuple val(meta), path("st_spatial_de.html"), emit: html - path("versions.yml") , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - """ - quarto render ${report} \ - -P input_adata_processed:${st_adata_processed} \ - -P n_top_spatial_degs:${params.st_n_top_spatial_degs} \ - -P output_spatial_degs:st_spatial_de.csv - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - quarto: \$(quarto -v) - leidenalg: \$(python -c "import leidenalg; print(leidenalg.version)") - scanpy: \$(python -c "import scanpy; print(scanpy.__version__)") - SpatialDE: \$(python -c "from importlib.metadata import version; print(version('SpatialDE'))") - END_VERSIONS - """ -} diff --git a/modules/nf-core/custom/dumpsoftwareversions/environment.yml b/modules/nf-core/custom/dumpsoftwareversions/environment.yml deleted file mode 100644 index 9b3272b..0000000 --- a/modules/nf-core/custom/dumpsoftwareversions/environment.yml +++ /dev/null @@ -1,7 +0,0 @@ -name: custom_dumpsoftwareversions -channels: - - conda-forge - - bioconda - - defaults -dependencies: - - bioconda::multiqc=1.19 diff --git a/modules/nf-core/custom/dumpsoftwareversions/main.nf b/modules/nf-core/custom/dumpsoftwareversions/main.nf deleted file mode 100644 index f218761..0000000 --- a/modules/nf-core/custom/dumpsoftwareversions/main.nf +++ /dev/null @@ -1,24 +0,0 @@ -process CUSTOM_DUMPSOFTWAREVERSIONS { - label 'process_single' - - // Requires `pyyaml` which does not have a dedicated container but is in the MultiQC container - conda "${moduleDir}/environment.yml" - container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.19--pyhdfd78af_0' : - 'biocontainers/multiqc:1.19--pyhdfd78af_0' }" - - input: - path versions - - output: - path "software_versions.yml" , emit: yml - path "software_versions_mqc.yml", emit: mqc_yml - path "versions.yml" , emit: versions - - when: - task.ext.when == null || task.ext.when - - script: - def args = task.ext.args ?: '' - template 'dumpsoftwareversions.py' -} diff --git a/modules/nf-core/custom/dumpsoftwareversions/meta.yml b/modules/nf-core/custom/dumpsoftwareversions/meta.yml deleted file mode 100644 index 5f15a5f..0000000 --- a/modules/nf-core/custom/dumpsoftwareversions/meta.yml +++ /dev/null @@ -1,37 +0,0 @@ -# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/meta-schema.json -name: custom_dumpsoftwareversions -description: Custom module used to dump software versions within the nf-core pipeline template -keywords: - - custom - - dump - - version -tools: - - custom: - description: Custom module used to dump software versions within the nf-core pipeline template - homepage: https://github.com/nf-core/tools - documentation: https://github.com/nf-core/tools - licence: ["MIT"] -input: - - versions: - type: file - description: YML file containing software versions - pattern: "*.yml" -output: - - yml: - type: file - description: Standard YML file containing software versions - pattern: "software_versions.yml" - - mqc_yml: - type: file - description: MultiQC custom content YML file containing software versions - pattern: "software_versions_mqc.yml" - - versions: - type: file - description: File containing software versions - pattern: "versions.yml" -authors: - - "@drpatelh" - - "@grst" -maintainers: - - "@drpatelh" - - "@grst" diff --git a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py b/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py deleted file mode 100755 index 4a99360..0000000 --- a/modules/nf-core/custom/dumpsoftwareversions/templates/dumpsoftwareversions.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python - - -"""Provide functions to merge multiple versions.yml files.""" - - -import yaml -import platform -from textwrap import dedent - - -def _make_versions_html(versions): - """Generate a tabular HTML output of all versions for MultiQC.""" - html = [ - dedent( - """\\ - - - - - - - - - - """ - ) - ] - for process, tmp_versions in sorted(versions.items()): - html.append("") - for i, (tool, version) in enumerate(sorted(tmp_versions.items())): - html.append( - dedent( - f"""\\ - - - - - - """ - ) - ) - html.append("") - html.append("
    Process Name Software Version
    {process if (i == 0) else ''}{tool}{version}
    ") - return "\\n".join(html) - - -def main(): - """Load all version files and generate merged output.""" - versions_this_module = {} - versions_this_module["${task.process}"] = { - "python": platform.python_version(), - "yaml": yaml.__version__, - } - - with open("$versions") as f: - versions_by_process = ( - yaml.load(f, Loader=yaml.BaseLoader) | versions_this_module - ) - - # aggregate versions by the module name (derived from fully-qualified process name) - versions_by_module = {} - for process, process_versions in versions_by_process.items(): - module = process.split(":")[-1] - try: - if versions_by_module[module] != process_versions: - raise AssertionError( - "We assume that software versions are the same between all modules. " - "If you see this error-message it means you discovered an edge-case " - "and should open an issue in nf-core/tools. " - ) - except KeyError: - versions_by_module[module] = process_versions - - versions_by_module["Workflow"] = { - "Nextflow": "$workflow.nextflow.version", - "$workflow.manifest.name": "$workflow.manifest.version", - } - - versions_mqc = { - "id": "software_versions", - "section_name": "${workflow.manifest.name} Software Versions", - "section_href": "https://github.com/${workflow.manifest.name}", - "plot_type": "html", - "description": "are collected at run time from the software output.", - "data": _make_versions_html(versions_by_module), - } - - with open("software_versions.yml", "w") as f: - yaml.dump(versions_by_module, f, default_flow_style=False) - with open("software_versions_mqc.yml", "w") as f: - yaml.dump(versions_mqc, f, default_flow_style=False) - - with open("versions.yml", "w") as f: - yaml.dump(versions_this_module, f, default_flow_style=False) - - -if __name__ == "__main__": - main() diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test deleted file mode 100644 index b1e1630..0000000 --- a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test +++ /dev/null @@ -1,43 +0,0 @@ -nextflow_process { - - name "Test Process CUSTOM_DUMPSOFTWAREVERSIONS" - script "../main.nf" - process "CUSTOM_DUMPSOFTWAREVERSIONS" - tag "modules" - tag "modules_nfcore" - tag "custom" - tag "dumpsoftwareversions" - tag "custom/dumpsoftwareversions" - - test("Should run without failures") { - when { - process { - """ - def tool1_version = ''' - TOOL1: - tool1: 0.11.9 - '''.stripIndent() - - def tool2_version = ''' - TOOL2: - tool2: 1.9 - '''.stripIndent() - - input[0] = Channel.of(tool1_version, tool2_version).collectFile() - """ - } - } - - then { - assertAll( - { assert process.success }, - { assert snapshot( - process.out.versions, - file(process.out.mqc_yml[0]).readLines()[0..10], - file(process.out.yml[0]).readLines()[0..7] - ).match() - } - ) - } - } -} diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap b/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap deleted file mode 100644 index 5f59a93..0000000 --- a/modules/nf-core/custom/dumpsoftwareversions/tests/main.nf.test.snap +++ /dev/null @@ -1,33 +0,0 @@ -{ - "Should run without failures": { - "content": [ - [ - "versions.yml:md5,76d454d92244589d32455833f7c1ba6d" - ], - [ - "data: \"\\n\\n \\n \\n \\n \\n \\n \\n \\n\\", - " \\n\\n\\n \\n \\n\\", - " \\ \\n\\n\\n\\n \\n \\", - " \\ \\n \\n\\n\\n\\n\\", - " \\n\\n \\n \\n\\", - " \\ \\n\\n\\n\\n\\n\\n \\n\\", - " \\ \\n \\n\\n\\n\\n\\", - " \\n\\n \\n \\n\\" - ], - [ - "CUSTOM_DUMPSOFTWAREVERSIONS:", - " python: 3.11.7", - " yaml: 5.4.1", - "TOOL1:", - " tool1: 0.11.9", - "TOOL2:", - " tool2: '1.9'", - "Workflow:" - ] - ], - "timestamp": "2024-01-09T23:01:18.710682" - } -} \ No newline at end of file diff --git a/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml b/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml deleted file mode 100644 index 405aa24..0000000 --- a/modules/nf-core/custom/dumpsoftwareversions/tests/tags.yml +++ /dev/null @@ -1,2 +0,0 @@ -custom/dumpsoftwareversions: - - modules/nf-core/custom/dumpsoftwareversions/** diff --git a/modules/nf-core/multiqc/environment.yml b/modules/nf-core/multiqc/environment.yml index 7625b75..ca39fb6 100644 --- a/modules/nf-core/multiqc/environment.yml +++ b/modules/nf-core/multiqc/environment.yml @@ -4,4 +4,4 @@ channels: - bioconda - defaults dependencies: - - bioconda::multiqc=1.19 + - bioconda::multiqc=1.21 diff --git a/modules/nf-core/multiqc/main.nf b/modules/nf-core/multiqc/main.nf index 1b9f7c4..47ac352 100644 --- a/modules/nf-core/multiqc/main.nf +++ b/modules/nf-core/multiqc/main.nf @@ -3,8 +3,8 @@ process MULTIQC { conda "${moduleDir}/environment.yml" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/multiqc:1.19--pyhdfd78af_0' : - 'biocontainers/multiqc:1.19--pyhdfd78af_0' }" + 'https://depot.galaxyproject.org/singularity/multiqc:1.21--pyhdfd78af_0' : + 'biocontainers/multiqc:1.21--pyhdfd78af_0' }" input: path multiqc_files, stageAs: "?/*" diff --git a/modules/nf-core/multiqc/tests/main.nf.test.snap b/modules/nf-core/multiqc/tests/main.nf.test.snap index 549ba79..7c3ff58 100644 --- a/modules/nf-core/multiqc/tests/main.nf.test.snap +++ b/modules/nf-core/multiqc/tests/main.nf.test.snap @@ -2,14 +2,14 @@ "multiqc_versions_single": { "content": [ [ - "versions.yml:md5,14e9a2661241abd828f4f06a7b5c222d" + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-31T17:43:40.529579" + "timestamp": "2024-02-29T08:48:55.657331" }, "multiqc_stub": { "content": [ @@ -17,25 +17,25 @@ "multiqc_report.html", "multiqc_data", "multiqc_plots", - "versions.yml:md5,14e9a2661241abd828f4f06a7b5c222d" + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-31T17:45:09.605359" + "timestamp": "2024-02-29T08:49:49.071937" }, "multiqc_versions_config": { "content": [ [ - "versions.yml:md5,14e9a2661241abd828f4f06a7b5c222d" + "versions.yml:md5,21f35ee29416b9b3073c28733efe4b7d" ] ], "meta": { "nf-test": "0.8.4", "nextflow": "23.10.1" }, - "timestamp": "2024-01-31T17:44:53.535994" + "timestamp": "2024-02-29T08:49:25.457567" } -} \ No newline at end of file +} diff --git a/modules/nf-core/quartonotebook/Dockerfile b/modules/nf-core/quartonotebook/Dockerfile new file mode 100644 index 0000000..78d2ab2 --- /dev/null +++ b/modules/nf-core/quartonotebook/Dockerfile @@ -0,0 +1,38 @@ +# +# First stage: Quarto installation +# +FROM ubuntu:20.04 as quarto +ARG QUARTO_VERSION=1.3.433 +ARG TARGETARCH +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + && apt-get clean + +RUN mkdir -p /opt/quarto \ + && curl -o quarto.tar.gz -L "https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-${TARGETARCH}.tar.gz" \ + && tar -zxvf quarto.tar.gz -C /opt/quarto/ --strip-components=1 \ + && rm quarto.tar.gz + +# +# Second stage: Conda environment +# +FROM condaforge/mambaforge:24.1.2-0@sha256:64c45c1a743737f61cf201f54cae974b5c853be94f9c1a84f5e82e0e854f0407 +COPY --from=quarto /opt/quarto /opt/quarto +ENV PATH="${PATH}:/opt/quarto/bin" + +# Install packages using Mamba; also remove static libraries, python bytecode +# files and javascript source maps that are not required for execution +COPY environment.yml ./ +RUN mamba env update --name base --file environment.yml \ + && mamba clean --all --force-pkgs-dirs --yes \ + && find /opt/conda -follow -type f -name '*.a' -delete \ + && find /opt/conda -follow -type f -name '*.pyc' -delete \ + && find /opt/conda -follow -type f -name '*.js.map' -delete + +CMD /bin/bash + +LABEL \ + authors = "Erik Fasterius" \ + description = "Dockerfile for the quartonotebook nf-core module" diff --git a/modules/nf-core/quartonotebook/environment.yml b/modules/nf-core/quartonotebook/environment.yml new file mode 100644 index 0000000..1084ec0 --- /dev/null +++ b/modules/nf-core/quartonotebook/environment.yml @@ -0,0 +1,12 @@ +name: quartonotebook + +channels: + - conda-forge + - bioconda + - defaults + +dependencies: + - conda-forge::jupyter=1.0.0 + - conda-forge::matplotlib=3.4.3 + - conda-forge::papermill=2.4.0 + - conda-forge::r-rmarkdown=2.25 diff --git a/modules/nf-core/quartonotebook/main.nf b/modules/nf-core/quartonotebook/main.nf new file mode 100644 index 0000000..2766dc5 --- /dev/null +++ b/modules/nf-core/quartonotebook/main.nf @@ -0,0 +1,103 @@ +include { dumpParamsYaml; indentCodeBlock } from "./parametrize" + +process QUARTONOTEBOOK { + tag "$meta.id" + label 'process_low' + + container "docker.io/erikfas/spatialtranscriptomics" + + input: + tuple val(meta), path(notebook) + val parameters + path input_files + path extensions + + output: + tuple val(meta), path("*.html") , emit: html + tuple val(meta), path("${notebook}"), emit: notebook + tuple val(meta), path("artifacts/*"), emit: artifacts, optional: true + tuple val(meta), path("params.yml") , emit: params_yaml, optional: true + tuple val(meta), path("_extensions"), emit: extensions, optional: true + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + // Exit if running this module with -profile conda / -profile mamba + // This is because of issues with getting a homogenous environment across + // both AMD64 and ARM64 architectures; please find more information at + // https://github.com/nf-core/modules/pull/4876#discussion_r1483541037. + if (workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1) { + exit 1, "The QUARTONOTEBOOK module does not support Conda/Mamba, please use Docker / Singularity / Podman instead." + } + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + def parametrize = (task.ext.parametrize == null) ? true : task.ext.parametrize + def implicit_params = (task.ext.implicit_params == null) ? true : task.ext.implicit_params + def meta_params = (task.ext.meta_params == null) ? true : task.ext.meta_params + + // Dump parameters to yaml file. + // Using a YAML file over using the CLI params because + // - No issue with escaping + // - Allows passing nested maps instead of just single values + // - Allows running with the language-agnostic `--execute-params` + def params_cmd = "" + def render_args = "" + if (parametrize) { + nb_params = [:] + if (implicit_params) { + nb_params["cpus"] = task.cpus + nb_params["artifact_dir"] = "artifacts" + nb_params["input_dir"] = "./" + } + if (meta_params) { + nb_params["meta"] = meta + } + nb_params += parameters + params_cmd = dumpParamsYaml(nb_params) + render_args = "--execute-params params.yml" + } + """ + # Dump .params.yml heredoc (section will be empty if parametrization is disabled) + ${indentCodeBlock(params_cmd, 4)} + + # Create output directory + mkdir artifacts + + # Set environment variables needed for Quarto rendering + export XDG_CACHE_HOME="./.xdg_cache_home" + export XDG_DATA_HOME="./.xdg_data_home" + + # Set parallelism for BLAS/MKL etc. to avoid over-booking of resources + export MKL_NUM_THREADS="$task.cpus" + export OPENBLAS_NUM_THREADS="$task.cpus" + export OMP_NUM_THREADS="$task.cpus" + export NUMBA_NUM_THREADS="$task.cpus" + + # Render notebook + quarto render \\ + ${notebook} \\ + ${render_args} \\ + ${args} \\ + --output ${prefix}.html + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + quarto: \$(quarto -v) + papermill: \$(papermill --version | cut -f1 -d' ') + END_VERSIONS + """ + + stub: + def args = task.ext.args ?: '' + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.html + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + quarto: \$(quarto -v) + END_VERSIONS + """ +} diff --git a/modules/nf-core/quartonotebook/meta.yml b/modules/nf-core/quartonotebook/meta.yml new file mode 100644 index 0000000..5d95e8b --- /dev/null +++ b/modules/nf-core/quartonotebook/meta.yml @@ -0,0 +1,83 @@ +name: "quartonotebook" +description: Render a Quarto notebook, including parametrization. +keywords: + - quarto + - notebook + - reports + - python + - r +tools: + - quartonotebook: + description: An open-source scientific and technical publishing system. + homepage: https://quarto.org/ + documentation: https://quarto.org/docs/reference/ + tool_dev_url: https://github.com/quarto-dev/quarto-cli + licence: ["MIT"] + - papermill: + description: Parameterize, execute, and analyze notebooks + homepage: https://github.com/nteract/papermill + documentation: http://papermill.readthedocs.io/en/latest/ + tool_dev_url: https://github.com/nteract/papermill + licence: ["BSD 3-clause"] + +input: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]`. + - notebook: + type: file + description: The Quarto notebook to be rendered. + pattern: "*.{qmd}" + - parameters: + type: map + description: | + Groovy map with notebook parameters which will be passed to Quarto to + generate parametrized reports. + - input_files: + type: file + description: One or multiple files serving as input data for the notebook. + pattern: "*" + - extensions: + type: file + description: | + A quarto `_extensions` directory with custom template(s) to be + available for rendering. + pattern: "*" + +output: + - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1', single_end:false ]`. + - html: + type: file + description: HTML report generated by Quarto. + pattern: "*.html" + - notebook: + type: file + description: The original, un-rendered notebook. + pattern: "*.[qmd,ipynb,rmd]" + - artifacts: + type: file + description: Artifacts generated during report rendering. + pattern: "*" + - params_yaml: + type: file + description: Parameters used during report rendering. + pattern: "*" + - extensions: + type: file + description: Quarto extensions used during report rendering. + pattern: "*" + - versions: + type: file + description: File containing software versions. + pattern: "versions.yml" + +authors: + - "@fasterius" +maintainers: + - "@fasterius" diff --git a/modules/nf-core/quartonotebook/parametrize.nf b/modules/nf-core/quartonotebook/parametrize.nf new file mode 100644 index 0000000..b3d8cea --- /dev/null +++ b/modules/nf-core/quartonotebook/parametrize.nf @@ -0,0 +1,36 @@ +import org.yaml.snakeyaml.Yaml +import org.yaml.snakeyaml.DumperOptions + + +/** + * Multiline code blocks need to have the same indentation level + * as the `script:` section. This function re-indents code to the specified level. + */ +def indentCodeBlock(code, n_spaces) { + def indent_str = " ".multiply(n_spaces) + return code.stripIndent().split("\n").join("\n" + indent_str) +} + +/** + * Create a config YAML file from a groovy map + * + * @params task The process' `task` variable + * @returns a line to be inserted in the bash script. + */ +def dumpParamsYaml(params) { + DumperOptions options = new DumperOptions(); + options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK); + def yaml = new Yaml(options) + def yaml_str = yaml.dump(params) + + // Writing the params.yml file directly as follows does not work. + // It only works in 'exec:', but not if there is a `script:` section: + // task.workDir.resolve('params.yml').text = yaml_str + + // Therefore, we inject it into the bash script: + return """\ + cat <<"END_PARAMS_SECTION" > ./params.yml + ${indentCodeBlock(yaml_str, 8)} + END_PARAMS_SECTION + """ +} diff --git a/modules/nf-core/quartonotebook/quartonotebook.diff b/modules/nf-core/quartonotebook/quartonotebook.diff new file mode 100644 index 0000000..018e646 --- /dev/null +++ b/modules/nf-core/quartonotebook/quartonotebook.diff @@ -0,0 +1,18 @@ +Changes in module 'nf-core/quartonotebook' +--- modules/nf-core/quartonotebook/main.nf ++++ modules/nf-core/quartonotebook/main.nf +@@ -4,11 +4,7 @@ + tag "$meta.id" + label 'process_low' + +- // NB: You'll likely want to override this with a container containing all +- // required dependencies for your analyses. You'll at least need Quarto +- // itself, Papermill and whatever language you are running your analyses on; +- // you can see an example in this module's Dockerfile. +- container "docker.io/erikfas/quartonotebook" ++ container "docker.io/erikfas/spatialtranscriptomics" + + input: + tuple val(meta), path(notebook) + +************************************************************ diff --git a/modules/nf-core/quartonotebook/tests/main.nf.test b/modules/nf-core/quartonotebook/tests/main.nf.test new file mode 100644 index 0000000..aeec8b1 --- /dev/null +++ b/modules/nf-core/quartonotebook/tests/main.nf.test @@ -0,0 +1,212 @@ +nextflow_process { + + name "Test Process QUARTONOTEBOOK" + script "../main.nf" + process "QUARTONOTEBOOK" + + tag "modules" + tag "modules_nfcore" + tag "quartonotebook" + + test("test notebook - [qmd:r]") { + + config "./no-parametrization.config" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['generic']['notebooks']['quarto_r'], checkIfExists: true) // Notebook + ] + input[1] = [:] // Parameters + input[2] = [] // Input files + input[3] = [] // Extensions + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + + } + + test("test notebook - [qmd:python]") { + + config "./no-parametrization.config" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['generic']['notebooks']['quarto_python'], checkIfExists: true) // Notebook + ] + input[1] = [] // Parameters + input[2] = [] // Input files + input[3] = [] // Extensions + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.versions, + process.out.artifacts, + process.out.params_yaml, + ).match() }, + { assert path(process.out.html[0][1]).readLines().any { it.contains('Hello world') } } + ) + } + + } + + test("test notebook - parametrized - [qmd:r]") { + + config "./with-parametrization.config" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['generic']['notebooks']['quarto_r'], checkIfExists: true) // Notebook + ] + input[1] = [input_filename: "hello.txt", n_iter: 12] // parameters + input[2] = file(params.test_data['generic']['txt']['hello'], checkIfExists: true) // Input files + input[3] = [] // Extensions + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + + } + + test("test notebook - parametrized - [qmd:python]") { + + config "./with-parametrization.config" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['generic']['notebooks']['quarto_python'], checkIfExists: true) // Notebook + ] + input[1] = [input_filename: "hello.txt", n_iter: 12] // parameters + input[2] = file(params.test_data['generic']['txt']['hello'], checkIfExists: true) // Input files + input[3] = [] // Extensions + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.versions, + process.out.artifacts, + process.out.params_yaml, + ).match() }, + { assert path(process.out.html[0][1]).readLines().any { it.contains('Hello world') } } + ) + } + + } + + test("test notebook - parametrized - [rmd]") { + + config "./with-parametrization.config" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['generic']['notebooks']['rmarkdown'], checkIfExists: true) // notebook + ] + input[1] = [input_filename: "hello.txt", n_iter: 12] // parameters + input[2] = file(params.test_data['generic']['txt']['hello'], checkIfExists: true) // Input files + input[3] = [] // Extensions + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + + } + + test("test notebook - parametrized - [ipynb]") { + + config "./with-parametrization.config" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['generic']['notebooks']['ipython_ipynb'], checkIfExists: true) // notebook + ] + input[1] = [input_filename: "hello.txt", n_iter: 12] // parameters + input[2] = file(params.test_data['generic']['txt']['hello'], checkIfExists: true) // Input files + input[3] = [] // Extensions + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + + } + + test("test notebook - stub - [qmd:r]") { + + config "./no-parametrization.config" + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test' ], // meta map + file(params.test_data['generic']['notebooks']['quarto_r'], checkIfExists: true) // Notebook + ] + input[1] = [:] // Parameters + input[2] = [] // Input files + input[3] = [] // Extensions + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() }, + ) + } + + } + +} \ No newline at end of file diff --git a/modules/nf-core/quartonotebook/tests/main.nf.test.snap b/modules/nf-core/quartonotebook/tests/main.nf.test.snap new file mode 100644 index 0000000..f0f04cb --- /dev/null +++ b/modules/nf-core/quartonotebook/tests/main.nf.test.snap @@ -0,0 +1,433 @@ +{ + "test notebook - stub - [qmd:r]": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "quarto_r.qmd:md5,b3fa8b456efae62495c0b278a4f7694c" + ] + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + "versions.yml:md5,93481281b24bb1b44ecc4387e0957a0e" + ], + "artifacts": [ + + ], + "extensions": [ + + ], + "html": [ + [ + { + "id": "test" + }, + "test.html:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "notebook": [ + [ + { + "id": "test" + }, + "quarto_r.qmd:md5,b3fa8b456efae62495c0b278a4f7694c" + ] + ], + "params_yaml": [ + + ], + "versions": [ + "versions.yml:md5,93481281b24bb1b44ecc4387e0957a0e" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-09T11:06:33.408525" + }, + "test notebook - [qmd:r]": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.html:md5,f09282296a5eee0154665975d842c07e" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "quarto_r.qmd:md5,b3fa8b456efae62495c0b278a4f7694c" + ] + ], + "2": [ + + ], + "3": [ + + ], + "4": [ + + ], + "5": [ + "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + ], + "artifacts": [ + + ], + "extensions": [ + + ], + "html": [ + [ + { + "id": "test" + }, + "test.html:md5,f09282296a5eee0154665975d842c07e" + ] + ], + "notebook": [ + [ + { + "id": "test" + }, + "quarto_r.qmd:md5,b3fa8b456efae62495c0b278a4f7694c" + ] + ], + "params_yaml": [ + + ], + "versions": [ + "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-09T11:05:50.985424" + }, + "test notebook - parametrized - [qmd:python]": { + "content": [ + [ + "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + ], + [ + [ + { + "id": "test" + }, + "artifact.txt:md5,8ddd8be4b179a529afa5f2ffae4b9858" + ] + ], + [ + [ + { + "id": "test" + }, + "params.yml:md5,efd62bc975f429e8749ba787a93042dd" + ] + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-09T09:21:18.194591" + }, + "test notebook - parametrized - [rmd]": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.html:md5,2b2026646ed8b59d49fdcbd54cb3a463" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "rmarkdown_notebook.Rmd:md5,1f5e4efbb41fd499b23c5bea2fc32e68" + ] + ], + "2": [ + [ + { + "id": "test" + }, + "artifact.txt:md5,b10a8db164e0754105b7a99be72e3fe5" + ] + ], + "3": [ + [ + { + "id": "test" + }, + "params.yml:md5,efd62bc975f429e8749ba787a93042dd" + ] + ], + "4": [ + + ], + "5": [ + "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + ], + "artifacts": [ + [ + { + "id": "test" + }, + "artifact.txt:md5,b10a8db164e0754105b7a99be72e3fe5" + ] + ], + "extensions": [ + + ], + "html": [ + [ + { + "id": "test" + }, + "test.html:md5,2b2026646ed8b59d49fdcbd54cb3a463" + ] + ], + "notebook": [ + [ + { + "id": "test" + }, + "rmarkdown_notebook.Rmd:md5,1f5e4efbb41fd499b23c5bea2fc32e68" + ] + ], + "params_yaml": [ + [ + { + "id": "test" + }, + "params.yml:md5,efd62bc975f429e8749ba787a93042dd" + ] + ], + "versions": [ + "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-09T11:06:25.046249" + }, + "test notebook - parametrized - [ipynb]": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.html:md5,d7378ec0d1fd83b44424a68bf03a8fc3" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "ipython_notebook.ipynb:md5,02a206bf6c66396827dd310e7443926d" + ] + ], + "2": [ + + ], + "3": [ + [ + { + "id": "test" + }, + "params.yml:md5,efd62bc975f429e8749ba787a93042dd" + ] + ], + "4": [ + + ], + "5": [ + "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + ], + "artifacts": [ + + ], + "extensions": [ + + ], + "html": [ + [ + { + "id": "test" + }, + "test.html:md5,d7378ec0d1fd83b44424a68bf03a8fc3" + ] + ], + "notebook": [ + [ + { + "id": "test" + }, + "ipython_notebook.ipynb:md5,02a206bf6c66396827dd310e7443926d" + ] + ], + "params_yaml": [ + [ + { + "id": "test" + }, + "params.yml:md5,efd62bc975f429e8749ba787a93042dd" + ] + ], + "versions": [ + "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-09T11:06:30.278412" + }, + "test notebook - [qmd:python]": { + "content": [ + [ + "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + ], + [ + + ], + [ + + ] + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-09T09:21:00.324109" + }, + "test notebook - parametrized - [qmd:r]": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.html:md5,a25cdff28851a163d28669d4e62655af" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "quarto_r.qmd:md5,b3fa8b456efae62495c0b278a4f7694c" + ] + ], + "2": [ + [ + { + "id": "test" + }, + "artifact.txt:md5,b10a8db164e0754105b7a99be72e3fe5" + ] + ], + "3": [ + [ + { + "id": "test" + }, + "params.yml:md5,efd62bc975f429e8749ba787a93042dd" + ] + ], + "4": [ + + ], + "5": [ + "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + ], + "artifacts": [ + [ + { + "id": "test" + }, + "artifact.txt:md5,b10a8db164e0754105b7a99be72e3fe5" + ] + ], + "extensions": [ + + ], + "html": [ + [ + { + "id": "test" + }, + "test.html:md5,a25cdff28851a163d28669d4e62655af" + ] + ], + "notebook": [ + [ + { + "id": "test" + }, + "quarto_r.qmd:md5,b3fa8b456efae62495c0b278a4f7694c" + ] + ], + "params_yaml": [ + [ + { + "id": "test" + }, + "params.yml:md5,efd62bc975f429e8749ba787a93042dd" + ] + ], + "versions": [ + "versions.yml:md5,55e1f767fbd72aae14cbbfb638e38a90" + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-09T11:06:08.013103" + } +} \ No newline at end of file diff --git a/modules/nf-core/quartonotebook/tests/no-parametrization.config b/modules/nf-core/quartonotebook/tests/no-parametrization.config new file mode 100644 index 0000000..f686514 --- /dev/null +++ b/modules/nf-core/quartonotebook/tests/no-parametrization.config @@ -0,0 +1,9 @@ +profiles { + docker { + docker.runOptions = '-u $(id -u):$(id -g)' + } +} + +process { + ext.parametrize = false +} diff --git a/modules/nf-core/quartonotebook/tests/tags.yml b/modules/nf-core/quartonotebook/tests/tags.yml new file mode 100644 index 0000000..638b0ce --- /dev/null +++ b/modules/nf-core/quartonotebook/tests/tags.yml @@ -0,0 +1,2 @@ +quartonotebook: + - "modules/nf-core/quartonotebook/**" diff --git a/modules/nf-core/quartonotebook/tests/with-parametrization.config b/modules/nf-core/quartonotebook/tests/with-parametrization.config new file mode 100644 index 0000000..ab7df66 --- /dev/null +++ b/modules/nf-core/quartonotebook/tests/with-parametrization.config @@ -0,0 +1,5 @@ +profiles { + docker { + docker.runOptions = '-u $(id -u):$(id -g)' + } +} diff --git a/modules/nf-core/spaceranger/count/main.nf b/modules/nf-core/spaceranger/count/main.nf index cac83e0..4f766cb 100644 --- a/modules/nf-core/spaceranger/count/main.nf +++ b/modules/nf-core/spaceranger/count/main.nf @@ -2,7 +2,7 @@ process SPACERANGER_COUNT { tag "$meta.id" label 'process_high' - container "docker.io/nfcore/spaceranger:2.1.0" + container "nf-core/spaceranger:3.0.0" input: tuple val(meta), path(reads), path(image), path(cytaimage), path(darkimage), path(colorizedimage), path(alignment), path(slidefile) diff --git a/modules/nf-core/spaceranger/count/tests/main.nf.test b/modules/nf-core/spaceranger/count/tests/main.nf.test index b751b07..7631d85 100644 --- a/modules/nf-core/spaceranger/count/tests/main.nf.test +++ b/modules/nf-core/spaceranger/count/tests/main.nf.test @@ -2,6 +2,7 @@ nextflow_process { name "Test Process SPACERANGER_COUNT" script "../main.nf" + config "./nextflow.config" process "SPACERANGER_COUNT" tag "modules" @@ -210,7 +211,8 @@ nextflow_process { 'molecule_info.h5', 'barcodes.tsv.gz', 'features.tsv.gz', - 'matrix.mtx.gz' + 'matrix.mtx.gz', + 'cloupe.cloupe' ]} ).match() }, diff --git a/modules/nf-core/spaceranger/count/tests/main.nf.test.snap b/modules/nf-core/spaceranger/count/tests/main.nf.test.snap index ece665f..c13496e 100644 --- a/modules/nf-core/spaceranger/count/tests/main.nf.test.snap +++ b/modules/nf-core/spaceranger/count/tests/main.nf.test.snap @@ -2,53 +2,89 @@ "spaceranger v1 (stub) - homo_sapiens - fasta - gtf - fastq - tif - csv": { "content": [ [ - "versions.yml:md5,038e17e049a72dd3d417d0e221dce732" + "versions.yml:md5,1539e8a9a3d63ce3653920721d1af509" ] ], - "timestamp": "2024-01-09T15:09:24.723008" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-04-02T09:29:02.205153668" }, "spaceranger v2 - homo_sapiens - fasta - gtf - fastq - tif - csv": { "content": [ [ - "versions.yml:md5,038e17e049a72dd3d417d0e221dce732" + "versions.yml:md5,1539e8a9a3d63ce3653920721d1af509" ], [ - "filtered_feature_bc_matrix.h5:md5,509e18ed6b218850e5095124ecc771c1", - "metrics_summary.csv:md5,412caff0fcd9f39cb54671147058de2f", - "possorted_genome_bam.bam:md5,23cd192fcc217d835b8c0afee0619f40", - "possorted_genome_bam.bam.bai:md5,baf623d3e554ba5008304f32414c9fb2", + "clusters.csv:md5,2cc2d0c94ec0af69f03db235f9ea6932", + "clusters.csv:md5,46c12f3845e28f27f2cd580cb004c0ea", + "clusters.csv:md5,4e5f082240b9c9903168842d1f9dbe34", + "clusters.csv:md5,e626eb7049baf591ea49f5d8c305621c", + "clusters.csv:md5,65cfb24fc937e4df903a742c1adf8b08", + "clusters.csv:md5,819a71787618945dacfa2d5301b953b1", + "clusters.csv:md5,5ae17ed02cdb9f61d7ceb0cd6922c9d4", + "clusters.csv:md5,641550bec22e02fff3611087f7fd6e07", + "clusters.csv:md5,9fbe5c79035175bc1899e9a7fc80f7ac", + "clusters.csv:md5,ed0c2dcca15c14a9983407ff9af0daaf", + "differential_expression.csv:md5,d37a8ef21699372ec4a4bdf0c43d71b7", + "differential_expression.csv:md5,ac3181524385c88d38a0fc17d3bdd526", + "differential_expression.csv:md5,557d6dfec7421c392aa6443725608cd1", + "differential_expression.csv:md5,1437fad68d701c97a4a46318aee45575", + "differential_expression.csv:md5,7a2f3d0e90782055580f4903617a7d27", + "differential_expression.csv:md5,41756e9570d07aee6aed710e6a965846", + "differential_expression.csv:md5,62ea7651c3f195d3c960c6c688dca477", + "differential_expression.csv:md5,b630542266c4abb71f4205922340498d", + "differential_expression.csv:md5,0deb97f0be7e72ad73e456092db31e6d", + "differential_expression.csv:md5,3bba8490f753507e7e2e29be759f218b", + "components.csv:md5,568bb9bcb6ee913356fcb4be3fea1911", + "dispersion.csv:md5,e2037b1db404f6e5d8b3144629f2500d", + "features_selected.csv:md5,3ba6d1315ae594963b306d94ba1180e7", + "projection.csv:md5,aef5d71381678d5245e471f3d5a8ab67", + "variance.csv:md5,475a95e51ce66e639ae21d801c455e2b", + "projection.csv:md5,928c0f68a9c773fba590941d3d5af7ca", + "projection.csv:md5,216dcc5589a083fcc27d981aa90fa2ab", + "filtered_feature_bc_matrix.h5:md5,f1a8f225c113974b47efffe08e70f367", + "metrics_summary.csv:md5,faa17487b479eab361050d3266da2efb", "probe_set.csv:md5,5bfb8f12319be1b2b6c14142537c3804", - "raw_feature_bc_matrix.h5:md5,2263d2c756785f86dc28f6b76fd61b73", + "raw_feature_bc_matrix.h5:md5,6e40ae93a116c6fc0adbe707b0eb415f", "raw_probe_bc_matrix.h5:md5,3d5e711d0891ca2caaf301a2c1fbda91", "aligned_fiducials.jpg:md5,51dcc3a32d3d5ca4704f664c8ede81ef", "cytassist_image.tiff:md5,0fb04a55e5658f4d158d986a334b034d", - "detected_tissue_image.jpg:md5,64d9adb4844ab91506131476d93b28dc", - "tissue_hires_image.png:md5,1c0f1e94522a886c19f56a629227e097", + "detected_tissue_image.jpg:md5,1d3ccc1e12c4fee091b006e48b9cc16a", + "spatial_enrichment.csv:md5,1117792553e82feb2b4b3934907a0136", + "tissue_hires_image.png:md5,834706fff299024fab48e6366afc9cb9", "tissue_lowres_image.png:md5,8c1fcb378f7f886301f49ffc4f84360a", - "tissue_positions.csv:md5,1b2df34f9e762e9e64aa226226b96c4b" + "tissue_positions.csv:md5,425601ef21661ec0126000f905ef044f" ] ], - "timestamp": "2024-01-11T17:49:27.776247" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-04-02T10:13:00.787792273" }, "spaceranger v1 - homo_sapiens - fasta - gtf - fastq - tif - csv": { "content": [ [ - "versions.yml:md5,038e17e049a72dd3d417d0e221dce732" + "versions.yml:md5,1539e8a9a3d63ce3653920721d1af509" ], [ - "filtered_feature_bc_matrix.h5:md5,f444a4816bf40d377271a6157b320556", - "metrics_summary.csv:md5,5e36f2f9b6987791e0b5eb2736d25115", - "molecule_info.h5:md5,b3d14dfbfc167bb8fc9b158f083efdb6", - "possorted_genome_bam.bam:md5,6ed7f3bb2f17322113f940989a3771ff", - "possorted_genome_bam.bam.bai:md5,08ce9ffd30d9b82091932b744873610b", - "raw_feature_bc_matrix.h5:md5,7e937b4863a98b0d3784f4e21c07c326", + "filtered_feature_bc_matrix.h5:md5,7e09d1cd2e1f497a698c5efde9e4af84", + "metrics_summary.csv:md5,07a6fcc2e20f854f8d3fcde2457a2f9a", + "molecule_info.h5:md5,1f2e0fd31d15509e7916e84f22632c9c", + "raw_feature_bc_matrix.h5:md5,5a4184a3bfaf722eec8d1a763a45906e", "aligned_fiducials.jpg:md5,f6217ddd707bb189e665f56b130c3da8", - "detected_tissue_image.jpg:md5,4a26b91db5aca179d627b86f352006e2", + "detected_tissue_image.jpg:md5,c1c7e8741701a576c1ec103c1aaf98ea", "tissue_hires_image.png:md5,d91f8f176ae35ab824ede87117ac0889", "tissue_lowres_image.png:md5,475a04208d193191c84d7a3b5d4eb287", - "tissue_positions.csv:md5,37d288d0e29e8572ea4c5bef292de4b6" + "tissue_positions.csv:md5,748bf590c445db409d7dbdf5a08e72e8" ] ], - "timestamp": "2024-01-11T20:34:18.669838" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-04-02T09:37:13.128424153" } } \ No newline at end of file diff --git a/modules/nf-core/spaceranger/count/tests/nextflow.config b/modules/nf-core/spaceranger/count/tests/nextflow.config new file mode 100644 index 0000000..fe9d61a --- /dev/null +++ b/modules/nf-core/spaceranger/count/tests/nextflow.config @@ -0,0 +1,5 @@ +process { + withName: SPACERANGER_COUNT { + ext.args = '--create-bam false' + } +} diff --git a/modules/nf-core/untar/tests/main.nf.test b/modules/nf-core/untar/tests/main.nf.test index 679e83c..2a7c97b 100644 --- a/modules/nf-core/untar/tests/main.nf.test +++ b/modules/nf-core/untar/tests/main.nf.test @@ -3,17 +3,12 @@ nextflow_process { name "Test Process UNTAR" script "../main.nf" process "UNTAR" - tag "modules" tag "modules_nfcore" tag "untar" - test("test_untar") { when { - params { - outdir = "$outputDir" - } process { """ input[0] = [ [], file(params.modules_testdata_base_path + 'genomics/sarscov2/genome/db/kraken2.tar.gz', checkIfExists: true) ] @@ -33,9 +28,6 @@ nextflow_process { test("test_untar_onlyfiles") { when { - params { - outdir = "$outputDir" - } process { """ input[0] = [ [], file(params.modules_testdata_base_path + 'generic/tar/hello.tar.gz', checkIfExists: true) ] diff --git a/modules/nf-core/untar/tests/main.nf.test.snap b/modules/nf-core/untar/tests/main.nf.test.snap index ace4257..6455029 100644 --- a/modules/nf-core/untar/tests/main.nf.test.snap +++ b/modules/nf-core/untar/tests/main.nf.test.snap @@ -12,7 +12,11 @@ ] ] ], - "timestamp": "2023-10-18T11:56:46.878844" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:49:41.320643" }, "test_untar": { "content": [ @@ -29,6 +33,10 @@ ] ] ], - "timestamp": "2023-10-18T11:56:08.16574" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T11:49:33.795172" } } \ No newline at end of file diff --git a/nextflow.config b/nextflow.config index 6d0f929..c622d42 100644 --- a/nextflow.config +++ b/nextflow.config @@ -18,19 +18,19 @@ params { spaceranger_save_reference = false // Quality controls and filtering - st_qc_min_counts = 500 - st_qc_min_genes = 250 - st_qc_min_spots = 1 - st_qc_mito_threshold = 20.0 - st_qc_ribo_threshold = 0.0 - st_qc_hb_threshold = 100.0 + qc_min_counts = 500 + qc_min_genes = 250 + qc_min_spots = 1 + qc_mito_threshold = 20.0 + qc_ribo_threshold = 0.0 + qc_hb_threshold = 100.0 // Clustering - st_cluster_n_hvgs = 2000 - st_cluster_resolution = 1.0 + cluster_n_hvgs = 2000 + cluster_resolution = 1.0 // Spatial differential expression - st_n_top_spatial_degs = 14 + n_top_svgs = 14 // MultiQC options multiqc_config = null @@ -54,12 +54,12 @@ params { version = false // Config options - custom_config_version = 'master' - custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" - config_profile_description = null - config_profile_contact = null - config_profile_url = null - config_profile_name = null + config_profile_name = null + config_profile_description = null + custom_config_version = 'master' + custom_config_base = "https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}" + config_profile_contact = null + config_profile_url = null // Max resource options // Defaults only, expecting to be overwritten diff --git a/nextflow_schema.json b/nextflow_schema.json index 2f9a6f4..ad63b5b 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -16,6 +16,7 @@ "type": "string", "format": "file-path", "exists": true, + "schema": "assets/schema_input.json", "mimetype": "text/csv", "pattern": "^\\S+\\.csv$", "description": "Path to comma-separated file containing information about the samples in the experiment.", @@ -95,57 +96,57 @@ "fa_icon": "fas fa-magnifying-glass-chart", "description": "Options related to the downstream analyses performed by the pipeline.", "properties": { - "st_qc_min_counts": { + "qc_min_counts": { "type": "integer", "default": 500, "description": "The minimum number of UMIs needed in a spot for that spot to pass the filtering.", "fa_icon": "fas fa-hashtag" }, - "st_qc_min_genes": { + "qc_min_genes": { "type": "integer", "default": 250, "description": "The minimum number of expressed genes in a spot needed for that spot to pass the filtering.", "fa_icon": "fas fa-hashtag" }, - "st_qc_min_spots": { + "qc_min_spots": { "type": "integer", "default": 1, "description": "The minimum number of spots in which a gene is expressed for that gene to pass the filtering.", "fa_icon": "fas fa-hashtag" }, - "st_qc_mito_threshold": { + "qc_mito_threshold": { "type": "number", "default": 20, "description": "The maximum proportion of mitochondrial content that a spot is allowed to have to pass the filtering.", "help_text": "If you do not wish to filter based on mitochondrial content, set this parameter to `100`.", "fa_icon": "fas fa-hashtag" }, - "st_qc_ribo_threshold": { + "qc_ribo_threshold": { "type": "number", "default": 0, - "description": "The minimum proportion of ribosomal content that a spot is needs to have to pass the filtering (no filtering is done by defeault).", + "description": "The minimum proportion of ribosomal content that a spot is needs to have to pass the filtering (no filtering is done by default).", "fa_icon": "fas fa-hashtag" }, - "st_qc_hb_threshold": { + "qc_hb_threshold": { "type": "number", "default": 100, - "description": "The maximum proportion of haemoglobin content that a spot is allowed to have to pass the filtering (no filtering is done by defeault).", + "description": "The maximum proportion of haemoglobin content that a spot is allowed to have to pass the filtering (no filtering is done by default).", "fa_icon": "fas fa-hashtag" }, - "st_cluster_n_hvgs": { + "cluster_n_hvgs": { "type": "integer", "default": 2000, "description": "The number of top highly variable genes to use for the analyses.", "fa_icon": "fas fa-hashtag" }, - "st_cluster_resolution": { + "cluster_resolution": { "type": "number", "default": 1, "description": "The resolution for the clustering of the spots.", "help_text": "The resolution controls the coarseness of the clustering, where a higher resolution leads to more clusters.", "fa_icon": "fas fa-circle-nodes" }, - "st_n_top_spatial_degs": { + "n_top_svgs": { "type": "integer", "default": 14, "description": "The number of top spatial differentially expressed genes to plot.", diff --git a/pyproject.toml b/pyproject.toml index 7d08e1c..5611062 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,11 +3,13 @@ [tool.ruff] line-length = 120 target-version = "py38" -select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] cache-dir = "~/.cache/ruff" -[tool.ruff.isort] +[tool.ruff.lint] +select = ["I", "E1", "E4", "E7", "E9", "F", "UP", "N"] + +[tool.ruff.lint.isort] known-first-party = ["nf_core"] -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "__init__.py" = ["E402", "F401"] diff --git a/subworkflows/local/downstream.nf b/subworkflows/local/downstream.nf new file mode 100644 index 0000000..77269ee --- /dev/null +++ b/subworkflows/local/downstream.nf @@ -0,0 +1,116 @@ +// +// Subworkflow for downstream analyses of ST data +// + +include { QUARTONOTEBOOK as QUALITY_CONTROLS } from '../../modules/nf-core/quartonotebook/main' +include { QUARTONOTEBOOK as SPATIALLY_VARIABLE_GENES } from '../../modules/nf-core/quartonotebook/main' +include { QUARTONOTEBOOK as CLUSTERING } from '../../modules/nf-core/quartonotebook/main' + +workflow DOWNSTREAM { + + take: + sdata_raw + + main: + + ch_versions = Channel.empty() + + // + // Quarto reports and extension files + // + quality_controls_notebook = file("${projectDir}/bin/quality_controls.qmd", checkIfExists: true) + clustering_notebook = file("${projectDir}/bin/clustering.qmd", checkIfExists: true) + spatially_variable_genes_notebook = file("${projectDir}/bin/spatially_variable_genes.qmd", checkIfExists: true) + extensions = Channel.fromPath("${projectDir}/assets/_extensions").collect() + + // + // Quality controls and filtering + // + ch_quality_controls_input_data = sdata_raw + .map { it -> it[1] } + ch_quality_controls_notebook = sdata_raw + .map { tuple(it[0], quality_controls_notebook) } + quality_controls_params = [ + input_sdata: "sdata_raw.zarr", + min_counts: params.qc_min_counts, + min_genes: params.qc_min_genes, + min_spots: params.qc_min_spots, + mito_threshold: params.qc_mito_threshold, + ribo_threshold: params.qc_ribo_threshold, + hb_threshold: params.qc_hb_threshold, + artifact_dir: "artifacts", + output_adata: "adata_filtered.h5ad", + output_sdata: "sdata_filtered.zarr", + ] + QUALITY_CONTROLS ( + ch_quality_controls_notebook, + quality_controls_params, + ch_quality_controls_input_data, + extensions + ) + ch_versions = ch_versions.mix(QUALITY_CONTROLS.out.versions) + + // + // Normalisation, dimensionality reduction and clustering + // + ch_clustering_input_data = QUALITY_CONTROLS.out.artifacts + .map { it -> it[1] } + ch_clustering_notebook = QUALITY_CONTROLS.out.artifacts + .map { tuple(it[0], clustering_notebook) } + clustering_params = [ + input_sdata: "sdata_filtered.zarr", + cluster_resolution: params.cluster_resolution, + n_hvgs: params.cluster_n_hvgs, + artifact_dir: "artifacts", + output_adata: "adata_processed.h5ad", + output_sdata: "sdata_processed.zarr", + ] + CLUSTERING ( + ch_clustering_notebook, + clustering_params, + ch_clustering_input_data, + extensions + ) + ch_versions = ch_versions.mix(CLUSTERING.out.versions) + + // + // Spatially variable genes + // + ch_spatially_variable_genes_input_data = CLUSTERING.out.artifacts + .map { it -> it[1] } + ch_spatially_variable_genes_notebook = CLUSTERING.out.artifacts + .map { tuple(it[0], spatially_variable_genes_notebook) } + spatially_variable_genes_params = [ + input_sdata: "sdata_processed.zarr", + n_top_svgs: params.n_top_svgs, + artifact_dir: "artifacts", + output_csv: "spatially_variable_genes.csv", + output_adata: "adata_spatially_variable_genes.h5ad", + output_sdata: "sdata.zarr", + ] + SPATIALLY_VARIABLE_GENES ( + ch_spatially_variable_genes_notebook, + spatially_variable_genes_params, + ch_spatially_variable_genes_input_data, + extensions + ) + ch_versions = ch_versions.mix(SPATIALLY_VARIABLE_GENES.out.versions) + + emit: + qc_html = QUALITY_CONTROLS.out.html // channel: [ meta, html ] + qc_sdata = QUALITY_CONTROLS.out.artifacts // channel: [ meta, h5ad ] + qc_nb = QUALITY_CONTROLS.out.notebook // channel: [ meta, qmd ] + qc_params = QUALITY_CONTROLS.out.params_yaml // channel: [ meta, yml ] + + clustering_html = CLUSTERING.out.html // channel: [ html ] + clustering_sdata = CLUSTERING.out.artifacts // channel: [ meta, h5ad] + clustering_nb = CLUSTERING.out.notebook // channel: [ meta, qmd ] + clustering_params = CLUSTERING.out.params_yaml // channel: [ meta, yml ] + + svg_html = SPATIALLY_VARIABLE_GENES.out.html // channel: [ meta, html ] + svg_csv = SPATIALLY_VARIABLE_GENES.out.artifacts // channel: [ meta, csv ] + svg_nb = SPATIALLY_VARIABLE_GENES.out.notebook // channel: [ meta, qmd ] + svg_params = SPATIALLY_VARIABLE_GENES.out.params_yaml // channel: [ meta, yml ] + + versions = ch_versions // channel: [ versions.yml ] +} diff --git a/subworkflows/local/input_check.nf b/subworkflows/local/input_check.nf index abd2391..06a9721 100644 --- a/subworkflows/local/input_check.nf +++ b/subworkflows/local/input_check.nf @@ -8,10 +8,13 @@ include { UNTAR as UNTAR_DOWNSTREAM_INPUT } from "../../modules/nf-core/untar" workflow INPUT_CHECK { take: - samplesheet // file: /path/to/samplesheet.csv + samplesheet // file: samplesheet read in from --input main: - ch_st = Channel.from(samplesheet) + + ch_versions = Channel.empty() + + ch_st = Channel.fromPath(samplesheet) .splitCsv ( header: true, sep: ',') .branch { spaceranger: !it.containsKey("spaceranger_dir") @@ -30,6 +33,7 @@ workflow INPUT_CHECK { // Extract tarballed inputs UNTAR_SPACERANGER_INPUT ( ch_spaceranger.tar ) + ch_versions = ch_versions.mix(UNTAR_SPACERANGER_INPUT.out.versions) // Combine extracted and directory inputs into one channel ch_spaceranger_combined = UNTAR_SPACERANGER_INPUT.out.untar @@ -51,6 +55,7 @@ workflow INPUT_CHECK { // Extract tarballed inputs UNTAR_DOWNSTREAM_INPUT ( ch_downstream.tar ) + ch_versions = ch_versions.mix(UNTAR_DOWNSTREAM_INPUT.out.versions) // Combine extracted and directory inputs into one channel ch_downstream_combined = UNTAR_DOWNSTREAM_INPUT.out.untar @@ -61,8 +66,9 @@ workflow INPUT_CHECK { ch_downstream_input = ch_downstream_combined.map { create_channel_downstream(it) } emit: - ch_spaceranger_input // channel: [ val(meta), [ st data ] ] - ch_downstream_input // channel: [ val(meta), [ st data ] ] + ch_spaceranger_input // channel: [ val(meta), [ st data ] ] + ch_downstream_input // channel: [ val(meta), [ st data ] ] + versions = ch_versions // channel: [ versions.yml ] } // Function to get list of [ meta, [ spaceranger_dir ]] @@ -72,12 +78,20 @@ def create_channel_downstream_tar(LinkedHashMap meta) { return [meta, spaceranger_dir] } + // Function to get list of [ meta, [ raw_feature_bc_matrix, tissue_positions, // scalefactors, hires_image, lowres_image ]] def create_channel_downstream(LinkedHashMap meta) { meta["id"] = meta.remove("sample") spaceranger_dir = file("${meta.remove('spaceranger_dir')}/**") - for (f in Utils.DOWNSTREAM_REQUIRED_SPACERANGER_FILES) { + DOWNSTREAM_REQUIRED_SPACERANGER_FILES = [ + "raw_feature_bc_matrix.h5", + "tissue_positions.csv", + "scalefactors_json.json", + "tissue_hires_image.png", + "tissue_lowres_image.png" + ] + for (f in DOWNSTREAM_REQUIRED_SPACERANGER_FILES) { if(!spaceranger_dir*.name.contains(f)) { error "The specified spaceranger output directory doesn't contain the required file `${f}` for sample `${meta.id}`" } diff --git a/subworkflows/local/spaceranger.nf b/subworkflows/local/spaceranger.nf index 8816074..3dab2bf 100644 --- a/subworkflows/local/spaceranger.nf +++ b/subworkflows/local/spaceranger.nf @@ -8,7 +8,7 @@ include { SPACERANGER_COUNT } from '../../modules/nf-core/spa workflow SPACERANGER { take: - ch_st_data // channel: [ val(meta), [ raw st data ] ] + ch_data // channel: [ val(meta), [ raw st data ] ] main: @@ -44,7 +44,7 @@ workflow SPACERANGER { // Run Space Ranger count // SPACERANGER_COUNT ( - ch_st_data, + ch_data, ch_reference, ch_probeset ) diff --git a/subworkflows/local/st_downstream.nf b/subworkflows/local/st_downstream.nf deleted file mode 100644 index 91b5e74..0000000 --- a/subworkflows/local/st_downstream.nf +++ /dev/null @@ -1,67 +0,0 @@ -// -// Subworkflow for downstream analyses of ST data -// - -include { ST_QUALITY_CONTROLS } from '../../modules/local/st_quality_controls' -include { ST_SPATIAL_DE } from '../../modules/local/st_spatial_de' -include { ST_CLUSTERING } from '../../modules/local/st_clustering' - -workflow ST_DOWNSTREAM { - - take: - st_adata_raw - - main: - - ch_versions = Channel.empty() - - // - // Report files - // - report_quality_controls = file("${projectDir}/bin/st_quality_controls.qmd") - report_clustering = file("${projectDir}/bin/st_clustering.qmd") - report_spatial_de = file("${projectDir}/bin/st_spatial_de.qmd") - report_template = Channel.fromPath("${projectDir}/assets/_extensions").collect() - - // - // Quality controls and filtering - // - ST_QUALITY_CONTROLS ( - report_quality_controls, - report_template, - st_adata_raw - ) - ch_versions = ch_versions.mix(ST_QUALITY_CONTROLS.out.versions) - - // - // Normalisation, dimensionality reduction and clustering - // - ST_CLUSTERING ( - report_clustering, - report_template, - ST_QUALITY_CONTROLS.out.st_adata_filtered - ) - ch_versions = ch_versions.mix(ST_CLUSTERING.out.versions) - - // - // Spatial differential expression - // - ST_SPATIAL_DE ( - report_spatial_de, - report_template, - ST_CLUSTERING.out.st_adata_processed - ) - ch_versions = ch_versions.mix(ST_SPATIAL_DE.out.versions) - - emit: - st_data_norm = ST_QUALITY_CONTROLS.out.st_adata_filtered // channel: [ meta, h5ad ] - html = ST_QUALITY_CONTROLS.out.html // channel: [ html ] - - st_adata_processed = ST_CLUSTERING.out.st_adata_processed // channel: [ meta, h5ad] - html = ST_CLUSTERING.out.html // channel: [ html ] - - degs = ST_SPATIAL_DE.out.degs // channel: [ meta, csv ] - html = ST_SPATIAL_DE.out.html // channel: [ html ] - - versions = ch_versions // channel: [ versions.yml ] -} diff --git a/subworkflows/local/utils_nfcore_spatialtranscriptomics_pipeline/main.nf b/subworkflows/local/utils_nfcore_spatialtranscriptomics_pipeline/main.nf new file mode 100644 index 0000000..c037da1 --- /dev/null +++ b/subworkflows/local/utils_nfcore_spatialtranscriptomics_pipeline/main.nf @@ -0,0 +1,184 @@ +// +// Subworkflow with functionality specific to the nf-core/spatialtranscriptomics pipeline +// + +/* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + IMPORT FUNCTIONS / MODULES / SUBWORKFLOWS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +*/ + +include { UTILS_NFVALIDATION_PLUGIN } from '../../nf-core/utils_nfvalidation_plugin' +include { paramsSummaryMap } from 'plugin/nf-validation' +include { fromSamplesheet } from 'plugin/nf-validation' +include { UTILS_NEXTFLOW_PIPELINE } from '../../nf-core/utils_nextflow_pipeline' +include { completionEmail } from '../../nf-core/utils_nfcore_pipeline' +include { completionSummary } from '../../nf-core/utils_nfcore_pipeline' +include { dashedLine } from '../../nf-core/utils_nfcore_pipeline' +include { nfCoreLogo } from '../../nf-core/utils_nfcore_pipeline' +include { imNotification } from '../../nf-core/utils_nfcore_pipeline' +include { UTILS_NFCORE_PIPELINE } from '../../nf-core/utils_nfcore_pipeline' +include { workflowCitation } from '../../nf-core/utils_nfcore_pipeline' + +/* +======================================================================================== + SUBWORKFLOW TO INITIALISE PIPELINE +======================================================================================== +*/ + +workflow PIPELINE_INITIALISATION { + + take: + version // boolean: Display version and exit + help // boolean: Display help text + validate_params // boolean: Boolean whether to validate parameters against the schema at runtime + monochrome_logs // boolean: Do not use coloured log outputs + nextflow_cli_args // array: List of positional nextflow CLI args + outdir // string: The output directory where the results will be saved + input // string: Path to input samplesheet + + main: + + ch_versions = Channel.empty() + + // + // Print version and exit if required and dump pipeline parameters to JSON file + // + UTILS_NEXTFLOW_PIPELINE ( + version, + true, + outdir, + workflow.profile.tokenize(',').intersect(['conda', 'mamba']).size() >= 1 + ) + + // + // Validate parameters and generate parameter summary to stdout + // + pre_help_text = nfCoreLogo(monochrome_logs) + post_help_text = '\n' + workflowCitation() + '\n' + dashedLine(monochrome_logs) + def String workflow_command = "nextflow run ${workflow.manifest.name} -profile --input samplesheet.csv --outdir " + UTILS_NFVALIDATION_PLUGIN ( + help, + workflow_command, + pre_help_text, + post_help_text, + validate_params, + "nextflow_schema.json" + ) + + // + // Check config provided to the pipeline + // + UTILS_NFCORE_PIPELINE ( + nextflow_cli_args + ) + + emit: + versions = ch_versions +} + +/* +======================================================================================== + SUBWORKFLOW FOR PIPELINE COMPLETION +======================================================================================== +*/ + +workflow PIPELINE_COMPLETION { + + take: + email // string: email address + email_on_fail // string: email address sent on pipeline failure + plaintext_email // boolean: Send plain-text email instead of HTML + outdir // path: Path to output directory where results will be published + monochrome_logs // boolean: Disable ANSI colour codes in log output + hook_url // string: hook URL for notifications + multiqc_report // string: Path to MultiQC report + + main: + + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + + // + // Completion email and summary + // + workflow.onComplete { + if (email || email_on_fail) { + completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs, multiqc_report.toList()) + } + + completionSummary(monochrome_logs) + + if (hook_url) { + imNotification(summary_params, hook_url) + } + } +} + +/* +======================================================================================== + FUNCTIONS +======================================================================================== +*/ + +// +// Generate methods description for MultiQC +// +def toolCitationText() { + + def citation_text = [ + "Tools used in the workflow included:", + "AnnData (Virshup et al. 2021),", + "FastQC (Andrews 2010),", + "MultiQC (Ewels et al. 2016),", + "Quarto (Allaire et al. 2022),", + "Scanpy (Wolf et al. 2018),", + "Space Ranger (10x Genomics)", + "SpatialData (Marconato et al. 2023) and", + "Squidpy (Palla et al. 2022)" + ].join(' ').trim() + + return citation_text +} + +def toolBibliographyText() { + + def reference_text = [ + '
  • Di Tommaso, P., Chatzou, M., Floden, E. W., Barja, P. P., Palumbo, E., & Notredame, C. (2017). Nextflow enables reproducible computational workflows. Nature Biotechnology, 35(4), 316-319. doi: 10.1038/nbt.3820
    • ', + '
  • Ewels, P. A., Peltzer, A., Fillinger, S., Patel, H., Alneberg, J., Wilm, A., Garcia, M. U., Di Tommaso, P., & Nahnsen, S. (2020). The nf-core framework for community-curated bioinformatics pipelines. Nature Biotechnology, 38(3), 276-278. doi: 10.1038/s41587-020-0439-x
    • ', + '
  • Grüning, B., Dale, R., Sjödin, A., Chapman, B. A., Rowe, J., Tomkins-Tinch, C. H., Valieris, R., Köster, J., & Bioconda Team. (2018). Bioconda: sustainable and comprehensive software distribution for the life sciences. Nature Methods, 15(7), 475–476. doi: 10.1038/s41592-018-0046-7
    • ', + '
  • da Veiga Leprevost, F., Grüning, B. A., Alves Aflitos, S., Röst, H. L., Uszkoreit, J., Barsnes, H., Vaudel, M., Moreno, P., Gatto, L., Weber, J., Bai, M., Jimenez, R. C., Sachsenberg, T., Pfeuffer, J., Vera Alvarez, R., Griss, J., Nesvizhskii, A. I., & Perez-Riverol, Y. (2017). BioContainers: an open-source and community-driven framework for software standardization. Bioinformatics (Oxford, England), 33(16), 2580–2582. doi: 10.1093/bioinformatics/btx192
    • ', + '
  • Virshup I, Rybakov S, Theis FJ, Angerer P, Wolf FA. bioRxiv 2021.12.16.473007. doi: 10.1101/2021.12.16.473007
    • ', + '
  • Andrews S. (2010). FastQC: A Quality Control Tool for High Throughput Sequence Data [Online]: bioinformatics.babraham.ak.uk/project/fastqc
    • ', + '
  • Ewels P, Magnusson M, Lundin S, Käller M. MultiQC: summarize analysis results for multiple tools and samples in a single report. Bioinformatics. 2016 Oct 1;32(19):3047-8. Epub 2016 Jun 16. PubMed PMID: 27312411; PubMed Central PMCID: PMC5039924. doi: 10.1093/bioinformatics/btw354
    • ', + '
  • Allaire J, Teague C, Scheidegger C, Xie Y, Dervieux C. Quarto (2022). doi: 10.5281/zenodo.5960048
    • ', + '
  • Wolf F, Angerer P, Theis F. SCANPY: large-scale single-cell gene expression data analysis. Genome Biol 19, 15 (2018). doi: 10.1186/s13059-017-1382-0
    • ', + '
  • 10x Genomics Space Ranger 2.1.0 [Online]: 10xgenomics.com/support/software/space-ranger
    • ', + '
  • Marconato L, Palla G, Yamauchi K, Virshup I, Heidari E, Treis T, Toth M, Shrestha R, Vöhringer H, Huber W, Gerstung M, Moore J, Theis F, Stegle O. SpatialData: an open and universal data framework for spatial omics. bioRxiv 2023.05.05.539647; doi: 10.1101/2023.05.05.539647
    • ', + '
  • Palla G, Spitzer H, Klein M et al. Squidpy: a scalable framework for spatial omics analysis. Nat Methods 19, 171–178 (2022). doi: 10.1038/s41592-021-01358-2
    • ', + ].join(' ').trim() + + return reference_text +} + +def methodsDescriptionText(mqc_methods_yaml) { + // Convert to a named map so can be used as with familar NXF ${workflow} variable syntax in the MultiQC YML file + def meta = [:] + meta.workflow = workflow.toMap() + meta["manifest_map"] = workflow.manifest.toMap() + + // Pipeline DOI + meta["doi_text"] = meta.manifest_map.doi ? "(doi: ${meta.manifest_map.doi})" : "" + meta["nodoi_text"] = meta.manifest_map.doi ? "": "
  • If available, make sure to update the text to include the Zenodo DOI of version of the pipeline used.
    • " + + // Tool references + meta["tool_citations"] = toolCitationText().replaceAll(", \\.", ".").replaceAll("\\. \\.", ".").replaceAll(", \\.", ".") + meta["tool_bibliography"] = toolBibliographyText() + + + def methods_text = mqc_methods_yaml.text + + def engine = new groovy.text.SimpleTemplateEngine() + def description_html = engine.createTemplate(methods_text).make(meta) + + return description_html.toString() +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/main.nf b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf new file mode 100644 index 0000000..ac31f28 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/main.nf @@ -0,0 +1,126 @@ +// +// Subworkflow with functionality that may be useful for any Nextflow pipeline +// + +import org.yaml.snakeyaml.Yaml +import groovy.json.JsonOutput +import nextflow.extension.FilesEx + +/* +======================================================================================== + SUBWORKFLOW DEFINITION +======================================================================================== +*/ + +workflow UTILS_NEXTFLOW_PIPELINE { + + take: + print_version // boolean: print version + dump_parameters // boolean: dump parameters + outdir // path: base directory used to publish pipeline results + check_conda_channels // boolean: check conda channels + + main: + + // + // Print workflow version and exit on --version + // + if (print_version) { + log.info "${workflow.manifest.name} ${getWorkflowVersion()}" + System.exit(0) + } + + // + // Dump pipeline parameters to a JSON file + // + if (dump_parameters && outdir) { + dumpParametersToJSON(outdir) + } + + // + // When running with Conda, warn if channels have not been set-up appropriately + // + if (check_conda_channels) { + checkCondaChannels() + } + + emit: + dummy_emit = true +} + +/* +======================================================================================== + FUNCTIONS +======================================================================================== +*/ + +// +// Generate version string +// +def getWorkflowVersion() { + String version_string = "" + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string +} + +// +// Dump pipeline parameters to a JSON file +// +def dumpParametersToJSON(outdir) { + def timestamp = new java.util.Date().format( 'yyyy-MM-dd_HH-mm-ss') + def filename = "params_${timestamp}.json" + def temp_pf = new File(workflow.launchDir.toString(), ".${filename}") + def jsonStr = JsonOutput.toJson(params) + temp_pf.text = JsonOutput.prettyPrint(jsonStr) + + FilesEx.copyTo(temp_pf.toPath(), "${outdir}/pipeline_info/params_${timestamp}.json") + temp_pf.delete() +} + +// +// When running with -profile conda, warn if channels have not been set-up appropriately +// +def checkCondaChannels() { + Yaml parser = new Yaml() + def channels = [] + try { + def config = parser.load("conda config --show channels".execute().text) + channels = config.channels + } catch(NullPointerException | IOException e) { + log.warn "Could not verify conda channel configuration." + return + } + + // Check that all channels are present + // This channel list is ordered by required channel priority. + def required_channels_in_order = ['conda-forge', 'bioconda', 'defaults'] + def channels_missing = ((required_channels_in_order as Set) - (channels as Set)) as Boolean + + // Check that they are in the right order + def channel_priority_violation = false + def n = required_channels_in_order.size() + for (int i = 0; i < n - 1; i++) { + channel_priority_violation |= !(channels.indexOf(required_channels_in_order[i]) < channels.indexOf(required_channels_in_order[i+1])) + } + + if (channels_missing | channel_priority_violation) { + log.warn "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n" + + " There is a problem with your Conda configuration!\n\n" + + " You will need to set-up the conda-forge and bioconda channels correctly.\n" + + " Please refer to https://bioconda.github.io/\n" + + " The observed channel order is \n" + + " ${channels}\n" + + " but the following channel order is required:\n" + + " ${required_channels_in_order}\n" + + "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml b/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml new file mode 100644 index 0000000..e5c3a0a --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/meta.yml @@ -0,0 +1,38 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NEXTFLOW_PIPELINE" +description: Subworkflow with functionality that may be useful for any Nextflow pipeline +keywords: + - utility + - pipeline + - initialise + - version +components: [] +input: + - print_version: + type: boolean + description: | + Print the version of the pipeline and exit + - dump_parameters: + type: boolean + description: | + Dump the parameters of the pipeline to a JSON file + - output_directory: + type: directory + description: Path to output dir to write JSON file to. + pattern: "results/" + - check_conda_channel: + type: boolean + description: | + Check if the conda channel priority is correct. +output: + - dummy_emit: + type: boolean + description: | + Dummy emit to make nf-core subworkflows lint happy +authors: + - "@adamrtalbot" + - "@drpatelh" +maintainers: + - "@adamrtalbot" + - "@drpatelh" + - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test new file mode 100644 index 0000000..68718e4 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test @@ -0,0 +1,54 @@ + +nextflow_function { + + name "Test Functions" + script "subworkflows/nf-core/utils_nextflow_pipeline/main.nf" + config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" + tag 'subworkflows' + tag 'utils_nextflow_pipeline' + tag 'subworkflows/utils_nextflow_pipeline' + + test("Test Function getWorkflowVersion") { + + function "getWorkflowVersion" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function dumpParametersToJSON") { + + function "dumpParametersToJSON" + + when { + function { + """ + // define inputs of the function here. Example: + input[0] = "$outputDir" + """.stripIndent() + } + } + + then { + assertAll( + { assert function.success } + ) + } + } + + test("Test Function checkCondaChannels") { + + function "checkCondaChannels" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 0000000..e3f0baf --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,20 @@ +{ + "Test Function getWorkflowVersion": { + "content": [ + "v9.9.9" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:05.308243" + }, + "Test Function checkCondaChannels": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:12.425833" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test new file mode 100644 index 0000000..ca964ce --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,111 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NEXTFLOW_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config" + workflow "UTILS_NEXTFLOW_PIPELINE" + tag 'subworkflows' + tag 'utils_nextflow_pipeline' + tag 'subworkflows/utils_nextflow_pipeline' + + test("Should run no inputs") { + + when { + workflow { + """ + print_version = false + dump_parameters = false + outdir = null + check_conda_channels = false + + input[0] = print_version + input[1] = dump_parameters + input[2] = outdir + input[3] = check_conda_channels + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should print version") { + + when { + workflow { + """ + print_version = true + dump_parameters = false + outdir = null + check_conda_channels = false + + input[0] = print_version + input[1] = dump_parameters + input[2] = outdir + input[3] = check_conda_channels + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.stdout.contains("nextflow_workflow v9.9.9") } + ) + } + } + + test("Should dump params") { + + when { + workflow { + """ + print_version = false + dump_parameters = true + outdir = 'results' + check_conda_channels = false + + input[0] = false + input[1] = true + input[2] = outdir + input[3] = false + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should not create params JSON if no output directory") { + + when { + workflow { + """ + print_version = false + dump_parameters = true + outdir = null + check_conda_channels = false + + input[0] = false + input[1] = true + input[2] = outdir + input[3] = false + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config new file mode 100644 index 0000000..d0a926b --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/nextflow.config @@ -0,0 +1,9 @@ +manifest { + name = 'nextflow_workflow' + author = """nf-core""" + homePage = 'https://127.0.0.1' + description = """Dummy pipeline""" + nextflowVersion = '!>=23.04.0' + version = '9.9.9' + doi = 'https://doi.org/10.5281/zenodo.5070524' +} diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml b/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml new file mode 100644 index 0000000..f847611 --- /dev/null +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nextflow_pipeline: + - subworkflows/nf-core/utils_nextflow_pipeline/** diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/main.nf b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf new file mode 100644 index 0000000..a8b55d6 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/main.nf @@ -0,0 +1,440 @@ +// +// Subworkflow with utility functions specific to the nf-core pipeline template +// + +import org.yaml.snakeyaml.Yaml +import nextflow.extension.FilesEx + +/* +======================================================================================== + SUBWORKFLOW DEFINITION +======================================================================================== +*/ + +workflow UTILS_NFCORE_PIPELINE { + + take: + nextflow_cli_args + + main: + valid_config = checkConfigProvided() + checkProfileProvided(nextflow_cli_args) + + emit: + valid_config +} + +/* +======================================================================================== + FUNCTIONS +======================================================================================== +*/ + +// +// Warn if a -profile or Nextflow config has not been provided to run the pipeline +// +def checkConfigProvided() { + valid_config = true + if (workflow.profile == 'standard' && workflow.configFiles.size() <= 1) { + log.warn "[$workflow.manifest.name] You are attempting to run the pipeline without any custom configuration!\n\n" + + "This will be dependent on your local compute environment but can be achieved via one or more of the following:\n" + + " (1) Using an existing pipeline profile e.g. `-profile docker` or `-profile singularity`\n" + + " (2) Using an existing nf-core/configs for your Institution e.g. `-profile crick` or `-profile uppmax`\n" + + " (3) Using your own local custom config e.g. `-c /path/to/your/custom.config`\n\n" + + "Please refer to the quick start section and usage docs for the pipeline.\n " + valid_config = false + } + return valid_config +} + +// +// Exit pipeline if --profile contains spaces +// +def checkProfileProvided(nextflow_cli_args) { + if (workflow.profile.endsWith(',')) { + error "The `-profile` option cannot end with a trailing comma, please remove it and re-run the pipeline!\n" + + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + } + if (nextflow_cli_args[0]) { + log.warn "nf-core pipelines do not accept positional arguments. The positional argument `${nextflow_cli_args[0]}` has been detected.\n" + + "HINT: A common mistake is to provide multiple values separated by spaces e.g. `-profile test, docker`.\n" + } +} + +// +// Citation string for pipeline +// +def workflowCitation() { + return "If you use ${workflow.manifest.name} for your analysis please cite:\n\n" + + "* The pipeline\n" + + " ${workflow.manifest.doi}\n\n" + + "* The nf-core framework\n" + + " https://doi.org/10.1038/s41587-020-0439-x\n\n" + + "* Software dependencies\n" + + " https://github.com/${workflow.manifest.name}/blob/master/CITATIONS.md" +} + +// +// Generate workflow version string +// +def getWorkflowVersion() { + String version_string = "" + if (workflow.manifest.version) { + def prefix_v = workflow.manifest.version[0] != 'v' ? 'v' : '' + version_string += "${prefix_v}${workflow.manifest.version}" + } + + if (workflow.commitId) { + def git_shortsha = workflow.commitId.substring(0, 7) + version_string += "-g${git_shortsha}" + } + + return version_string +} + +// +// Get software versions for pipeline +// +def processVersionsFromYAML(yaml_file) { + Yaml yaml = new Yaml() + versions = yaml.load(yaml_file).collectEntries { k, v -> [ k.tokenize(':')[-1], v ] } + return yaml.dumpAsMap(versions).trim() +} + +// +// Get workflow version for pipeline +// +def workflowVersionToYAML() { + return """ + Workflow: + $workflow.manifest.name: ${getWorkflowVersion()} + Nextflow: $workflow.nextflow.version + """.stripIndent().trim() +} + +// +// Get channel of software versions used in pipeline in YAML format +// +def softwareVersionsToYAML(ch_versions) { + return ch_versions + .unique() + .map { processVersionsFromYAML(it) } + .unique() + .mix(Channel.of(workflowVersionToYAML())) +} + +// +// Get workflow summary for MultiQC +// +def paramsSummaryMultiqc(summary_params) { + def summary_section = '' + for (group in summary_params.keySet()) { + def group_params = summary_params.get(group) // This gets the parameters of that particular group + if (group_params) { + summary_section += "

    $group

    \n" + summary_section += "
    \n" + for (param in group_params.keySet()) { + summary_section += "
    $param
    ${group_params.get(param) ?: 'N/A'}
    \n" + } + summary_section += "
    \n" + } + } + + String yaml_file_text = "id: '${workflow.manifest.name.replace('/','-')}-summary'\n" + yaml_file_text += "description: ' - this information is collected when the pipeline is started.'\n" + yaml_file_text += "section_name: '${workflow.manifest.name} Workflow Summary'\n" + yaml_file_text += "section_href: 'https://github.com/${workflow.manifest.name}'\n" + yaml_file_text += "plot_type: 'html'\n" + yaml_file_text += "data: |\n" + yaml_file_text += "${summary_section}" + + return yaml_file_text +} + +// +// nf-core logo +// +def nfCoreLogo(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + String.format( + """\n + ${dashedLine(monochrome_logs)} + ${colors.green},--.${colors.black}/${colors.green},-.${colors.reset} + ${colors.blue} ___ __ __ __ ___ ${colors.green}/,-._.--~\'${colors.reset} + ${colors.blue} |\\ | |__ __ / ` / \\ |__) |__ ${colors.yellow}} {${colors.reset} + ${colors.blue} | \\| | \\__, \\__/ | \\ |___ ${colors.green}\\`-._,-`-,${colors.reset} + ${colors.green}`._,._,\'${colors.reset} + ${colors.purple} ${workflow.manifest.name} ${getWorkflowVersion()}${colors.reset} + ${dashedLine(monochrome_logs)} + """.stripIndent() + ) +} + +// +// Return dashed line +// +def dashedLine(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + return "-${colors.dim}----------------------------------------------------${colors.reset}-" +} + +// +// ANSII colours used for terminal logging +// +def logColours(monochrome_logs=true) { + Map colorcodes = [:] + + // Reset / Meta + colorcodes['reset'] = monochrome_logs ? '' : "\033[0m" + colorcodes['bold'] = monochrome_logs ? '' : "\033[1m" + colorcodes['dim'] = monochrome_logs ? '' : "\033[2m" + colorcodes['underlined'] = monochrome_logs ? '' : "\033[4m" + colorcodes['blink'] = monochrome_logs ? '' : "\033[5m" + colorcodes['reverse'] = monochrome_logs ? '' : "\033[7m" + colorcodes['hidden'] = monochrome_logs ? '' : "\033[8m" + + // Regular Colors + colorcodes['black'] = monochrome_logs ? '' : "\033[0;30m" + colorcodes['red'] = monochrome_logs ? '' : "\033[0;31m" + colorcodes['green'] = monochrome_logs ? '' : "\033[0;32m" + colorcodes['yellow'] = monochrome_logs ? '' : "\033[0;33m" + colorcodes['blue'] = monochrome_logs ? '' : "\033[0;34m" + colorcodes['purple'] = monochrome_logs ? '' : "\033[0;35m" + colorcodes['cyan'] = monochrome_logs ? '' : "\033[0;36m" + colorcodes['white'] = monochrome_logs ? '' : "\033[0;37m" + + // Bold + colorcodes['bblack'] = monochrome_logs ? '' : "\033[1;30m" + colorcodes['bred'] = monochrome_logs ? '' : "\033[1;31m" + colorcodes['bgreen'] = monochrome_logs ? '' : "\033[1;32m" + colorcodes['byellow'] = monochrome_logs ? '' : "\033[1;33m" + colorcodes['bblue'] = monochrome_logs ? '' : "\033[1;34m" + colorcodes['bpurple'] = monochrome_logs ? '' : "\033[1;35m" + colorcodes['bcyan'] = monochrome_logs ? '' : "\033[1;36m" + colorcodes['bwhite'] = monochrome_logs ? '' : "\033[1;37m" + + // Underline + colorcodes['ublack'] = monochrome_logs ? '' : "\033[4;30m" + colorcodes['ured'] = monochrome_logs ? '' : "\033[4;31m" + colorcodes['ugreen'] = monochrome_logs ? '' : "\033[4;32m" + colorcodes['uyellow'] = monochrome_logs ? '' : "\033[4;33m" + colorcodes['ublue'] = monochrome_logs ? '' : "\033[4;34m" + colorcodes['upurple'] = monochrome_logs ? '' : "\033[4;35m" + colorcodes['ucyan'] = monochrome_logs ? '' : "\033[4;36m" + colorcodes['uwhite'] = monochrome_logs ? '' : "\033[4;37m" + + // High Intensity + colorcodes['iblack'] = monochrome_logs ? '' : "\033[0;90m" + colorcodes['ired'] = monochrome_logs ? '' : "\033[0;91m" + colorcodes['igreen'] = monochrome_logs ? '' : "\033[0;92m" + colorcodes['iyellow'] = monochrome_logs ? '' : "\033[0;93m" + colorcodes['iblue'] = monochrome_logs ? '' : "\033[0;94m" + colorcodes['ipurple'] = monochrome_logs ? '' : "\033[0;95m" + colorcodes['icyan'] = monochrome_logs ? '' : "\033[0;96m" + colorcodes['iwhite'] = monochrome_logs ? '' : "\033[0;97m" + + // Bold High Intensity + colorcodes['biblack'] = monochrome_logs ? '' : "\033[1;90m" + colorcodes['bired'] = monochrome_logs ? '' : "\033[1;91m" + colorcodes['bigreen'] = monochrome_logs ? '' : "\033[1;92m" + colorcodes['biyellow'] = monochrome_logs ? '' : "\033[1;93m" + colorcodes['biblue'] = monochrome_logs ? '' : "\033[1;94m" + colorcodes['bipurple'] = monochrome_logs ? '' : "\033[1;95m" + colorcodes['bicyan'] = monochrome_logs ? '' : "\033[1;96m" + colorcodes['biwhite'] = monochrome_logs ? '' : "\033[1;97m" + + return colorcodes +} + +// +// Attach the multiqc report to email +// +def attachMultiqcReport(multiqc_report) { + def mqc_report = null + try { + if (workflow.success) { + mqc_report = multiqc_report.getVal() + if (mqc_report.getClass() == ArrayList && mqc_report.size() >= 1) { + if (mqc_report.size() > 1) { + log.warn "[$workflow.manifest.name] Found multiple reports from process 'MULTIQC', will use only one" + } + mqc_report = mqc_report[0] + } + } + } catch (all) { + if (multiqc_report) { + log.warn "[$workflow.manifest.name] Could not attach MultiQC report to summary email" + } + } + return mqc_report +} + +// +// Construct and send completion email +// +def completionEmail(summary_params, email, email_on_fail, plaintext_email, outdir, monochrome_logs=true, multiqc_report=null) { + + // Set up the e-mail variables + def subject = "[$workflow.manifest.name] Successful: $workflow.runName" + if (!workflow.success) { + subject = "[$workflow.manifest.name] FAILED: $workflow.runName" + } + + def summary = [:] + for (group in summary_params.keySet()) { + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['Date Started'] = workflow.start + misc_fields['Date Completed'] = workflow.complete + misc_fields['Pipeline script file path'] = workflow.scriptFile + misc_fields['Pipeline script hash ID'] = workflow.scriptId + if (workflow.repository) misc_fields['Pipeline repository Git URL'] = workflow.repository + if (workflow.commitId) misc_fields['Pipeline repository Git Commit'] = workflow.commitId + if (workflow.revision) misc_fields['Pipeline Git branch/tag'] = workflow.revision + misc_fields['Nextflow Version'] = workflow.nextflow.version + misc_fields['Nextflow Build'] = workflow.nextflow.build + misc_fields['Nextflow Compile Timestamp'] = workflow.nextflow.timestamp + + def email_fields = [:] + email_fields['version'] = getWorkflowVersion() + email_fields['runName'] = workflow.runName + email_fields['success'] = workflow.success + email_fields['dateComplete'] = workflow.complete + email_fields['duration'] = workflow.duration + email_fields['exitStatus'] = workflow.exitStatus + email_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + email_fields['errorReport'] = (workflow.errorReport ?: 'None') + email_fields['commandLine'] = workflow.commandLine + email_fields['projectDir'] = workflow.projectDir + email_fields['summary'] = summary << misc_fields + + // On success try attach the multiqc report + def mqc_report = attachMultiqcReport(multiqc_report) + + // Check if we are only sending emails on failure + def email_address = email + if (!email && email_on_fail && !workflow.success) { + email_address = email_on_fail + } + + // Render the TXT template + def engine = new groovy.text.GStringTemplateEngine() + def tf = new File("${workflow.projectDir}/assets/email_template.txt") + def txt_template = engine.createTemplate(tf).make(email_fields) + def email_txt = txt_template.toString() + + // Render the HTML template + def hf = new File("${workflow.projectDir}/assets/email_template.html") + def html_template = engine.createTemplate(hf).make(email_fields) + def email_html = html_template.toString() + + // Render the sendmail template + def max_multiqc_email_size = (params.containsKey('max_multiqc_email_size') ? params.max_multiqc_email_size : 0) as nextflow.util.MemoryUnit + def smail_fields = [ email: email_address, subject: subject, email_txt: email_txt, email_html: email_html, projectDir: "${workflow.projectDir}", mqcFile: mqc_report, mqcMaxSize: max_multiqc_email_size.toBytes() ] + def sf = new File("${workflow.projectDir}/assets/sendmail_template.txt") + def sendmail_template = engine.createTemplate(sf).make(smail_fields) + def sendmail_html = sendmail_template.toString() + + // Send the HTML e-mail + Map colors = logColours(monochrome_logs) + if (email_address) { + try { + if (plaintext_email) { throw GroovyException('Send plaintext e-mail, not HTML') } + // Try to send HTML e-mail using sendmail + def sendmail_tf = new File(workflow.launchDir.toString(), ".sendmail_tmp.html") + sendmail_tf.withWriter { w -> w << sendmail_html } + [ 'sendmail', '-t' ].execute() << sendmail_html + log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (sendmail)-" + } catch (all) { + // Catch failures and try with plaintext + def mail_cmd = [ 'mail', '-s', subject, '--content-type=text/html', email_address ] + mail_cmd.execute() << email_html + log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Sent summary e-mail to $email_address (mail)-" + } + } + + // Write summary e-mail HTML to a file + def output_hf = new File(workflow.launchDir.toString(), ".pipeline_report.html") + output_hf.withWriter { w -> w << email_html } + FilesEx.copyTo(output_hf.toPath(), "${outdir}/pipeline_info/pipeline_report.html"); + output_hf.delete() + + // Write summary e-mail TXT to a file + def output_tf = new File(workflow.launchDir.toString(), ".pipeline_report.txt") + output_tf.withWriter { w -> w << email_txt } + FilesEx.copyTo(output_tf.toPath(), "${outdir}/pipeline_info/pipeline_report.txt"); + output_tf.delete() +} + +// +// Print pipeline summary on completion +// +def completionSummary(monochrome_logs=true) { + Map colors = logColours(monochrome_logs) + if (workflow.success) { + if (workflow.stats.ignoredCount == 0) { + log.info "-${colors.purple}[$workflow.manifest.name]${colors.green} Pipeline completed successfully${colors.reset}-" + } else { + log.info "-${colors.purple}[$workflow.manifest.name]${colors.yellow} Pipeline completed successfully, but with errored process(es) ${colors.reset}-" + } + } else { + log.info "-${colors.purple}[$workflow.manifest.name]${colors.red} Pipeline completed with errors${colors.reset}-" + } +} + +// +// Construct and send a notification to a web server as JSON e.g. Microsoft Teams and Slack +// +def imNotification(summary_params, hook_url) { + def summary = [:] + for (group in summary_params.keySet()) { + summary << summary_params[group] + } + + def misc_fields = [:] + misc_fields['start'] = workflow.start + misc_fields['complete'] = workflow.complete + misc_fields['scriptfile'] = workflow.scriptFile + misc_fields['scriptid'] = workflow.scriptId + if (workflow.repository) misc_fields['repository'] = workflow.repository + if (workflow.commitId) misc_fields['commitid'] = workflow.commitId + if (workflow.revision) misc_fields['revision'] = workflow.revision + misc_fields['nxf_version'] = workflow.nextflow.version + misc_fields['nxf_build'] = workflow.nextflow.build + misc_fields['nxf_timestamp'] = workflow.nextflow.timestamp + + def msg_fields = [:] + msg_fields['version'] = getWorkflowVersion() + msg_fields['runName'] = workflow.runName + msg_fields['success'] = workflow.success + msg_fields['dateComplete'] = workflow.complete + msg_fields['duration'] = workflow.duration + msg_fields['exitStatus'] = workflow.exitStatus + msg_fields['errorMessage'] = (workflow.errorMessage ?: 'None') + msg_fields['errorReport'] = (workflow.errorReport ?: 'None') + msg_fields['commandLine'] = workflow.commandLine.replaceFirst(/ +--hook_url +[^ ]+/, "") + msg_fields['projectDir'] = workflow.projectDir + msg_fields['summary'] = summary << misc_fields + + // Render the JSON template + def engine = new groovy.text.GStringTemplateEngine() + // Different JSON depending on the service provider + // Defaults to "Adaptive Cards" (https://adaptivecards.io), except Slack which has its own format + def json_path = hook_url.contains("hooks.slack.com") ? "slackreport.json" : "adaptivecard.json" + def hf = new File("${workflow.projectDir}/assets/${json_path}") + def json_template = engine.createTemplate(hf).make(msg_fields) + def json_message = json_template.toString() + + // POST + def post = new URL(hook_url).openConnection(); + post.setRequestMethod("POST") + post.setDoOutput(true) + post.setRequestProperty("Content-Type", "application/json") + post.getOutputStream().write(json_message.getBytes("UTF-8")); + def postRC = post.getResponseCode(); + if (! postRC.equals(200)) { + log.warn(post.getErrorStream().getText()); + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml new file mode 100644 index 0000000..d08d243 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/meta.yml @@ -0,0 +1,24 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NFCORE_PIPELINE" +description: Subworkflow with utility functions specific to the nf-core pipeline template +keywords: + - utility + - pipeline + - initialise + - version +components: [] +input: + - nextflow_cli_args: + type: list + description: | + Nextflow CLI positional arguments +output: + - success: + type: boolean + description: | + Dummy output to indicate success +authors: + - "@adamrtalbot" +maintainers: + - "@adamrtalbot" + - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test new file mode 100644 index 0000000..1dc317f --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test @@ -0,0 +1,134 @@ + +nextflow_function { + + name "Test Functions" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Test Function checkConfigProvided") { + + function "checkConfigProvided" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function checkProfileProvided") { + + function "checkProfileProvided" + + when { + function { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function workflowCitation") { + + function "workflowCitation" + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function nfCoreLogo") { + + function "nfCoreLogo" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function dashedLine") { + + function "dashedLine" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function without logColours") { + + function "logColours" + + when { + function { + """ + input[0] = true + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } + + test("Test Function with logColours") { + function "logColours" + + when { + function { + """ + input[0] = false + """ + } + } + + then { + assertAll( + { assert function.success }, + { assert snapshot(function.result).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap new file mode 100644 index 0000000..1037232 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.function.nf.test.snap @@ -0,0 +1,166 @@ +{ + "Test Function checkProfileProvided": { + "content": null, + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:03.360873" + }, + "Test Function checkConfigProvided": { + "content": [ + true + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:02:59.729647" + }, + "Test Function nfCoreLogo": { + "content": [ + "\n\n-\u001b[2m----------------------------------------------------\u001b[0m-\n \u001b[0;32m,--.\u001b[0;30m/\u001b[0;32m,-.\u001b[0m\n\u001b[0;34m ___ __ __ __ ___ \u001b[0;32m/,-._.--~'\u001b[0m\n\u001b[0;34m |\\ | |__ __ / ` / \\ |__) |__ \u001b[0;33m} {\u001b[0m\n\u001b[0;34m | \\| | \\__, \\__/ | \\ |___ \u001b[0;32m\\`-._,-`-,\u001b[0m\n \u001b[0;32m`._,._,'\u001b[0m\n\u001b[0;35m nextflow_workflow v9.9.9\u001b[0m\n-\u001b[2m----------------------------------------------------\u001b[0m-\n" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:10.562934" + }, + "Test Function workflowCitation": { + "content": [ + "If you use nextflow_workflow for your analysis please cite:\n\n* The pipeline\n https://doi.org/10.5281/zenodo.5070524\n\n* The nf-core framework\n https://doi.org/10.1038/s41587-020-0439-x\n\n* Software dependencies\n https://github.com/nextflow_workflow/blob/master/CITATIONS.md" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:07.019761" + }, + "Test Function without logColours": { + "content": [ + { + "reset": "", + "bold": "", + "dim": "", + "underlined": "", + "blink": "", + "reverse": "", + "hidden": "", + "black": "", + "red": "", + "green": "", + "yellow": "", + "blue": "", + "purple": "", + "cyan": "", + "white": "", + "bblack": "", + "bred": "", + "bgreen": "", + "byellow": "", + "bblue": "", + "bpurple": "", + "bcyan": "", + "bwhite": "", + "ublack": "", + "ured": "", + "ugreen": "", + "uyellow": "", + "ublue": "", + "upurple": "", + "ucyan": "", + "uwhite": "", + "iblack": "", + "ired": "", + "igreen": "", + "iyellow": "", + "iblue": "", + "ipurple": "", + "icyan": "", + "iwhite": "", + "biblack": "", + "bired": "", + "bigreen": "", + "biyellow": "", + "biblue": "", + "bipurple": "", + "bicyan": "", + "biwhite": "" + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:17.969323" + }, + "Test Function dashedLine": { + "content": [ + "-\u001b[2m----------------------------------------------------\u001b[0m-" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:14.366181" + }, + "Test Function with logColours": { + "content": [ + { + "reset": "\u001b[0m", + "bold": "\u001b[1m", + "dim": "\u001b[2m", + "underlined": "\u001b[4m", + "blink": "\u001b[5m", + "reverse": "\u001b[7m", + "hidden": "\u001b[8m", + "black": "\u001b[0;30m", + "red": "\u001b[0;31m", + "green": "\u001b[0;32m", + "yellow": "\u001b[0;33m", + "blue": "\u001b[0;34m", + "purple": "\u001b[0;35m", + "cyan": "\u001b[0;36m", + "white": "\u001b[0;37m", + "bblack": "\u001b[1;30m", + "bred": "\u001b[1;31m", + "bgreen": "\u001b[1;32m", + "byellow": "\u001b[1;33m", + "bblue": "\u001b[1;34m", + "bpurple": "\u001b[1;35m", + "bcyan": "\u001b[1;36m", + "bwhite": "\u001b[1;37m", + "ublack": "\u001b[4;30m", + "ured": "\u001b[4;31m", + "ugreen": "\u001b[4;32m", + "uyellow": "\u001b[4;33m", + "ublue": "\u001b[4;34m", + "upurple": "\u001b[4;35m", + "ucyan": "\u001b[4;36m", + "uwhite": "\u001b[4;37m", + "iblack": "\u001b[0;90m", + "ired": "\u001b[0;91m", + "igreen": "\u001b[0;92m", + "iyellow": "\u001b[0;93m", + "iblue": "\u001b[0;94m", + "ipurple": "\u001b[0;95m", + "icyan": "\u001b[0;96m", + "iwhite": "\u001b[0;97m", + "biblack": "\u001b[1;90m", + "bired": "\u001b[1;91m", + "bigreen": "\u001b[1;92m", + "biyellow": "\u001b[1;93m", + "biblue": "\u001b[1;94m", + "bipurple": "\u001b[1;95m", + "bicyan": "\u001b[1;96m", + "biwhite": "\u001b[1;97m" + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:21.714424" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test new file mode 100644 index 0000000..8940d32 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test @@ -0,0 +1,29 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFCORE_PIPELINE" + script "../main.nf" + config "subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config" + workflow "UTILS_NFCORE_PIPELINE" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "utils_nfcore_pipeline" + tag "subworkflows/utils_nfcore_pipeline" + + test("Should run without failures") { + + when { + workflow { + """ + input[0] = [] + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert snapshot(workflow.out).match() } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap new file mode 100644 index 0000000..859d103 --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/main.workflow.nf.test.snap @@ -0,0 +1,19 @@ +{ + "Should run without failures": { + "content": [ + { + "0": [ + true + ], + "valid_config": [ + true + ] + } + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-02-28T12:03:25.726491" + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config b/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config new file mode 100644 index 0000000..d0a926b --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/nextflow.config @@ -0,0 +1,9 @@ +manifest { + name = 'nextflow_workflow' + author = """nf-core""" + homePage = 'https://127.0.0.1' + description = """Dummy pipeline""" + nextflowVersion = '!>=23.04.0' + version = '9.9.9' + doi = 'https://doi.org/10.5281/zenodo.5070524' +} diff --git a/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml b/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml new file mode 100644 index 0000000..ac8523c --- /dev/null +++ b/subworkflows/nf-core/utils_nfcore_pipeline/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nfcore_pipeline: + - subworkflows/nf-core/utils_nfcore_pipeline/** diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf b/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf new file mode 100644 index 0000000..2585b65 --- /dev/null +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/main.nf @@ -0,0 +1,62 @@ +// +// Subworkflow that uses the nf-validation plugin to render help text and parameter summary +// + +/* +======================================================================================== + IMPORT NF-VALIDATION PLUGIN +======================================================================================== +*/ + +include { paramsHelp } from 'plugin/nf-validation' +include { paramsSummaryLog } from 'plugin/nf-validation' +include { validateParameters } from 'plugin/nf-validation' + +/* +======================================================================================== + SUBWORKFLOW DEFINITION +======================================================================================== +*/ + +workflow UTILS_NFVALIDATION_PLUGIN { + + take: + print_help // boolean: print help + workflow_command // string: default commmand used to run pipeline + pre_help_text // string: string to be printed before help text and summary log + post_help_text // string: string to be printed after help text and summary log + validate_params // boolean: validate parameters + schema_filename // path: JSON schema file, null to use default value + + main: + + log.debug "Using schema file: ${schema_filename}" + + // Default values for strings + pre_help_text = pre_help_text ?: '' + post_help_text = post_help_text ?: '' + workflow_command = workflow_command ?: '' + + // + // Print help message if needed + // + if (print_help) { + log.info pre_help_text + paramsHelp(workflow_command, parameters_schema: schema_filename) + post_help_text + System.exit(0) + } + + // + // Print parameter summary to stdout + // + log.info pre_help_text + paramsSummaryLog(workflow, parameters_schema: schema_filename) + post_help_text + + // + // Validate parameters relative to the parameter JSON schema + // + if (validate_params){ + validateParameters(parameters_schema: schema_filename) + } + + emit: + dummy_emit = true +} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml b/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml new file mode 100644 index 0000000..3d4a6b0 --- /dev/null +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/meta.yml @@ -0,0 +1,44 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "UTILS_NFVALIDATION_PLUGIN" +description: Use nf-validation to initiate and validate a pipeline +keywords: + - utility + - pipeline + - initialise + - validation +components: [] +input: + - print_help: + type: boolean + description: | + Print help message and exit + - workflow_command: + type: string + description: | + The command to run the workflow e.g. "nextflow run main.nf" + - pre_help_text: + type: string + description: | + Text to print before the help message + - post_help_text: + type: string + description: | + Text to print after the help message + - validate_params: + type: boolean + description: | + Validate the parameters and error if invalid. + - schema_filename: + type: string + description: | + The filename of the schema to validate against. +output: + - dummy_emit: + type: boolean + description: | + Dummy emit to make nf-core subworkflows lint happy +authors: + - "@adamrtalbot" +maintainers: + - "@adamrtalbot" + - "@maxulysse" diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test new file mode 100644 index 0000000..5784a33 --- /dev/null +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/main.nf.test @@ -0,0 +1,200 @@ +nextflow_workflow { + + name "Test Workflow UTILS_NFVALIDATION_PLUGIN" + script "../main.nf" + workflow "UTILS_NFVALIDATION_PLUGIN" + tag "subworkflows" + tag "subworkflows_nfcore" + tag "plugin/nf-validation" + tag "'plugin/nf-validation'" + tag "utils_nfvalidation_plugin" + tag "subworkflows/utils_nfvalidation_plugin" + + test("Should run nothing") { + + when { + + params { + monochrome_logs = true + test_data = '' + } + + workflow { + """ + help = false + workflow_command = null + pre_help_text = null + post_help_text = null + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.success } + ) + } + } + + test("Should run help") { + + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = true + workflow_command = null + pre_help_text = null + post_help_text = null + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.exitStatus == 0 }, + { assert workflow.stdout.any { it.contains('Input/output options') } }, + { assert workflow.stdout.any { it.contains('--outdir') } } + ) + } + } + + test("Should run help with command") { + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = true + workflow_command = "nextflow run noorg/doesntexist" + pre_help_text = null + post_help_text = null + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.exitStatus == 0 }, + { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, + { assert workflow.stdout.any { it.contains('Input/output options') } }, + { assert workflow.stdout.any { it.contains('--outdir') } } + ) + } + } + + test("Should run help with extra text") { + + + when { + + params { + monochrome_logs = true + test_data = '' + } + workflow { + """ + help = true + workflow_command = "nextflow run noorg/doesntexist" + pre_help_text = "pre-help-text" + post_help_text = "post-help-text" + validate_params = false + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.success }, + { assert workflow.exitStatus == 0 }, + { assert workflow.stdout.any { it.contains('pre-help-text') } }, + { assert workflow.stdout.any { it.contains('nextflow run noorg/doesntexist') } }, + { assert workflow.stdout.any { it.contains('Input/output options') } }, + { assert workflow.stdout.any { it.contains('--outdir') } }, + { assert workflow.stdout.any { it.contains('post-help-text') } } + ) + } + } + + test("Should validate params") { + + when { + + params { + monochrome_logs = true + test_data = '' + outdir = 1 + } + workflow { + """ + help = false + workflow_command = null + pre_help_text = null + post_help_text = null + validate_params = true + schema_filename = "$moduleTestDir/nextflow_schema.json" + + input[0] = help + input[1] = workflow_command + input[2] = pre_help_text + input[3] = post_help_text + input[4] = validate_params + input[5] = schema_filename + """ + } + } + + then { + assertAll( + { assert workflow.failed }, + { assert workflow.stdout.any { it.contains('ERROR ~ ERROR: Validation of pipeline parameters failed!') } } + ) + } + } +} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json new file mode 100644 index 0000000..7626c1c --- /dev/null +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/nextflow_schema.json @@ -0,0 +1,96 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/./master/nextflow_schema.json", + "title": ". pipeline parameters", + "description": "", + "type": "object", + "definitions": { + "input_output_options": { + "title": "Input/output options", + "type": "object", + "fa_icon": "fas fa-terminal", + "description": "Define where the pipeline should find input data and save output data.", + "required": ["outdir"], + "properties": { + "validate_params": { + "type": "boolean", + "description": "Validate parameters?", + "default": true, + "hidden": true + }, + "outdir": { + "type": "string", + "format": "directory-path", + "description": "The output directory where the results will be saved. You have to use absolute paths to storage on Cloud infrastructure.", + "fa_icon": "fas fa-folder-open" + }, + "test_data_base": { + "type": "string", + "default": "https://raw.githubusercontent.com/nf-core/test-datasets/modules", + "description": "Base for test data directory", + "hidden": true + }, + "test_data": { + "type": "string", + "description": "Fake test data param", + "hidden": true + } + } + }, + "generic_options": { + "title": "Generic options", + "type": "object", + "fa_icon": "fas fa-file-import", + "description": "Less common options for the pipeline, typically set in a config file.", + "help_text": "These options are common to all nf-core pipelines and allow you to customise some of the core preferences for how the pipeline runs.\n\nTypically these options would be set in a Nextflow config file loaded for all pipeline runs, such as `~/.nextflow/config`.", + "properties": { + "help": { + "type": "boolean", + "description": "Display help text.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "version": { + "type": "boolean", + "description": "Display version and exit.", + "fa_icon": "fas fa-question-circle", + "hidden": true + }, + "logo": { + "type": "boolean", + "default": true, + "description": "Display nf-core logo in console output.", + "fa_icon": "fas fa-image", + "hidden": true + }, + "singularity_pull_docker_container": { + "type": "boolean", + "description": "Pull Singularity container from Docker?", + "hidden": true + }, + "publish_dir_mode": { + "type": "string", + "default": "copy", + "description": "Method used to save pipeline results to output directory.", + "help_text": "The Nextflow `publishDir` option specifies which intermediate files should be saved to the output directory. This option tells the pipeline what method should be used to move these files. See [Nextflow docs](https://www.nextflow.io/docs/latest/process.html#publishdir) for details.", + "fa_icon": "fas fa-copy", + "enum": ["symlink", "rellink", "link", "copy", "copyNoFollow", "move"], + "hidden": true + }, + "monochrome_logs": { + "type": "boolean", + "description": "Use monochrome_logs", + "hidden": true + } + } + } + }, + "allOf": [ + { + "$ref": "#/definitions/input_output_options" + }, + { + "$ref": "#/definitions/generic_options" + } + ] +} diff --git a/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml new file mode 100644 index 0000000..60b1cff --- /dev/null +++ b/subworkflows/nf-core/utils_nfvalidation_plugin/tests/tags.yml @@ -0,0 +1,2 @@ +subworkflows/utils_nfvalidation_plugin: + - subworkflows/nf-core/utils_nfvalidation_plugin/** diff --git a/tests/pipeline/lib/UTILS.groovy b/tests/pipeline/lib/UTILS.groovy index 311403c..deacb58 100644 --- a/tests/pipeline/lib/UTILS.groovy +++ b/tests/pipeline/lib/UTILS.groovy @@ -2,7 +2,7 @@ class UTILS { public static String removeNextflowVersion(outputDir) { - def softwareVersions = path("$outputDir/pipeline_info/software_versions.yml").yaml + def softwareVersions = path("$outputDir/pipeline_info/nf_core_pipeline_software_mqc_versions.yml").yaml if (softwareVersions.containsKey("Workflow")) { softwareVersions.Workflow.remove("Nextflow") } diff --git a/tests/pipeline/test_downstream.nf.test b/tests/pipeline/test_downstream.nf.test index dbfc38a..6233756 100644 --- a/tests/pipeline/test_downstream.nf.test +++ b/tests/pipeline/test_downstream.nf.test @@ -12,33 +12,36 @@ nextflow_pipeline { spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" // Parameters - st_qc_min_counts = 5 - st_qc_min_genes = 3 + qc_min_counts = 5 + qc_min_genes = 3 outdir = "$outputDir" } } then { assertAll( + // Workflow { assert workflow.success }, - { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("nf_core_pipeline_software_mqc_versions.yml") }, // Data - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/st_adata_processed.h5ad").exists() }, - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2.2/data/st_adata_processed.h5ad").exists() }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/adata_processed.h5ad").exists() }, + { assert path("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/sdata_processed.zarr").exists() }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2.2/data/adata_processed.h5ad").exists() }, + { assert path("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2.2/data/sdata_processed.zarr").exists() }, // Reports - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_quality_controls.html").text.contains("final results of all the filtering") }, - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_clustering.html").text.contains("spatial distribution of clusters") }, - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_spatial_de.html").text.contains("plot the top spatially variable genes") }, - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2.2/reports/st_quality_controls.html").text.contains("final results of all the filtering") }, - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2.2/reports/st_clustering.html").text.contains("spatial distribution of clusters") }, - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2.2/reports/st_spatial_de.html").text.contains("plot the top spatially variable genes") }, - - // DEGs - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/degs/st_spatial_de.csv").exists() }, - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2.2/degs/st_spatial_de.csv").exists() }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/quality_controls.html").text.contains("final results of all the filtering") }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/clustering.html").text.contains("spatial distribution of clusters") }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/spatially_variable_genes.html").text.contains("Spatial transcriptomics data can give insight") }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2.2/reports/quality_controls.html").text.contains("final results of all the filtering") }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2.2/reports/clustering.html").text.contains("spatial distribution of clusters") }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2.2/reports/spatially_variable_genes.html").text.contains("Spatial transcriptomics data can give insight") }, + + // Spatially variable genes + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/spatially_variable_genes.csv").exists() }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2.2/data/spatially_variable_genes.csv").exists() }, // MultiQC { assert file("$outputDir/multiqc/multiqc_report.html").exists() } diff --git a/tests/pipeline/test_downstream.nf.test.snap b/tests/pipeline/test_downstream.nf.test.snap index 75f12d8..cdacf62 100644 --- a/tests/pipeline/test_downstream.nf.test.snap +++ b/tests/pipeline/test_downstream.nf.test.snap @@ -1,8 +1,12 @@ { - "software_versions": { + "nf_core_pipeline_software_mqc_versions.yml": { "content": [ - "{CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.7, yaml=5.4.1}, SPACERANGER_UNTAR_REFERENCE={untar=1.30}, ST_CLUSTERING={quarto=1.3.302, scanpy=1.9.3}, ST_QUALITY_CONTROLS={quarto=1.3.302, scanpy=1.9.3}, ST_READ_DATA={scanpy=1.7.2}, ST_SPATIAL_DE={SpatialDE=1.1.3, leidenalg=0.9.1, quarto=1.3.302, scanpy=1.9.3}, Workflow={nf-core/spatialtranscriptomics=1.0dev}}" + "{CLUSTERING={quarto=1.3.450, papermill=null}, QUALITY_CONTROLS={quarto=1.3.450, papermill=null}, READ_DATA={spatialdata_io=0.1.2}, SPACERANGER_UNTAR_REFERENCE={untar=1.3}, SPATIALLY_VARIABLE_GENES={quarto=1.3.450, papermill=null}, UNTAR_DOWNSTREAM_INPUT={untar=1.3}, Workflow={nf-core/spatialtranscriptomics=v1.0dev}}" ], - "timestamp": "2024-01-15T16:00:03.485826" + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, + "timestamp": "2024-03-19T18:46:59.035976" } } diff --git a/tests/pipeline/test_spaceranger_ffpe_v1.nf.test b/tests/pipeline/test_spaceranger_ffpe_v1.nf.test index fc02eaf..3ee63d2 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v1.nf.test +++ b/tests/pipeline/test_spaceranger_ffpe_v1.nf.test @@ -9,8 +9,8 @@ nextflow_pipeline { input = 'https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-ovarian-cancer-1-standard_v1_ffpe/samplesheet_spaceranger.csv' spaceranger_probeset = 'https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/human-ovarian-cancer-1-standard_v1_ffpe/Visium_Human_Transcriptome_Probe_Set_v1.0_GRCh38-2020-A.csv' spaceranger_reference = "https://raw.githubusercontent.com/nf-core/test-datasets/spatialtranscriptomics/testdata/homo_sapiens_chr22_reference.tar.gz" - st_qc_min_counts = 5 - st_qc_min_genes = 3 + qc_min_counts = 5 + qc_min_genes = 3 outdir = "$outputDir" } } @@ -20,18 +20,19 @@ nextflow_pipeline { // Workflow { assert workflow.success }, - { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("nf_core_pipeline_software_mqc_versions.yml") }, // Data - { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/data/st_adata_processed.h5ad").exists() }, + { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/data/adata_processed.h5ad").exists() }, + { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/data/sdata_processed.zarr").exists() }, // Reports - { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/reports/st_quality_controls.html").text.contains("final results of all the filtering") }, - { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/reports/st_clustering.html").text.contains("spatial distribution of clusters") }, - { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/reports/st_spatial_de.html").text.contains("plot the top spatially variable genes") }, + { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/reports/quality_controls.html").text.contains("final results of all the filtering") }, + { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/reports/clustering.html").text.contains("spatial distribution of clusters") }, + { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/reports/spatially_variable_genes.html").text.contains("Spatial transcriptomics data can give insight") }, - // DEGs - { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/degs/st_spatial_de.csv").exists() }, + // Spatially variable genes + { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/data/spatially_variable_genes.csv").exists() }, // Space Ranger { assert file("$outputDir/Visium_FFPE_Human_Ovarian_Cancer/spaceranger/outs/web_summary.html").exists() }, diff --git a/tests/pipeline/test_spaceranger_ffpe_v1.nf.test.snap b/tests/pipeline/test_spaceranger_ffpe_v1.nf.test.snap index 441c711..11b5370 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v1.nf.test.snap +++ b/tests/pipeline/test_spaceranger_ffpe_v1.nf.test.snap @@ -1,8 +1,22 @@ { "software_versions": { "content": [ - "{CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.7, yaml=5.4.1}, FASTQC={fastqc=0.12.1}, SPACERANGER_COUNT={spaceranger=2.1.0}, SPACERANGER_UNTAR_REFERENCE={untar=1.30}, ST_CLUSTERING={quarto=1.3.302, scanpy=1.9.3}, ST_QUALITY_CONTROLS={quarto=1.3.302, scanpy=1.9.3}, ST_READ_DATA={scanpy=1.7.2}, ST_SPATIAL_DE={SpatialDE=1.1.3, leidenalg=0.9.1, quarto=1.3.302, scanpy=1.9.3}, Workflow={nf-core/spatialtranscriptomics=1.0dev}}" + "{CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.7, yaml=5.4.1}, FASTQC={fastqc=0.12.1}, SPACERANGER_COUNT={spaceranger=2.1.0}, SPACERANGER_UNTAR_REFERENCE={untar=1.30}, CLUSTERING={quarto=1.3.302, scanpy=1.9.3}, QUALITY_CONTROLS={quarto=1.3.302, scanpy=1.9.3}, READ_DATA={scanpy=1.7.2}, SPATIAL_DE={SpatialDE=1.1.3, leidenalg=0.9.1, quarto=1.3.302, scanpy=1.9.3}, Workflow={nf-core/spatialtranscriptomics=1.0dev}}" ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "23.10.1" + }, "timestamp": "2024-01-15T13:44:40.789425" + }, + "nf_core_pipeline_software_mqc_versions.yml": { + "content": [ + "{CLUSTERING={quarto=1.3.450, papermill=null}, FASTQC={fastqc=0.12.1}, QUALITY_CONTROLS={quarto=1.3.450, papermill=null}, READ_DATA={spatialdata_io=0.1.2}, SPACERANGER_COUNT={spaceranger=3.0.0}, SPACERANGER_UNTAR_REFERENCE={untar=1.3}, SPATIALLY_VARIABLE_GENES={quarto=1.3.450, papermill=null}, UNTAR_SPACERANGER_INPUT={untar=1.3}, Workflow={nf-core/spatialtranscriptomics=v1.0dev}}" + ], + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.02.0" + }, + "timestamp": "2024-04-04T10:08:16.975105" } } diff --git a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test index ab7a967..124ebe3 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test +++ b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test @@ -16,18 +16,19 @@ nextflow_pipeline { // Workflow { assert workflow.success }, - { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("software_versions") }, + { assert snapshot(UTILS.removeNextflowVersion("$outputDir")).match("nf_core_pipeline_software_mqc_versions.yml") }, // Data - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/st_adata_processed.h5ad").exists() }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/adata_processed.h5ad").exists() }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/sdata_processed.zarr").exists() }, // Reports - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_quality_controls.html").text.contains("final results of all the filtering") }, - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_clustering.html").text.contains("spatial distribution of clusters") }, - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/st_spatial_de.html").text.contains("plot the top spatially variable genes") }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/quality_controls.html").text.contains("final results of all the filtering") }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/clustering.html").text.contains("spatial distribution of clusters") }, + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/reports/spatially_variable_genes.html").text.contains("Spatial transcriptomics data can give insight") }, - // DEGs - { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/degs/st_spatial_de.csv").exists() }, + // Spatially variable genes + { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/data/spatially_variable_genes.csv").exists() }, // Space Ranger { assert file("$outputDir/CytAssist_11mm_FFPE_Human_Glioblastoma_2/spaceranger/outs/web_summary.html").exists() }, diff --git a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test.snap b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test.snap index 1dad690..be972cc 100644 --- a/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test.snap +++ b/tests/pipeline/test_spaceranger_ffpe_v2_cytassist.nf.test.snap @@ -1,8 +1,12 @@ { - "software_versions": { + "nf_core_pipeline_software_mqc_versions.yml": { "content": [ - "{CUSTOM_DUMPSOFTWAREVERSIONS={python=3.11.7, yaml=5.4.1}, FASTQC={fastqc=0.12.1}, SPACERANGER_COUNT={spaceranger=2.1.0}, SPACERANGER_UNTAR_REFERENCE={untar=1.30}, ST_CLUSTERING={quarto=1.3.302, scanpy=1.9.3}, ST_QUALITY_CONTROLS={quarto=1.3.302, scanpy=1.9.3}, ST_READ_DATA={scanpy=1.7.2}, ST_SPATIAL_DE={SpatialDE=1.1.3, leidenalg=0.9.1, quarto=1.3.302, scanpy=1.9.3}, Workflow={nf-core/spatialtranscriptomics=1.0dev}}" + "{CLUSTERING={quarto=1.3.450, papermill=null}, FASTQC={fastqc=0.12.1}, QUALITY_CONTROLS={quarto=1.3.450, papermill=null}, READ_DATA={spatialdata_io=0.1.2}, SPACERANGER_COUNT={spaceranger=3.0.0}, SPACERANGER_UNTAR_REFERENCE={untar=1.3}, SPATIALLY_VARIABLE_GENES={quarto=1.3.450, papermill=null}, UNTAR_SPACERANGER_INPUT={untar=1.3}, Workflow={nf-core/spatialtranscriptomics=v1.0dev}}" ], - "timestamp": "2024-01-15T15:42:52.651007" + "meta": { + "nf-test": "0.8.4", + "nextflow": "24.02.0" + }, + "timestamp": "2024-04-04T10:42:54.76102" } } diff --git a/workflows/spatialtranscriptomics.nf b/workflows/spatialtranscriptomics.nf index 37047b9..3b2fb75 100644 --- a/workflows/spatialtranscriptomics.nf +++ b/workflows/spatialtranscriptomics.nf @@ -1,99 +1,43 @@ /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - PRINT PARAMS SUMMARY + IMPORT MODULES / SUBWORKFLOWS / FUNCTIONS ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -include { paramsSummaryLog; paramsSummaryMap } from 'plugin/nf-validation' - -def logo = NfcoreTemplate.logo(workflow, params.monochrome_logs) -def citation = '\n' + WorkflowMain.citation(workflow) + '\n' -def summary_params = paramsSummaryMap(workflow) - -// Print parameter summary log to screen -log.info logo + paramsSummaryLog(workflow) + citation - -WorkflowSpatialtranscriptomics.initialise(params, log) - -// Check input path parameters to see if they exist -log.info """\ - Project directory: ${projectDir} - """ - .stripIndent() - -def checkPathParamList = [ - params.input, - params.spaceranger_reference, - params.spaceranger_probeset -] -for (param in checkPathParamList) { if (param) { file(param, checkIfExists: true) } } - -// Check mandatory parameters -if (params.input) { ch_input = file(params.input) } else { exit 1, 'Input samplesheet not specified!' } - - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - CONFIG FILES -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) -ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath( params.multiqc_config, checkIfExists: true ) : Channel.empty() -ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath( params.multiqc_logo, checkIfExists: true ) : Channel.empty() -ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) - - -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - IMPORT LOCAL MODULES/SUBWORKFLOWS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -// -// MODULE: Loaded from modules/local/ -// -include { ST_READ_DATA } from '../modules/local/st_read_data' - -// -// SUBWORKFLOW: Consisting of a mix of local and nf-core/modules -// -include { INPUT_CHECK } from '../subworkflows/local/input_check' -include { SPACERANGER } from '../subworkflows/local/spaceranger' -include { ST_DOWNSTREAM } from '../subworkflows/local/st_downstream' +include { READ_DATA } from '../modules/local/read_data' +include { FASTQC } from '../modules/nf-core/fastqc/main' +include { MULTIQC } from '../modules/nf-core/multiqc/main' +include { paramsSummaryMap } from 'plugin/nf-validation' +include { INPUT_CHECK } from '../subworkflows/local/input_check' +include { SPACERANGER } from '../subworkflows/local/spaceranger' +include { DOWNSTREAM } from '../subworkflows/local/downstream' +include { paramsSummaryMultiqc } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { softwareVersionsToYAML } from '../subworkflows/nf-core/utils_nfcore_pipeline' +include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_spatialtranscriptomics_pipeline' /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - IMPORT NF-CORE MODULES/SUBWORKFLOWS + RUN MAIN WORKFLOW ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -// -// MODULE: Installed directly from nf-core/modules -// -include { FASTQC } from "../modules/nf-core/fastqc/main" -include { MULTIQC } from "../modules/nf-core/multiqc/main" -include { CUSTOM_DUMPSOFTWAREVERSIONS } from '../modules/nf-core/custom/dumpsoftwareversions/main' +workflow SPATIALTRANSCRIPTOMICS { -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - RUN MAIN WORKFLOW -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ + take: + samplesheet // file: samplesheet read in from --input -// -// Spatial transcriptomics workflow -// -workflow ST { + main: ch_versions = Channel.empty() + ch_multiqc_files = Channel.empty() // // SUBWORKFLOW: Read and validate samplesheet // INPUT_CHECK ( - ch_input + samplesheet ) + ch_versions = ch_versions.mix(INPUT_CHECK.out.versions) // // MODULE: FastQC @@ -102,55 +46,63 @@ workflow ST { INPUT_CHECK.out.ch_spaceranger_input.map{ it -> [it[0] /* meta */, it[1] /* reads */]} ) ch_versions = ch_versions.mix(FASTQC.out.versions) + ch_multiqc_files = ch_multiqc_files.mix(FASTQC.out.zip.collect{it[1]}) // // SUBWORKFLOW: Space Ranger raw data processing // + DOWNSTREAM_REQUIRED_SPACERANGER_FILES = [ + "raw_feature_bc_matrix.h5", + "tissue_positions.csv", + "scalefactors_json.json", + "tissue_hires_image.png", + "tissue_lowres_image.png" + ] SPACERANGER ( INPUT_CHECK.out.ch_spaceranger_input ) ch_versions = ch_versions.mix(SPACERANGER.out.versions) + ch_multiqc_files = ch_multiqc_files.mix(SPACERANGER.out.sr_dir.collect{it[1]}) ch_downstream_input = INPUT_CHECK.out.ch_downstream_input.concat(SPACERANGER.out.sr_dir).map{ - meta, outs -> [meta, outs.findAll{ it -> Utils.DOWNSTREAM_REQUIRED_SPACERANGER_FILES.contains(it.name) }] + meta, outs -> [meta, outs.findAll{ it -> DOWNSTREAM_REQUIRED_SPACERANGER_FILES.contains(it.name) }] } // - // MODULE: Read ST data and save as `anndata` + // MODULE: Read ST data and save as `SpatialData` // - ST_READ_DATA ( + READ_DATA ( ch_downstream_input ) - ch_versions = ch_versions.mix(ST_READ_DATA.out.versions) + ch_versions = ch_versions.mix(READ_DATA.out.versions) // // SUBWORKFLOW: Downstream analyses of ST data // - ST_DOWNSTREAM ( - ST_READ_DATA.out.st_adata_raw + DOWNSTREAM ( + READ_DATA.out.sdata_raw ) - ch_versions = ch_versions.mix(ST_DOWNSTREAM.out.versions) + ch_versions = ch_versions.mix(DOWNSTREAM.out.versions) // - // MODULE: Pipeline reporting + // Collate and save software versions // - CUSTOM_DUMPSOFTWAREVERSIONS ( - ch_versions.unique().collectFile(name: 'collated_versions.yml') - ) + softwareVersionsToYAML(ch_versions) + .collectFile(storeDir: "${params.outdir}/pipeline_info", name: 'nf_core_pipeline_software_mqc_versions.yml', sort: true, newLine: true) + .set { ch_collated_versions } // // MODULE: MultiQC // - workflow_summary = WorkflowSpatialtranscriptomics.paramsSummaryMultiqc(workflow, summary_params) - ch_workflow_summary = Channel.value(workflow_summary) - - methods_description = WorkflowSpatialtranscriptomics.methodsDescriptionText(workflow, ch_multiqc_custom_methods_description, params) - ch_methods_description = Channel.value(methods_description) - - ch_multiqc_files = ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml').mix( - ch_methods_description.collectFile(name: 'methods_description_mqc.yaml'), - CUSTOM_DUMPSOFTWAREVERSIONS.out.mqc_yml.collect(), - FASTQC.out.zip.collect{ meta, qcfile -> qcfile } - ) + ch_multiqc_config = Channel.fromPath("$projectDir/assets/multiqc_config.yml", checkIfExists: true) + ch_multiqc_custom_config = params.multiqc_config ? Channel.fromPath(params.multiqc_config, checkIfExists: true) : Channel.empty() + ch_multiqc_logo = params.multiqc_logo ? Channel.fromPath(params.multiqc_logo, checkIfExists: true) : Channel.empty() + summary_params = paramsSummaryMap(workflow, parameters_schema: "nextflow_schema.json") + ch_workflow_summary = Channel.value(paramsSummaryMultiqc(summary_params)) + ch_multiqc_custom_methods_description = params.multiqc_methods_description ? file(params.multiqc_methods_description, checkIfExists: true) : file("$projectDir/assets/methods_description_template.yml", checkIfExists: true) + ch_methods_description = Channel.value(methodsDescriptionText(ch_multiqc_custom_methods_description)) + ch_multiqc_files = ch_multiqc_files.mix(ch_workflow_summary.collectFile(name: 'workflow_summary_mqc.yaml')) + ch_multiqc_files = ch_multiqc_files.mix(ch_collated_versions) + ch_multiqc_files = ch_multiqc_files.mix(ch_methods_description.collectFile(name: 'methods_description_mqc.yaml', sort: false)) MULTIQC ( ch_multiqc_files.collect(), @@ -158,31 +110,10 @@ workflow ST { ch_multiqc_custom_config.toList(), ch_multiqc_logo.toList() ) - multiqc_report = MULTIQC.out.report.toList() -} -/* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - COMPLETION EMAIL AND SUMMARY -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -*/ - -workflow.onComplete { - if (params.email || params.email_on_fail) { - NfcoreTemplate.email(workflow, params, summary_params, projectDir, log, multiqc_report) - } - NfcoreTemplate.dump_parameters(workflow, params) - NfcoreTemplate.summary(workflow, params, log) - if (params.hook_url) { - NfcoreTemplate.IM_notification(workflow, params, summary_params, projectDir, log) - } -} - -workflow.onError { - if (workflow.errorReport.contains("Process requirement exceeds available memory")) { - println("🛑 Default resources exceed availability 🛑 ") - println("💡 See here on how to configure pipeline: https://nf-co.re/docs/usage/configuration#tuning-workflow-resources 💡") - } + emit: + multiqc_report = MULTIQC.out.report.toList() // channel: /path/to/multiqc_report.html + versions = ch_versions // channel: [ path(versions.yml) ] } /*
    Process Name \\", - " \\ Software Version
    CUSTOM_DUMPSOFTWAREVERSIONSpython3.11.7
    yaml5.4.1
    TOOL1tool10.11.9
    TOOL2tool21.9
    WorkflowNextflow