Skip to content
Run tests

Added nf-test for vrhyme/vrhyme #14641

Sign in to view logs

Added nf-test for vrhyme/vrhyme

Added nf-test for vrhyme/vrhyme #14641

Workflow file for this run

.github/workflows/test.yml at 758f64e
name: Run tests
on:
pull_request:
branches: [master]
merge_group:
types: [checks_requested]
branches: [master]
workflow_dispatch:
inputs:
runners:
description: "Runners to test on"
type: choice
options:
- "ubuntu-latest"
- "self-hosted"
default: "self-hosted"
# Cancel if a newer run is started
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
# FIXME Flip this off once we get to less than a couple hundred. Adding
# this so it will only run against changed files. It'll make it much
# easier to fix these as they come up rather than everything at once.
with:
extra_args: ""
prettier:
runs-on: ubuntu-latest
steps:
- name: Check out repository
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: Install NodeJS
uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4
with:
node-version: "20"
- name: Install Prettier
run: npm install -g [email protected]
- name: Run Prettier --check
run: prettier --check .
editorconfig:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4
with:
node-version: "20"
- name: Install editorconfig-checker
run: npm install -g editorconfig-checker
- name: Run ECLint check
run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test)
pytest-changes:
name: pytest-changes
runs-on: ubuntu-latest
outputs:
tags: ${{ steps.filter.outputs.changes }}
modules: ${{ steps.tags.outputs.modules }}
subworkflows: ${{ steps.tags.outputs.subworkflows }}
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
with:
fetch-depth: 2 # To retrieve the preceding commit.
# TODO: change back to using dorny/paths-filter when https://github.com/dorny/paths-filter/pull/133 is implemented
- uses: mirpedrol/paths-filter@main
id: filter
with:
filters: "tests/config/pytest_modules.yml"
token: ""
- name: Fetch module tags
id: tags
run: |
echo modules=$(echo '${{ steps.filter.outputs.changes }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/"; ""))') >> $GITHUB_OUTPUT
echo subworkflows=$(echo '${{ steps.filter.outputs.changes }}' | jq '. | map(select(contains("subworkflow"))) | map(gsub("subworkflows/"; ""))') >> $GITHUB_OUTPUT
- name: debug
run: |
echo ${{ steps.tags.outputs.modules }}
echo ${{ steps.tags.outputs.subworkflows }}
nf-test-changes:
name: nf-test-changes
runs-on: ubuntu-latest
outputs:
# Expose detected tags as 'modules' and 'workflows' output variables
paths: ${{ steps.list.outputs.components }}
modules: ${{ steps.outputs.outputs.modules }}
subworkflows: ${{ steps.outputs.outputs.subworkflows}}
# Prod for version bumping
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
with:
fetch-depth: 0
- name: List nf-test files
id: list
uses: adamrtalbot/detect-nf-test-changes@7c8be3ffd0d6538312b363c8c949dbbf5f26c3dd # v0.0.4
with:
head: ${{ github.sha }}
base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
n_parents: 2
- name: Separate modules and subworkflows
id: outputs
run: |
echo modules=$(echo '${{ steps.list.outputs.components }}' | jq -c '. | map(select(contains("modules"))) | map(gsub("modules/nf-core/"; ""))') >> $GITHUB_OUTPUT
echo subworkflows=$(echo '${{ steps.list.outputs.components }}' | jq '. | map(select(contains("subworkflows"))) | map(gsub("subworkflows/nf-core/"; ""))') >> $GITHUB_OUTPUT
- name: debug
run: |
echo ${{ steps.outputs.outputs.modules }}
echo ${{ steps.outputs.outputs.subworkflows }}
nf-core-lint-modules:
runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
name: nf-core-lint-modules
needs: [pytest-changes, nf-test-changes]
if: ${{ (needs.pytest-changes.outputs.modules != '[]') || ( needs.nf-test-changes.outputs.modules != '[]') }}
strategy:
fail-fast: false
matrix:
tags:
[
"${{ fromJson(needs.pytest-changes.outputs.modules) }}",
"${{ fromJson(needs.nf-test-changes.outputs.modules) }}",
]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: Set up Python
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
with:
python-version: "3.11"
- uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4
id: cache-pip
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip
restore-keys: |
${{ runner.os }}-pip
- name: Install pip
run: python -m pip install --upgrade pip
- uses: actions/setup-java@6a0805fcefea3d4657a47ac4c165951e33482018 # v4
with:
distribution: "temurin"
java-version: "17"
- name: Setup Nextflow
uses: nf-core/setup-nextflow@v2
- name: Install nf-core tools development version
run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
- name: Lint module ${{ matrix.tags }}
run: nf-core modules lint ${{ matrix.tags }}
nf-core-lint-subworkflows:
runs-on: ubuntu-latest
name: nf-core-lint-modules
needs: [pytest-changes, nf-test-changes]
if: ${{ (needs.pytest-changes.outputs.subworkflows != '[]') || ( needs.nf-test-changes.outputs.subworkflows != '[]') }}
strategy:
fail-fast: false
matrix:
tags:
[
"${{ fromJson(needs.pytest-changes.outputs.subworkflows) }}",
"${{ fromJson(needs.nf-test-changes.outputs.subworkflows) }}",
]
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: Set up Python
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
with:
python-version: "3.11"
- name: Install pip
run: python -m pip install --upgrade pip
- uses: actions/setup-java@6a0805fcefea3d4657a47ac4c165951e33482018 # v4
with:
distribution: "temurin"
java-version: "17"
- name: Setup Nextflow
uses: nf-core/setup-nextflow@561fcfc7146dcb12e3871909b635ab092a781f34 # v2
- name: Install nf-core tools development version
run: python -m pip install --upgrade --force-reinstall git+https://github.com/nf-core/tools.git@dev
- name: Lint module ${{ matrix.tags }}
run: nf-core subworkflows lint ${{ matrix.tags }}
pytest:
runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
name: pytest
needs: [pytest-changes]
if: needs.pytest-changes.outputs.tags != '[]'
strategy:
fail-fast: false
matrix:
tags: ["${{ fromJson(needs.pytest-changes.outputs.tags) }}"]
profile: [conda, docker, singularity]
exclude:
- tags: nf-test
- profile: conda
tags: backsub
- profile: conda
tags: bases2fastq
- profile: singularity
tags: bases2fastq
- profile: conda
tags: basicpy
- profile: conda
tags: bcl2fastq
- profile: conda
tags: bclconvert
- profile: conda
tags: bwa/aln
- profile: conda
tags: bwa/index
- profile: conda
tags: bwa/mem
- profile: conda
tags: bwa/sampe
- profile: conda
tags: bwa/samse
- profile: conda
tags: cellpose
- profile: conda
tags: cellrangerarc/count
- profile: conda
tags: cellrangerarc/mkfastq
- profile: conda
tags: cellrangerarc/mkref
- profile: conda
tags: cellrangeratac/count
- profile: conda
tags: cellrangeratac/mkfastq
- profile: conda
tags: cellrangeratac/mkref
- profile: conda
tags: checkm2/databasedownload
- profile: conda
tags: checkm2/predict
- profile: conda
tags: controlfreec/makegraph2
- profile: conda
tags: coreograph
- profile: conda
tags: deepcell/mesmer
- profile: conda
tags: deepvariant
- profile: conda
tags: fastani
- profile: conda
tags: fastk/fastk
- profile: conda
tags: fastk/histex
- profile: conda
tags: fastk/merge
- profile: conda
tags: fcs/fcsadaptor
- profile: conda
tags: fcs/fcsgx
- profile: conda
tags: gatk4/cnnscorevariants
- profile: conda
tags: gatk4/determinegermlinecontigploidy
- profile: singularity
tags: gatk4/determinegermlinecontigploidy
- profile: conda
tags: gatk4/germlinecnvcaller
- profile: conda
tags: gatk4/postprocessgermlinecnvcalls
- profile: conda
tags: genescopefk
- profile: conda
tags: happy/sompy
- profile: conda
tags: hlala/preparegraph
- profile: conda
tags: ilastik/multicut
- profile: conda
tags: ilastik/pixelclassification
- profile: conda
tags: imputeme/vcftoprs
- profile: conda
tags: islandpath
- profile: conda
tags: manta/convertinversion
- profile: conda
tags: mcquant
- profile: conda
tags: medaka
- profile: conda
tags: merquryfk/katcomp
- profile: conda
tags: merquryfk/katgc
- profile: conda
tags: merquryfk/merquryfk
- profile: conda
tags: merquryfk/ploidyplot
- profile: conda
tags: minimap2/align
- profile: conda
tags: mitohifi/findmitoreference
- profile: conda
tags: mitohifi/mitohifi
- profile: conda
tags: nanoplot
- profile: conda
tags: ncbitools/vecscreen
- profile: conda
tags: parabricks/applybqsr
- profile: conda
tags: parabricks/dbsnp
- profile: conda
tags: parabricks/deepvariant
- profile: conda
tags: parabricks/fq2bam
- profile: conda
tags: parabricks/genotypegvcf
- profile: conda
tags: parabricks/haplotypecaller
- profile: conda
tags: parabricks/indexgvcf
- profile: conda
tags: parabricks/mutectcaller
- profile: conda
tags: picard/collecthsmetrics
- profile: conda
tags: picard/collectwgsmetrics
- profile: conda
tags: sentieon/applyvarcal
- profile: conda
tags: sentieon/datametrics
- profile: conda
tags: sentieon/dnamodelapply
- profile: conda
tags: sentieon/dnascope
- profile: conda
tags: sentieon/readwriter
- profile: conda
tags: sentieon/tnfilter
- profile: conda
tags: sentieon/tnhaplotyper2
- profile: conda
tags: sentieon/tnscope
- profile: conda
tags: sentieon/varcal
- profile: conda
tags: sentieon/wgsmetrics
- profile: conda
tags: subworkflows/bam_qc_picard
- profile: conda
tags: subworkflows/bcl_demultiplex
- profile: conda
tags: subworkflows/fasta_clean_fcs
- profile: conda
tags: svanalyzer/svbenchmark
- profile: conda
tags: universc
- profile: singularity
tags: universc
- profile: conda
tags: vt/decompose
env:
NXF_ANSI_LOG: false
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- name: Set up Python
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
with:
python-version: "3.11"
- uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4
id: cache-pip-pytest
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-pytest
restore-keys: |
${{ runner.os }}-pip-pytest
- name: Install Python dependencies
run: python -m pip install --upgrade pip pytest-workflow cryptography
- uses: actions/setup-java@6a0805fcefea3d4657a47ac4c165951e33482018 # v4
with:
distribution: "temurin"
java-version: "17"
- name: Setup Nextflow ${{ matrix.NXF_VER }}
uses: nf-core/setup-nextflow@v2
with:
version: "${{ matrix.NXF_VER }}"
- name: Setup apptainer
if: matrix.profile == 'singularity'
uses: eWaterCycle/setup-apptainer@main
- name: Set up Singularity
if: matrix.profile == 'singularity'
run: |
mkdir -p $NXF_SINGULARITY_CACHEDIR
mkdir -p $NXF_SINGULARITY_LIBRARYDIR
- name: Set up miniconda
uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
with:
miniconda-version: "latest"
channels: conda-forge,bioconda,defaults
python-version: ${{ matrix.python-version }}
- name: Conda setup
run: |
conda clean -a
conda install -n base conda-libmamba-solver
conda config --set solver libmamba
echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
echo $(realpath python) >> $GITHUB_PATH
# Test the module
- name: Run pytest-workflow
# only use one thread for pytest-workflow to avoid race condition on conda cache.
run: TMPDIR=~ PROFILE=${{ matrix.profile }} pytest --tag ${{ matrix.tags }} --symlink --kwdof --git-aware --color=yes
- name: Output log on failure
if: failure()
run: |
sudo apt-get update > /dev/null
sudo apt-get install bat > /dev/null
batcat --decorations=always --color=always /home/ubuntu/pytest_workflow_*/*/log.{out,err}
- name: Setting global variables
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
id: parsed
with:
script: |
return '${{ matrix.tags }}'.toLowerCase().replaceAll(/\//g, '-').trim('-').trim('"')
result-encoding: string
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4
with:
name: logs-${{ matrix.profile }}-${{ steps.parsed.outputs.result }}
path: |
/home/ubuntu/pytest_workflow_*/*/.nextflow.log
/home/ubuntu/pytest_workflow_*/*/log.out
/home/ubuntu/pytest_workflow_*/*/log.err
/home/ubuntu/pytest_workflow_*/*/work
!/home/ubuntu/pytest_workflow_*/*/work/conda
!/home/ubuntu/pytest_workflow_*/*/work/singularity
!${{ github.workspace }}/.singularity
nf-test:
runs-on: ${{ github.event.inputs.runners || 'self-hosted' }}
name: nf-test
needs: [nf-test-changes]
if: ( needs.nf-test-changes.outputs.paths != '[]' )
strategy:
fail-fast: false
matrix:
path: ["${{ fromJson(needs.nf-test-changes.outputs.paths) }}"]
profile: [conda, docker_self_hosted, singularity]
exclude:
- path: modules/nf-core/nf-test
- profile: conda
path: modules/nf-core/angsd/gl
- profile: conda
path: modules/nf-core/annotsv/installannotations
- profile: conda
path: modules/nf-core/happy/sompy
- profile: conda
path: modules/nf-core/backsub
- profile: conda
path: modules/nf-core/bakta/bakta
- profile: conda
path: modules/nf-core/bakta/baktadbdownload
- profile: conda
path: modules/nf-core/bases2fastq
- profile: conda
path: modules/nf-core/bcl2fastq
- profile: conda
path: modules/nf-core/bclconvert
- profile: conda
path: modules/nf-core/celesta
- profile: conda
path: modules/nf-core/cellpose
- profile: conda
path: modules/nf-core/cellranger/count
- profile: conda
path: modules/nf-core/cellranger/mkfastq
- profile: conda
path: modules/nf-core/cellranger/mkgtf
- profile: conda
path: modules/nf-core/cellranger/mkref
- profile: conda
path: modules/nf-core/cellranger/mkvdjref
- profile: conda
path: modules/nf-core/cellranger/multi
- profile: conda
path: modules/nf-core/cellranger/vdj
- profile: conda
path: modules/nf-core/checkqc
- profile: conda
path: modules/nf-core/custom/dumpsoftwareversions
- profile: conda
path: modules/nf-core/deepcell/mesmer
- profile: conda
path: modules/nf-core/deepvariant
- profile: conda
path: modules/nf-core/ensemblvep/vep
- profile: conda
path: modules/nf-core/fastk/fastk
- profile: conda
path: modules/nf-core/cellrangerarc/mkgtf
- profile: conda
path: modules/nf-core/fastk/histex
- profile: conda
path: modules/nf-core/fastk/merge
- profile: conda
path: modules/nf-core/fcs/fcsadaptor
- profile: conda
path: modules/nf-core/fcs/fcsgx
- profile: conda
path: modules/nf-core/ganon/buildcustom
- profile: conda
path: modules/nf-core/ganon/classify
- profile: conda
path: modules/nf-core/ganon/report
- profile: conda
path: modules/nf-core/ganon/table
- profile: conda
path: modules/nf-core/gatk4/cnnscorevariants
- profile: conda
path: modules/nf-core/gatk4/determinegermlinecontigploidy
- profile: conda
path: modules/nf-core/genescopefk
- profile: conda
path: modules/nf-core/ilastik/multicut
- profile: conda
path: modules/nf-core/ilastik/pixelclassification
- profile: conda
path: modules/nf-core/imputeme/vcftoprs
- profile: conda
path: modules/nf-core/mcquant
- profile: conda
path: modules/nf-core/merquryfk/katcomp
- profile: conda
path: modules/nf-core/merquryfk/katgc
- profile: conda
path: modules/nf-core/merquryfk/merquryfk
- profile: conda
path: modules/nf-core/merquryfk/ploidyplot
- profile: conda
path: modules/nf-core/molkartgarage/clahe
- profile: conda
path: modules/nf-core/quartonotebook
- profile: conda
path: modules/nf-core/scimap/spatiallda
- profile: conda
path: modules/nf-core/sentieon/bwaindex
- profile: conda
path: modules/nf-core/sentieon/bwamem
- profile: conda
path: modules/nf-core/sentieon/dedup
- profile: conda
path: modules/nf-core/sentieon/qualcal
- profile: conda
path: modules/nf-core/spaceranger/mkgtf
- profile: conda
path: modules/nf-core/spaceranger/mkref
- profile: conda
path: modules/nf-core/spaceranger/count
- profile: conda
path: modules/nf-core/spotiflow
- profile: conda
path: modules/nf-core/svanalyzer/svbenchmark
- profile: conda
path: modules/nf-core/universc
- profile: singularity
path: modules/nf-core/universc
- profile: conda
path: modules/nf-core/vt/decompose
- profile: singularity
path: modules/nf-core/bases2fastq
- profile: conda
path: modules/nf-core/wittyer
- profile: conda
path: modules/nf-core/islandpath
- profile: conda
path: modules/nf-core/scimap/mcmicro
- profile: conda
path: subworkflows/nf-core/vcf_annotate_ensemblvep
- profile: conda
path: subworkflows/nf-core/bcl_demultiplex
- profile: conda
path: subworkflows/nf-core/fastq_align_bamcmp_bwa
- profile: conda
path: subworkflows/nf-core/fastq_align_bwa
env:
NXF_ANSI_LOG: false
NFTEST_VER: "0.9.0"
SENTIEON_LICENSE_MESSAGE: ${{ secrets.SENTIEON_LICENSE_MESSAGE }}
SENTIEON_ENCRYPTION_KEY: ${{ secrets.SENTIEON_ENCRYPTION_KEY }}
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- uses: actions/setup-java@6a0805fcefea3d4657a47ac4c165951e33482018 # v4
with:
distribution: "temurin"
java-version: "17"
- name: Setup Nextflow
uses: nf-core/setup-nextflow@v2
- name: Install nf-test
uses: nf-core/setup-nf-test@v1
with:
version: ${{ env.NFTEST_VER }}
- name: Setup apptainer
if: matrix.profile == 'singularity'
uses: eWaterCycle/setup-apptainer@main
- name: Set up Singularity
if: matrix.profile == 'singularity'
run: |
mkdir -p $NXF_SINGULARITY_CACHEDIR
mkdir -p $NXF_SINGULARITY_LIBRARYDIR
- name: Set up Python
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5
with:
python-version: "3.11"
- uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4
id: cache-pip-pdiff
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-pdiff
restore-keys: |
${{ runner.os }}-pip-pdiff
- name: Install Python dependencies
run: python -m pip install --upgrade pip pdiff cryptography
- name: Set up miniconda
if: matrix.profile == 'conda'
uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
with:
miniconda-version: "latest"
auto-update-conda: true
channels: conda-forge,bioconda,defaults
- name: Conda setup
if: matrix.profile == 'conda'
run: |
conda clean -a
conda install -n base conda-libmamba-solver
conda config --set solver libmamba
echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
echo $(realpath python) >> $GITHUB_PATH
# Set up secrets
- name: Set up nextflow secrets
# TODO Only run if the tag includes `sentieon`
if: env.SENTIEON_ENCRYPTION_KEY != null && env.SENTIEON_LICENSE_MESSAGE != null
run: |
nextflow secrets set SENTIEON_AUTH_DATA $(python3 tests/modules/nf-core/sentieon/license_message.py encrypt --key "${{ secrets.SENTIEON_ENCRYPTION_KEY }}" --message "${{ secrets.SENTIEON_LICENSE_MESSAGE }}")
# Test the module
- name: Run nf-test
env:
NFT_DIFF: "pdiff"
NFT_DIFF_ARGS: "--line-numbers --width 120 --expand-tabs=2"
SENTIEON_LICSRVR_IP: ${{ secrets.SENTIEON_LICSRVR_IP }}
SENTIEON_AUTH_MECH: "GitHub Actions - token"
run: |
# use "docker_self_hosted" if it runs on self-hosted runner and matrix.profile=docker
if [ "${{ matrix.profile }}" == "docker" ]; then
PROFILE="docker_self_hosted"
else
PROFILE=${{ matrix.profile }}
fi
NFT_WORKDIR=~ \
nf-test test \
--profile=${{ matrix.profile }} \
--tap=test.tap \
--verbose \
${{ matrix.path }}
- uses: pcolby/tap-summary@0959cbe1d4422e62afc65778cdaea6716c41d936 # v1
with:
path: >-
test.tap
- name: Clean up
if: always()
run: |
sudo rm -rf /home/ubuntu/tests/
confirm-pass:
runs-on: ubuntu-latest
needs:
[
prettier,
editorconfig,
pytest-changes,
nf-core-lint-modules,
nf-core-lint-subworkflows,
pytest,
nf-test-changes,
nf-test,
]
if: always()
steps:
- name: All tests ok
if: ${{ success() || !contains(needs.*.result, 'failure') }}
run: exit 0
- name: One or more tests failed
if: ${{ contains(needs.*.result, 'failure') }}
run: exit 1
- name: debug-print
if: always()
run: |
echo "toJSON(needs) = ${{ toJSON(needs) }}"
echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"