Should reduce concurrent burden at the expense of time. Code is requi…

…red to lint before running. This will help us during hackathons etc when there is a high burden on the GHA runners, but will be slower in general use. Changes: - linting and testing in one GHA file - Linting is required to pass before pytest or nf-test runs - Check for changes only happens once
adamrtalbot · Jul 28, 2023 · c194267 · c194267
1 parent 54eaddb
commit c194267
Show file tree

Hide file tree

Showing 5 changed files with 268 additions and 8 deletions.
diff --git a/.github/workflows/pytest-workflow.yml → .github/workflows/test.yml b/.github/workflows/pytest-workflow.yml → .github/workflows/test.yml
@@ -1,5 +1,4 @@
-name: pytest
-run-name: pytest
+name: test
 on:
   push:
     branches: [master]
@@ -15,9 +14,38 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  changes:
-    name: check-changes
+  Prettier:
     runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
+
+      - name: Install NodeJS
+        uses: actions/setup-node@v3
+
+      - name: Install Prettier
+        run: npm install -g prettier
+
+      - name: Run Prettier --check
+        run: prettier --check .
+
+  EditorConfig:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+
+      - name: Install editorconfig-checker
+        run: npm install -g editorconfig-checker
+
+      - name: Run ECLint check
+        run: editorconfig-checker -exclude README.md $(git ls-files | grep -v test)
+
+  pytest-changes:
+    name: pytest-changes
+    runs-on: ubuntu-latest
+    needs: [Prettier, EditorConfig]
     outputs:
       # Expose matched filters as job 'modules' output variable
       modules: ${{ steps.filter.outputs.changes }}
@@ -29,17 +57,74 @@ jobs:
         with:
           filters: "tests/config/pytest_modules.yml"
 
+  nf-core-lint:
+    runs-on: ubuntu-20.04
+    name: nf-core-lint
+    needs: pytest-changes
+    if: needs.pytest-changes.outputs.modules != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: ["${{ fromJson(needs.pytest-changes.outputs.modules) }}"]
+        exclude:
+          - tags: "nf-test"
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+
+      - name: Install pip
+        run: python -m pip install --upgrade pip
+
+      - name: Install nf-core tools
+        run: python -m pip install nf-core
+
+      - name: Install Nextflow
+        env:
+          CAPSULE_LOG: none
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+
+      - name: Lint ${{ matrix.tags }}
+        run: nf-core modules lint ${{ matrix.tags }}
+        # HACK
+        if: startsWith( matrix.tags, 'subworkflow' ) != true
+
+      - uses: actions/cache@v3
+        with:
+          path: /usr/local/bin/nextflow
+          key: ${{ runner.os }}
+          restore-keys: |
+            ${{ runner.os }}
+
   pytest:
     runs-on: ubuntu-20.04
     name: pytest
-    needs: changes
+    needs: [changes, Prettier, EditorConfig, nf_core_lint]
     if: needs.changes.outputs.modules != '[]'
     strategy:
       fail-fast: false
       matrix:
-        tags: ["${{ fromJson(needs.changes.outputs.modules) }}"]
+        tags: ["${{ fromJson(needs.pytest-changes.outputs.modules) }}"]
         profile: ["docker", "singularity", "conda"]
         exclude:
+          - tags: "nf-test"
           - profile: "conda"
             tags: annotsv
           - profile: "conda"
@@ -250,3 +335,172 @@ jobs:
             /home/runner/pytest_workflow_*/*/work
             !/home/runner/pytest_workflow_*/*/work/conda
             !/home/runner/pytest_workflow_*/*/work/singularity
+
+  nf-test-changes:
+    name: nf-test-changes
+    runs-on: ubuntu-latest
+    needs: [Prettier, EditorConfig]
+    outputs:
+      # Expose matched filters as job 'modules' output variable
+      modules: ${{ steps.filter.outputs.changes }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: "tests/config/pytest_modules.yml"
+
+  nf-test:
+    runs-on: ubuntu-20.04
+    name: nf-test
+    needs: [nf-test-changes, Prettier, EditorConfig, nf_core_lint]
+    if: needs.nf-test-changes.outputs.modules != '[]'
+    strategy:
+      fail-fast: false
+      matrix:
+        tags: ["${{ fromJson(needs.nf-test-changes.outputs.modules) }}"]
+        profile: ["docker", "singularity", "conda"]
+        exclude:
+          - tags: "nf-test"
+          - profile: "conda"
+            tags: annotsv
+          - profile: "conda"
+            tags: bases2fastq
+          - profile: "conda"
+            tags: bcl2fastq
+          - profile: "conda"
+            tags: bclconvert
+          - profile: "conda"
+            tags: cellranger/count
+          - profile: "conda"
+            tags: cellranger/mkfastq
+          - profile: "conda"
+            tags: cellranger/mkgtf
+          - profile: "conda"
+            tags: cellranger/mkref
+          - profile: "conda"
+            tags: deepvariant
+          - profile: "conda"
+            tags: ensemblvep/vep
+          - profile: "conda"
+            tags: fastk/fastk
+          - profile: "conda"
+            tags: fastk/histex
+          - profile: "conda"
+            tags: fastk/merge
+          - profile: "conda"
+            tags: fcs/fcsadaptor
+          - profile: "conda"
+            tags: fcs/fcsgx
+          - profile: "conda"
+            tags: gatk4/cnnscorevariants
+          - profile: "conda"
+            tags: gatk4/determinegermlinecontigploidy
+          - profile: "conda"
+            tags: genescopefk
+          - profile: "conda"
+            tags: ilastik/multicut
+          - profile: "conda"
+            tags: ilastik/pixelclassification
+          - profile: "conda"
+            tags: imputeme/vcftoprs
+          - profile: "conda"
+            tags: merquryfk/katcomp
+          - profile: "conda"
+            tags: merquryfk/katgc
+          - profile: "conda"
+            tags: merquryfk/merquryfk
+          - profile: "conda"
+            tags: merquryfk/ploidyplot
+          - profile: "conda"
+            tags: sentieon/bwaindex
+          - profile: "conda"
+            tags: sentieon/bwamem
+          - profile: "conda"
+            tags: universc
+          - profile: "singularity"
+            tags: universc
+          - profile: "conda"
+            tags: subworkflows/vcf_annotate_ensemblvep
+    env:
+      NXF_ANSI_LOG: false
+      SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/cache@v3
+        with:
+          path: /usr/local/bin/nextflow
+          key: ${{ runner.os }}
+          restore-keys: |
+            ${{ runner.os }}-nextflow-
+
+      - name: Install Nextflow
+        env:
+          CAPSULE_LOG: none
+        run: |
+          wget -qO- get.nextflow.io | bash
+          sudo mv nextflow /usr/local/bin/
+
+      - name: Cache nf-test installation
+        id: cache-software
+        uses: actions/cache@v3
+        with:
+          path: |
+            /usr/local/bin/nf-test
+            /home/runner/.nf-test/nf-test.jar
+          key: ${{ runner.os }}-nftest
+
+      - name: Install nf-test
+        if: steps.cache-software.outputs.cache-hit != 'true'
+        run: |
+          wget -qO- https://code.askimed.com/install/nf-test | bash
+          sudo mv nf-test /usr/local/bin/
+
+      - name: Set up Singularity
+        if: matrix.profile == 'singularity'
+        uses: eWaterCycle/setup-singularity@v5
+        with:
+          singularity-version: 3.7.1
+
+      - name: Set up miniconda
+        if: matrix.profile == 'conda'
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          channels: conda-forge,bioconda,defaults
+          python-version: ${{ matrix.python-version }}
+
+      - name: Conda setup
+        if: matrix.profile == 'conda'
+        run: |
+          conda clean -a
+          conda install -n base conda-libmamba-solver
+          conda config --set solver libmamba
+
+      # Set up secrets
+      - name: Set up nextflow secrets
+        if: env.SENTIEON_LICENSE_BASE64 != null
+        run: |
+          nextflow secrets set SENTIEON_LICENSE_BASE64 ${{ secrets.SENTIEON_LICENSE_BASE64 }}
+          nextflow secrets set SENTIEON_AUTH_MECH_BASE64 ${{ secrets.SENTIEON_AUTH_MECH_BASE64 }}
+          SENTIEON_ENCRYPTION_KEY=$(echo -n "${{ secrets.ENCRYPTION_KEY_BASE64 }}" | base64 -d)
+          SENTIEON_LICENSE_MESSAGE=$(echo -n "${{ secrets.LICENSE_MESSAGE_BASE64 }}" | base64 -d)
+          SENTIEON_AUTH_DATA=$(python tests/modules/nf-core/sentieon/license_message.py encrypt --key "$SENTIEON_ENCRYPTION_KEY" --message "$SENTIEON_LICENSE_MESSAGE")
+          SENTIEON_AUTH_DATA_BASE64=$(echo -n "$SENTIEON_AUTH_DATA" | base64 -w 0)
+          nextflow secrets set SENTIEON_AUTH_DATA_BASE64 $SENTIEON_AUTH_DATA_BASE64
+
+      # Test the module
+      - name: Run nf-test
+        run: |
+          nf-test test \
+            --profile=${{ matrix.profile }} \
+            --tag ${{ matrix.tags }} \
+            --tap=test.tap
+
+      - uses: pcolby/tap-summary@v1
+        with:
+          path: >-
+            test.tap
diff --git a/modules/nf-core/gatk4/haplotypecaller/main.nf b/modules/nf-core/gatk4/haplotypecaller/main.nf
@@ -5,7 +5,7 @@ process GATK4_HAPLOTYPECALLER {
     conda "bioconda::gatk4=4.4.0.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0':
-        'quay.io/biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
+        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
 
     input:
     tuple val(meta), path(input), path(input_index), path(intervals), path(dragstr_model)
@@ -50,6 +50,8 @@ process GATK4_HAPLOTYPECALLER {
         --tmp-dir . \\
         $args
 
+    # Fake stuff
+
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         gatk4: \$(echo \$(gatk --version 2>&1) | sed 's/^.*(GATK) v//; s/ .*\$//')

diff --git a/modules/nf-core/samtools/index/main.nf b/modules/nf-core/samtools/index/main.nf
@@ -28,6 +28,8 @@ process SAMTOOLS_INDEX {
         $args \\
         $input
 
+    # Fake stuff
+
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         samtools: \$(echo \$(samtools --version 2>&1) | sed 's/^.*samtools //; s/Using.*\$//')

diff --git a/modules/nf-core/samtools/sort/main.nf b/modules/nf-core/samtools/sort/main.nf
@@ -5,7 +5,7 @@ process SAMTOOLS_SORT {
     conda "bioconda::samtools=1.17"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/samtools:1.17--h00cdaf9_0' :
-        'quay.io/biocontainers/samtools:1.17--h00cdaf9_0' }"
+        'biocontainers/samtools:1.17--h00cdaf9_0' }"
 
     input:
     tuple val(meta), path(bam)

diff --git a/tests/modules/nf-core/fastqc/main.nf.test b/tests/modules/nf-core/fastqc/main.nf.test
@@ -5,6 +5,8 @@ nextflow_process {
     process "FASTQC"
     tag "fastqc"
 
+    // Fake stuff
+
     test("Single-Read") {
 
         when {