Merge pull request #249 from reagento/coverage-report

add coverage report generating in pull request

.github/workflows/lint_and_test.yml

@@ -18,7 +18,7 @@ concurrency:
 
 jobs:
   linting:
-    name: Run linters
+    name: Linting
     runs-on: ubuntu-latest
     timeout-minutes: 3
 
@@ -44,22 +44,22 @@ jobs:
           just lint
 
   testing:
-    name: Run tests
+    name: Testing (${{ matrix.python_version.tox }}, ${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     needs: linting
 
     strategy:
       fail-fast: true
       matrix:
         python_version:
-          - { setup: '3.8',      tox: 'py38'    }
-          - { setup: '3.9',      tox: 'py39'    }
-          - { setup: '3.10',     tox: 'py310'   }
-          - { setup: '3.11',     tox: 'py311'   }
-          - { setup: '3.12',     tox: 'py312'   }
-          - { setup: 'pypy3.8',  tox: 'pypy38'  }
-          - { setup: 'pypy3.9',  tox: 'pypy39'  }
-          - { setup: 'pypy3.10', tox: 'pypy310' }
+          - { setup: '3.8',      tox: 'py38',    cov: true  }
+          - { setup: '3.9',      tox: 'py39',    cov: true  }
+          - { setup: '3.10',     tox: 'py310',   cov: true  }
+          - { setup: '3.11',     tox: 'py311',   cov: true  }
+          - { setup: '3.12',     tox: 'py312',   cov: true  }
+          - { setup: 'pypy3.8',  tox: 'pypy38',  cov: false }
+          - { setup: 'pypy3.9',  tox: 'pypy39',  cov: false }
+          - { setup: 'pypy3.10', tox: 'pypy310', cov: false }
 
         os: ['ubuntu-latest']
 
@@ -80,6 +80,49 @@ jobs:
         run:
           just setup-runner
 
-      - name: Run tests
+      - name: Run tests with coverage
+        if: ${{ matrix.python_version.cov }}
         run:
-          just test-on ${{ matrix.python_version.tox }}
+          just inv test-on-ci
+            --py-target ${{ matrix.python_version.tox }}
+            --cov-output .coverage.${{ matrix.python_version.tox }}
+
+      - name: Store coverage file
+        uses: actions/upload-artifact@v4
+        if: ${{  (matrix.python_version.cov && github.event_name == 'pull_request') }}
+        with:
+          name: coverage-${{ matrix.python_version.tox }}
+          path: .coverage.${{ matrix.python_version.tox }}
+          if-no-files-found: error
+
+      - name: Run tests without coverage
+        if: ${{ !(matrix.python_version.cov && github.event_name == 'pull_request') }}
+        run:
+          just inv test-on-ci
+            --py-target ${{ matrix.python_version.tox }}
+
+  coverage:
+    name: Coverage
+    runs-on: ubuntu-latest
+    needs: testing
+    permissions:
+      pull-requests: write
+      contents: write
+    if: ${{ github.event_name == 'pull_request' }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/download-artifact@v4
+        id: download
+        with:
+          pattern: coverage-*
+          merge-multiple: true
+
+      - name: Coverage comment
+        id: coverage_comment
+        uses: py-cov-action/python-coverage-comment-action@v3
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          MERGE_COVERAGE_FILES: true
+          MINIMUM_GREEN: 90

justfile

@@ -17,11 +17,6 @@ set windows-powershell := true
     pip-sync requirements/pre.txt requirements/dev.txt
     pip install -e .
 
-[private]
-@setup-runner:
-    pip install -r requirements/pre.txt
-    pip install -r requirements/runner.txt
-
 # run all linters
 @lint:
     tox -e lint
@@ -38,14 +33,13 @@ set windows-powershell := true
 @test-all:
     tox -e $(tox list --no-desc | grep '^py' | sort -r | tr '\n' ',') -p auto
 
-# run all tests on specific python version
-@test-on target:
-    tox -e $(tox list --no-desc | grep '^{{ target }}' | sort -r | tr '\n' ',')
-
 inv := "inv -r scripts -c invoke_tasks"
 
-@cov:
-    {{ inv }} cov
+@cov output='coverage.xml':
+    {{ inv }} cov \
+      --env-list $(tox list --no-desc | grep -e '^py' | grep -v '^pypy' | sort -r | tr '\n' ',') \
+      --output {{ output }} \
+      --parallel
 
 @deps-compile:
     {{ inv }} deps-compile
@@ -64,3 +58,15 @@ doc_target := "docs-build"
 # clean generated documentation and build cache
 @doc-clean:
     sphinx-build -M clean {{ doc_source }} {{ doc_target }}
+
+
+# Continious integration
+
+[private]
+@setup-runner:
+    pip install -r requirements/pre.txt
+    pip install -r requirements/runner.txt
+
+[private]
+@inv *ARGS:
+    {{ inv }} {{ ARGS }}

pyproject.toml

@@ -156,6 +156,7 @@ keep-runtime-typing = true
 [tool.ruff.lint.flake8-pytest-style]
 parametrize-names-type = "list"
 
+
 [tool.towncrier]
 package = 'adaptix'
 filename = 'docs/changelog/changelog_body.rst'
@@ -170,3 +171,9 @@ type = [
   { name = "Bug Fixes",        directory = "bugfix",      showcontent = true },
   { name = "Other",            directory = "other",       showcontent = true },
 ]
+
+
+[tool.coverage.run]
+branch = true
+relative_files = true
+include = ["src/**"]

requirements/dev.txt

@@ -52,8 +52,10 @@ colorama==0.4.6
     #   tox
 contourpy==1.2.0
     # via matplotlib
-coverage==7.3.3
-    # via -r requirements/raw/dev.txt
+coverage==7.4.3
+    # via
+    #   -r requirements/raw/runner.txt
+    #   -r requirements/raw/test_extra_none.txt
 cycler==0.12.1
     # via matplotlib
 dataclass-factory==2.16

requirements/lint.txt

@@ -40,6 +40,8 @@ colorama==0.4.6
     # via radon
 contourpy==1.2.0
     # via matplotlib
+coverage==7.4.3
+    # via -r requirements/raw/test_extra_none.txt
 cycler==0.12.1
     # via matplotlib
 dataclass-factory==2.16

requirements/raw/dev.txt

@@ -1,4 +1,3 @@
-coverage==7.3.3
 pip-tools==7.3.0
 setuptools==69.0.0
 towncrier==23.11.0

requirements/raw/runner.txt

@@ -1,2 +1,3 @@
 tox==4.11.4
 invoke==2.2.0
+coverage==7.4.3

requirements/raw/test_extra_none.txt

@@ -8,3 +8,4 @@ dirty-equals==0.7.1.post0
 
 typing-extensions==4.9.0
 greenlet==3.0.2
+coverage==7.4.3

requirements/runner.txt

@@ -10,6 +10,8 @@ chardet==5.2.0
     # via tox
 colorama==0.4.6
     # via tox
+coverage==7.4.3
+    # via -r requirements/raw/runner.txt
 distlib==0.3.8
     # via virtualenv
 filelock==3.13.1

requirements/test_extra_new.txt

@@ -8,6 +8,8 @@
     # via -r requirements/raw/test_extra_none.txt
 attrs==23.2.0
     # via -r requirements/raw/test_extra_new.txt
+coverage==7.4.3
+    # via -r requirements/raw/test_extra_none.txt
 dirty-equals==0.7.1.post0
     # via -r requirements/raw/test_extra_none.txt
 greenlet==3.0.2

requirements/test_extra_none.txt

@@ -6,6 +6,8 @@
 #
 -e ./tests/tests_helpers
     # via -r requirements/raw/test_extra_none.txt
+coverage==7.4.3
+    # via -r requirements/raw/test_extra_none.txt
 dirty-equals==0.7.1.post0
     # via -r requirements/raw/test_extra_none.txt
 greenlet==3.0.2

requirements/test_extra_old.txt

@@ -8,6 +8,8 @@
     # via -r requirements/raw/test_extra_none.txt
 attrs==21.3.0
     # via -r requirements/raw/test_extra_old.txt
+coverage==7.4.3
+    # via -r requirements/raw/test_extra_none.txt
 dirty-equals==0.7.1.post0
     # via -r requirements/raw/test_extra_none.txt
 greenlet==3.0.2

scripts/invoke_tasks.py

@@ -11,35 +11,39 @@ def q(value: Union[Path, str]) -> str:
     return shlex.quote(str(value))
 
 
+def if_str(flag: bool, value: str) -> str:  # noqa: FBT001
+    return value if flag else ""
+
+
 @task
-def cov(c: Context):
+def cov(c: Context, env_list, output="coverage.xml", parallel=False):
     inner_bash_command = q(
         "coverage run"
-        " --branch"
         " --data-file=.tox/cov-storage/.coverage.$TOX_ENV_NAME"
         " -m pytest",
     )
     tox_commands = f"bash -c '{q(inner_bash_command)}'"
     c.run(
-        r"tox -e $(tox -l | grep -e '^py' | grep -v 'bench' | sort -r | tr '\n' ',')"
-        " -p auto"
+        f"tox -e {q(env_list)}"
+        + if_str(parallel, " -p auto") +
         "  --override 'testenv.allowlist_externals=bash'"
         f" --override 'testenv.commands={tox_commands}'",
         pty=True,
     )
     c.run("coverage combine --data-file .tox/cov-storage/.coverage .tox/cov-storage")
-    c.run("coverage xml --data-file .tox/cov-storage/.coverage")
+    if output.endswith(".xml"):
+        c.run(f"coverage xml --data-file .tox/cov-storage/.coverage -o {output}")
+    else:
+        c.run(f"cp .tox/cov-storage/.coverage {output}")
 
 
 @task
 def deps_compile(c: Context, upgrade=False):
-    extra = ""
-    if upgrade:
-        extra += " --upgrade"
     promises = [
         c.run(
             f'pip-compile {req} -o {Path("requirements") / req.name}'
-            ' -q --allow-unsafe --strip-extras' + extra,
+            ' -q --allow-unsafe --strip-extras'
+            + if_str(upgrade, " --upgrade"),
             asynchronous=True,
         )
         for req in Path(".").glob("requirements/raw/*.txt")
@@ -51,3 +55,12 @@ def deps_compile(c: Context, upgrade=False):
     for file in Path(".").glob("requirements/*.txt"):
         c.run(fr'sed -i -E "s/-e file:.+\/tests\/tests_helpers/-e .\/tests\/tests_helpers/" {file}')
         c.run(fr'sed -i -E "s/-e file:.+\/benchmarks/-e .\/benchmarks/" {file}')
+
+
+@task
+def test_on_ci(c: Context, py_target, cov_output=None):
+    env_list = c.run(fr"tox list --no-desc | grep '^{py_target}' | sort -r | tr '\n' ','", hide=True).stdout
+    if cov_output is None:
+        c.run(fr"tox -e {env_list}", pty=True)
+    else:
+        cov(c, env_list=env_list, output=cov_output)