ci - update cruft

.cruft.json

@@ -1,6 +1,6 @@
 {
   "template": "https://github.com/MartinBernstorff/swift-python-cookiecutter",
-  "commit": "bbabcc2c8860575a995699e32cf5f663e1406029",
+  "commit": "eed18f71d1d7f6078fb79b8e979d9902716f5cab",
   "checkout": null,
   "context": {
     "cookiecutter": {

.github/workflows/static_type_checks.yml

@@ -17,39 +17,37 @@ jobs:
     concurrency:
       group: "${{ github.workflow }} @ ${{ github.ref }}"
       cancel-in-progress: true
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.9"]
     steps:
       - uses: actions/checkout@v2
 
-      - name: Cache venv
+      - name: Cache tox
         uses: actions/[email protected]
-        id: cache_venv
+        id: cache_tox
         with:
           path: |
-            .venv
-          key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }}
+            .tox
+          key: ${{ runner.os }}-${{ matrix.python-version }}-static-type-checks
 
       - name: Set up Python
         uses: actions/setup-python@v4
         id: setup_python
-        if: steps.cache_venv.outputs.cache-hit != 'true'
         with:
-          python-version: "3.9"
+          python-version: ${{ matrix.python-version}}
 
       - name: Install dependencies
         shell: bash
-        if: steps.cache_venv.outputs.cache-hit != 'true'
         run: |
-          python -m venv .venv
-          source .venv/bin/activate
-          python -m pip install --upgrade pip
-          pip install -e .[dev]
+          pip install invoke tox
 
       - name: Run static type checker
         id: pyright
         continue-on-error: true
         run: |
-          source .venv/bin/activate
-          if pyright .; then
+          if inv static-type-checks; then
             echo "pyright check passed"
             echo "pyright_failed=0" >> $GITHUB_OUTPUT
           else
@@ -96,5 +94,4 @@ jobs:
         id: fail_run
         if: ${{steps.pyright.outputs.pyright_failed == 1}}
         run: |
-          source .venv/bin/activate
-          pyright . # Rerunning pyright isn't optimal computationally, but typically takes no more than a couple of seconds, and this ensures that the errors are in the failing step
+          inv static-type-checks # Rerunning pyright isn't optimal computationally, but typically takes no more than a couple of seconds, and this ensures that the errors are in the failing step

.github/workflows/tests.yml

@@ -30,43 +30,30 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      - name: Cache venv
+      - name: Cache tox
         uses: actions/[email protected]
-        id: cache_venv
+        id: cache_tox
         with:
           path: |
-            .venv
-          key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('**/pyproject.toml') }}
+            .tox
+          key: ${{ runner.os }}-${{ matrix.python-version }}-tests
 
       - name: Set up Python
         uses: actions/setup-python@v4
-        id: setup_python
-        if: steps.cache_venv.outputs.cache-hit != 'true'
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install dependencies
         shell: bash
         run: |
-          python3 -m venv .venv
-          if [ "${{ matrix.os }}" == "windows-latest" ]; then
-            source .venv/Scripts/activate
-          else
-            source .venv/bin/activate
-          fi
-          python3 -m pip install --upgrade pip
-          pip install -e .[tests]
+          pip install invoke tox
 
       - name: Run and write pytest
         shell: bash
         run: |
-          # activate venv dependent on the os
-          if [ "${{ matrix.os }}" == "windows-latest" ]; then
-            source .venv/Scripts/activate
-          else
-            source .venv/bin/activate
-          fi
-          pytest --durations=0 -n 2 -x --junitxml=pytest.xml --cov-report=term-missing --cov=src/ tests/
+          # Specifying two sets of "--pytest-args" is required for invoke to parse it as a list
+          inv test --pytest-args="--durations=0" --pytest-args="--junitxml=pytest.xml --cov-report=term-missing --cov=src/"
+
 
       - name: Test report on failures
         uses: EnricoMi/publish-unit-test-result-action@v2

.gitignore

@@ -6,6 +6,10 @@
 /src/*.egg-info/
 __pycache__/
 .*venv*
+_build/
+.tox
+
+#
 
 # VSCode
 .vscode/

pyproject.toml

@@ -13,7 +13,6 @@ classifiers = [
   "Operating System :: POSIX :: Linux",
   "Operating System :: MacOS :: MacOS X",
   "Operating System :: Microsoft :: Windows",
-  "Programming Language :: Python :: 3",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10"
 ]
@@ -72,7 +71,7 @@ repository = "https://github.com/MartinBernstorff/psycop-model-evaluation"
 documentation = "https://MartinBernstorff.github.io/psycop-model-evaluation/"
 
 [tool.pyright]
-exclude = [".*venv*"]
+exclude = [".*venv*", ".tox"]
 pythonPlatform = "Darwin"
 
 [tool.ruff]
@@ -167,3 +166,33 @@ build_command = "python -m pip install build; python -m build"
 
 [tool.setuptools]
 include-package-data = true
+
+
+[tool.tox]
+legacy_tox_ini = """
+[tox]
+envlist = py{39,310}
+
+[testenv]
+description: run unit tests
+extras = tests
+use_develop = true
+commands =
+  pytest -n auto {posargs:test}
+
+[testenv:type]
+description: run type checks
+extras = test, dev
+basepython = py39 # Setting these explicitly avoid recreating env if your shell is set to a different version
+use_develop = true
+commands =
+  pyright .
+
+[testenv:docs]
+description: build docs
+extras = docs
+basepython = py39 # Setting these explicitly avoid recreating env if your shell is set to a different version
+use_develop = true
+commands =
+  sphinx-build -b html docs docs/_build/html
+"""

pyproject.toml.rej

@@ -0,0 +1,9 @@
+diff a/pyproject.toml b/pyproject.toml	(rejected hunks)
+@@ -41,6 +40,7 @@ tests = [
+   "pytest-cov>=3.0.0,<3.1.0",
+   "pytest-xdist>=3.0.0,<3.2.0",
+   "pytest-sugar>=0.9.4,<0.10.0",
++  "tox",
+ ]
+ docs = [
+   "sphinx>=5.3.0,<5.4.0",

tasks.py

@@ -20,10 +20,17 @@
 import re
 import shutil
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
 
 from invoke import Context, Result, task
 
+# Extract supported python versions from the pyproject.toml classifiers key
+SUPPORTED_PYTHON_VERSIONS = [
+    line.split("::")[-1].strip().replace('"', "").replace(",", "")
+    for line in Path("pyproject.toml").read_text().splitlines()
+    if "Programming Language :: Python ::" in line
+]
+
 NOT_WINDOWS = platform.system() != "Windows"
 
 
@@ -205,13 +212,15 @@ def update_pr(c: Context):
             c.run("gh pr view --web", pty=NOT_WINDOWS)
 
 
-def exit_if_remaining_errors(result: Result):
+def exit_if_error_in_stdout(result: Result):
     # Find N remaining using regex
 
     if "error" in result.stdout:
-        errors_remaining = re.findall(r"\d+(?=( remaining))", result.stdout)[0]
+        errors_remaining = re.findall(r"\d+(?=( remaining))", result.stdout)[
+            0
+        ]  # testing
         if errors_remaining != "0":
-            exit(1)
+            exit(0)
 
 
 def pre_commit(c: Context, auto_fix: bool):
@@ -229,14 +238,14 @@ def pre_commit(c: Context, auto_fix: bool):
     pre_commit_cmd = "pre-commit run --all-files"
     result = c.run(pre_commit_cmd, pty=NOT_WINDOWS, warn=True)
 
-    exit_if_remaining_errors(result)
+    exit_if_error_in_stdout(result)
 
     if ("fixed" in result.stdout or "reformatted" in result.stdout) and auto_fix:
         _add_commit(c, msg="style: Auto-fixes from pre-commit")
 
         print(f"{msg_type.DOING} Fixed errors, re-running pre-commit checks")
         second_result = c.run(pre_commit_cmd, pty=NOT_WINDOWS, warn=True)
-        exit_if_remaining_errors(second_result)
+        exit_if_error_in_stdout(second_result)
     else:
         if result.return_code != 0:
             print(f"{msg_type.FAIL} Pre-commit checks failed")
@@ -246,7 +255,7 @@ def pre_commit(c: Context, auto_fix: bool):
 @task
 def static_type_checks(c: Context):
     echo_header(f"{msg_type.CLEAN} Running static type checks")
-    c.run("pyright-polite .", pty=NOT_WINDOWS)
+    c.run("tox -e type", pty=NOT_WINDOWS)
 
 
 @task
@@ -317,25 +326,47 @@ def update(c: Context):
     install(c, pip_args="--upgrade", msg=False)
 
 
-@task
-def test(c: Context):
+@task(iterable="pytest_args")
+def test(
+    c: Context,
+    python_versions: List[str] = (SUPPORTED_PYTHON_VERSIONS[0],),  # noqa # type: ignore
+    pytest_args: List[str] = [],  # noqa
+):
     """Run tests"""
+    # Invoke requires lists as type hints, but does not support lists as default arguments.
+    # Hence this super weird type hint and default argument for the python_versions arg.
     echo_header(f"{msg_type.TEST} Running tests")
+
+    python_version_strings = [f"py{v.replace('.', '')}" for v in python_versions]
+    python_version_arg_string = ",".join(python_version_strings)
+
+    if not pytest_args:
+        pytest_args = [
+            "tests",
+            "-n auto",
+            "-rfE",
+            "--failed-first",
+            "-p no:cov",
+            "--disable-warnings",
+            "-q",
+        ]
+
+    pytest_arg_str = " ".join(pytest_args)
+
     test_result: Result = c.run(
-        "pytest tests/ -n auto -rfE --failed-first -p no:cov --disable-warnings -q",
+        f"tox -e {python_version_arg_string} -- {pytest_arg_str}",
         warn=True,
         pty=NOT_WINDOWS,
     )
 
     # If "failed" in the pytest results
-    if "failed" in test_result.stdout:
+    failed_tests = [line for line in test_result.stdout if line.startswith("FAILED")]
+
+    if len(failed_tests) > 0:
+        print("\n\n\n")
+        echo_header("Failed tests")
         print("\n\n\n")
         echo_header("Failed tests")
-
-        # Get lines with "FAILED" in them from the .pytest_results file
-        failed_tests = [
-            line for line in test_result.stdout if line.startswith("FAILED")
-        ]
 
         for line in failed_tests:
             # Remove from start of line until /test_
@@ -374,7 +405,7 @@ def pr(c: Context, auto_fix: bool = False):
     """Run all checks and update the PR."""
     add_and_commit(c)
     lint(c, auto_fix=auto_fix)
-    test(c)
+    test(c, python_versions=SUPPORTED_PYTHON_VERSIONS)
     update_branch(c)
     update_pr(c)
 
@@ -386,11 +417,12 @@ def docs(c: Context, view: bool = False, view_only: bool = False):
     """
     if not view_only:
         echo_header(f"{msg_type.DOING}: Building docs")
-        c.run("sphinx-build -b html docs docs/_build/html")
+        c.run("tox -e docs")
+
     if view or view_only:
         echo_header(f"{msg_type.EXAMINE}: Opening docs in browser")
         # check the OS and open the docs in the browser
-        if NOT_WINDOWS:
-            c.run("open docs/_build/html/index.html")
-        else:
+        if platform.system() == "Windows":
             c.run("start docs/_build/html/index.html")
+        else:
+            c.run("open docs/_build/html/index.html")

tests/first_test.py

@@ -0,0 +1,5 @@
+"""Start of the test suite for psycop-model-evaluation. This file is required for pytest to not throw an 'no tests found' error."""
+
+
+def test_first():
+    pass