From c5ec0738db1e3f0ac5bc0b918b75dbb5aec24328 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 7 Apr 2024 00:06:07 -0400 Subject: [PATCH 1/6] feat: support PEP 723 with a toml load function Signed-off-by: Henry Schreiner --- .pre-commit-config.yaml | 1 + docs/tutorial.rst | 40 +++++++++++++++ nox/__init__.py | 11 ++++- nox/_toml.py | 61 +++++++++++++++++++++++ pyproject.toml | 1 + tests/test_toml.py | 107 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 nox/_toml.py create mode 100644 tests/test_toml.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec349577..72e48e79 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,6 +40,7 @@ repos: - jinja2 - packaging - importlib_metadata + - tomli - uv - repo: https://github.com/codespell-project/codespell diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 4b3131f1..1aea1f42 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -169,6 +169,46 @@ dependency (e.g. ``libfoo``) is available during installation: These commands will run even if you are only installing, and will not run if the environment is being reused without reinstallation. + +Loading dependencies from pyproject.toml or scripts +--------------------------------------------------- + +One common need is loading a dependency list from a ``pyproject.toml`` file +(say to prepare an environment without installing the package) or supporting +`PEP 723 `_ scripts. Nox provides a helper to +load these with ``nox.load_toml``. It can be passed a filepath to a toml file +or to a script file following PEP 723. For example, if you have the following +``peps.py``: + + +.. code-block:: python + + # /// script + # requires-python = ">=3.11" + # dependencies = [ + # "requests<3", + # "rich", + # ] + # /// + + import requests + from rich.pretty import pprint + + resp = requests.get("https://peps.python.org/api/peps.json") + data = resp.json() + pprint([(k, v["title"]) for k, v in data.items()][:10]) + +You can make a session for it like this: + +.. code-block:: python + + @nox.session + def peps(session): + requirements = nox.load_toml("peps.py")["dependencies"] + session.install(*requirements) + session.run("peps.py") + + Running commands ---------------- diff --git a/nox/__init__.py b/nox/__init__.py index 1afe51a7..bb311db9 100644 --- a/nox/__init__.py +++ b/nox/__init__.py @@ -17,9 +17,18 @@ from nox._options import noxfile_options as options from nox._parametrize import Param as param from nox._parametrize import parametrize_decorator as parametrize +from nox._toml import load_toml from nox.registry import session_decorator as session from nox.sessions import Session needs_version: str | None = None -__all__ = ["needs_version", "parametrize", "param", "session", "options", "Session"] +__all__ = [ + "needs_version", + "parametrize", + "param", + "session", + "options", + "Session", + "load_toml", +] diff --git a/nox/_toml.py b/nox/_toml.py new file mode 100644 index 00000000..d3df54d6 --- /dev/null +++ b/nox/_toml.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import os +import re +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any + +if sys.version_info < (3, 11): + import tomli as tomllib +else: + import tomllib + +# Note: the implementation (including this regex) taken from PEP 723 +# https://peps.python.org/pep-0723 + +REGEX = re.compile( + r"(?m)^# /// (?P[a-zA-Z0-9-]+)$\s(?P(^#(| .*)$\s)+)^# ///$" +) + + +def load_toml(filename: os.PathLike[str] | str) -> dict[str, Any]: + """ + Load a toml file or a script with a PEP 723 script block. + + The file must have a ``.toml`` extension to be considered a toml file or a + ``.py`` extension / no extension to be considered a script. Other file + extensions are not valid in this function. + """ + filepath = Path(filename) + if filepath.suffix == ".toml": + return _load_toml_file(filepath) + if filepath.suffix in {".py", ""}: + return _load_script_block(filepath) + msg = f"Extension must be .py or .toml, got {filepath.suffix}" + raise ValueError(msg) + + +def _load_toml_file(filepath: Path) -> dict[str, Any]: + with filepath.open("rb") as f: + return tomllib.load(f) + + +def _load_script_block(filepath: Path) -> dict[str, Any]: + name = "script" + script = filepath.read_text(encoding="utf-8") + matches = list(filter(lambda m: m.group("type") == name, REGEX.finditer(script))) + + if not matches: + raise ValueError(f"No {name} block found in {filepath}") + if len(matches) > 1: + raise ValueError(f"Multiple {name} blocks found in {filepath}") + + content = "".join( + line[2:] if line.startswith("# ") else line[1:] + for line in matches[0].group("content").splitlines(keepends=True) + ) + return tomllib.loads(content) diff --git a/pyproject.toml b/pyproject.toml index 38b6ec87..aee23e67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ "colorlog<7.0.0,>=2.6.1", 'importlib-metadata; python_version < "3.8"', "packaging>=20.9", + 'tomli>=1; python_version < "3.11"', 'typing-extensions>=3.7.4; python_version < "3.8"', "virtualenv>=20.14.1", ] diff --git a/tests/test_toml.py b/tests/test_toml.py new file mode 100644 index 00000000..7bda84c5 --- /dev/null +++ b/tests/test_toml.py @@ -0,0 +1,107 @@ +import textwrap +from pathlib import Path + +import pytest + +import nox + + +def test_load_pyproject(tmp_path: Path) -> None: + filepath = tmp_path / "example.toml" + filepath.write_text( + """ + [project] + name = "hi" + version = "1.0" + dependencies = ["numpy", "requests"] + """ + ) + + toml = nox.load_toml(filepath) + assert toml["project"]["dependencies"] == ["numpy", "requests"] + + +@pytest.mark.parametrize("example", ["example.py", "example"]) +def test_load_script_block(tmp_path: Path, example: str) -> None: + filepath = tmp_path / example + filepath.write_text( + textwrap.dedent( + """\ + #!/usr/bin/env pipx run + # /// script + # requires-python = ">=3.11" + # dependencies = [ + # "requests<3", + # "rich", + # ] + # /// + + import requests + from rich.pretty import pprint + + resp = requests.get("https://peps.python.org/api/peps.json") + data = resp.json() + pprint([(k, v["title"]) for k, v in data.items()][:10]) + """ + ) + ) + + toml = nox.load_toml(filepath) + assert toml["dependencies"] == ["requests<3", "rich"] + + +def test_load_no_script_block(tmp_path: Path) -> None: + filepath = tmp_path / "example.py" + filepath.write_text( + textwrap.dedent( + """\ + #!/usr/bin/python + + import requests + from rich.pretty import pprint + + resp = requests.get("https://peps.python.org/api/peps.json") + data = resp.json() + pprint([(k, v["title"]) for k, v in data.items()][:10]) + """ + ) + ) + + with pytest.raises(ValueError, match="No script block found"): + nox.load_toml(filepath) + + +def test_load_multiple_script_block(tmp_path: Path) -> None: + filepath = tmp_path / "example.py" + filepath.write_text( + textwrap.dedent( + """\ + # /// script + # dependencies = [ + # "requests<3", + # "rich", + # ] + # /// + + # /// script + # requires-python = ">=3.11" + # /// + + import requests + from rich.pretty import pprint + + resp = requests.get("https://peps.python.org/api/peps.json") + data = resp.json() + pprint([(k, v["title"]) for k, v in data.items()][:10]) + """ + ) + ) + + with pytest.raises(ValueError, match="Multiple script blocks found"): + nox.load_toml(filepath) + + +def test_load_non_recongnised_extension(): + msg = "Extension must be .py or .toml, got .txt" + with pytest.raises(ValueError, match=msg): + nox.load_toml("some.txt") From 61cefde57b34a400d031c87054cf549b619e0911 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 7 Apr 2024 13:16:19 -0400 Subject: [PATCH 2/6] refactor: nox.toml.load Signed-off-by: Henry Schreiner --- docs/tutorial.rst | 4 ++-- nox/__init__.py | 4 ++-- nox/{_toml.py => toml.py} | 10 +++++++++- tests/test_toml.py | 10 +++++----- 4 files changed, 18 insertions(+), 10 deletions(-) rename nox/{_toml.py => toml.py} (92%) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 1aea1f42..6e6fd151 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -176,7 +176,7 @@ Loading dependencies from pyproject.toml or scripts One common need is loading a dependency list from a ``pyproject.toml`` file (say to prepare an environment without installing the package) or supporting `PEP 723 `_ scripts. Nox provides a helper to -load these with ``nox.load_toml``. It can be passed a filepath to a toml file +load these with ``nox.toml.load``. It can be passed a filepath to a toml file or to a script file following PEP 723. For example, if you have the following ``peps.py``: @@ -204,7 +204,7 @@ You can make a session for it like this: @nox.session def peps(session): - requirements = nox.load_toml("peps.py")["dependencies"] + requirements = nox.toml.load("peps.py")["dependencies"] session.install(*requirements) session.run("peps.py") diff --git a/nox/__init__.py b/nox/__init__.py index bb311db9..7b318c24 100644 --- a/nox/__init__.py +++ b/nox/__init__.py @@ -14,10 +14,10 @@ from __future__ import annotations +from nox import toml from nox._options import noxfile_options as options from nox._parametrize import Param as param from nox._parametrize import parametrize_decorator as parametrize -from nox._toml import load_toml from nox.registry import session_decorator as session from nox.sessions import Session @@ -30,5 +30,5 @@ "session", "options", "Session", - "load_toml", + "toml", ] diff --git a/nox/_toml.py b/nox/toml.py similarity index 92% rename from nox/_toml.py rename to nox/toml.py index d3df54d6..13a1f216 100644 --- a/nox/_toml.py +++ b/nox/toml.py @@ -14,6 +14,14 @@ else: import tomllib + +__all__ = ["load"] + + +def __dir__() -> list[str]: + return __all__ + + # Note: the implementation (including this regex) taken from PEP 723 # https://peps.python.org/pep-0723 @@ -22,7 +30,7 @@ ) -def load_toml(filename: os.PathLike[str] | str) -> dict[str, Any]: +def load(filename: os.PathLike[str] | str) -> dict[str, Any]: """ Load a toml file or a script with a PEP 723 script block. diff --git a/tests/test_toml.py b/tests/test_toml.py index 7bda84c5..a461f5fe 100644 --- a/tests/test_toml.py +++ b/tests/test_toml.py @@ -17,7 +17,7 @@ def test_load_pyproject(tmp_path: Path) -> None: """ ) - toml = nox.load_toml(filepath) + toml = nox.toml.load(filepath) assert toml["project"]["dependencies"] == ["numpy", "requests"] @@ -46,7 +46,7 @@ def test_load_script_block(tmp_path: Path, example: str) -> None: ) ) - toml = nox.load_toml(filepath) + toml = nox.toml.load(filepath) assert toml["dependencies"] == ["requests<3", "rich"] @@ -68,7 +68,7 @@ def test_load_no_script_block(tmp_path: Path) -> None: ) with pytest.raises(ValueError, match="No script block found"): - nox.load_toml(filepath) + nox.toml.load(filepath) def test_load_multiple_script_block(tmp_path: Path) -> None: @@ -98,10 +98,10 @@ def test_load_multiple_script_block(tmp_path: Path) -> None: ) with pytest.raises(ValueError, match="Multiple script blocks found"): - nox.load_toml(filepath) + nox.toml.load(filepath) def test_load_non_recongnised_extension(): msg = "Extension must be .py or .toml, got .txt" with pytest.raises(ValueError, match=msg): - nox.load_toml("some.txt") + nox.toml.load("some.txt") From 4aaa9472993e812c0724cab78b854949f238d556 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 8 Apr 2024 00:28:56 -0400 Subject: [PATCH 3/6] tests: fix coverage for module dir's Signed-off-by: Henry Schreiner --- noxfile.py | 2 +- pyproject.toml | 2 +- requirements-test.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index a151e5b1..0b3b054f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -96,7 +96,7 @@ def cover(session: nox.Session) -> None: if ON_WINDOWS_CI: return - session.install("coverage[toml]>=5.3") + session.install("coverage[toml]>=7.3") session.run("coverage", "combine") session.run("coverage", "report", "--fail-under=100", "--show-missing") session.run("coverage", "erase") diff --git a/pyproject.toml b/pyproject.toml index aee23e67..913ae1dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -108,7 +108,7 @@ relative_files = true source_pkgs = [ "nox" ] [tool.coverage.report] -exclude_lines = [ "pragma: no cover", "if TYPE_CHECKING:", "@overload" ] +exclude_also = [ "def __dir__()", "if TYPE_CHECKING:", "@overload" ] [tool.mypy] files = [ "nox/**/*.py", "noxfile.py" ] diff --git a/requirements-test.txt b/requirements-test.txt index eb05f07f..b4240b7e 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,4 @@ -coverage[toml]>=5.3 +coverage[toml]>=7.2 flask myst-parser pytest>=6.0 From 6ca708860f3d3793ab510767c636bc2b2434ea6f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 8 Apr 2024 00:32:50 -0400 Subject: [PATCH 4/6] refactor: nox.project.load_toml Signed-off-by: Henry Schreiner --- docs/tutorial.rst | 8 ++++---- nox/__init__.py | 4 ++-- nox/{toml.py => project.py} | 4 ++-- tests/test_toml.py | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) rename nox/{toml.py => project.py} (95%) diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 6e6fd151..e9030c50 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -176,9 +176,9 @@ Loading dependencies from pyproject.toml or scripts One common need is loading a dependency list from a ``pyproject.toml`` file (say to prepare an environment without installing the package) or supporting `PEP 723 `_ scripts. Nox provides a helper to -load these with ``nox.toml.load``. It can be passed a filepath to a toml file -or to a script file following PEP 723. For example, if you have the following -``peps.py``: +load these with ``nox.project.load_toml``. It can be passed a filepath to a toml +file or to a script file following PEP 723. For example, if you have the +following ``peps.py``: .. code-block:: python @@ -204,7 +204,7 @@ You can make a session for it like this: @nox.session def peps(session): - requirements = nox.toml.load("peps.py")["dependencies"] + requirements = nox.project.load_toml("peps.py")["dependencies"] session.install(*requirements) session.run("peps.py") diff --git a/nox/__init__.py b/nox/__init__.py index 7b318c24..2e47ad57 100644 --- a/nox/__init__.py +++ b/nox/__init__.py @@ -14,7 +14,7 @@ from __future__ import annotations -from nox import toml +from nox import project from nox._options import noxfile_options as options from nox._parametrize import Param as param from nox._parametrize import parametrize_decorator as parametrize @@ -30,5 +30,5 @@ "session", "options", "Session", - "toml", + "project", ] diff --git a/nox/toml.py b/nox/project.py similarity index 95% rename from nox/toml.py rename to nox/project.py index 13a1f216..20a76aa7 100644 --- a/nox/toml.py +++ b/nox/project.py @@ -15,7 +15,7 @@ import tomllib -__all__ = ["load"] +__all__ = ["load_toml"] def __dir__() -> list[str]: @@ -30,7 +30,7 @@ def __dir__() -> list[str]: ) -def load(filename: os.PathLike[str] | str) -> dict[str, Any]: +def load_toml(filename: os.PathLike[str] | str) -> dict[str, Any]: """ Load a toml file or a script with a PEP 723 script block. diff --git a/tests/test_toml.py b/tests/test_toml.py index a461f5fe..833f1d12 100644 --- a/tests/test_toml.py +++ b/tests/test_toml.py @@ -17,7 +17,7 @@ def test_load_pyproject(tmp_path: Path) -> None: """ ) - toml = nox.toml.load(filepath) + toml = nox.project.load_toml(filepath) assert toml["project"]["dependencies"] == ["numpy", "requests"] @@ -46,7 +46,7 @@ def test_load_script_block(tmp_path: Path, example: str) -> None: ) ) - toml = nox.toml.load(filepath) + toml = nox.project.load_toml(filepath) assert toml["dependencies"] == ["requests<3", "rich"] @@ -68,7 +68,7 @@ def test_load_no_script_block(tmp_path: Path) -> None: ) with pytest.raises(ValueError, match="No script block found"): - nox.toml.load(filepath) + nox.project.load_toml(filepath) def test_load_multiple_script_block(tmp_path: Path) -> None: @@ -98,10 +98,10 @@ def test_load_multiple_script_block(tmp_path: Path) -> None: ) with pytest.raises(ValueError, match="Multiple script blocks found"): - nox.toml.load(filepath) + nox.project.load_toml(filepath) def test_load_non_recongnised_extension(): msg = "Extension must be .py or .toml, got .txt" with pytest.raises(ValueError, match=msg): - nox.toml.load("some.txt") + nox.project.load_toml("some.txt") From 2524576f6fe9dd521bfca2ae2f00af87571ad80e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 12 Apr 2024 17:12:53 -0400 Subject: [PATCH 5/6] Update requirements-test.txt Co-authored-by: Claudio Jolowicz --- requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index b4240b7e..09ab4d23 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,4 @@ -coverage[toml]>=7.2 +coverage[toml]>=7.3 flask myst-parser pytest>=6.0 From 57fa7f1303ca2340e818573cb4d2d25ca1ed430c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 12 Apr 2024 17:21:30 -0400 Subject: [PATCH 6/6] Update requirements-test.txt --- requirements-test.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-test.txt b/requirements-test.txt index 09ab4d23..b4240b7e 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,4 @@ -coverage[toml]>=7.3 +coverage[toml]>=7.2 flask myst-parser pytest>=6.0