diff --git a/nox/_decorators.py b/nox/_decorators.py index 25f13de6..bd210cb2 100644 --- a/nox/_decorators.py +++ b/nox/_decorators.py @@ -62,7 +62,7 @@ class Func(FunctionDecorator): def __init__( self, func: Callable[..., Any], - python: _typing.Python = None, + python: _typing.Python | None = None, reuse_venv: bool | None = None, name: str | None = None, venv_backend: Any = None, @@ -71,6 +71,7 @@ def __init__( tags: Sequence[str] | None = None, *, default: bool = True, + requires: Sequence[str] | None = None, ) -> None: self.func = func self.python = python @@ -81,6 +82,7 @@ def __init__( self.should_warn = dict(should_warn or {}) self.tags = list(tags or []) self.default = default + self.requires = list(requires or []) def __call__(self, *args: Any, **kwargs: Any) -> Any: return self.func(*args, **kwargs) @@ -98,6 +100,28 @@ def copy(self, name: str | None = None) -> Func: self.should_warn, self.tags, default=self.default, + requires=self._requires, + ) + + @property + def requires(self) -> list[str]: + # Compute dynamically on lookup since ``self.python`` can be modified after + # creation (e.g. on an instance from ``self.copy``). + return list(map(self.format_dependency, self._requires)) + + @requires.setter + def requires(self, value: Sequence[str]) -> None: + self._requires = list(value) + + def format_dependency(self, dependency: str) -> str: + if isinstance(self.python, (bool, str)): + formatted = dependency.format(python=self.python, py=self.python) + if isinstance(self.python, bool) and formatted != dependency: + msg = "Cannot parametrize requires with {python} when python is None or a bool." + raise ValueError(msg) + return formatted + raise TypeError( + "The requires of a not-yet-parametrized session cannot be parametrized." ) @@ -130,6 +154,7 @@ def __init__(self, func: Func, param_spec: Param) -> None: func.should_warn, func.tags, default=func.default, + requires=func.requires, ) self.call_spec = call_spec self.session_signature = session_signature diff --git a/nox/_resolver.py b/nox/_resolver.py new file mode 100644 index 00000000..880ab415 --- /dev/null +++ b/nox/_resolver.py @@ -0,0 +1,205 @@ +# Copyright 2022 Alethea Katherine Flowers +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import itertools +from collections import OrderedDict +from typing import Hashable, Iterable, Iterator, Mapping, TypeVar + +Node = TypeVar("Node", bound=Hashable) + + +class CycleError(ValueError): + """An exception indicating that a cycle was encountered in a graph.""" + + +def lazy_stable_topo_sort( + dependencies: Mapping[Node, Iterable[Node]], + root: Node, + drop_root: bool = True, +) -> Iterator[Node]: + """Returns the "lazy, stable" topological sort of a dependency graph. + + The sort returned will be a topological sort of the subgraph containing only + ``root`` and its (recursive) dependencies. ``root`` will not be included in the + output sort if ``drop_root`` is ``True``. + + The sort returned is "lazy" in the sense that a node will not appear any earlier in + the output sort than is necessitated by its dependents. + + The sort returned is "stable" in the sense that the relative order of two nodes in + ``dependencies[node]`` is preserved in the output sort, except when doing so would + prevent the output sort from being either topological or lazy. The order of nodes in + ``dependencies[node]`` allows the caller to exert a preference on the order of the + output sort. + + For example, consider: + + >>> list( + ... lazy_stable_topo_sort( + ... dependencies = { + ... "a": ["c", "b"], + ... "b": [], + ... "c": [], + ... "d": ["e"], + ... "e": ["c"], + ... "root": ["a", "d"], + ... }, + ... "root", + ... drop_root=False, + ... ) + ... ) + ["c", "b", "a", "e", "d", "root"] + + Notice that: + + 1. This is a topological sort of the dependency graph. That is, nodes only + occur in the sort after all of their dependencies occur. + + 2. Had we also included a node ``"f": ["b"]`` but kept ``dependencies["root"]`` + the same, the output would not have changed. This is because ``"f"`` was not + requested directly by including it in ``dependencies["root"]`` or + transitively as a (recursive) dependency of a node in + ``dependencies["root"]``. + + 3. ``"e"`` occurs no earlier than was required by its dependents ``{"d"}``. + This is an example of the sort being "lazy". If ``"e"`` had occurred in the + output any earlier---for example, just before ``"a"``---the sort would not + have been lazy, but (in this example) the output would still have been a + topological sort. + + 4. Because the topological order between ``"a"`` and ``"d"`` is undefined and + because it is possible to do so without making the output sort non-lazy, + ``"a"`` and ``"d"`` are kept in the relative order that they have in + ``dependencies["root"]``. This is an example of the sort being stable + between pairs in ``dependencies[node]`` whenever possible. If ``"a"``'s + dependency list was instead ``["d"]``, however, the relative order between + ``"a"`` and ``"d"`` in ``dependencies["root"]`` would have been ignored to + satisfy this dependency. + + Similarly, ``"b"`` and ``"c"`` are kept in the relative order that they have + in ``dependencies["a"]``. If ``"c"``'s dependency list was instead + ``["b"]``, however, the relative order between ``"b"`` and ``"c"`` in + ``dependencies["a"]`` would have been ignored to satisfy this dependency. + + This implementation of this function is recursive and thus should not be used on + large dependency graphs, but it is suitable for noxfile-sized dependency graphs. + + Args: + dependencies (Mapping[~nox._resolver.Node, Iterable[~nox._resolver.Node]]): + A mapping from each node in the graph to the (ordered) list of nodes that it + depends on. Using a mapping type with O(1) lookup (e.g. `dict`) is strongly + suggested. + root (~nox._resolver.Node): + The root node to start the sort at. If ``drop_root`` is not ``True``, + ``root`` will be the last element of the output. + drop_root (bool): + If ``True``, ``root`` will be not be included in the output sort. Defaults + to ``True``. + + + Returns: + Iterator[~nox._resolver.Node]: The "lazy, stable" topological sort of the + subgraph containing ``root`` and its dependencies. + + Raises: + ~nox._resolver.CycleError: If a dependency cycle is encountered. + """ + + visited = {node: False for node in dependencies} + + def prepended_by_dependencies( + node: Node, + walk: OrderedDict[Node, None] | None = None, + ) -> Iterator[Node]: + """Yields a node's dependencies depth-first, followed by the node itself. + + A dependency will be skipped if has already been yielded by another call of + ``prepended_by_dependencies``. Since ``prepended_by_dependencies`` is recursive, + this means that each node will only be yielded once, and only the deepest + occurrence of a node will be yielded. + + Args: + node (~nox._resolver.Node): + A node in the dependency graph. + walk (OrderedDict[~nox._resolver.Node, None] | None): + An ``OrderedDict`` whose keys are the nodes traversed when walking a + path leading up to ``node`` on the reversed-edge dependency graph. + Defaults to ``OrderedDict()``. + + Yields: + ~nox._resolver.Node: ``node``'s direct dependencies, each + prepended by their own direct dependencies, and so forth recursively, + depth-first, followed by ``node``. + + Raises: + ValueError: If a dependency cycle is encountered. + """ + nonlocal visited + # We would like for ``walk`` to be an ordered set so that we get (a) O(1) ``node + # in walk`` and (b) so that we can use the order to report to the user what the + # dependency cycle is, if one is encountered. The standard library does not have + # an ordered set type, so we instead use the keys of an ``OrderedDict[Node, + # None]`` as an ordered set. + walk = walk or OrderedDict() + walk = extend_walk(walk, node) + if not visited[node]: + visited[node] = True + # Recurse for each node in dependencies[node] in order so that we adhere to + # the ``dependencies[node]`` order preference if doing so is possible. + yield from itertools.chain.from_iterable( + prepended_by_dependencies(dependency, walk) + for dependency in dependencies[node] + ) + yield node + else: + return + + def extend_walk( + walk: OrderedDict[Node, None], node: Node + ) -> OrderedDict[Node, None]: + """Extend a walk by a node, checking for dependency cycles. + + Args: + walk (OrderedDict[~nox._resolver.Node, None]): + See ``prepended_by_dependencies``. + nodes (~nox._resolver.Node): + A node to extend the walk with. + + Returns: + OrderedDict[~nox._resolver.Node, None]: ``walk``, extended by + ``node``. + + Raises: + ValueError: If extending ``walk`` by ``node`` introduces a cycle into the + represented walk on the dependency graph. + """ + walk = walk.copy() + if node in walk: + # Dependency cycle found. + walk_list = list(walk) + cycle = walk_list[walk_list.index(node) :] + [node] + raise CycleError("Nodes are in a dependency cycle", tuple(cycle)) + else: + walk[node] = None + return walk + + sort = prepended_by_dependencies(root) + if drop_root: + return filter( + lambda node: not (node == root and hash(node) == hash(root)), sort + ) + else: + return sort diff --git a/nox/manifest.py b/nox/manifest.py index 63e55148..4bbe5ac9 100644 --- a/nox/manifest.py +++ b/nox/manifest.py @@ -17,11 +17,13 @@ import argparse import ast import itertools +import operator from collections import OrderedDict -from collections.abc import Iterable, Iterator, Sequence -from typing import Any, Mapping +from collections.abc import Iterable, Iterator, Mapping, Sequence +from typing import Any, cast from nox._decorators import Call, Func +from nox._resolver import CycleError, lazy_stable_topo_sort from nox.sessions import Session, SessionRunner WARN_PYTHONS_IGNORED = "python_ignored" @@ -104,6 +106,32 @@ def list_all_sessions(self) -> Iterator[tuple[SessionRunner, bool]]: for session in self._all_sessions: yield session, session in self._queue + @property + def all_sessions_by_signature(self) -> dict[str, SessionRunner]: + return { + signature: session + for session in self._all_sessions + for signature in session.signatures + } + + @property + def parametrized_sessions_by_name(self) -> dict[str, list[SessionRunner]]: + """Returns a mapping from names to all sessions that are parameterizations of + the ``@session`` with each name. + + The sessions in each returned list will occur in the same order as they occur in + ``self._all_sessions``. + """ + parametrized_sessions = filter(operator.attrgetter("multi"), self._all_sessions) + key = operator.attrgetter("name") + # Note that ``sorted`` uses a stable sorting algorithm. + return { + name: list(sessions_parametrizing_name) + for name, sessions_parametrizing_name in itertools.groupby( + sorted(parametrized_sessions, key=key), key + ) + } + def add_session(self, session: SessionRunner) -> None: """Add the given session to the manifest. @@ -192,6 +220,56 @@ def filter_by_tags(self, tags: Iterable[str]) -> None: """ self._queue = [x for x in self._queue if set(x.tags).intersection(tags)] + def add_dependencies(self) -> None: + """Add direct and recursive dependencies to the queue. + + Raises: + KeyError: If any depended-on sessions are not found. + ~nox._resolver.CycleError: If a dependency cycle is encountered. + """ + sessions_by_id = self.all_sessions_by_signature + # For each session that was parametrized from a list of Pythons, create a fake + # parent session that depends on it. + parent_sessions: set[SessionRunner] = set() + for ( + parent_name, + parametrized_sessions, + ) in self.parametrized_sessions_by_name.items(): + parent_func = _null_session_func.copy() + parent_func.requires = [ + session.signatures[0] for session in parametrized_sessions + ] + parent_session = SessionRunner( + parent_name, [], parent_func, self._config, self, False + ) + parent_sessions.add(parent_session) + sessions_by_id[parent_name] = parent_session + + # Construct the dependency graph. Note that this is done lazily with iterators + # so that we won't raise if a session that doesn't actually need to run declares + # missing/improper dependencies. + dependency_graph = { + session: session.get_direct_dependencies(sessions_by_id) + for session in sessions_by_id.values() + } + + # Resolve the dependency graph. + root = cast(SessionRunner, object()) # sentinel + try: + resolved_graph = list( + lazy_stable_topo_sort({**dependency_graph, root: self._queue}, root) + ) + except CycleError as exc: + raise CycleError( + "Sessions are in a dependency cycle: " + + " -> ".join(session.name for session in exc.args[1]) + ) from exc + + # Remove fake parent sessions from the resolved graph. + self._queue = [ + session for session in resolved_graph if session not in parent_sessions + ] + def make_session( self, name: str, func: Func, multi: bool = False ) -> list[SessionRunner]: @@ -259,7 +337,7 @@ def make_session( if func.python: long_names.append(f"{name}-{func.python}") - return [SessionRunner(name, long_names, func, self._config, self)] + return [SessionRunner(name, long_names, func, self._config, self, multi)] # Since this function is parametrized, we need to add a distinct # session for each permutation. @@ -274,13 +352,15 @@ def make_session( # Ensure that specifying session-python will run all parameterizations. long_names.append(f"{name}-{func.python}") - sessions.append(SessionRunner(name, long_names, call, self._config, self)) + sessions.append( + SessionRunner(name, long_names, call, self._config, self, multi) + ) # Edge case: If the parameters made it such that there were no valid # calls, add an empty, do-nothing session. if not calls: sessions.append( - SessionRunner(name, [], _null_session_func, self._config, self) + SessionRunner(name, [], _null_session_func, self._config, self, multi) ) # Return the list of sessions. diff --git a/nox/registry.py b/nox/registry.py index 4e5c72aa..8dcd79fa 100644 --- a/nox/registry.py +++ b/nox/registry.py @@ -45,6 +45,7 @@ def session_decorator( tags: Sequence[str] | None = ..., *, default: bool = ..., + requires: Sequence[str] | None = ..., ) -> Callable[[F], F]: ... @@ -60,6 +61,7 @@ def session_decorator( tags: Sequence[str] | None = None, *, default: bool = True, + requires: Sequence[str] | None = None, ) -> F | Callable[[F], F]: """Designate the decorated function as a session.""" # If `func` is provided, then this is the decorator call with the function @@ -80,6 +82,7 @@ def session_decorator( venv_params=venv_params, tags=tags, default=default, + requires=requires, ) if py is not None and python is not None: @@ -92,6 +95,7 @@ def session_decorator( python = py final_name = name or func.__name__ + fn = Func( func, python, @@ -101,8 +105,9 @@ def session_decorator( venv_params, tags=tags, default=default, + requires=requires, ) - _REGISTRY[final_name] = fn + _REGISTRY[name or func.__name__] = fn return fn diff --git a/nox/sessions.py b/nox/sessions.py index a9b361d2..fbfdcb4a 100644 --- a/nox/sessions.py +++ b/nox/sessions.py @@ -29,6 +29,7 @@ Callable, Generator, Iterable, + Iterator, Mapping, Sequence, ) @@ -869,6 +870,7 @@ def __init__( func: Func, global_config: argparse.Namespace, manifest: Manifest, + multi: bool = False, ) -> None: self.name = name self.signatures = signatures @@ -877,6 +879,8 @@ def __init__( self.manifest = manifest self.venv: ProcessEnv | None = None self.posargs: list[str] = global_config.posargs[:] + self.result: Result | None = None + self.multi = multi @property def description(self) -> str | None: @@ -902,6 +906,43 @@ def tags(self) -> list[str]: def envdir(self) -> str: return _normalize_path(self.global_config.envdir, self.friendly_name) + def get_direct_dependencies( + self, sessions_by_id: Mapping[str, SessionRunner] | None = None + ) -> Iterator[SessionRunner]: + """Yields the sessions of the session's direct dependencies. + + Args: + sessions_by_id (Mapping[str, ~nox.sessions.SessionRunner] | None): An + optional mapping from both dependency signatures and names to + corresponding ``SessionRunner``s. If this is not provided, + ``self.manifest.all_sessions_by_signature`` will be used to find the + sessions corresponding to signatures in ``self.func.requires``, and + non-signature names (i.e. names of sessions that were parameterized with + multiple Pythons) in ``self.func.requires`` will be resolved via + ``self.manifest.parametrized_sessions_by_name``. + + Returns: + Iterator[~nox.session.SessionRunner] + + Raises: + KeyError: If a dependency's session could not be found. + """ + try: + if sessions_by_id is None: + sessions_by_signature = self.manifest.all_sessions_by_signature + parametrized_sessions_by_name = ( + self.manifest.parametrized_sessions_by_name + ) + for requirement in self.func.requires: + if requirement in sessions_by_signature: + yield sessions_by_signature[requirement] + else: + yield from parametrized_sessions_by_name[requirement] + else: + yield from map(sessions_by_id.__getitem__, self.func.requires) + except KeyError as exc: + raise KeyError(f"Session not found: {exc.args[0]}") from exc + def _create_venv(self) -> None: reuse_existing = self.reuse_existing_venv() @@ -991,6 +1032,18 @@ def reuse_existing_venv(self) -> bool: def execute(self) -> Result: logger.warning(f"Running session {self.friendly_name}") + for dependency in self.get_direct_dependencies(): + if not dependency.result: + self.result = Result( + self, + Status.ABORTED, + reason=( + f"Prerequisite session {dependency.friendly_name} was not" + " successful" + ), + ) + return self.result + try: cwd = os.path.realpath(os.path.dirname(self.global_config.noxfile)) @@ -1001,25 +1054,25 @@ def execute(self) -> Result: self.func(session) # Nothing went wrong; return a success. - return Result(self, Status.SUCCESS) + self.result = Result(self, Status.SUCCESS) except nox.virtualenv.InterpreterNotFound as exc: if self.global_config.error_on_missing_interpreters: - return Result(self, Status.FAILED, reason=str(exc)) + self.result = Result(self, Status.FAILED, reason=str(exc)) else: logger.warning( "Missing interpreters will error by default on CI systems." ) - return Result(self, Status.SKIPPED, reason=str(exc)) + self.result = Result(self, Status.SKIPPED, reason=str(exc)) except _SessionQuit as exc: - return Result(self, Status.ABORTED, reason=str(exc)) + self.result = Result(self, Status.ABORTED, reason=str(exc)) except _SessionSkip as exc: - return Result(self, Status.SKIPPED, reason=str(exc)) + self.result = Result(self, Status.SKIPPED, reason=str(exc)) except nox.command.CommandFailed: - return Result(self, Status.FAILED) + self.result = Result(self, Status.FAILED) except KeyboardInterrupt: logger.error(f"Session {self.friendly_name} interrupted.") @@ -1027,7 +1080,9 @@ def execute(self) -> Result: except Exception as exc: logger.exception(f"Session {self.friendly_name} raised exception {exc!r}") - return Result(self, Status.FAILED) + self.result = Result(self, Status.FAILED) + + return self.result class Result: diff --git a/nox/tasks.py b/nox/tasks.py index 818dc935..49d7b031 100644 --- a/nox/tasks.py +++ b/nox/tasks.py @@ -27,6 +27,7 @@ import nox from nox import _options, registry +from nox._resolver import CycleError from nox._version import InvalidVersionSpecifier, VersionCheckFailed, check_nox_version from nox.logger import logger from nox.manifest import WARN_PYTHONS_IGNORED, Manifest @@ -221,6 +222,14 @@ def filter_manifest(manifest: Manifest, global_config: Namespace) -> Manifest | logger.error("No sessions selected after filtering by keyword.") return 3 + # Add dependencies. + try: + manifest.add_dependencies() + except (KeyError, CycleError) as exc: + logger.error("Error while resolving session dependencies.") + logger.error(exc.args[0]) + return 3 + # Return the modified manifest. return manifest