From 4ffd90300e6af43775813dccec80f0451aecde55 Mon Sep 17 00:00:00 2001 From: Andy Kluger Date: Thu, 2 Nov 2023 18:02:56 -0400 Subject: [PATCH] Reorganize canonicalize_ireq and PIP_VERSION into utils canonicalize_ireq now returns a copy ireq instead of modifying in place --- piptools/_compat/__init__.py | 6 ------ piptools/_compat/pip_compat.py | 31 ++++++------------------------- piptools/build.py | 18 +++++++++--------- piptools/resolver.py | 3 ++- piptools/scripts/compile.py | 10 ++++++++-- piptools/utils.py | 24 +++++++++++++++++++++++- tests/conftest.py | 3 ++- 7 files changed, 50 insertions(+), 45 deletions(-) diff --git a/piptools/_compat/__init__.py b/piptools/_compat/__init__.py index ccc61eec..cded6776 100644 --- a/piptools/_compat/__init__.py +++ b/piptools/_compat/__init__.py @@ -1,21 +1,15 @@ from __future__ import annotations from .pip_compat import ( - PIP_VERSION, Distribution, - canonicalize_ireq, create_wheel_cache, get_dev_pkgs, - install_req_from_line, parse_requirements, ) __all__ = [ - "PIP_VERSION", "Distribution", "parse_requirements", - "install_req_from_line", "create_wheel_cache", "get_dev_pkgs", - "canonicalize_ireq", ] diff --git a/piptools/_compat/pip_compat.py b/piptools/_compat/pip_compat.py index 6cf1eb2b..9d961e21 100644 --- a/piptools/_compat/pip_compat.py +++ b/piptools/_compat/pip_compat.py @@ -2,9 +2,8 @@ import optparse from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Iterable, Iterator, Set, cast +from typing import TYPE_CHECKING, Iterable, Iterator, Set, cast -import pip from pip._internal.cache import WheelCache from pip._internal.index.package_finder import PackageFinder from pip._internal.metadata import BaseDistribution @@ -13,16 +12,9 @@ from pip._internal.network.session import PipSession from pip._internal.req import InstallRequirement from pip._internal.req import parse_requirements as _parse_requirements -from pip._internal.req.constructors import ( - install_req_from_line as _install_req_from_line, -) from pip._internal.req.constructors import install_req_from_parsed_requirement -from pip._vendor.packaging.utils import canonicalize_name -from pip._vendor.packaging.version import parse as parse_version from pip._vendor.pkg_resources import Requirement -PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split("."))) - # The Distribution interface has changed between pkg_resources and # importlib.metadata, so this compat layer allows for a consistent access # pattern. In pip 22.1, importlib.metadata became the default on Python 3.11 @@ -30,6 +22,8 @@ if TYPE_CHECKING: from pip._internal.metadata.importlib import Distribution as _ImportLibDist +from ..utils import PIP_VERSION, canonicalize_ireq + @dataclass(frozen=True) class Distribution: @@ -67,19 +61,6 @@ def _from_importlib(cls, dist: _ImportLibDist) -> Distribution: return cls(dist._dist.name, dist._dist.version, requires, dist.direct_url) -def canonicalize_ireq(ireq: InstallRequirement) -> None: - if hasattr(ireq.req, "extras") and ireq.req.extras: - ireq.req.extras = set(map(canonicalize_name, ireq.req.extras)) - if hasattr(ireq, "extras") and ireq.extras: - ireq.extras = set(map(canonicalize_name, ireq.extras)) - - -def install_req_from_line(*args: Any, **kwargs: Any) -> InstallRequirement: - ireq = _install_req_from_line(*args, **kwargs) - canonicalize_ireq(ireq) - return ireq - - def parse_requirements( filename: str, session: PipSession, @@ -91,9 +72,9 @@ def parse_requirements( for parsed_req in _parse_requirements( filename, session, finder=finder, options=options, constraint=constraint ): - ireq = install_req_from_parsed_requirement(parsed_req, isolated=isolated) - canonicalize_ireq(ireq) - yield ireq + yield canonicalize_ireq( + install_req_from_parsed_requirement(parsed_req, isolated=isolated) + ) def create_wheel_cache(cache_dir: str, format_control: str | None = None) -> WheelCache: diff --git a/piptools/build.py b/piptools/build.py index 73867f6a..08fd0cde 100644 --- a/piptools/build.py +++ b/piptools/build.py @@ -15,7 +15,7 @@ from pip._internal.req import InstallRequirement from pip._internal.req.constructors import parse_req_from_line -from ._compat import canonicalize_ireq, install_req_from_line +from .utils import canonicalize_ireq, install_req_from_line PYPROJECT_TOML = "pyproject.toml" @@ -141,15 +141,15 @@ def _prepare_requirements( replaced_package_name = req.replace(package_name, str(package_dir), 1) parts = parse_req_from_line(replaced_package_name, comes_from) - ireq = InstallRequirement( - parts.requirement, - comes_from, - link=parts.link, - markers=parts.markers, - extras=parts.extras, + yield canonicalize_ireq( + InstallRequirement( + parts.requirement, + comes_from, + link=parts.link, + markers=parts.markers, + extras=parts.extras, + ) ) - canonicalize_ireq(ireq) - yield ireq def _prepare_build_requirements( diff --git a/piptools/resolver.py b/piptools/resolver.py index 5c26ba33..29a1b85e 100644 --- a/piptools/resolver.py +++ b/piptools/resolver.py @@ -26,7 +26,7 @@ from piptools.cache import DependencyCache from piptools.repositories.base import BaseRepository -from ._compat import create_wheel_cache, install_req_from_line +from ._compat import create_wheel_cache from .exceptions import PipToolsError from .logging import log from .utils import ( @@ -35,6 +35,7 @@ copy_install_requirement, format_requirement, format_specifier, + install_req_from_line, is_pinned_requirement, is_url_requirement, key_from_ireq, diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 5838f048..8e425d01 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -14,7 +14,7 @@ from pip._internal.req import InstallRequirement from pip._internal.utils.misc import redact_auth_from_url -from .._compat import install_req_from_line, parse_requirements +from .._compat import parse_requirements from ..build import build_project_metadata from ..cache import DependencyCache from ..exceptions import NoCandidateFound, PipToolsError @@ -22,7 +22,13 @@ from ..repositories import LocalRequirementsRepository, PyPIRepository from ..repositories.base import BaseRepository from ..resolver import BacktrackingResolver, LegacyResolver -from ..utils import dedup, drop_extras, is_pinned_requirement, key_from_ireq +from ..utils import ( + dedup, + drop_extras, + install_req_from_line, + is_pinned_requirement, + key_from_ireq, +) from ..writer import OutputWriter from . import options from .options import BuildTargetT diff --git a/piptools/utils.py b/piptools/utils.py index d5ccd9c2..e0c9fe20 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -20,8 +20,12 @@ import tomli as tomllib import click +import pip from click.utils import LazyFile from pip._internal.req import InstallRequirement +from pip._internal.req.constructors import ( + install_req_from_line as _install_req_from_line, +) from pip._internal.resolution.resolvelib.base import Requirement as PipRequirement from pip._internal.utils.misc import redact_auth_from_url from pip._internal.vcs import is_url @@ -30,9 +34,9 @@ from pip._vendor.packaging.specifiers import SpecifierSet from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.packaging.version import Version +from pip._vendor.packaging.version import parse as parse_version from pip._vendor.pkg_resources import get_distribution -from piptools._compat import PIP_VERSION, install_req_from_line from piptools.locations import DEFAULT_CONFIG_FILE_NAMES from piptools.subprocess_utils import run_python_snippet @@ -41,6 +45,8 @@ _T = TypeVar("_T") _S = TypeVar("_S") +PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split("."))) + UNSAFE_PACKAGES = {"setuptools", "distribute", "pip"} COMPILE_EXCLUDE_OPTIONS = { "--dry-run", @@ -87,6 +93,22 @@ def comment(text: str) -> str: return click.style(text, fg="green") +def canonicalize_ireq(ireq: InstallRequirement) -> InstallRequirement: + """ + Return a copy of ireq with canonicalized extras strings + """ + ireq = copy_install_requirement( + ireq, extras=set(map(canonicalize_name, ireq.extras)) + ) + if ireq.req: + ireq.req.extras = set(ireq.extras) + return ireq + + +def install_req_from_line(*args: Any, **kwargs: Any) -> InstallRequirement: + return canonicalize_ireq(_install_req_from_line(*args, **kwargs)) + + def make_install_requirement( name: str, version: str | Version, ireq: InstallRequirement ) -> InstallRequirement: diff --git a/tests/conftest.py b/tests/conftest.py index 1f380383..c181fe7e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,7 +28,7 @@ from pip._vendor.packaging.version import Version from pip._vendor.pkg_resources import Requirement -from piptools._compat import PIP_VERSION, Distribution +from piptools._compat import Distribution from piptools.cache import DependencyCache from piptools.exceptions import NoCandidateFound from piptools.locations import DEFAULT_CONFIG_FILE_NAMES @@ -37,6 +37,7 @@ from piptools.repositories.base import BaseRepository from piptools.resolver import BacktrackingResolver, LegacyResolver from piptools.utils import ( + PIP_VERSION, as_tuple, is_url_requirement, key_from_ireq,