Sync direct references with hashes (#1885)

piptools/_compat/__init__.py

@@ -1,5 +1,10 @@
 from __future__ import annotations
 
-from .pip_compat import PIP_VERSION, create_wheel_cache, parse_requirements
+from .pip_compat import (
+    PIP_VERSION,
+    Distribution,
+    create_wheel_cache,
+    parse_requirements,
+)
 
-__all__ = ["PIP_VERSION", "parse_requirements", "create_wheel_cache"]
+__all__ = ["PIP_VERSION", "Distribution", "parse_requirements", "create_wheel_cache"]

piptools/_compat/pip_compat.py

@@ -1,11 +1,15 @@
 from __future__ import annotations
 
 import optparse
-from typing import Callable, Iterable, Iterator, cast
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Iterable, Iterator
 
 import pip
 from pip._internal.cache import WheelCache
 from pip._internal.index.package_finder import PackageFinder
+from pip._internal.metadata import BaseDistribution
+from pip._internal.metadata.pkg_resources import Distribution as _PkgResourcesDist
+from pip._internal.models.direct_url import DirectUrl
 from pip._internal.network.session import PipSession
 from pip._internal.req import InstallRequirement
 from pip._internal.req import parse_requirements as _parse_requirements
@@ -15,11 +19,48 @@
 
 PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split(".")))
 
-__all__ = [
-    "dist_requires",
-    "uses_pkg_resources",
-    "Distribution",
-]
+# The Distribution interface has changed between pkg_resources and
+# importlib.metadata, so this compat layer allows for a consistent access
+# pattern. In pip 22.1, importlib.metadata became the default on Python 3.11
+# (and later), but is overridable. `select_backend` returns what's being used.
+if TYPE_CHECKING:
+    from pip._internal.metadata.importlib import Distribution as _ImportLibDist
+
+
+@dataclass(frozen=True)
+class Distribution:
+    key: str
+    version: str
+    requires: Iterable[Requirement]
+    direct_url: DirectUrl | None
+
+    @classmethod
+    def from_pip_distribution(cls, dist: BaseDistribution) -> Distribution:
+        # TODO: Use only the BaseDistribution protocol properties and methods
+        # instead of specializing by type.
+        if isinstance(dist, _PkgResourcesDist):
+            return cls._from_pkg_resources(dist)
+        else:
+            return cls._from_importlib(dist)
+
+    @classmethod
+    def _from_pkg_resources(cls, dist: _PkgResourcesDist) -> Distribution:
+        return cls(
+            dist._dist.key, dist._dist.version, dist._dist.requires(), dist.direct_url
+        )
+
+    @classmethod
+    def _from_importlib(cls, dist: _ImportLibDist) -> Distribution:
+        """Mimics pkg_resources.Distribution.requires for the case of no
+        extras. This doesn't fulfill that API's `extras` parameter but
+        satisfies the needs of pip-tools."""
+        reqs = (Requirement.parse(req) for req in (dist._dist.requires or ()))
+        requires = [
+            req
+            for req in reqs
+            if not req.marker or req.marker.evaluate({"extra": None})
+        ]
+        return cls(dist._dist.name, dist._dist.version, requires, dist.direct_url)
 
 
 def parse_requirements(
@@ -36,46 +77,6 @@ def parse_requirements(
         yield install_req_from_parsed_requirement(parsed_req, isolated=isolated)
 
 
-# The Distribution interface has changed between pkg_resources and
-# importlib.metadata, so this compat layer allows for a consistent access
-# pattern. In pip 22.1, importlib.metadata became the default on Python 3.11
-# (and later), but is overridable. `select_backend` returns what's being used.
-
-
-def _uses_pkg_resources() -> bool:
-    from pip._internal.metadata import select_backend
-    from pip._internal.metadata.pkg_resources import Distribution as _Dist
-
-    return select_backend().Distribution is _Dist
-
-
-uses_pkg_resources = _uses_pkg_resources()
-
-if uses_pkg_resources:
-    from operator import methodcaller
-
-    from pip._vendor.pkg_resources import Distribution
-
-    dist_requires = cast(
-        Callable[[Distribution], Iterable[Requirement]], methodcaller("requires")
-    )
-else:
-    from pip._internal.metadata import select_backend
-
-    Distribution = select_backend().Distribution
-
-    def dist_requires(dist: Distribution) -> Iterable[Requirement]:
-        """Mimics pkg_resources.Distribution.requires for the case of no
-        extras. This doesn't fulfill that API's `extras` parameter but
-        satisfies the needs of pip-tools."""
-        reqs = (Requirement.parse(req) for req in (dist.requires or ()))
-        return [
-            req
-            for req in reqs
-            if not req.marker or req.marker.evaluate({"extra": None})
-        ]
-
-
 def create_wheel_cache(cache_dir: str, format_control: str | None = None) -> WheelCache:
     kwargs: dict[str, str | None] = {"cache_dir": cache_dir}
     if PIP_VERSION[:2] <= (23, 0):

piptools/scripts/sync.py

@@ -15,8 +15,7 @@
 from pip._internal.metadata import get_environment
 
 from .. import sync
-from .._compat import parse_requirements
-from .._compat.pip_compat import Distribution
+from .._compat import Distribution, parse_requirements
 from ..exceptions import PipToolsError
 from ..locations import CONFIG_FILE_NAME
 from ..logging import log
@@ -303,4 +302,4 @@ def _get_installed_distributions(
         user_only=user_only,
         skip=[],
     )
-    return [cast(Distribution, dist)._dist for dist in dists]
+    return [Distribution.from_pip_distribution(dist) for dist in dists]

piptools/sync.py

@@ -9,10 +9,15 @@
 
 import click
 from pip._internal.commands.freeze import DEV_PKGS
+from pip._internal.models.direct_url import ArchiveInfo
 from pip._internal.req import InstallRequirement
 from pip._internal.utils.compat import stdlib_pkgs
+from pip._internal.utils.direct_url_helpers import (
+    direct_url_as_pep440_direct_reference,
+    direct_url_from_link,
+)
 
-from ._compat.pip_compat import Distribution, dist_requires
+from ._compat import Distribution
 from .exceptions import IncompatibleRequirements
 from .logging import log
 from .utils import (
@@ -55,13 +60,13 @@ def dependency_tree(
 
     while queue:
         v = queue.popleft()
-        key = key_from_req(v)
+        key = v.key
         if key in dependencies:
             continue
 
         dependencies.add(key)
 
-        for dep_specifier in dist_requires(v):
+        for dep_specifier in v.requires:
             dep_name = key_from_req(dep_specifier)
             if dep_name in installed_keys:
                 dep = installed_keys[dep_name]
@@ -81,7 +86,7 @@ def get_dists_to_ignore(installed: Iterable[Distribution]) -> list[str]:
     locally, click should also be installed/uninstalled depending on the given
     requirements.
     """
-    installed_keys = {key_from_req(r): r for r in installed}
+    installed_keys = {r.key: r for r in installed}
     return list(
         flat_map(lambda req: dependency_tree(installed_keys, req), PACKAGES_TO_IGNORE)
     )
@@ -120,22 +125,36 @@ def diff_key_from_ireq(ireq: InstallRequirement) -> str:
     """
     Calculate a key for comparing a compiled requirement with installed modules.
     For URL requirements, only provide a useful key if the url includes
-    #egg=name==version, which will set ireq.req.name and ireq.specifier.
+    a hash, e.g. #sha1=..., in any of the supported hash algorithms.
     Otherwise return ireq.link so the key will not match and the package will
     reinstall. Reinstall is necessary to ensure that packages will reinstall
-    if the URL is changed but the version is not.
+    if the contents at the URL have changed but the version has not.
     """
     if is_url_requirement(ireq):
-        if (
-            ireq.req
-            and (getattr(ireq.req, "key", None) or getattr(ireq.req, "name", None))
-            and ireq.specifier
-        ):
-            return key_from_ireq(ireq)
+        if getattr(ireq.req, "name", None) and ireq.link.has_hash:
+            return str(
+                direct_url_as_pep440_direct_reference(
+                    direct_url_from_link(ireq.link), ireq.req.name
+                )
+            )
+        # TODO: Also support VCS and editable installs.
         return str(ireq.link)
     return key_from_ireq(ireq)
 
 
+def diff_key_from_req(req: Distribution) -> str:
+    """Get a unique key for the requirement."""
+    key = req.key
+    if (
+        req.direct_url
+        and isinstance(req.direct_url.info, ArchiveInfo)
+        and req.direct_url.info.hash
+    ):
+        key = direct_url_as_pep440_direct_reference(req.direct_url, key)
+    # TODO: Also support VCS and editable installs.
+    return key
+
+
 def diff(
     compiled_requirements: Iterable[InstallRequirement],
     installed_dists: Iterable[Distribution],
@@ -152,7 +171,7 @@ def diff(
 
     pkgs_to_ignore = get_dists_to_ignore(installed_dists)
     for dist in installed_dists:
-        key = key_from_req(dist)
+        key = diff_key_from_req(dist)
         if key not in requirements_lut or not requirements_lut[key].match_markers():
             to_uninstall.add(key)
         elif requirements_lut[key].specifier.contains(dist.version):

piptools/utils.py

@@ -23,10 +23,11 @@
 from pip._internal.utils.misc import redact_auth_from_url
 from pip._internal.vcs import is_url
 from pip._vendor.packaging.markers import Marker
+from pip._vendor.packaging.requirements import Requirement
 from pip._vendor.packaging.specifiers import SpecifierSet
 from pip._vendor.packaging.utils import canonicalize_name
 from pip._vendor.packaging.version import Version
-from pip._vendor.pkg_resources import Distribution, Requirement, get_distribution
+from pip._vendor.pkg_resources import get_distribution
 
 from piptools._compat import PIP_VERSION
 from piptools.locations import CONFIG_FILE_NAME
@@ -64,15 +65,9 @@ def key_from_ireq(ireq: InstallRequirement) -> str:
         return key_from_req(ireq.req)
 
 
-def key_from_req(req: InstallRequirement | Distribution | Requirement) -> str:
+def key_from_req(req: InstallRequirement | Requirement) -> str:
     """Get an all-lowercase version of the requirement's name."""
-    if hasattr(req, "key"):
-        # from pkg_resources, such as installed dists for pip-sync
-        key = req.key
-    else:
-        # from packaging, such as install requirements from requirements.txt
-        key = req.name
-    return str(canonicalize_name(key))
+    return str(canonicalize_name(req.name))
 
 
 def comment(text: str) -> str:

tests/conftest.py

@@ -18,15 +18,17 @@
 from pip._internal.commands.install import InstallCommand
 from pip._internal.index.package_finder import PackageFinder
 from pip._internal.models.candidate import InstallationCandidate
+from pip._internal.models.link import Link
 from pip._internal.network.session import PipSession
 from pip._internal.req.constructors import (
     install_req_from_editable,
     install_req_from_line,
 )
+from pip._internal.utils.direct_url_helpers import direct_url_from_link
 from pip._vendor.packaging.version import Version
 from pip._vendor.pkg_resources import Requirement
 
-from piptools._compat.pip_compat import PIP_VERSION, uses_pkg_resources
+from piptools._compat import PIP_VERSION, Distribution
 from piptools.cache import DependencyCache
 from piptools.exceptions import NoCandidateFound
 from piptools.locations import CONFIG_FILE_NAME
@@ -38,7 +40,6 @@
     as_tuple,
     is_url_requirement,
     key_from_ireq,
-    key_from_req,
     make_install_requirement,
 )
 
@@ -125,34 +126,6 @@ def command(self) -> InstallCommand:
         """Not used"""
 
 
-class FakeInstalledDistribution:
-    def __init__(self, line, deps=None):
-        if deps is None:
-            deps = []
-        self.dep_strs = deps
-        self.deps = [Requirement.parse(d) for d in deps]
-
-        self.req = Requirement.parse(line)
-
-        self.key = key_from_req(self.req)
-        self.specifier = self.req.specifier
-
-        self.version = line.split("==")[1]
-
-    # The Distribution interface has changed between pkg_resources and
-    # importlib.metadata.
-    if uses_pkg_resources:
-
-        def requires(self):
-            return self.deps
-
-    else:
-
-        @property
-        def requires(self):
-            return self.dep_strs
-
-
 def pytest_collection_modifyitems(config, items):
     for item in items:
         # Mark network tests as flaky
@@ -162,7 +135,22 @@ def pytest_collection_modifyitems(config, items):
 
 @pytest.fixture
 def fake_dist():
-    return FakeInstalledDistribution
+    def _fake_dist(line, deps=None):
+        if deps is None:
+            deps = []
+        req = Requirement.parse(line)
+        key = req.key
+        if "==" in line:
+            version = line.split("==")[1]
+        else:
+            version = "0+unknown"
+        requires = [Requirement.parse(d) for d in deps]
+        direct_url = None
+        if req.url:
+            direct_url = direct_url_from_link(Link(req.url))
+        return Distribution(key, version, requires, direct_url)
+
+    return _fake_dist
 
 
 @pytest.fixture