feat(twine): support 'bzlmod' users out of the box (bazelbuild#1572)

Implements a test that starts a [`pypiserver`] and checks
that the publishing with the new machinery still works.

Fixes bazelbuild#1369

[pypiserver]: https://github.com/pypiserver/pypiserver

CHANGELOG.md

@@ -43,6 +43,9 @@ A brief description of the categories of changes:
 * (wheel) Add support for `data_files` attributes in py_wheel rule
   ([#1777](https://github.com/bazelbuild/rules_python/issues/1777))
 
+* (py_wheel) `bzlmod` installations now provide a `twine` setup for the default
+  Python toolchain in `rules_python` for version 3.11.
+
 [0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
 [python_default_visibility]: gazelle/README.md#directive-python_default_visibility
 
@@ -269,7 +272,6 @@ A brief description of the categories of changes:
   attribute for every target in the package. This is enabled through a separate
   directive `python_generation_mode_per_file_include_init`.
 
-
 ## [0.27.0] - 2023-11-16
 
 [0.27.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.27.0

MODULE.bazel

@@ -48,11 +48,24 @@ python.toolchain(
     is_default = True,
     python_version = "3.11",
 )
-use_repo(python, "pythons_hub")
+use_repo(python, "python_versions", "pythons_hub")
 
 # This call registers the Python toolchains.
 register_toolchains("@pythons_hub//:all")
 
+#####################
+# Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.
+
+pip = use_extension("//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "rules_python_publish_deps",
+    python_version = "3.11",
+    requirements_darwin = "//tools/publish:requirements_darwin.txt",
+    requirements_lock = "//tools/publish:requirements.txt",
+    requirements_windows = "//tools/publish:requirements_windows.txt",
+)
+use_repo(pip, "rules_python_publish_deps")
+
 # ===== DEV ONLY DEPS AND SETUP BELOW HERE =====
 bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")
 bazel_dep(name = "rules_bazel_integration_test", version = "0.20.0", dev_dependency = True)
@@ -84,24 +97,12 @@ dev_pip.parse(
     python_version = "3.11",
     requirements_lock = "//docs/sphinx:requirements.txt",
 )
-
-#####################
-# Install twine for our own runfiles wheel publishing.
-# Eventually we might want to install twine automatically for users too, see:
-# https://github.com/bazelbuild/rules_python/issues/1016.
-
 dev_pip.parse(
-    hub_name = "publish_deps",
+    hub_name = "pypiserver",
     python_version = "3.11",
-    requirements_darwin = "//tools/publish:requirements_darwin.txt",
-    requirements_lock = "//tools/publish:requirements.txt",
-    requirements_windows = "//tools/publish:requirements_windows.txt",
-)
-use_repo(
-    dev_pip,
-    "dev_pip",
-    publish_deps_twine = "publish_deps_311_twine",
+    requirements_lock = "//examples/wheel:requirements_server.txt",
 )
+use_repo(dev_pip, "dev_pip", "pypiserver")
 
 # Bazel integration test setup below
 

WORKSPACE

@@ -94,17 +94,27 @@ load("@python//3.11.8:defs.bzl", "interpreter")
 load("@rules_python//python:pip.bzl", "pip_parse")
 
 pip_parse(
-    name = "publish_deps",
+    name = "rules_python_publish_deps",
     python_interpreter_target = interpreter,
     requirements_darwin = "//tools/publish:requirements_darwin.txt",
     requirements_lock = "//tools/publish:requirements.txt",
     requirements_windows = "//tools/publish:requirements_windows.txt",
 )
 
-load("@publish_deps//:requirements.bzl", "install_deps")
+load("@rules_python_publish_deps//:requirements.bzl", "install_deps")
 
 install_deps()
 
+pip_parse(
+    name = "pypiserver",
+    python_interpreter_target = interpreter,
+    requirements_lock = "//examples/wheel:requirements_server.txt",
+)
+
+load("@pypiserver//:requirements.bzl", install_pypiserver = "install_deps")
+
+install_pypiserver()
+
 #####################
 # Install sphinx for doc generation.
 

examples/wheel/BUILD.bazel

@@ -17,7 +17,10 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("//examples/wheel/private:wheel_utils.bzl", "directory_writer", "make_variable_tags")
 load("//python:defs.bzl", "py_library", "py_test")
 load("//python:packaging.bzl", "py_package", "py_wheel")
+load("//python:pip.bzl", "compile_pip_requirements")
 load("//python:versions.bzl", "gen_python_config_settings")
+load("//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary")
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")  # buildifier: disable=bzl-visibility
 
 package(default_visibility = ["//visibility:public"])
 
@@ -56,6 +59,10 @@ py_wheel(
     # Package data. We're building "example_minimal_library-0.0.1-py3-none-any.whl"
     distribution = "example_minimal_library",
     python_tag = "py3",
+    # NOTE: twine_binary = "//tools/publish:twine" does not work on non-bzlmod
+    # setups because the `//tools/publish:twine` produces multiple files and is
+    # unsuitable as the `src` to the underlying native_binary rule.
+    twine = None if BZLMOD_ENABLED else "@rules_python_publish_deps_twine//:pkg",
     version = "0.0.1",
     deps = [
         "//examples/wheel/lib:module_with_data",
@@ -348,3 +355,39 @@ py_test(
         "//python/runfiles",
     ],
 )
+
+# Test wheel publishing
+
+compile_pip_requirements(
+    name = "requirements_server",
+    src = "requirements_server.in",
+)
+
+py_test(
+    name = "test_publish",
+    srcs = ["test_publish.py"],
+    data = [
+        ":minimal_with_py_library",
+        ":minimal_with_py_library.publish",
+        ":pypiserver",
+    ],
+    env = {
+        "PUBLISH_PATH": "$(location :minimal_with_py_library.publish)",
+        "SERVER_PATH": "$(location :pypiserver)",
+        "WHEEL_PATH": "$(rootpath :minimal_with_py_library)",
+    },
+    target_compatible_with = select({
+        "@platforms//os:linux": [],
+        "@platforms//os:macos": [],
+        "//conditions:default": ["@platforms//:incompatible"],
+    }),
+    deps = [
+        "@pypiserver//pypiserver",
+    ],
+)
+
+py_console_script_binary(
+    name = "pypiserver",
+    pkg = "@pypiserver//pypiserver",
+    script = "pypi-server",
+)

examples/wheel/requirements_server.in

@@ -0,0 +1,2 @@
+# This is for running publishing tests
+pypiserver

examples/wheel/requirements_server.txt

@@ -0,0 +1,16 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    bazel run //examples/wheel:requirements_server.update
+#
+pypiserver==2.0.1 \
+    --hash=sha256:1dd98fb99d2da4199fb44c7284e57d69a9f7fda2c6c8dc01975c151c592677bf \
+    --hash=sha256:7b58fbd54468235f79e4de07c4f7a9ff829e7ac6869bef47ec11e0710138e162
+    # via -r examples/wheel/requirements_server.in
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==24.0 \
+    --hash=sha256:ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc \
+    --hash=sha256:ea9bd1a847e8c5774a5777bb398c19e80bcd4e2aa16a4b301b718fe6f593aba2
+    # via pypiserver

examples/wheel/test_publish.py

@@ -0,0 +1,117 @@
+import os
+import socket
+import subprocess
+import textwrap
+import time
+import unittest
+from contextlib import closing
+from pathlib import Path
+from urllib.request import urlopen
+
+
+def find_free_port():
+    with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        s.bind(("", 0))
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        return s.getsockname()[1]
+
+
+class TestTwineUpload(unittest.TestCase):
+    def setUp(self):
+        self.maxDiff = 1000
+        self.port = find_free_port()
+        self.url = f"http://localhost:{self.port}"
+        self.dir = Path(os.environ["TEST_TMPDIR"])
+
+        self.log_file = self.dir / "pypiserver-log.txt"
+        self.log_file.touch()
+        _storage_dir = self.dir / "data"
+        for d in [_storage_dir]:
+            d.mkdir(exist_ok=True)
+
+        print("Starting PyPI server...")
+        self._server = subprocess.Popen(
+            [
+                str(Path(os.environ["SERVER_PATH"])),
+                "run",
+                "--verbose",
+                "--log-file",
+                str(self.log_file),
+                "--host",
+                "localhost",
+                "--port",
+                str(self.port),
+                # Allow unauthenticated access
+                "--authenticate",
+                ".",
+                "--passwords",
+                ".",
+                str(_storage_dir),
+            ],
+        )
+
+        line = "Hit Ctrl-C to quit"
+        interval = 0.1
+        wait_seconds = 40
+        for _ in range(int(wait_seconds / interval)):  # 40 second timeout
+            current_logs = self.log_file.read_text()
+            if line in current_logs:
+                print(current_logs.strip())
+                print("...")
+                break
+
+            time.sleep(0.1)
+        else:
+            raise RuntimeError(
+                f"Could not get the server running fast enough, waited for {wait_seconds}s"
+            )
+
+    def tearDown(self):
+        self._server.terminate()
+        print(f"Stopped PyPI server, all logs:\n{self.log_file.read_text()}")
+
+    def test_upload_and_query_simple_api(self):
+        # Given
+        script_path = Path(os.environ["PUBLISH_PATH"])
+        whl = Path(os.environ["WHEEL_PATH"])
+
+        # When I publish a whl to a package registry
+        subprocess.check_output(
+            [
+                str(script_path),
+                "--no-color",
+                "upload",
+                str(whl),
+                "--verbose",
+                "--non-interactive",
+                "--disable-progress-bar",
+            ],
+            env={
+                "TWINE_REPOSITORY_URL": self.url,
+                "TWINE_USERNAME": "dummy",
+                "TWINE_PASSWORD": "dummy",
+            },
+        )
+
+        # Then I should be able to get its contents
+        with urlopen(self.url + "/example-minimal-library/") as response:
+            got_content = response.read().decode("utf-8")
+            want_content = """
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>Links for example-minimal-library</title>
+    </head>
+    <body>
+        <h1>Links for example-minimal-library</h1>
+             <a href="/packages/example_minimal_library-0.0.1-py3-none-any.whl#sha256=79a4e9c1838c0631d5d8fa49a26efd6e9a364f6b38d9597c0f6df112271a0e28">example_minimal_library-0.0.1-py3-none-any.whl</a><br>
+    </body>
+</html>"""
+            self.assertEqual(
+                textwrap.dedent(want_content).strip(),
+                textwrap.dedent(got_content).strip(),
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

python/BUILD.bazel

@@ -76,11 +76,13 @@ bzl_library(
     srcs = ["packaging.bzl"],
     deps = [
         ":py_binary_bzl",
+        "//python/private:bzlmod_enabled_bzl",
         "//python/private:py_package.bzl",
         "//python/private:py_wheel_bzl",
         "//python/private:py_wheel_normalize_pep440.bzl",
         "//python/private:stamp_bzl",
         "//python/private:util_bzl",
+        "@bazel_skylib//rules:native_binary",
     ],
 )