Add option to specify a package server mirror; clean up hardcoded URL…

…s throughout codebase (#540) * Add in new --pypi-mirror-url and --pypi-metadata-url options and deprecate --pypi-url * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Line length --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Marcelo Duarte Trevisani <[email protected]>
conda · Aug 4, 2024 · ede342b · ede342b
1 parent fa0b1a3
commit ede342b
Show file tree

Hide file tree

Showing 5 changed files with 82 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -76,6 +76,27 @@ grayskull pypi ./pytest-5.3.5.tar.gz
 Note that such a recipe isn't really portable as it will depend on the local path of the
 sdist file. It can be useful if you want to automatically generate a conda package.
 
+### Use Grayskull with an internal package index
+
+Grayskull can create recipes that point to any Python Package Index. Supply the `--pypi-mirror-url` keyword.
+
+* Example:
+```bash
+grayskull pypi --pypi-mirror-url https://pypi.example.com pytest
+```
+
+The above will source packages from `https://pypi.example.com/packages/source/...`
+
+This assumes that the mirror follows the same API as pypi _including_ hosting metadata at the `/pypi/{package_name}/json` endpoint.
+To specify an alternate metadata location use the `--pypi-metadata-url` option.
+
+* Example:
+```bash
+grayskull pypi --pypi-mirror-url https://pypi.example.com --pypi-metadata-url https://pypi_meta.example.com pytest
+```
+
+> *Note:* `--pypi-metadata-url` is a replacement for `--pypi-url`; `--pypi-url` is deprecated and will be removed in a future release.
+
 ### Online Grayskull
 
 It is also possible to use Grayskull without any installation. You can go to this website [marcelotrevisani.com/grayskull](https://www.marcelotrevisani.com/grayskull) and inform the name and the version (optional) of the package and it will create the recipe for you.

diff --git a/grayskull/config.py b/grayskull/config.py
@@ -4,6 +4,9 @@
 from grayskull.cli.parser import parse_pkg_name_version
 from grayskull.utils import PyVer
 
+DEFAULT_PYPI_URL = "https://pypi.org"
+DEFAULT_PYPI_META_URL = "https://pypi.org/pypi"
+
 
 @dataclass
 class Configuration:
@@ -37,7 +40,8 @@ class Configuration:
         default_factory=lambda: ("cython", "cython-blis", "blis")
     )
     pkg_need_cxx_compiler: Tuple = field(default_factory=lambda: ("pybind11",))
-    url_pypi_metadata: str = "https://pypi.org/pypi/{pkg_name}/json"
+    url_pypi: str = DEFAULT_PYPI_URL
+    url_pypi_metadata: str = DEFAULT_PYPI_META_URL
     download: bool = False
     is_arch: bool = False
     repo_github: Optional[str] = None
@@ -103,11 +107,12 @@ def get_py_version_available(
         return py_ver_enabled
 
     def __post_init__(self):
+        if not self.url_pypi_metadata.endswith("/{pkg_name}/json"):
+            self.url_pypi_metadata = (
+                self.url_pypi_metadata.rstrip("/") + "/{pkg_name}/json"
+            )
         if self.from_local_sdist:
             self.local_sdist = self.local_sdist or self.name
-        if self.url_pypi_metadata != "https://pypi.org/pypi/{pkg_name}/json":
-            prefix = "" if self.url_pypi_metadata.endswith("/") else "/"
-            self.url_pypi_metadata += f"{prefix}{{pkg_name}}/json"
         pkg_repo, pkg_name, pkg_version = parse_pkg_name_version(self.name)
         if pkg_repo:
             prefix = "" if pkg_repo.endswith("/") else "/"

diff --git a/grayskull/main.py b/grayskull/main.py
@@ -13,7 +13,7 @@
 from grayskull.base.github import get_git_current_user
 from grayskull.cli import CLIConfig
 from grayskull.cli.stdout import print_msg
-from grayskull.config import Configuration
+from grayskull.config import DEFAULT_PYPI_META_URL, DEFAULT_PYPI_URL, Configuration
 from grayskull.utils import generate_recipe, origin_is_github, origin_is_local_sdist
 
 init(autoreset=True)
@@ -170,10 +170,26 @@ def init_parser():
         help="It will generate the recipes strict for the conda-forge channel.",
     )
     pypi_parser.add_argument(
-        "--pypi-url",
-        default="https://pypi.org/pypi/",
+        "--pypi-metadata-url",
+        default=DEFAULT_PYPI_META_URL,
         dest="url_pypi_metadata",
-        help="Pypi url server",
+        help=(
+            "Pypi url server metadata endpoint;"
+            + "will be appended with '{pkgname}/json'"
+        ),
+    )
+    pypi_parser.add_argument(
+        "--pypi-mirror-url",
+        default=DEFAULT_PYPI_URL,
+        dest="url_pypi_mirror",
+        help="Pypi mirror URL; assumed to have same API as pypi.org",
+    )
+    # TODO: Remove before 3.0 release
+    pypi_parser.add_argument(
+        "--pypi-url",
+        default=None,
+        dest="url_pypi_metadata_deprecated",
+        help="DEPRECATED: use --pypi-metadata-url instead",
     )
     pypi_parser.add_argument(
         "--recursive",
@@ -315,11 +331,37 @@ def generate_recipes_from_list(list_pkgs, args):
         if Path(pkg_name).is_file() and (not from_local_sdist):
             args.output = pkg_name
         try:
+            # TODO: Remove before 3.0 release
+            if args.url_pypi_metadata_deprecated and args.url_pypi_metadata:
+                raise RuntimeError(
+                    "--pypi-url is deprecated in favor of --pypi-url-metadata "
+                    + "and may not be passed in conjunction with --pypi-url-metadata"
+                )
+
+            # TODO: Remove before 3.0 release
+            if args.url_pypi_metadata_deprecated is not None:
+                logging.warning(
+                    "--pypi-url is deprecated; use --pypi-url-metadata instead"
+                )
+                args.url_pypi_metadata = args.url_pypi_metadata_deprecated
+
+            # If a PYPI mirror is selected, but the metadata URL is not
+            # explicitly passed, assume the mirror can handle the standard
+            # metadata endpoint and coerce the metadata URL appropriately in a
+            # way that respects the DEFAULT settings from config.
+            if (args.url_pypi_mirror.rstrip("/") != DEFAULT_PYPI_URL) and (
+                args.url_pypi_metadata.rstrip("/") == DEFAULT_PYPI_META_URL
+            ):
+                args.url_pypi_metadata = DEFAULT_PYPI_META_URL.replace(
+                    DEFAULT_PYPI_URL, args.url_pypi_mirror.rstrip("/")
+                )
+
             recipe, config = create_python_recipe(
                 pkg_name,
                 is_strict_cf=args.is_strict_conda_forge,
                 download=args.download,
-                url_pypi_metadata=args.url_pypi_metadata,
+                url_pypi=args.url_pypi_mirror.rstrip("/"),
+                url_pypi_metadata=args.url_pypi_metadata.rstrip("/"),
                 sections_populate=args.sections_populate,
                 from_local_sdist=from_local_sdist,
                 extras_require_test=args.extras_require_test,

diff --git a/grayskull/strategy/pypi.py b/grayskull/strategy/pypi.py
@@ -247,14 +247,14 @@ def get_pypi_metadata(config: Configuration) -> dict:
     """
     print_msg("Recovering metadata from pypi...")
     if config.version:
-        url_pypi = config.url_pypi_metadata.format(
+        url_pypi_metadata = config.url_pypi_metadata.format(
             pkg_name=f"{config.name}/{config.version}"
         )
     else:
         log.info(f"Version for {config.name} not specified.\nGetting the latest one.")
-        url_pypi = config.url_pypi_metadata.format(pkg_name=config.name)
+        url_pypi_metadata = config.url_pypi_metadata.format(pkg_name=config.name)
 
-    metadata = requests.get(url=url_pypi, timeout=5)
+    metadata = requests.get(url=url_pypi_metadata, timeout=5)
     if metadata.status_code != 200:
         raise requests.HTTPError(
             f"It was not possible to recover package metadata for {config.name}.\n"
@@ -288,7 +288,7 @@ def get_pypi_metadata(config: Configuration) -> dict:
         "url": info.get("home_page"),
         "license": info.get("license"),
         "source": {
-            "url": "https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/"
+            "url": config.url_pypi + "/packages/source/{{ name[0] }}/{{ name }}/"
             f"{get_url_filename(metadata)}",
             "sha256": get_sha256_from_pypi_metadata(metadata),
         },

diff --git a/tests/cli/test_cli_cmds.py b/tests/cli/test_cli_cmds.py
@@ -76,6 +76,7 @@ def test_change_pypi_url(mocker):
         "pytest=5.3.2",
         is_strict_cf=False,
         download=False,
+        url_pypi="https://pypi.org",
         url_pypi_metadata="http://url_pypi.com/abc",
         sections_populate=None,
         from_local_sdist=False,