Merge branch 'main' into fix-license-test

CODE_OF_CONDUCT.md

@@ -1,7 +1,7 @@
 # Conda Organization Code of Conduct
 
 > [!NOTE]
-> Below is the short version of our CoC, see the long version [here](https://github.com/conda-incubator/governance/blob/main/CODE_OF_CONDUCT.md).
+> Below is the short version of our CoC, see the long version [here](https://github.com/conda/governance/blob/main/CODE_OF_CONDUCT.md).
 
 # The Short Version
 

README.md

@@ -76,6 +76,27 @@ grayskull pypi ./pytest-5.3.5.tar.gz
 Note that such a recipe isn't really portable as it will depend on the local path of the
 sdist file. It can be useful if you want to automatically generate a conda package.
 
+### Use Grayskull with an internal package index
+
+Grayskull can create recipes that point to any Python Package Index. Supply the `--pypi-mirror-url` keyword.
+
+* Example:
+```bash
+grayskull pypi --pypi-mirror-url https://pypi.example.com pytest
+```
+
+The above will source packages from `https://pypi.example.com/packages/source/...`
+
+This assumes that the mirror follows the same API as pypi _including_ hosting metadata at the `/pypi/{package_name}/json` endpoint.
+To specify an alternate metadata location use the `--pypi-metadata-url` option.
+
+* Example:
+```bash
+grayskull pypi --pypi-mirror-url https://pypi.example.com --pypi-metadata-url https://pypi_meta.example.com pytest
+```
+
+> *Note:* `--pypi-metadata-url` is a replacement for `--pypi-url`; `--pypi-url` is deprecated and will be removed in a future release.
+
 ### Online Grayskull
 
 It is also possible to use Grayskull without any installation. You can go to this website [marcelotrevisani.com/grayskull](https://www.marcelotrevisani.com/grayskull) and inform the name and the version (optional) of the package and it will create the recipe for you.

grayskull/config.py

@@ -4,6 +4,9 @@
 from grayskull.cli.parser import parse_pkg_name_version
 from grayskull.utils import PyVer
 
+DEFAULT_PYPI_URL = "https://pypi.org"
+DEFAULT_PYPI_META_URL = "https://pypi.org/pypi"
+
 
 @dataclass
 class Configuration:
@@ -37,7 +40,8 @@ class Configuration:
         default_factory=lambda: ("cython", "cython-blis", "blis")
     )
     pkg_need_cxx_compiler: Tuple = field(default_factory=lambda: ("pybind11",))
-    url_pypi_metadata: str = "https://pypi.org/pypi/{pkg_name}/json"
+    url_pypi: str = DEFAULT_PYPI_URL
+    url_pypi_metadata: str = DEFAULT_PYPI_META_URL
     download: bool = False
     is_arch: bool = False
     repo_github: Optional[str] = None
@@ -103,11 +107,12 @@ def get_py_version_available(
         return py_ver_enabled
 
     def __post_init__(self):
+        if not self.url_pypi_metadata.endswith("/{pkg_name}/json"):
+            self.url_pypi_metadata = (
+                self.url_pypi_metadata.rstrip("/") + "/{pkg_name}/json"
+            )
         if self.from_local_sdist:
             self.local_sdist = self.local_sdist or self.name
-        if self.url_pypi_metadata != "https://pypi.org/pypi/{pkg_name}/json":
-            prefix = "" if self.url_pypi_metadata.endswith("/") else "/"
-            self.url_pypi_metadata += f"{prefix}{{pkg_name}}/json"
         pkg_repo, pkg_name, pkg_version = parse_pkg_name_version(self.name)
         if pkg_repo:
             prefix = "" if pkg_repo.endswith("/") else "/"

grayskull/main.py

@@ -13,7 +13,7 @@
 from grayskull.base.github import get_git_current_user
 from grayskull.cli import CLIConfig
 from grayskull.cli.stdout import print_msg
-from grayskull.config import Configuration
+from grayskull.config import DEFAULT_PYPI_META_URL, DEFAULT_PYPI_URL, Configuration
 from grayskull.utils import generate_recipe, origin_is_github, origin_is_local_sdist
 
 init(autoreset=True)
@@ -170,10 +170,26 @@ def init_parser():
         help="It will generate the recipes strict for the conda-forge channel.",
     )
     pypi_parser.add_argument(
-        "--pypi-url",
-        default="https://pypi.org/pypi/",
+        "--pypi-metadata-url",
+        default=DEFAULT_PYPI_META_URL,
         dest="url_pypi_metadata",
-        help="Pypi url server",
+        help=(
+            "Pypi url server metadata endpoint;"
+            + "will be appended with '{pkgname}/json'"
+        ),
+    )
+    pypi_parser.add_argument(
+        "--pypi-mirror-url",
+        default=DEFAULT_PYPI_URL,
+        dest="url_pypi_mirror",
+        help="Pypi mirror URL; assumed to have same API as pypi.org",
+    )
+    # TODO: Remove before 3.0 release
+    pypi_parser.add_argument(
+        "--pypi-url",
+        default=None,
+        dest="url_pypi_metadata_deprecated",
+        help="DEPRECATED: use --pypi-metadata-url instead",
     )
     pypi_parser.add_argument(
         "--recursive",
@@ -315,11 +331,37 @@ def generate_recipes_from_list(list_pkgs, args):
         if Path(pkg_name).is_file() and (not from_local_sdist):
             args.output = pkg_name
         try:
+            # TODO: Remove before 3.0 release
+            if args.url_pypi_metadata_deprecated and args.url_pypi_metadata:
+                raise RuntimeError(
+                    "--pypi-url is deprecated in favor of --pypi-url-metadata "
+                    + "and may not be passed in conjunction with --pypi-url-metadata"
+                )
+
+            # TODO: Remove before 3.0 release
+            if args.url_pypi_metadata_deprecated is not None:
+                logging.warning(
+                    "--pypi-url is deprecated; use --pypi-url-metadata instead"
+                )
+                args.url_pypi_metadata = args.url_pypi_metadata_deprecated
+
+            # If a PYPI mirror is selected, but the metadata URL is not
+            # explicitly passed, assume the mirror can handle the standard
+            # metadata endpoint and coerce the metadata URL appropriately in a
+            # way that respects the DEFAULT settings from config.
+            if (args.url_pypi_mirror.rstrip("/") != DEFAULT_PYPI_URL) and (
+                args.url_pypi_metadata.rstrip("/") == DEFAULT_PYPI_META_URL
+            ):
+                args.url_pypi_metadata = DEFAULT_PYPI_META_URL.replace(
+                    DEFAULT_PYPI_URL, args.url_pypi_mirror.rstrip("/")
+                )
+
             recipe, config = create_python_recipe(
                 pkg_name,
                 is_strict_cf=args.is_strict_conda_forge,
                 download=args.download,
-                url_pypi_metadata=args.url_pypi_metadata,
+                url_pypi=args.url_pypi_mirror.rstrip("/"),
+                url_pypi_metadata=args.url_pypi_metadata.rstrip("/"),
                 sections_populate=args.sections_populate,
                 from_local_sdist=from_local_sdist,
                 extras_require_test=args.extras_require_test,

grayskull/strategy/pypi.py

@@ -247,14 +247,14 @@ def get_pypi_metadata(config: Configuration) -> dict:
     """
     print_msg("Recovering metadata from pypi...")
     if config.version:
-        url_pypi = config.url_pypi_metadata.format(
+        url_pypi_metadata = config.url_pypi_metadata.format(
             pkg_name=f"{config.name}/{config.version}"
         )
     else:
         log.info(f"Version for {config.name} not specified.\nGetting the latest one.")
-        url_pypi = config.url_pypi_metadata.format(pkg_name=config.name)
+        url_pypi_metadata = config.url_pypi_metadata.format(pkg_name=config.name)
 
-    metadata = requests.get(url=url_pypi, timeout=5)
+    metadata = requests.get(url=url_pypi_metadata, timeout=5)
     if metadata.status_code != 200:
         raise requests.HTTPError(
             f"It was not possible to recover package metadata for {config.name}.\n"
@@ -288,7 +288,7 @@ def get_pypi_metadata(config: Configuration) -> dict:
         "url": info.get("home_page"),
         "license": info.get("license"),
         "source": {
-            "url": "https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/"
+            "url": config.url_pypi + "/packages/source/{{ name[0] }}/{{ name }}/"
             f"{get_url_filename(metadata)}",
             "sha256": get_sha256_from_pypi_metadata(metadata),
         },
@@ -347,9 +347,14 @@ def get_metadata(recipe, config) -> dict:
     """Method responsible to get the whole metadata available. It will
     merge metadata from multiple sources (pypi, setup.py, setup.cfg)
     """
-    name = config.name
     sdist_metadata, pypi_metadata = get_origin_wise_metadata(config)
     metadata = merge_pypi_sdist_metadata(pypi_metadata, sdist_metadata, config)
+    if config.from_local_sdist:
+        # Overwrite package name from sdist filename with name from metadata
+        # sdist filename is normalized by setuptools since version 69.3.0
+        # See https://github.com/pypa/setuptools/issues/3593
+        config.name = metadata["name"]
+    name = config.name
     log.debug(f"Data merged from pypi, setup.cfg and setup.py: {metadata}")
     if metadata.get("scripts") is not None:
         config.is_arch = True
@@ -505,6 +510,9 @@ def update_recipe(recipe: Recipe, config: Configuration, all_sections: List[str]
             if section == "package":
                 package_metadata = dict(metadata[section])
                 if package_metadata["name"].lower() == config.name.lower():
+                    if config.from_local_sdist:
+                        # Initial name set in the recipe came from the sdist filename
+                        set_global_jinja_var(recipe, "name", package_metadata["name"])
                     package_metadata.pop("name")
                 else:
                     package_metadata["name"] = package_metadata["name"].replace(

pyproject.toml

@@ -26,7 +26,7 @@ dependencies = [
     "ruamel.yaml >=0.16.10",
     "ruamel.yaml.jinja2",
     "setuptools >=30.3.0",
-    "semver~=3.0.0",
+    "semver >=3.0.0,==3.0.*",
     "stdlib-list",
     "tomli",
     "tomli-w",

tests/cli/test_cli_cmds.py

@@ -76,6 +76,7 @@ def test_change_pypi_url(mocker):
         "pytest=5.3.2",
         is_strict_cf=False,
         download=False,
+        url_pypi="https://pypi.org",
         url_pypi_metadata="http://url_pypi.com/abc",
         sections_populate=None,
         from_local_sdist=False,

tests/conftest.py

@@ -15,7 +15,9 @@ def data_dir() -> str:
 @fixture(scope="session")
 def pkg_pytest(tmpdir_factory) -> str:
     folder = tmpdir_factory.mktemp("test-download-pkg")
-    dest_pkg = str(folder / "PYTEST-PKG.tar.gz")
+    # Use different package name and version for the sdist archive on purpose
+    # Correct info should be extracted from the metadata and not filename
+    dest_pkg = str(folder / "PYTEST-PKG-1.0.0.tar.gz")
     download_sdist_pkg(
         "https://pypi.io/packages/source/p/pytest/pytest-5.3.5.tar.gz", dest_pkg
     )

tests/test_pypi.py

@@ -6,6 +6,7 @@
 
 import pytest
 from colorama import Fore, Style
+from souschef.jinja_expression import get_global_jinja_var
 from souschef.recipe import Recipe
 
 from grayskull.base.factory import GrayskullFactory
@@ -1257,6 +1258,8 @@ def test_create_recipe_from_local_sdist(pkg_pytest):
     assert recipe["about"]["summary"] == "pytest: simple powerful testing with Python"
     assert recipe["about"]["license"] == "MIT"
     assert recipe["about"]["license_file"] == "LICENSE"
+    assert get_global_jinja_var(recipe, "name") == "pytest"
+    assert get_global_jinja_var(recipe, "version") == "5.3.5"
 
 
 @patch("grayskull.strategy.py_base.get_all_toml_info", return_value={})