From fa0b1a3a5c3d81c6256448b4a3335d2d521ebdbf Mon Sep 17 00:00:00 2001 From: Benjamin Bertrand Date: Sun, 4 Aug 2024 12:31:49 +0200 Subject: [PATCH 1/4] Fix package name based on local sdist archive (#536) * Add test on local sdist package name * Fix package name based on local sdist archive When using a local sdist as source, the package name of the conda recipe was based on the sdist filename. This used to be the same but since setuptools 69.3.0, the sdist filename is now normalized. For example, if a package name contains a dash, it's converted to underscore. The package name should be extracted from the metadata and not the sdist filename. --- grayskull/strategy/pypi.py | 10 +++++++++- tests/conftest.py | 4 +++- tests/test_pypi.py | 3 +++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/grayskull/strategy/pypi.py b/grayskull/strategy/pypi.py index 1bed893b7..8fb43d114 100644 --- a/grayskull/strategy/pypi.py +++ b/grayskull/strategy/pypi.py @@ -347,9 +347,14 @@ def get_metadata(recipe, config) -> dict: """Method responsible to get the whole metadata available. It will merge metadata from multiple sources (pypi, setup.py, setup.cfg) """ - name = config.name sdist_metadata, pypi_metadata = get_origin_wise_metadata(config) metadata = merge_pypi_sdist_metadata(pypi_metadata, sdist_metadata, config) + if config.from_local_sdist: + # Overwrite package name from sdist filename with name from metadata + # sdist filename is normalized by setuptools since version 69.3.0 + # See https://github.com/pypa/setuptools/issues/3593 + config.name = metadata["name"] + name = config.name log.debug(f"Data merged from pypi, setup.cfg and setup.py: {metadata}") if metadata.get("scripts") is not None: config.is_arch = True @@ -505,6 +510,9 @@ def update_recipe(recipe: Recipe, config: Configuration, all_sections: List[str] if section == "package": package_metadata = dict(metadata[section]) if package_metadata["name"].lower() == config.name.lower(): + if config.from_local_sdist: + # Initial name set in the recipe came from the sdist filename + set_global_jinja_var(recipe, "name", package_metadata["name"]) package_metadata.pop("name") else: package_metadata["name"] = package_metadata["name"].replace( diff --git a/tests/conftest.py b/tests/conftest.py index 9556de8fc..429f65043 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,7 +15,9 @@ def data_dir() -> str: @fixture(scope="session") def pkg_pytest(tmpdir_factory) -> str: folder = tmpdir_factory.mktemp("test-download-pkg") - dest_pkg = str(folder / "PYTEST-PKG.tar.gz") + # Use different package name and version for the sdist archive on purpose + # Correct info should be extracted from the metadata and not filename + dest_pkg = str(folder / "PYTEST-PKG-1.0.0.tar.gz") download_sdist_pkg( "https://pypi.io/packages/source/p/pytest/pytest-5.3.5.tar.gz", dest_pkg ) diff --git a/tests/test_pypi.py b/tests/test_pypi.py index 1564ee152..2b04ce725 100644 --- a/tests/test_pypi.py +++ b/tests/test_pypi.py @@ -6,6 +6,7 @@ import pytest from colorama import Fore, Style +from souschef.jinja_expression import get_global_jinja_var from souschef.recipe import Recipe from grayskull.base.factory import GrayskullFactory @@ -1257,6 +1258,8 @@ def test_create_recipe_from_local_sdist(pkg_pytest): assert recipe["about"]["summary"] == "pytest: simple powerful testing with Python" assert recipe["about"]["license"] == "MIT" assert recipe["about"]["license_file"] == "LICENSE" + assert get_global_jinja_var(recipe, "name") == "pytest" + assert get_global_jinja_var(recipe, "version") == "5.3.5" @patch("grayskull.strategy.py_base.get_all_toml_info", return_value={}) From ede342b276a69585c029f584bd4b4cf718a7c2b7 Mon Sep 17 00:00:00 2001 From: Faustin Carter Date: Sun, 4 Aug 2024 03:33:53 -0700 Subject: [PATCH 2/4] Add option to specify a package server mirror; clean up hardcoded URLs throughout codebase (#540) * Add in new --pypi-mirror-url and --pypi-metadata-url options and deprecate --pypi-url * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Line length --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Marcelo Duarte Trevisani --- README.md | 21 +++++++++++++++ grayskull/config.py | 13 +++++++--- grayskull/main.py | 52 ++++++++++++++++++++++++++++++++++---- grayskull/strategy/pypi.py | 8 +++--- tests/cli/test_cli_cmds.py | 1 + 5 files changed, 82 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 16122e991..bdd0f2367 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,27 @@ grayskull pypi ./pytest-5.3.5.tar.gz Note that such a recipe isn't really portable as it will depend on the local path of the sdist file. It can be useful if you want to automatically generate a conda package. +### Use Grayskull with an internal package index + +Grayskull can create recipes that point to any Python Package Index. Supply the `--pypi-mirror-url` keyword. + +* Example: +```bash +grayskull pypi --pypi-mirror-url https://pypi.example.com pytest +``` + +The above will source packages from `https://pypi.example.com/packages/source/...` + +This assumes that the mirror follows the same API as pypi _including_ hosting metadata at the `/pypi/{package_name}/json` endpoint. +To specify an alternate metadata location use the `--pypi-metadata-url` option. + +* Example: +```bash +grayskull pypi --pypi-mirror-url https://pypi.example.com --pypi-metadata-url https://pypi_meta.example.com pytest +``` + +> *Note:* `--pypi-metadata-url` is a replacement for `--pypi-url`; `--pypi-url` is deprecated and will be removed in a future release. + ### Online Grayskull It is also possible to use Grayskull without any installation. You can go to this website [marcelotrevisani.com/grayskull](https://www.marcelotrevisani.com/grayskull) and inform the name and the version (optional) of the package and it will create the recipe for you. diff --git a/grayskull/config.py b/grayskull/config.py index 544095028..e9df19ca3 100644 --- a/grayskull/config.py +++ b/grayskull/config.py @@ -4,6 +4,9 @@ from grayskull.cli.parser import parse_pkg_name_version from grayskull.utils import PyVer +DEFAULT_PYPI_URL = "https://pypi.org" +DEFAULT_PYPI_META_URL = "https://pypi.org/pypi" + @dataclass class Configuration: @@ -37,7 +40,8 @@ class Configuration: default_factory=lambda: ("cython", "cython-blis", "blis") ) pkg_need_cxx_compiler: Tuple = field(default_factory=lambda: ("pybind11",)) - url_pypi_metadata: str = "https://pypi.org/pypi/{pkg_name}/json" + url_pypi: str = DEFAULT_PYPI_URL + url_pypi_metadata: str = DEFAULT_PYPI_META_URL download: bool = False is_arch: bool = False repo_github: Optional[str] = None @@ -103,11 +107,12 @@ def get_py_version_available( return py_ver_enabled def __post_init__(self): + if not self.url_pypi_metadata.endswith("/{pkg_name}/json"): + self.url_pypi_metadata = ( + self.url_pypi_metadata.rstrip("/") + "/{pkg_name}/json" + ) if self.from_local_sdist: self.local_sdist = self.local_sdist or self.name - if self.url_pypi_metadata != "https://pypi.org/pypi/{pkg_name}/json": - prefix = "" if self.url_pypi_metadata.endswith("/") else "/" - self.url_pypi_metadata += f"{prefix}{{pkg_name}}/json" pkg_repo, pkg_name, pkg_version = parse_pkg_name_version(self.name) if pkg_repo: prefix = "" if pkg_repo.endswith("/") else "/" diff --git a/grayskull/main.py b/grayskull/main.py index bcf77459f..25c83cdb3 100644 --- a/grayskull/main.py +++ b/grayskull/main.py @@ -13,7 +13,7 @@ from grayskull.base.github import get_git_current_user from grayskull.cli import CLIConfig from grayskull.cli.stdout import print_msg -from grayskull.config import Configuration +from grayskull.config import DEFAULT_PYPI_META_URL, DEFAULT_PYPI_URL, Configuration from grayskull.utils import generate_recipe, origin_is_github, origin_is_local_sdist init(autoreset=True) @@ -170,10 +170,26 @@ def init_parser(): help="It will generate the recipes strict for the conda-forge channel.", ) pypi_parser.add_argument( - "--pypi-url", - default="https://pypi.org/pypi/", + "--pypi-metadata-url", + default=DEFAULT_PYPI_META_URL, dest="url_pypi_metadata", - help="Pypi url server", + help=( + "Pypi url server metadata endpoint;" + + "will be appended with '{pkgname}/json'" + ), + ) + pypi_parser.add_argument( + "--pypi-mirror-url", + default=DEFAULT_PYPI_URL, + dest="url_pypi_mirror", + help="Pypi mirror URL; assumed to have same API as pypi.org", + ) + # TODO: Remove before 3.0 release + pypi_parser.add_argument( + "--pypi-url", + default=None, + dest="url_pypi_metadata_deprecated", + help="DEPRECATED: use --pypi-metadata-url instead", ) pypi_parser.add_argument( "--recursive", @@ -315,11 +331,37 @@ def generate_recipes_from_list(list_pkgs, args): if Path(pkg_name).is_file() and (not from_local_sdist): args.output = pkg_name try: + # TODO: Remove before 3.0 release + if args.url_pypi_metadata_deprecated and args.url_pypi_metadata: + raise RuntimeError( + "--pypi-url is deprecated in favor of --pypi-url-metadata " + + "and may not be passed in conjunction with --pypi-url-metadata" + ) + + # TODO: Remove before 3.0 release + if args.url_pypi_metadata_deprecated is not None: + logging.warning( + "--pypi-url is deprecated; use --pypi-url-metadata instead" + ) + args.url_pypi_metadata = args.url_pypi_metadata_deprecated + + # If a PYPI mirror is selected, but the metadata URL is not + # explicitly passed, assume the mirror can handle the standard + # metadata endpoint and coerce the metadata URL appropriately in a + # way that respects the DEFAULT settings from config. + if (args.url_pypi_mirror.rstrip("/") != DEFAULT_PYPI_URL) and ( + args.url_pypi_metadata.rstrip("/") == DEFAULT_PYPI_META_URL + ): + args.url_pypi_metadata = DEFAULT_PYPI_META_URL.replace( + DEFAULT_PYPI_URL, args.url_pypi_mirror.rstrip("/") + ) + recipe, config = create_python_recipe( pkg_name, is_strict_cf=args.is_strict_conda_forge, download=args.download, - url_pypi_metadata=args.url_pypi_metadata, + url_pypi=args.url_pypi_mirror.rstrip("/"), + url_pypi_metadata=args.url_pypi_metadata.rstrip("/"), sections_populate=args.sections_populate, from_local_sdist=from_local_sdist, extras_require_test=args.extras_require_test, diff --git a/grayskull/strategy/pypi.py b/grayskull/strategy/pypi.py index 8fb43d114..ff60f2596 100644 --- a/grayskull/strategy/pypi.py +++ b/grayskull/strategy/pypi.py @@ -247,14 +247,14 @@ def get_pypi_metadata(config: Configuration) -> dict: """ print_msg("Recovering metadata from pypi...") if config.version: - url_pypi = config.url_pypi_metadata.format( + url_pypi_metadata = config.url_pypi_metadata.format( pkg_name=f"{config.name}/{config.version}" ) else: log.info(f"Version for {config.name} not specified.\nGetting the latest one.") - url_pypi = config.url_pypi_metadata.format(pkg_name=config.name) + url_pypi_metadata = config.url_pypi_metadata.format(pkg_name=config.name) - metadata = requests.get(url=url_pypi, timeout=5) + metadata = requests.get(url=url_pypi_metadata, timeout=5) if metadata.status_code != 200: raise requests.HTTPError( f"It was not possible to recover package metadata for {config.name}.\n" @@ -288,7 +288,7 @@ def get_pypi_metadata(config: Configuration) -> dict: "url": info.get("home_page"), "license": info.get("license"), "source": { - "url": "https://pypi.io/packages/source/{{ name[0] }}/{{ name }}/" + "url": config.url_pypi + "/packages/source/{{ name[0] }}/{{ name }}/" f"{get_url_filename(metadata)}", "sha256": get_sha256_from_pypi_metadata(metadata), }, diff --git a/tests/cli/test_cli_cmds.py b/tests/cli/test_cli_cmds.py index 009c64af0..49d473899 100644 --- a/tests/cli/test_cli_cmds.py +++ b/tests/cli/test_cli_cmds.py @@ -76,6 +76,7 @@ def test_change_pypi_url(mocker): "pytest=5.3.2", is_strict_cf=False, download=False, + url_pypi="https://pypi.org", url_pypi_metadata="http://url_pypi.com/abc", sections_populate=None, from_local_sdist=False, From 894b9c855b6ef4f64a02644845ad66d430f3e707 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Sun, 4 Aug 2024 12:34:29 +0200 Subject: [PATCH 3/4] Expand out the "compatible release" operator for semver dep (#544) Co-authored-by: Marcelo Duarte Trevisani --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 85c87b4e0..645bc6e40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "ruamel.yaml >=0.16.10", "ruamel.yaml.jinja2", "setuptools >=30.3.0", - "semver~=3.0.0", + "semver >=3.0.0,==3.0.*", "stdlib-list", "tomli", "tomli-w", From e42c36b521d72156869d233ea74c6ac735a6349f Mon Sep 17 00:00:00 2001 From: conda-bot <18747875+conda-bot@users.noreply.github.com> Date: Sun, 4 Aug 2024 05:34:45 -0500 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=94=84=20synced=20file(s)=20with=20co?= =?UTF-8?q?nda/infrastructure=20(#546)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Conda Bot --- CODE_OF_CONDUCT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 663464fe8..320cddbbd 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,7 +1,7 @@ # Conda Organization Code of Conduct > [!NOTE] -> Below is the short version of our CoC, see the long version [here](https://github.com/conda-incubator/governance/blob/main/CODE_OF_CONDUCT.md). +> Below is the short version of our CoC, see the long version [here](https://github.com/conda/governance/blob/main/CODE_OF_CONDUCT.md). # The Short Version