From 938888999593c54ef439da9f0df6f6bbb029522b Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 09:36:34 +0200 Subject: [PATCH 01/15] Fix type hints --- grayskull/strategy/py_toml.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/py_toml.py index 8edbaa25d..2f54da9ab 100644 --- a/grayskull/strategy/py_toml.py +++ b/grayskull/strategy/py_toml.py @@ -31,7 +31,7 @@ class InvalidPoetryDependency(BaseException): pass -def parse_version(version: str) -> Dict[str, Optional[str]]: +def parse_version(version: str) -> Dict[str, Optional[int]]: """ Parses a version string (not necessarily semver) to a dictionary with keys "major", "minor", and "patch". "minor" and "patch" are possibly None. @@ -46,7 +46,7 @@ def parse_version(version: str) -> Dict[str, Optional[str]]: } -def vdict_to_vinfo(version_dict: Dict[str, Optional[str]]) -> semver.VersionInfo: +def vdict_to_vinfo(version_dict: Dict[str, Optional[int]]) -> semver.VersionInfo: """ Coerces version dictionary to a semver.VersionInfo object. If minor or patch numbers are missing, 0 is substituted in their place. @@ -154,7 +154,7 @@ def encode_poetry_version(poetry_specifier: str) -> str: @singledispatch -def get_constrained_dep(dep_spec, dep_name): +def get_constrained_dep(dep_spec: Union[str, dict], dep_name: str) -> str: raise InvalidPoetryDependency( "Expected Poetry dependency specification to be of type str or dict, " f"received {type(dep_spec).__name__}" @@ -162,13 +162,13 @@ def get_constrained_dep(dep_spec, dep_name): @get_constrained_dep.register -def __get_constrained_dep_dict(dep_spec: dict, dep_name: str): +def __get_constrained_dep_dict(dep_spec: dict, dep_name: str) -> str: conda_version = encode_poetry_version(dep_spec.get("version", "")) return f"{dep_name} {conda_version}".strip() @get_constrained_dep.register -def __get_constrained_dep_str(dep_spec: str, dep_name: str): +def __get_constrained_dep_str(dep_spec: str, dep_name: str) -> str: conda_version = encode_poetry_version(dep_spec) return f"{dep_name} {conda_version}" From 513aec089eb3c94c956a9910e54bd6f1425488b8 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 09:37:37 +0200 Subject: [PATCH 02/15] Split nested function calls in coerce_to_semver --- grayskull/strategy/py_toml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/py_toml.py index 2f54da9ab..413584d1f 100644 --- a/grayskull/strategy/py_toml.py +++ b/grayskull/strategy/py_toml.py @@ -62,7 +62,9 @@ def coerce_to_semver(version: str) -> str: if semver.VersionInfo.is_valid(version): return version - return str(vdict_to_vinfo(parse_version(version))) + parsed_version = parse_version(version) + vinfo = vdict_to_vinfo(parsed_version) + return str(vinfo) def get_caret_ceiling(target: str) -> str: From 6fb7fa21e9c3b4231d0bf95b850ac890ba3755e9 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 10:29:32 +0200 Subject: [PATCH 03/15] Add more tests to fail on invalid semver --- tests/test_py_toml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_py_toml.py b/tests/test_py_toml.py index 498fcbb6c..ceb198786 100644 --- a/tests/test_py_toml.py +++ b/tests/test_py_toml.py @@ -40,7 +40,9 @@ def test_parse_version_success(version, major, minor, patch): assert parse_version(version) == {"major": major, "minor": minor, "patch": patch} -@pytest.mark.parametrize("invalid_version", ["asdf", "", "."]) +@pytest.mark.parametrize( + "invalid_version", ["asdf", "", ".", "x.2.3", "1.x.3", "1.2.x"] +) def test_parse_version_failure(invalid_version): with pytest.raises(InvalidVersion): parse_version(invalid_version) From 780dcf495f78ec6a939594c0c2ee8dd26fe1cbaf Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 09:47:57 +0200 Subject: [PATCH 04/15] Fix semver regex Ensure that the regex applies to the entire string. --- grayskull/strategy/py_toml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/py_toml.py index 413584d1f..674c85d67 100644 --- a/grayskull/strategy/py_toml.py +++ b/grayskull/strategy/py_toml.py @@ -10,14 +10,14 @@ from grayskull.utils import nested_dict VERSION_REGEX = re.compile( - r"""[vV]? + r"""^[vV]? (?P0|[1-9]\d*) (\. (?P0|[1-9]\d*) (\. (?P0|[1-9]\d*) )? - )? + )?$ """, re.VERBOSE, ) From 17049572fe279e08795e486d528969e12853eafd Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 11:39:07 +0200 Subject: [PATCH 05/15] Split history py_toml.py to parse_poetry_version.py - rename file to target-name --- grayskull/strategy/{py_toml.py => parse_poetry_version.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename grayskull/strategy/{py_toml.py => parse_poetry_version.py} (100%) diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/parse_poetry_version.py similarity index 100% rename from grayskull/strategy/py_toml.py rename to grayskull/strategy/parse_poetry_version.py From 3aab8ffea20a029bce8312e7501fba7ec9eb9b0a Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 11:39:07 +0200 Subject: [PATCH 06/15] Split history py_toml.py to parse_poetry_version.py - rename source-file to temp --- grayskull/strategy/{py_toml.py => temp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename grayskull/strategy/{py_toml.py => temp} (100%) diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/temp similarity index 100% rename from grayskull/strategy/py_toml.py rename to grayskull/strategy/temp From 111e39625a88f523367fb720ae29f38303756b88 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 11:39:08 +0200 Subject: [PATCH 07/15] Split history py_toml.py to parse_poetry_version.py - restore name of source-file --- grayskull/strategy/{temp => py_toml.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename grayskull/strategy/{temp => py_toml.py} (100%) diff --git a/grayskull/strategy/temp b/grayskull/strategy/py_toml.py similarity index 100% rename from grayskull/strategy/temp rename to grayskull/strategy/py_toml.py From 539e0f77c48f590e8aa386d959d2b709ec6d88d7 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 11:40:25 +0200 Subject: [PATCH 08/15] Complete the splitoff into parse_poetry_version --- grayskull/strategy/parse_poetry_version.py | 251 +-------------------- grayskull/strategy/py_toml.py | 146 +----------- tests/test_py_toml.py | 12 +- 3 files changed, 10 insertions(+), 399 deletions(-) diff --git a/grayskull/strategy/parse_poetry_version.py b/grayskull/strategy/parse_poetry_version.py index 674c85d67..0c44dbfaf 100644 --- a/grayskull/strategy/parse_poetry_version.py +++ b/grayskull/strategy/parse_poetry_version.py @@ -1,13 +1,7 @@ import re -from collections import defaultdict -from functools import singledispatch -from pathlib import Path -from typing import Dict, Optional, Tuple, Union +from typing import Dict, Optional import semver -import tomli - -from grayskull.utils import nested_dict VERSION_REGEX = re.compile( r"""^[vV]? @@ -27,10 +21,6 @@ class InvalidVersion(BaseException): pass -class InvalidPoetryDependency(BaseException): - pass - - def parse_version(version: str) -> Dict[str, Optional[int]]: """ Parses a version string (not necessarily semver) to a dictionary with keys @@ -153,242 +143,3 @@ def encode_poetry_version(poetry_specifier: str) -> str: conda_clauses.append(poetry_clause) return ",".join(conda_clauses) - - -@singledispatch -def get_constrained_dep(dep_spec: Union[str, dict], dep_name: str) -> str: - raise InvalidPoetryDependency( - "Expected Poetry dependency specification to be of type str or dict, " - f"received {type(dep_spec).__name__}" - ) - - -@get_constrained_dep.register -def __get_constrained_dep_dict(dep_spec: dict, dep_name: str) -> str: - conda_version = encode_poetry_version(dep_spec.get("version", "")) - return f"{dep_name} {conda_version}".strip() - - -@get_constrained_dep.register -def __get_constrained_dep_str(dep_spec: str, dep_name: str) -> str: - conda_version = encode_poetry_version(dep_spec) - return f"{dep_name} {conda_version}" - - -def encode_poetry_deps(poetry_deps: dict) -> Tuple[list, list]: - run = [] - run_constrained = [] - for dep_name, dep_spec in poetry_deps.items(): - constrained_dep = get_constrained_dep(dep_spec, dep_name) - try: - assert dep_spec.get("optional", False) - run_constrained.append(constrained_dep) - except (AttributeError, AssertionError): - run.append(constrained_dep) - return run, run_constrained - - -def add_poetry_metadata(metadata: dict, toml_metadata: dict) -> dict: - if not is_poetry_present(toml_metadata): - return metadata - - poetry_metadata = toml_metadata["tool"]["poetry"] - poetry_deps = poetry_metadata.get("dependencies", {}) - req_run, req_run_constrained = encode_poetry_deps(poetry_deps) - - # add dependencies - metadata["requirements"].setdefault("run", []) - metadata["requirements"]["run"].extend(req_run) - - # add optional dependencies - if len(req_run_constrained): - metadata["requirements"].setdefault("run_constrained", []) - metadata["requirements"]["run_constrained"].extend(req_run_constrained) - - host_metadata = metadata["requirements"].get("host", []) - if toml_metadata["tool"].get("poetry", {}).get("scripts"): - metadata["build"]["entry_points"] = [] - for entry_name, entry_path in toml_metadata["tool"]["poetry"][ - "scripts" - ].items(): - metadata["build"]["entry_points"].append(f"{entry_name} = {entry_path}") - if "poetry" not in host_metadata and "poetry-core" not in host_metadata: - metadata["requirements"]["host"] = host_metadata + ["poetry-core"] - - poetry_test_deps = ( - poetry_metadata.get("group", {}).get("test", {}).get("dependencies", {}) - ) - # add required test dependencies and ignore optional test dependencies, as - # there doesn't appear to be a way to specify them in Conda recipe metadata. - test_reqs, _ = encode_poetry_deps(poetry_test_deps) - metadata["test"].get("requires", []).extend(test_reqs) - return metadata - - -def is_poetry_present(toml_metadata: dict) -> bool: - return "poetry" in toml_metadata.get("tool", {}) - - -def is_flit_present(toml_metadata: dict) -> bool: - return "flit" in toml_metadata.get("tool", {}) - - -def add_flit_metadata(metadata: dict, toml_metadata: dict) -> dict: - if not is_flit_present(toml_metadata): - return metadata - - flit_metadata = toml_metadata["tool"]["flit"] - flit_scripts = flit_metadata.get("scripts", {}) - for entry_name, entry_path in flit_scripts.items(): - if "build" not in metadata: - metadata["build"] = {} - if "entry_points" not in metadata["build"]: - metadata["build"]["entry_points"] = [] - metadata["build"]["entry_points"].append(f"{entry_name} = {entry_path}") - return metadata - - -def is_pep725_present(toml_metadata: dict): - return "external" in toml_metadata - - -def get_pep725_mapping(purl: str): - """This function maps a PURL to the name in the conda ecosystem. It is expected - that this will be provided on a per-ecosystem basis (such as by conda-forge)""" - - package_mapping = { - "virtual:compiler/c": "{{ compiler('c') }}", - "virtual:compiler/cpp": "{{ compiler('cxx') }}", - "virtual:compiler/fortran": "{{ compiler('fortran') }}", - "virtual:compiler/rust": "{{ compiler('rust') }}", - "virtual:interface/blas": "{{ blas }}", - "pkg:generic/boost": "boost-cpp", - "pkg:generic/brial": "brial", - "pkg:generic/cddlib": "cddlib", - "pkg:generic/cliquer": "cliquer", - "pkg:generic/ecl": "ecl", - "pkg:generic/eclib": "eclib", - "pkg:generic/ecm": "ecm", - "pkg:generic/fflas-ffpack": "fflas-ffpack", - "pkg:generic/fplll": "fplll", - "pkg:generic/flint": "libflint", - "pkg:generic/libgd": "libgd", - "pkg:generic/gap": "gap-defaults", - "pkg:generic/gfan": "gfan", - "pkg:generic/gmp": "gmp", - "pkg:generic/giac": "giac", - "pkg:generic/givaro": "givaro", - "pkg:generic/pkg-config": "pkg-config", - "pkg:generic/glpk": "glpk", - "pkg:generic/gsl": "gsl", - "pkg:generic/iml": "iml", - "pkg:generic/lcalc": "lcalc", - "pkg:generic/libbraiding": "libbraiding", - "pkg:generic/libhomfly": "libhomfly", - "pkg:generic/lrcalc": "lrcalc", - "pkg:generic/libpng": "libpng", - "pkg:generic/linbox": "linbox", - "pkg:generic/m4ri": "m4ri", - "pkg:generic/m4rie": "m4rie", - "pkg:generic/mpc": "mpc", - "pkg:generic/mpfi": "mpfi", - "pkg:generic/mpfr": "mpfr", - "pkg:generic/maxima": "maxima", - "pkg:generic/nauty": "nauty", - "pkg:generic/ntl": "ntl", - "pkg:generic/pari": "pari", - "pkg:generic/pari-elldata": "pari-elldata", - "pkg:generic/pari-galdata": "pari-galdata", - "pkg:generic/pari-seadata": "pari-seadata", - "pkg:generic/palp": "palp", - "pkg:generic/planarity": "planarity", - "pkg:generic/ppl": "ppl", - "pkg:generic/primesieve": "primesieve", - "pkg:generic/primecount": "primecount", - "pkg:generic/qhull": "qhull", - "pkg:generic/rw": "rw", - "pkg:generic/singular": "singular", - "pkg:generic/symmetrica": "symmetrica", - "pkg:generic/sympow": "sympow", - } - return package_mapping.get(purl, purl) - - -def add_pep725_metadata(metadata: dict, toml_metadata: dict): - if not is_pep725_present(toml_metadata): - return metadata - - externals = toml_metadata["external"] - # each of these is a list of PURLs. For each one we find, - # we need to map it to the the conda ecosystem - requirements = metadata.get("requirements", {}) - section_map = ( - ("build", "build-requires"), - ("host", "host-requires"), - ("run", "dependencies"), - ) - for conda_section, pep725_section in section_map: - requirements.setdefault(conda_section, []) - requirements[conda_section].extend( - [get_pep725_mapping(purl) for purl in externals.get(pep725_section, [])] - ) - # TODO: handle optional dependencies properly - optional_features = toml_metadata.get(f"optional-{pep725_section}", {}) - for feature_name, feature_deps in optional_features.items(): - requirements[conda_section].append( - f'# OPTIONAL dependencies from feature "{feature_name}"' - ) - requirements[conda_section].extend(feature_deps) - if not requirements[conda_section]: - del requirements[conda_section] - - if requirements: - metadata["requirements"] = requirements - return metadata - - -def get_all_toml_info(path_toml: Union[Path, str]) -> dict: - with open(path_toml, "rb") as f: - toml_metadata = tomli.load(f) - toml_metadata = defaultdict(dict, toml_metadata) - metadata = nested_dict() - toml_project = toml_metadata.get("project", {}) or {} - metadata["requirements"]["host"] = toml_metadata["build-system"].get("requires", []) - metadata["requirements"]["run"] = toml_project.get("dependencies", []) - license = toml_project.get("license") - if isinstance(license, dict): - license = license.get("text", "") - metadata["about"]["license"] = license - optional_deps = toml_project.get("optional-dependencies", {}) - metadata["test"]["requires"] = ( - optional_deps.get("testing", []) - or optional_deps.get("test", []) - or optional_deps.get("tests", []) - ) - - tom_urls = toml_project.get("urls", {}) - if homepage := tom_urls.get("Homepage"): - metadata["about"]["home"] = homepage - if dev_url := tom_urls.get("Source"): - metadata["about"]["dev_url"] = dev_url - - if toml_project.get("requires-python"): - py_constrain = f"python {toml_project['requires-python']}" - metadata["requirements"]["host"].append(py_constrain) - metadata["requirements"]["run"].append(py_constrain) - - if toml_project.get("scripts"): - metadata["build"]["entry_points"] = [] - for entry_name, entry_path in toml_project.get("scripts", {}).items(): - metadata["build"]["entry_points"].append(f"{entry_name} = {entry_path}") - if all_urls := toml_project.get("urls"): - metadata["about"]["dev_url"] = all_urls.get("Source", None) - metadata["about"]["home"] = all_urls.get("Homepage", None) - metadata["about"]["summary"] = toml_project.get("description") - metadata["name"] = metadata.get("name") or toml_project.get("name") - - add_poetry_metadata(metadata, toml_metadata) - add_flit_metadata(metadata, toml_metadata) - add_pep725_metadata(metadata, toml_metadata) - - return metadata diff --git a/grayskull/strategy/py_toml.py b/grayskull/strategy/py_toml.py index 674c85d67..8681dd203 100644 --- a/grayskull/strategy/py_toml.py +++ b/grayskull/strategy/py_toml.py @@ -1,160 +1,18 @@ -import re from collections import defaultdict from functools import singledispatch from pathlib import Path -from typing import Dict, Optional, Tuple, Union +from typing import Tuple, Union -import semver import tomli +from grayskull.strategy.parse_poetry_version import encode_poetry_version from grayskull.utils import nested_dict -VERSION_REGEX = re.compile( - r"""^[vV]? - (?P0|[1-9]\d*) - (\. - (?P0|[1-9]\d*) - (\. - (?P0|[1-9]\d*) - )? - )?$ - """, - re.VERBOSE, -) - - -class InvalidVersion(BaseException): - pass - class InvalidPoetryDependency(BaseException): pass -def parse_version(version: str) -> Dict[str, Optional[int]]: - """ - Parses a version string (not necessarily semver) to a dictionary with keys - "major", "minor", and "patch". "minor" and "patch" are possibly None. - """ - match = VERSION_REGEX.search(version) - if not match: - raise InvalidVersion(f"Could not parse version {version}.") - - return { - key: None if value is None else int(value) - for key, value in match.groupdict().items() - } - - -def vdict_to_vinfo(version_dict: Dict[str, Optional[int]]) -> semver.VersionInfo: - """ - Coerces version dictionary to a semver.VersionInfo object. If minor or patch - numbers are missing, 0 is substituted in their place. - """ - ver = {key: 0 if value is None else value for key, value in version_dict.items()} - return semver.VersionInfo(**ver) - - -def coerce_to_semver(version: str) -> str: - """ - Coerces a version string to a semantic version. - """ - if semver.VersionInfo.is_valid(version): - return version - - parsed_version = parse_version(version) - vinfo = vdict_to_vinfo(parsed_version) - return str(vinfo) - - -def get_caret_ceiling(target: str) -> str: - """ - Accepts a Poetry caret target and returns the exclusive version ceiling. - - Targets that are invalid semver strings (e.g. "1.2", "0") are handled - according to the Poetry caret requirements specification, which is based on - whether the major version is 0: - - - If the major version is 0, the ceiling is determined by bumping the - rightmost specified digit and then coercing it to semver. - Example: 0 => 1.0.0, 0.1 => 0.2.0, 0.1.2 => 0.1.3 - - - If the major version is not 0, the ceiling is determined by - coercing it to semver and then bumping the major version. - Example: 1 => 2.0.0, 1.2 => 2.0.0, 1.2.3 => 2.0.0 - """ - if not semver.VersionInfo.is_valid(target): - target_dict = parse_version(target) - - if target_dict["major"] == 0: - if target_dict["minor"] is None: - target_dict["major"] += 1 - elif target_dict["patch"] is None: - target_dict["minor"] += 1 - else: - target_dict["patch"] += 1 - return str(vdict_to_vinfo(target_dict)) - - vdict_to_vinfo(target_dict) - return str(vdict_to_vinfo(target_dict).bump_major()) - - target_vinfo = semver.VersionInfo.parse(target) - - if target_vinfo.major == 0: - if target_vinfo.minor == 0: - return str(target_vinfo.bump_patch()) - else: - return str(target_vinfo.bump_minor()) - else: - return str(target_vinfo.bump_major()) - - -def get_tilde_ceiling(target: str) -> str: - """ - Accepts a Poetry tilde target and returns the exclusive version ceiling. - """ - target_dict = parse_version(target) - if target_dict["minor"]: - return str(vdict_to_vinfo(target_dict).bump_minor()) - - return str(vdict_to_vinfo(target_dict).bump_major()) - - -def encode_poetry_version(poetry_specifier: str) -> str: - """ - Encodes Poetry version specifier as a Conda version specifier. - - Example: ^1 => >=1.0.0,<2.0.0 - """ - poetry_clauses = poetry_specifier.split(",") - - conda_clauses = [] - for poetry_clause in poetry_clauses: - poetry_clause = poetry_clause.replace(" ", "") - if poetry_clause.startswith("^"): - # handle ^ operator - target = poetry_clause[1:] - floor = coerce_to_semver(target) - ceiling = get_caret_ceiling(target) - conda_clauses.append(">=" + floor) - conda_clauses.append("<" + ceiling) - continue - - if poetry_clause.startswith("~"): - # handle ~ operator - target = poetry_clause[1:] - floor = coerce_to_semver(target) - ceiling = get_tilde_ceiling(target) - conda_clauses.append(">=" + floor) - conda_clauses.append("<" + ceiling) - continue - - # other poetry clauses should be conda-compatible - conda_clauses.append(poetry_clause) - - return ",".join(conda_clauses) - - @singledispatch def get_constrained_dep(dep_spec: Union[str, dict], dep_name: str) -> str: raise InvalidPoetryDependency( diff --git a/tests/test_py_toml.py b/tests/test_py_toml.py index ceb198786..0ea0f57b5 100644 --- a/tests/test_py_toml.py +++ b/tests/test_py_toml.py @@ -6,17 +6,19 @@ import pytest from grayskull.main import generate_recipes_from_list, init_parser -from grayskull.strategy.py_toml import ( +from grayskull.strategy.parse_poetry_version import ( InvalidVersion, + encode_poetry_version, + get_caret_ceiling, + get_tilde_ceiling, + parse_version, +) +from grayskull.strategy.py_toml import ( add_flit_metadata, add_pep725_metadata, add_poetry_metadata, - encode_poetry_version, get_all_toml_info, - get_caret_ceiling, get_constrained_dep, - get_tilde_ceiling, - parse_version, ) From fa9dacbb3b26ebef8e308e67d29c46f80512a2bf Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 11:41:04 +0200 Subject: [PATCH 09/15] Split history test_py_toml.py to test_parse_poetry_version.py - rename file to target-name --- tests/{test_py_toml.py => test_parse_poetry_version.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_py_toml.py => test_parse_poetry_version.py} (100%) diff --git a/tests/test_py_toml.py b/tests/test_parse_poetry_version.py similarity index 100% rename from tests/test_py_toml.py rename to tests/test_parse_poetry_version.py From d13b90b90460dd365ba87084b360e7beccdede34 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 11:41:04 +0200 Subject: [PATCH 10/15] Split history test_py_toml.py to test_parse_poetry_version.py - rename source-file to temp --- tests/{test_py_toml.py => temp} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_py_toml.py => temp} (100%) diff --git a/tests/test_py_toml.py b/tests/temp similarity index 100% rename from tests/test_py_toml.py rename to tests/temp From 48c512bdf5937efd45fccba3c1ddaf7b64fa58a3 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 11:41:05 +0200 Subject: [PATCH 11/15] Split history test_py_toml.py to test_parse_poetry_version.py - restore name of source-file --- tests/{temp => test_py_toml.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{temp => test_py_toml.py} (100%) diff --git a/tests/temp b/tests/test_py_toml.py similarity index 100% rename from tests/temp rename to tests/test_py_toml.py From c0a63a1646c818aec32d5d6a9a87889f1bf5bfa9 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 11:45:27 +0200 Subject: [PATCH 12/15] Complete the splitoff of the tests --- tests/test_parse_poetry_version.py | 146 +---------------------------- tests/test_py_toml.py | 87 ----------------- 2 files changed, 1 insertion(+), 232 deletions(-) diff --git a/tests/test_parse_poetry_version.py b/tests/test_parse_poetry_version.py index 0ea0f57b5..0c0a60b7e 100644 --- a/tests/test_parse_poetry_version.py +++ b/tests/test_parse_poetry_version.py @@ -1,11 +1,7 @@ -"""Unit and integration tests for recipifying Poetry projects.""" - -import filecmp -from pathlib import Path +"""Unit tests for parsing Poetry versions.""" import pytest -from grayskull.main import generate_recipes_from_list, init_parser from grayskull.strategy.parse_poetry_version import ( InvalidVersion, encode_poetry_version, @@ -13,20 +9,6 @@ get_tilde_ceiling, parse_version, ) -from grayskull.strategy.py_toml import ( - add_flit_metadata, - add_pep725_metadata, - add_poetry_metadata, - get_all_toml_info, - get_constrained_dep, -) - - -def test_add_flit_metadata(): - metadata = {"build": {"entry_points": []}} - toml_metadata = {"tool": {"flit": {"scripts": {"key": "value"}}}} - result = add_flit_metadata(metadata, toml_metadata) - assert result == {"build": {"entry_points": ["key = value"]}} @pytest.mark.parametrize( @@ -107,129 +89,3 @@ def test_get_tilde_ceiling(version, ceiling_version): ) def test_encode_poetry_version(version, encoded_version): assert encode_poetry_version(version) == encoded_version - - -def test_add_poetry_metadata(): - toml_metadata = { - "tool": { - "poetry": { - "dependencies": {"tomli": ">=1.0.0", "requests": ">=1.0.0"}, - "group": { - "test": {"dependencies": {"tox": ">=1.0.0", "pytest": ">=1.0.0"}} - }, - } - } - } - metadata = { - "requirements": { - "host": ["pkg_host1 >=1.0.0", "pkg_host2"], - "run": ["pkg_run1", "pkg_run2 >=2.0.0"], - }, - "test": {"requires": ["mock", "pkg_test >=1.0.0"]}, - } - assert add_poetry_metadata(metadata, toml_metadata) == { - "requirements": { - "host": ["pkg_host1 >=1.0.0", "pkg_host2", "poetry-core"], - "run": [ - "pkg_run1", - "pkg_run2 >=2.0.0", - "tomli >=1.0.0", - "requests >=1.0.0", - ], - }, - "test": { - "requires": ["mock", "pkg_test >=1.0.0", "tox >=1.0.0", "pytest >=1.0.0"] - }, - } - - -def test_poetry_dependencies(): - toml_path = Path(__file__).parent / "data" / "poetry" / "poetry.toml" - result = get_all_toml_info(toml_path) - - assert result["test"]["requires"] == ["cachy 0.3.0", "deepdiff >=6.2.0,<7.0.0"] - assert result["requirements"]["host"] == ["setuptools>=1.1.0", "poetry-core"] - assert result["requirements"]["run"] == [ - "python >=3.7.0,<4.0.0", - "cleo >=2.0.0,<3.0.0", - "html5lib >=1.0.0,<2.0.0", - "urllib3 >=1.26.0,<2.0.0", - ] - - -def test_poetry_langchain_snapshot(tmpdir): - """Snapshot test that asserts correct recipifying of an example Poetry project.""" - snapshot_path = ( - Path(__file__).parent / "data" / "poetry" / "langchain-expected.yaml" - ) - output_path = tmpdir / "langchain" / "meta.yaml" - - parser = init_parser() - args = parser.parse_args( - ["pypi", "langchain==0.0.119", "-o", str(tmpdir), "-m", "AddYourGitHubIdHere"] - ) - - generate_recipes_from_list(args.pypi_packages, args) - assert filecmp.cmp(snapshot_path, output_path, shallow=False) - - -def test_poetry_get_constrained_dep_version_not_present(): - assert ( - get_constrained_dep( - {"git": "https://codeberg.org/hjacobs/pytest-kind.git"}, "pytest-kind" - ) - == "pytest-kind" - ) - - -def test_poetry_entrypoints(): - poetry = { - "requirements": {"host": ["setuptools"], "run": ["python"]}, - "build": {}, - "test": {}, - } - toml_metadata = { - "tool": { - "poetry": { - "scripts": { - "grayskull": "grayskull.main:main", - "grayskull-recipe": "grayskull.main:recipe", - } - } - } - } - assert add_poetry_metadata(poetry, toml_metadata) == { - "requirements": { - "host": ["setuptools", "poetry-core"], - "run": ["python"], - }, - "build": { - "entry_points": [ - "grayskull = grayskull.main:main", - "grayskull-recipe = grayskull.main:recipe", - ] - }, - "test": {}, - } - - -@pytest.mark.parametrize( - "conda_section, pep725_section", - [("build", "build-requires"), ("host", "host-requires"), ("run", "dependencies")], -) -@pytest.mark.parametrize( - "purl, purl_translated", - [ - ("virtual:compiler/c", "{{ compiler('c') }}"), - ("pkg:alice/bob", "pkg:alice/bob"), - ], -) -def test_pep725_section_lookup(conda_section, pep725_section, purl, purl_translated): - toml_metadata = { - "external": { - pep725_section: [purl], - } - } - assert add_pep725_metadata({}, toml_metadata) == { - "requirements": {conda_section: [purl_translated]} - } diff --git a/tests/test_py_toml.py b/tests/test_py_toml.py index 0ea0f57b5..1c3cdf9c0 100644 --- a/tests/test_py_toml.py +++ b/tests/test_py_toml.py @@ -6,13 +6,6 @@ import pytest from grayskull.main import generate_recipes_from_list, init_parser -from grayskull.strategy.parse_poetry_version import ( - InvalidVersion, - encode_poetry_version, - get_caret_ceiling, - get_tilde_ceiling, - parse_version, -) from grayskull.strategy.py_toml import ( add_flit_metadata, add_pep725_metadata, @@ -29,86 +22,6 @@ def test_add_flit_metadata(): assert result == {"build": {"entry_points": ["key = value"]}} -@pytest.mark.parametrize( - "version, major, minor, patch", - [ - ("0", 0, None, None), - ("1", 1, None, None), - ("1.2", 1, 2, None), - ("1.2.3", 1, 2, 3), - ], -) -def test_parse_version_success(version, major, minor, patch): - assert parse_version(version) == {"major": major, "minor": minor, "patch": patch} - - -@pytest.mark.parametrize( - "invalid_version", ["asdf", "", ".", "x.2.3", "1.x.3", "1.2.x"] -) -def test_parse_version_failure(invalid_version): - with pytest.raises(InvalidVersion): - parse_version(invalid_version) - - -@pytest.mark.parametrize( - "version, ceiling_version", - [ - ("0", "1.0.0"), - ("0.0", "0.1.0"), - ("0.0.3", "0.0.4"), - ("0.2.3", "0.3.0"), - ("1", "2.0.0"), - ("1.2", "2.0.0"), - ("1.2.3", "2.0.0"), - ], -) -def test_get_caret_ceiling(version, ceiling_version): - # examples from Poetry docs - assert get_caret_ceiling(version) == ceiling_version - - -@pytest.mark.parametrize( - "version, ceiling_version", - [("1", "2.0.0"), ("1.2", "1.3.0"), ("1.2.3", "1.3.0")], -) -def test_get_tilde_ceiling(version, ceiling_version): - # examples from Poetry docs - assert get_tilde_ceiling(version) == ceiling_version - - -@pytest.mark.parametrize( - "version, encoded_version", - [ - # should be unchanged - ("1.*", "1.*"), - (">=1,<2", ">=1,<2"), - ("==1.2.3", "==1.2.3"), - ("!=1.2.3", "!=1.2.3"), - # strip spaces - (">= 1, < 2", ">=1,<2"), - # handle exact version specifiers correctly - ("1.2.3", "1.2.3"), - ("==1.2.3", "==1.2.3"), - # handle caret operator correctly - # examples from Poetry docs - ("^0", ">=0.0.0,<1.0.0"), - ("^0.0", ">=0.0.0,<0.1.0"), - ("^0.0.3", ">=0.0.3,<0.0.4"), - ("^0.2.3", ">=0.2.3,<0.3.0"), - ("^1", ">=1.0.0,<2.0.0"), - ("^1.2", ">=1.2.0,<2.0.0"), - ("^1.2.3", ">=1.2.3,<2.0.0"), - # handle tilde operator correctly - # examples from Poetry docs - ("~1", ">=1.0.0,<2.0.0"), - ("~1.2", ">=1.2.0,<1.3.0"), - ("~1.2.3", ">=1.2.3,<1.3.0"), - ], -) -def test_encode_poetry_version(version, encoded_version): - assert encode_poetry_version(version) == encoded_version - - def test_add_poetry_metadata(): toml_metadata = { "tool": { From 0736b95ef317f728c072a37cefdfb08649c2bfeb Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 13:26:40 +0200 Subject: [PATCH 13/15] Convert most parse_poetry_version tests to doctests This makes it more self-contained --- grayskull/strategy/parse_poetry_version.py | 79 +++++++++++++++++++++ tests/test_parse_poetry_version.py | 80 +--------------------- 2 files changed, 80 insertions(+), 79 deletions(-) diff --git a/grayskull/strategy/parse_poetry_version.py b/grayskull/strategy/parse_poetry_version.py index 0c44dbfaf..65aba425f 100644 --- a/grayskull/strategy/parse_poetry_version.py +++ b/grayskull/strategy/parse_poetry_version.py @@ -25,6 +25,15 @@ def parse_version(version: str) -> Dict[str, Optional[int]]: """ Parses a version string (not necessarily semver) to a dictionary with keys "major", "minor", and "patch". "minor" and "patch" are possibly None. + + >>> parse_version("0") + {'major': 0, 'minor': None, 'patch': None} + >>> parse_version("1") + {'major': 1, 'minor': None, 'patch': None} + >>> parse_version("1.2") + {'major': 1, 'minor': 2, 'patch': None} + >>> parse_version("1.2.3") + {'major': 1, 'minor': 2, 'patch': 3} """ match = VERSION_REGEX.search(version) if not match: @@ -72,6 +81,22 @@ def get_caret_ceiling(target: str) -> str: - If the major version is not 0, the ceiling is determined by coercing it to semver and then bumping the major version. Example: 1 => 2.0.0, 1.2 => 2.0.0, 1.2.3 => 2.0.0 + + # Examples from Poetry docs + >>> get_caret_ceiling("0") + '1.0.0' + >>> get_caret_ceiling("0.0") + '0.1.0' + >>> get_caret_ceiling("0.0.3") + '0.0.4' + >>> get_caret_ceiling("0.2.3") + '0.3.0' + >>> get_caret_ceiling("1") + '2.0.0' + >>> get_caret_ceiling("1.2") + '2.0.0' + >>> get_caret_ceiling("1.2.3") + '2.0.0' """ if not semver.VersionInfo.is_valid(target): target_dict = parse_version(target) @@ -102,6 +127,14 @@ def get_caret_ceiling(target: str) -> str: def get_tilde_ceiling(target: str) -> str: """ Accepts a Poetry tilde target and returns the exclusive version ceiling. + + # Examples from Poetry docs + >>> get_tilde_ceiling("1") + '2.0.0' + >>> get_tilde_ceiling("1.2") + '1.3.0' + >>> get_tilde_ceiling("1.2.3") + '1.3.0' """ target_dict = parse_version(target) if target_dict["minor"]: @@ -115,6 +148,52 @@ def encode_poetry_version(poetry_specifier: str) -> str: Encodes Poetry version specifier as a Conda version specifier. Example: ^1 => >=1.0.0,<2.0.0 + + # should be unchanged + >>> encode_poetry_version("1.*") + '1.*' + >>> encode_poetry_version(">=1,<2") + '>=1,<2' + >>> encode_poetry_version("==1.2.3") + '==1.2.3' + >>> encode_poetry_version("!=1.2.3") + '!=1.2.3' + + # strip spaces + >>> encode_poetry_version(">= 1, < 2") + '>=1,<2' + + # handle exact version specifiers correctly + >>> encode_poetry_version("1.2.3") + '1.2.3' + >>> encode_poetry_version("==1.2.3") + '==1.2.3' + + # handle caret operator correctly + # examples from Poetry docs + >>> encode_poetry_version("^0") + '>=0.0.0,<1.0.0' + >>> encode_poetry_version("^0.0") + '>=0.0.0,<0.1.0' + >>> encode_poetry_version("^0.0.3") + '>=0.0.3,<0.0.4' + >>> encode_poetry_version("^0.2.3") + '>=0.2.3,<0.3.0' + >>> encode_poetry_version("^1") + '>=1.0.0,<2.0.0' + >>> encode_poetry_version("^1.2") + '>=1.2.0,<2.0.0' + >>> encode_poetry_version("^1.2.3") + '>=1.2.3,<2.0.0' + + # handle tilde operator correctly + # examples from Poetry docs + >>> encode_poetry_version("~1") + '>=1.0.0,<2.0.0' + >>> encode_poetry_version("~1.2") + '>=1.2.0,<1.3.0' + >>> encode_poetry_version("~1.2.3") + '>=1.2.3,<1.3.0' """ poetry_clauses = poetry_specifier.split(",") diff --git a/tests/test_parse_poetry_version.py b/tests/test_parse_poetry_version.py index 0c0a60b7e..96fdeb1bb 100644 --- a/tests/test_parse_poetry_version.py +++ b/tests/test_parse_poetry_version.py @@ -2,26 +2,7 @@ import pytest -from grayskull.strategy.parse_poetry_version import ( - InvalidVersion, - encode_poetry_version, - get_caret_ceiling, - get_tilde_ceiling, - parse_version, -) - - -@pytest.mark.parametrize( - "version, major, minor, patch", - [ - ("0", 0, None, None), - ("1", 1, None, None), - ("1.2", 1, 2, None), - ("1.2.3", 1, 2, 3), - ], -) -def test_parse_version_success(version, major, minor, patch): - assert parse_version(version) == {"major": major, "minor": minor, "patch": patch} +from grayskull.strategy.parse_poetry_version import InvalidVersion, parse_version @pytest.mark.parametrize( @@ -30,62 +11,3 @@ def test_parse_version_success(version, major, minor, patch): def test_parse_version_failure(invalid_version): with pytest.raises(InvalidVersion): parse_version(invalid_version) - - -@pytest.mark.parametrize( - "version, ceiling_version", - [ - ("0", "1.0.0"), - ("0.0", "0.1.0"), - ("0.0.3", "0.0.4"), - ("0.2.3", "0.3.0"), - ("1", "2.0.0"), - ("1.2", "2.0.0"), - ("1.2.3", "2.0.0"), - ], -) -def test_get_caret_ceiling(version, ceiling_version): - # examples from Poetry docs - assert get_caret_ceiling(version) == ceiling_version - - -@pytest.mark.parametrize( - "version, ceiling_version", - [("1", "2.0.0"), ("1.2", "1.3.0"), ("1.2.3", "1.3.0")], -) -def test_get_tilde_ceiling(version, ceiling_version): - # examples from Poetry docs - assert get_tilde_ceiling(version) == ceiling_version - - -@pytest.mark.parametrize( - "version, encoded_version", - [ - # should be unchanged - ("1.*", "1.*"), - (">=1,<2", ">=1,<2"), - ("==1.2.3", "==1.2.3"), - ("!=1.2.3", "!=1.2.3"), - # strip spaces - (">= 1, < 2", ">=1,<2"), - # handle exact version specifiers correctly - ("1.2.3", "1.2.3"), - ("==1.2.3", "==1.2.3"), - # handle caret operator correctly - # examples from Poetry docs - ("^0", ">=0.0.0,<1.0.0"), - ("^0.0", ">=0.0.0,<0.1.0"), - ("^0.0.3", ">=0.0.3,<0.0.4"), - ("^0.2.3", ">=0.2.3,<0.3.0"), - ("^1", ">=1.0.0,<2.0.0"), - ("^1.2", ">=1.2.0,<2.0.0"), - ("^1.2.3", ">=1.2.3,<2.0.0"), - # handle tilde operator correctly - # examples from Poetry docs - ("~1", ">=1.0.0,<2.0.0"), - ("~1.2", ">=1.2.0,<1.3.0"), - ("~1.2.3", ">=1.2.3,<1.3.0"), - ], -) -def test_encode_poetry_version(version, encoded_version): - assert encode_poetry_version(version) == encoded_version From bdd0393c503b45bb48639134578e6a6f20a7e664 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 6 Sep 2024 14:13:29 +0200 Subject: [PATCH 14/15] Update failing CRAN test Please review carefully. The results changed but I know neither R nor the intention. --- grayskull/strategy/cran.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/grayskull/strategy/cran.py b/grayskull/strategy/cran.py index 0ebc64ff1..3ec559034 100644 --- a/grayskull/strategy/cran.py +++ b/grayskull/strategy/cran.py @@ -76,23 +76,24 @@ def dict_from_cran_lines(lines): def remove_package_line_continuations(chunk): """ >>> chunk = [ - 'Package: A3', - 'Version: 0.9.2', - 'Depends: R (>= 2.15.0), xtable, pbapply', - 'Suggests: randomForest, e1071', - 'Imports: MASS, R.methodsS3 (>= 1.5.2), R.oo (>= 1.15.8), R.utils (>=', - ' 1.27.1), matrixStats (>= 0.8.12), R.filesets (>= 2.3.0), ', - ' sampleSelection, scatterplot3d, strucchange, systemfit', - 'License: GPL (>= 2)', - 'NeedsCompilation: no'] - >>> remove_package_line_continuations(chunk) + ... 'Package: A3', + ... 'Version: 0.9.2', + ... 'Depends: R (>= 2.15.0), xtable, pbapply', + ... 'Suggests: randomForest, e1071', + ... 'Imports: MASS, R.methodsS3 (>= 1.5.2), R.oo (>= 1.15.8), R.utils (>=', + ... ' 1.27.1), matrixStats (>= 0.8.12), R.filesets (>= 2.3.0), ', + ... ' sampleSelection, scatterplot3d, strucchange, systemfit', + ... 'License: GPL (>= 2)', + ... 'NeedsCompilation: no'] + >>> remove_package_line_continuations(chunk) # doctest: +NORMALIZE_WHITESPACE ['Package: A3', 'Version: 0.9.2', 'Depends: R (>= 2.15.0), xtable, pbapply', 'Suggests: randomForest, e1071', - 'Imports: MASS, R.methodsS3 (>= 1.5.2), R.oo (>= 1.15.8), R.utils (>= 1.27.1), matrixStats (>= 0.8.12), R.filesets (>= 2.3.0), sampleSelection, scatterplot3d, strucchange, systemfit, rgl,' + 'Imports: MASS, R.methodsS3 (>= 1.5.2), R.oo (>= 1.15.8), R.utils (>= 1.27.1), matrixStats (>= 0.8.12), R.filesets (>= 2.3.0), sampleSelection, scatterplot3d, strucchange, systemfit', 'License: GPL (>= 2)', - 'NeedsCompilation: no'] + 'NeedsCompilation: no', + ''] """ # NOQA continuation = (" ", "\t") continued_ix = None From 6954797a0e3dfee829805edab3b208031e785972 Mon Sep 17 00:00:00 2001 From: Ben Mares Date: Fri, 20 Sep 2024 09:23:17 +0200 Subject: [PATCH 15/15] Enable doctests --- .github/workflows/tests.yml | 15 +++++++++++++++ pytest.ini | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9cd83fd05..d413d8a7a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,6 +35,21 @@ jobs: conda info --all conda list + - name: Running doctests + shell: bash -l {0} + run: | + pytest grayskull \ + -vv \ + -n 0 \ + --color=yes \ + --cov=./ \ + --cov-append \ + --cov-report html:coverage-serial-html \ + --cov-report xml:coverage-serial.xml \ + --cov-config=.coveragerc \ + --junit-xml=Linux-py${{ matrix.py_ver }}-serial.xml \ + --junit-prefix=Linux-py${{ matrix.py_ver }}-serial + - name: Running serial tests shell: bash -l {0} run: | diff --git a/pytest.ini b/pytest.ini index 048f43276..9e80a78d6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,8 +1,8 @@ [pytest] junit_family=xunit2 junit_duration_report=call -addopts = -ra -q -testpaths = tests +addopts = -ra -q --doctest-modules +testpaths = grayskull tests markers = serial: Mark for tests which cannot be executed in parallel github: Tests which need to communicate with github and might reach the limit of github requisitions