diff --git a/scripts/populate_tox/populate_tox.py b/scripts/populate_tox/populate_tox.py index 01497cf987..bff3b90d3a 100644 --- a/scripts/populate_tox/populate_tox.py +++ b/scripts/populate_tox/populate_tox.py @@ -1,16 +1,14 @@ import configparser import functools +import time from datetime import datetime, timedelta from pathlib import Path import requests -from ..split_tox_gh_actions.split_tox_gh_actions import GROUPS - -print(GROUPS) # Only consider package versions going back this far -CUTOFF = datetime.now() - timedelta(days=365 * 3) +CUTOFF = datetime.now() - timedelta(days=365 * 5) LOWEST_SUPPORTED_PY_VERSION = "3.6" TOX_FILE = Path(__file__).resolve().parent.parent.parent / "tox.ini" @@ -18,6 +16,8 @@ PYPI_PROJECT_URL = "https://pypi.python.org/pypi/{project}/json" PYPI_VERSION_URL = "https://pypi.python.org/pypi/{project}/{version}/json" +CLASSIFIER_PREFIX = "Programming Language :: Python :: " + EXCLUDE = { "common", } @@ -27,7 +27,7 @@ @functools.total_ordering class Version: - def __init__(self, version, metadata): + def __init__(self, version, metadata=None): self.raw = version self.metadata = metadata @@ -36,10 +36,14 @@ def __init__(self, version, metadata): self.patch = None self.parsed = None + self.python_versions = [] + try: parsed = version.split(".") - if parsed[2].isnumeric(): + if len(parsed) == 3 and parsed[2].isnumeric(): self.major, self.minor, self.patch = (int(p) for p in parsed) + elif len(parsed) == 2 and parsed[1].isnumeric(): + self.major, self.minor = (int(p) for p in parsed) except Exception: # This will fail for e.g. prereleases, but we don't care about those # for now @@ -97,50 +101,125 @@ def parse_tox(): print(f"ERROR reading line {line}") -def fetch_metadata(package): +def fetch_package(package: str) -> dict: + """Fetch package metadata from PYPI.""" url = PYPI_PROJECT_URL.format(project=package) pypi_data = requests.get(url) if pypi_data.status_code != 200: print(f"{package} not found") - import pprint - - pprint.pprint(package) - pprint.pprint(pypi_data.json()) return pypi_data.json() -def parse_metadata(data): - package = data["info"]["name"] +def get_releases(pypi_data: dict) -> list[Version]: + package = pypi_data["info"]["name"] - majors = {} + versions = [] + + for release, metadata in pypi_data["releases"].items(): + if not metadata: + continue - for release, metadata in data["releases"].items(): meta = metadata[0] if datetime.fromisoformat(meta["upload_time"]) < CUTOFF: continue version = Version(release, meta) if not version.valid: - print(f"Failed to parse version {release} of package {package}") + print( + f"Failed to parse version {release} of package {package}. Ignoring..." + ) continue - if version.major not in majors: - # 0 -> [min 0.x version, max 0.x version] - majors[version.major] = [version, version] - continue + versions.append(version) - if version < majors[version.major][0]: - majors[version.major][0] = version - if version > majors[version.major][1]: - majors[version.major][1] = version + return sorted(versions) + + +def pick_releases_to_test(releases: list[Version]) -> list[Version]: + indexes = [ + 0, # oldest version younger than CUTOFF + len(releases) // 3, + len(releases) // 3 * 2, + -1, # latest + ] + return [releases[i] for i in indexes] + + +def fetch_release(package: str, version: Version) -> dict: + url = PYPI_VERSION_URL.format(project=package, version=version) + pypi_data = requests.get(url) + + if pypi_data.status_code != 200: + print(f"{package} not found") + + return pypi_data.json() + + +def determine_python_versions( + package: str, version: Version, pypi_data: dict +) -> list[str]: + try: + classifiers = pypi_data["info"]["classifiers"] + except (AttributeError, IndexError): + print(f"{package} {version} has no classifiers") + return [] + + python_versions = [] + for classifier in classifiers: + if classifier.startswith(CLASSIFIER_PREFIX): + python_version = classifier[len(CLASSIFIER_PREFIX) :] + if "." in python_version: + # we don't care about stuff like + # Programming Language :: Python :: 3 :: Only + # Programming Language :: Python :: 3 + # etc., we're only interested in specific versions like 3.13 + python_versions.append(python_version) + + python_versions = [ + version + for version in python_versions + if Version(version) >= Version(LOWEST_SUPPORTED_PY_VERSION) + ] + + return python_versions + + +def write_tox_file(package, versions): + for version in versions: + print( + "{python_versions}-{package}-v{version}".format( + python_versions=",".join([f"py{v}" for v in version.python_versions]), + package=package, + version=version, + ) + ) + + print() + + for version in versions: + print(f"{package}-v{version}: {package}=={version.raw}") + + +if __name__ == "__main__": + for package in ("celery", "django"): + pypi_data = fetch_package(package) + releases = get_releases(pypi_data) + test_releases = pick_releases_to_test(releases) + for release in test_releases: + release_pypi_data = fetch_release(package, release) + release.python_versions = determine_python_versions( + package, release, release_pypi_data + ) + # XXX if no supported python versions -> delete - print(release, "not too old", meta["upload_time"]) + print(release, " on ", release.python_versions) + time.sleep(0.1) - return majors + print(releases) + print(test_releases) + write_tox_file(package, test_releases) -print(parse_tox()) -print(packages) -print(parse_metadata(fetch_metadata("celery"))) + print(parse_tox())