From f041671c105c81134eaf1c35ed4aec5a4c40a16a Mon Sep 17 00:00:00 2001 From: antazoey Date: Fri, 22 Sep 2023 07:48:58 -0500 Subject: [PATCH] feat!: upgrade to pydantic v2 (#36) --- .pre-commit-config.yaml | 10 ++--- README.md | 8 +--- pyproject.toml | 3 +- setup.cfg | 1 + setup.py | 18 ++++---- tests/functional/test_schema_fuzzing.py | 6 ++- tests/functional/test_uniswap_examples.py | 50 ++++++++++++++++++--- tests/integration/test_cli.py | 2 +- tokenlists/_cli.py | 4 +- tokenlists/manager.py | 12 +++-- tokenlists/typing.py | 54 ++++++++++++----------- 11 files changed, 106 insertions(+), 62 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 38a4406..2001bc4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.4.0 hooks: - id: check-yaml @@ -10,24 +10,24 @@ repos: - id: isort - repo: https://github.com/psf/black - rev: 23.3.0 + rev: 23.9.1 hooks: - id: black name: black - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 6.1.0 hooks: - id: flake8 - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.991 + rev: v1.5.1 hooks: - id: mypy additional_dependencies: [types-setuptools, types-requests] - repo: https://github.com/executablebooks/mdformat - rev: 0.7.14 + rev: 0.7.17 hooks: - id: mdformat additional_dependencies: [mdformat-gfm, mdformat-frontmatter] diff --git a/README.md b/README.md index 58eb8b0..4f1ef14 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ## Dependencies -- [python3](https://www.python.org/downloads/release/python-368/) version 3.7.2 or greater, python3-dev +- [python3](https://www.python.org/downloads/release/python-368/) version 3.8 or greater, python3-dev ## Installation @@ -40,12 +40,6 @@ python3 setup.py install ['1inch'] ``` -## Development - -This project is in early development and should be considered an alpha. -Things might not work, breaking changes are likely. -Comments, questions, criticisms and pull requests are welcomed. - ## License This project is licensed under the [MIT license](LICENSE). diff --git a/pyproject.toml b/pyproject.toml index 57885b7..508992d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=51.1.1", "wheel", "setuptools_scm[toml]>=5.0"] +requires = ["setuptools>=51.1.1", "wheel", "setuptools_scm[toml]>=5.0,<8"] [tool.mypy] exclude = "build/" @@ -19,7 +19,6 @@ include = '\.pyi?$' [tool.pytest.ini_options] addopts = """ - -n auto -p no:ape_test --cov-branch --cov-report term diff --git a/setup.cfg b/setup.cfg index 6c5adf0..58fc753 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,3 +5,4 @@ exclude = .tox docs build + ./tokenlists/version.py diff --git a/setup.py b/setup.py index cd5cd40..5faacd6 100644 --- a/setup.py +++ b/setup.py @@ -1,24 +1,25 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from setuptools import find_packages, setup # type: ignore +from setuptools import find_packages, setup extras_require = { "test": [ # `test` GitHub Action jobs uses this "pytest>=6.0", # Core testing package "pytest-xdist", # multi-process runner "pytest-cov", # Coverage analyzer plugin - "hypothesis>=6.2.0,<7", # Strategy-based fuzzer + "hypothesis>=6.86.2,<7", # Strategy-based fuzzer "PyGithub>=1.54,<2", # Necessary to pull official schema from github "hypothesis-jsonschema==0.19.0", # Fuzzes based on a json schema ], "lint": [ - "black>=23.3.0,<24", # auto-formatter and linter - "mypy>=0.991,<1", # Static type analyzer + "black>=23.9.1,<24", # auto-formatter and linter + "mypy>=1.5.1,<2", # Static type analyzer "types-requests", # Needed due to mypy typeshed - "flake8>=6.0.0,<7", # Style linter + "types-setuptools", # Needed due to mypy typeshed + "flake8>=6.1.0,<7", # Style linter "isort>=5.10.1", # Import sorting linter - "mdformat>=0.7.16", # Auto-formatter for markdown + "mdformat>=0.7.17", # Auto-formatter for markdown "mdformat-gfm>=0.3.5", # Needed for formatting GitHub-flavored markdown "mdformat-frontmatter>=0.4.1", # Needed for frontmatters-style headers in issue templates ], @@ -68,9 +69,8 @@ install_requires=[ "importlib-metadata ; python_version<'3.8'", "click>=8.1.3,<9", - "pydantic>=1.9.2,<2", + "pydantic>=2.3.0,<3", "pyyaml>=6.0,<7", - "semantic-version>=2.10.0,<3", "requests>=2.28.1,<3", ], entry_points={"console_scripts": ["tokenlists=tokenlists._cli:cli"]}, @@ -82,7 +82,7 @@ packages=find_packages(exclude=["tests", "tests.*"]), package_data={"ape_tokens": ["py.typed"]}, classifiers=[ - "Development Status :: 3 - Alpha", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Natural Language :: English", diff --git a/tests/functional/test_schema_fuzzing.py b/tests/functional/test_schema_fuzzing.py index c87c6e2..9e6aaf4 100644 --- a/tests/functional/test_schema_fuzzing.py +++ b/tests/functional/test_schema_fuzzing.py @@ -1,5 +1,5 @@ import pytest -import requests # type: ignore[import] +import requests from hypothesis import HealthCheck, given, settings from hypothesis_jsonschema import from_schema from pydantic import ValidationError @@ -27,7 +27,9 @@ def clean_data(tl: dict) -> dict: @settings(suppress_health_check=(HealthCheck.too_slow,)) def test_schema(token_list): try: - assert TokenList.parse_obj(token_list).dict() == clean_data(token_list) + assert TokenList.model_validate(token_list).model_dump(mode="json") == clean_data( + token_list + ) except (ValidationError, ValueError): pass # Expect these kinds of errors diff --git a/tests/functional/test_uniswap_examples.py b/tests/functional/test_uniswap_examples.py index f733daa..5a8ccc8 100644 --- a/tests/functional/test_uniswap_examples.py +++ b/tests/functional/test_uniswap_examples.py @@ -1,8 +1,9 @@ import os +from typing import Any, Optional import github import pytest -import requests # type: ignore[import] +import requests from pydantic import ValidationError from tokenlists import TokenList @@ -27,9 +28,46 @@ def test_uniswap_tokenlists(token_list_name): # https://github.com/Uniswap/token-lists/pull/420 token_list.pop("tokenMap") - if "invalid" not in token_list_name: - assert TokenList.parse_obj(token_list).dict() == token_list - - else: + if "invalid" in token_list_name: with pytest.raises((ValidationError, ValueError)): - TokenList.parse_obj(token_list).dict() + TokenList.model_validate(token_list) + else: + actual = TokenList.model_validate(token_list).model_dump(mode="json") + + def assert_tokenlists(_actual: Any, _expected: Any, parent_key: Optional[str] = None): + parent_key = parent_key or "__root__" + assert type(_actual) is type(_expected) + + if isinstance(_actual, list): + for idx, (actual_item, expected_item) in enumerate(zip(_actual, _expected)): + assert_tokenlists( + actual_item, expected_item, parent_key=f"{parent_key}_index_{idx}" + ) + + elif isinstance(_actual, dict): + unexpected = {} + handled = set() + for key, actual_value in _actual.items(): + if key not in _expected: + unexpected[key] = actual_value + continue + + expected_value = _expected[key] + assert type(actual_value) is type(expected_value) + assert_tokenlists(actual_value, expected_value, parent_key=key) + handled.add(key) + + handled_str = ", ".join(list(handled)) or "" + missing = {f"{x}" for x in _expected if x not in handled} + unexpected_str = ", ".join([f"{k}={v}" for k, v in unexpected.items()]) + assert not unexpected, f"Unexpected keys: {unexpected_str}, Parent: {parent_key}" + assert not missing, ( + f"Missing keys: '{', '.join(list(missing))}'; " + f"handled: '{handled_str}', " + f"Parent: {parent_key}." + ) + + else: + assert _actual == _expected + + assert_tokenlists(actual, token_list) diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index 1ef5707..76bc50c 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -21,7 +21,7 @@ def test_install(runner, cli): assert "No tokenlists exist" in result.output result = runner.invoke(cli, ["install", TEST_URI]) - assert result.exit_code == 0 + assert result.exit_code == 0, result.output result = runner.invoke(cli, ["list"]) assert result.exit_code == 0 diff --git a/tokenlists/_cli.py b/tokenlists/_cli.py index a642c02..7137d8b 100644 --- a/tokenlists/_cli.py +++ b/tokenlists/_cli.py @@ -77,7 +77,7 @@ def list_tokens(search, tokenlist_name, chain_id): lambda t: pattern.match(t.symbol), manager.get_tokens(tokenlist_name, chain_id), ): - click.echo("{address} ({symbol})".format(**token_info.dict())) + click.echo("{address} ({symbol})".format(**token_info.model_dump(mode="json"))) @cli.command(short_help="Display the info for a particular token") @@ -92,7 +92,7 @@ def token_info(symbol, tokenlist_name, chain_id, case_insensitive): raise click.ClickException("No tokenlists available!") token_info = manager.get_token_info(symbol, tokenlist_name, chain_id, case_insensitive) - token_info = token_info.dict() + token_info = token_info.model_dump(mode="json") if "tags" not in token_info: token_info["tags"] = "" diff --git a/tokenlists/manager.py b/tokenlists/manager.py index d52bfda..3ca7704 100644 --- a/tokenlists/manager.py +++ b/tokenlists/manager.py @@ -1,3 +1,4 @@ +from json import JSONDecodeError from typing import Iterator, List, Optional import requests @@ -15,7 +16,7 @@ def __init__(self): # Load all the ones cached on disk self.installed_tokenlists = {} for path in self.cache_folder.glob("*.json"): - tokenlist = TokenList.parse_file(path) + tokenlist = TokenList.model_validate_json(path.read_text()) self.installed_tokenlists[tokenlist.name] = tokenlist self.default_tokenlist = config.DEFAULT_TOKENLIST @@ -42,13 +43,18 @@ def install_tokenlist(self, uri: str) -> str: # Load and store the tokenlist response = requests.get(uri) response.raise_for_status() - tokenlist = TokenList.parse_obj(response.json()) + try: + response_json = response.json() + except JSONDecodeError as err: + raise ValueError(f"Invalid response: {response.text}") from err + + tokenlist = TokenList.model_validate(response_json) self.installed_tokenlists[tokenlist.name] = tokenlist # Cache it on disk for later instances self.cache_folder.mkdir(exist_ok=True) token_list_file = self.cache_folder.joinpath(f"{tokenlist.name}.json") - token_list_file.write_text(tokenlist.json()) + token_list_file.write_text(tokenlist.model_dump_json()) return tokenlist.name diff --git a/tokenlists/typing.py b/tokenlists/typing.py index 6c0197b..bc9afee 100644 --- a/tokenlists/typing.py +++ b/tokenlists/typing.py @@ -1,16 +1,12 @@ -from datetime import datetime from itertools import chain from typing import Any, Dict, List, Optional from pydantic import AnyUrl from pydantic import BaseModel as _BaseModel -from pydantic import validator -from semantic_version import Version # type: ignore +from pydantic import PastDatetime, field_validator ChainId = int - TagId = str - TokenAddress = str TokenName = str TokenDecimals = int @@ -18,11 +14,11 @@ class BaseModel(_BaseModel): - def dict(self, *args, **kwargs): + def model_dump(self, *args, **kwargs): if "exclude_unset" not in kwargs: kwargs["exclude_unset"] = True - return super().dict(*args, **kwargs) + return super().model_dump(*args, **kwargs) class Config: froze = True @@ -44,17 +40,17 @@ class TokenInfo(BaseModel): tags: Optional[List[TagId]] = None extensions: Optional[Dict[str, Any]] = None - @validator("logoURI") + @field_validator("logoURI") def validate_uri(cls, v: Optional[str]) -> Optional[str]: if v is None: return v - if "://" not in v or not AnyUrl(v, scheme=v.split("://")[0]): + if "://" not in v or not AnyUrl(v): raise ValueError(f"'{v}' is not a valid URI") return v - @validator("extensions", pre=True) + @field_validator("extensions", mode="before") def parse_extensions(cls, v: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: # 1. Check extension depth first def extension_depth(obj: Optional[Dict[str, Any]]) -> int: @@ -68,12 +64,17 @@ def extension_depth(obj: Optional[Dict[str, Any]]) -> int: # 2. Parse valid extensions if v and "bridgeInfo" in v: - raw_bridge_info = v.pop("bridgeInfo") - v["bridgeInfo"] = {int(k): BridgeInfo.parse_obj(v) for k, v in raw_bridge_info.items()} + # NOTE: Avoid modifying `v`. + return { + **v, + "bridgeInfo": { + int(k): BridgeInfo.model_validate(v) for k, v in v["bridgeInfo"].items() + }, + } return v - @validator("extensions") + @field_validator("extensions") def extensions_must_contain_allowed_types( cls, d: Optional[Dict[str, Any]] ) -> Optional[Dict[str, Any]]: @@ -92,11 +93,11 @@ def extensions_must_contain_allowed_types( @property def bridge_info(self) -> Optional[BridgeInfo]: if self.extensions and "bridgeInfo" in self.extensions: - return self.extensions["bridgeInfo"] # type: ignore + return self.extensions["bridgeInfo"] return None - @validator("address") + @field_validator("address") def address_must_hex(cls, v: str): if not v.startswith("0x") or set(v) > set("x0123456789abcdefABCDEF") or len(v) % 2 != 0: raise ValueError("Address is not hex") @@ -108,7 +109,7 @@ def address_must_hex(cls, v: str): return v - @validator("decimals") + @field_validator("decimals") def decimals_must_be_uint8(cls, v: TokenDecimals): if not (0 <= v < 256): raise ValueError(f"Invalid token decimals: {v}") @@ -121,12 +122,12 @@ class Tag(BaseModel): description: str -class TokenListVersion(BaseModel, Version): +class TokenListVersion(BaseModel): major: int minor: int patch: int - @validator("*") + @field_validator("*") def no_negative_version_numbers(cls, v: int): if v < 0: raise ValueError("Invalid version number") @@ -150,7 +151,7 @@ def __str__(self) -> str: class TokenList(BaseModel): name: str - timestamp: datetime + timestamp: PastDatetime version: TokenListVersion tokens: List[TokenInfo] keywords: Optional[List[str]] = None @@ -180,18 +181,21 @@ class Config: # NOTE: Not frozen as we may need to dynamically modify this froze = False - @validator("logoURI") + @field_validator("logoURI") def validate_uri(cls, v: Optional[str]) -> Optional[str]: if v is None: return v - if "://" not in v or not AnyUrl(v, scheme=v.split("://")[0]): + if "://" not in v or not AnyUrl(v): raise ValueError(f"'{v}' is not a valid URI") return v - def dict(self, *args, **kwargs) -> dict: - data = super().dict(*args, **kwargs) - # NOTE: This was the easiest way to make sure this property returns isoformat - data["timestamp"] = self.timestamp.isoformat() + def model_dump(self, *args, **kwargs) -> Dict: + data = super().model_dump(*args, **kwargs) + + if kwargs.get("mode", "").lower() == "json": + # NOTE: This was the easiest way to make sure this property returns isoformat + data["timestamp"] = self.timestamp.isoformat() + return data