From 1f84eebb9f8daf3706caf134094abf5e75fb714f Mon Sep 17 00:00:00 2001 From: Just some guy <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 16 May 2022 22:54:15 -0400 Subject: [PATCH 01/12] WIP --- tokenlists/typing.py | 53 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/tokenlists/typing.py b/tokenlists/typing.py index 8636e2f..93b5209 100644 --- a/tokenlists/typing.py +++ b/tokenlists/typing.py @@ -1,6 +1,6 @@ from datetime import datetime from itertools import chain -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from pydantic import AnyUrl from pydantic import BaseModel as _BaseModel @@ -28,6 +28,23 @@ class Config: froze = True +class BridgeInfoItem(BaseModel): + tokenAddress: TokenAddress + originBridgeAddress: TokenAddress + destBridgeAddress: TokenAddress + + +class BridgeInfo(BaseModel): + __root__: Dict[str, BridgeInfoItem] + + @validator("__root__") + def validate_keys_are_chainIds(cls, v): + for chain_id in v: + int(chain_id) + + return v + + class TokenInfo(BaseModel): chainId: ChainId address: TokenAddress @@ -36,7 +53,34 @@ class TokenInfo(BaseModel): symbol: TokenSymbol logoURI: Optional[AnyUrl] = None tags: Optional[List[TagId]] = None - extensions: Optional[dict] = None + extensions: Optional[Dict[str, Any]] = None + + @validator("extensions") + def parse_extensions(cls, v: dict): + if "bridgeInfo" in v: + v["bridgeInfo"] = BridgeInfo.parse_obj(v.pop("bridgeInfo")) + + return v + + @validator("extensions") + def check_extension_depth(cls, v: dict): + def extension_depth(obj: dict) -> int: + depth = 0 + for v in obj.values(): + if isinstance(v, dict): + depth = max(depth, extension_depth(v)) + + return depth + 1 + + assert extension_depth(v) < 3 + return v + + @property + def bridge_info(self) -> Optional[BridgeInfo]: + if self.extensions and "bridgeInfo" in self.extensions: + return self.extensions["bridgeInfo"] # type: ignore + + return None @validator("address") def address_must_hex(cls, v: str): @@ -63,7 +107,10 @@ def extensions_must_contain_simple_types(cls, d: Optional[dict]) -> Optional[dic return d # `extensions` is `Dict[str, Union[str, int, bool, None]]`, but pydantic mutates entries - for val in d.values(): + for key, val in d.items(): + if key in "bridgeInfo": + continue # don't parse valid extensions + if not isinstance(val, (str, int, bool)) and val is not None: raise ValueError(f"Incorrect extension field value: {val}") From 1d512331960bdbba4c37c75599ea26eaaefeff5c Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 17 Jul 2023 19:34:47 -0400 Subject: [PATCH 02/12] fix: bridge addresses are optional --- tokenlists/typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tokenlists/typing.py b/tokenlists/typing.py index 93b5209..1b04c57 100644 --- a/tokenlists/typing.py +++ b/tokenlists/typing.py @@ -30,8 +30,8 @@ class Config: class BridgeInfoItem(BaseModel): tokenAddress: TokenAddress - originBridgeAddress: TokenAddress - destBridgeAddress: TokenAddress + originBridgeAddress: Optional[TokenAddress] = None + destBridgeAddress: Optional[TokenAddress] = None class BridgeInfo(BaseModel): From a0d338f084946485e96961ad1d63c64a5ab97c3b Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 17 Jul 2023 19:35:02 -0400 Subject: [PATCH 03/12] fix: update validators for extensions to function correctly --- tokenlists/typing.py | 53 ++++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/tokenlists/typing.py b/tokenlists/typing.py index 1b04c57..b85991d 100644 --- a/tokenlists/typing.py +++ b/tokenlists/typing.py @@ -55,25 +55,39 @@ class TokenInfo(BaseModel): tags: Optional[List[TagId]] = None extensions: Optional[Dict[str, Any]] = None - @validator("extensions") - def parse_extensions(cls, v: dict): - if "bridgeInfo" in v: + @validator("extensions", pre=True) + def parse_extensions(cls, v: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + # 1. Check extension depth first + def extension_depth(obj: Optional[Dict[str, Any]]) -> int: + if not isinstance(obj, dict) or len(obj) == 0: + return 0 + + return 1 + max(extension_depth(v) for v in obj.values()) + + if (depth := extension_depth(v)) > 3: + raise ValueError(f"Extension depth is greater than 3: {depth}") + + # 2. Parse valid extensions + if v and "bridgeInfo" in v: v["bridgeInfo"] = BridgeInfo.parse_obj(v.pop("bridgeInfo")) return v @validator("extensions") - def check_extension_depth(cls, v: dict): - def extension_depth(obj: dict) -> int: - depth = 0 - for v in obj.values(): - if isinstance(v, dict): - depth = max(depth, extension_depth(v)) + def extensions_must_contain_allowed_types( + cls, d: Optional[Dict[str, Any]] + ) -> Optional[Dict[str, Any]]: + if not d: + return d - return depth + 1 + # NOTE: `extensions` is mapping from `str` to either: + # - a parsed `dict` type (e.g. `BaseModel`) + # - a "simple" type (e.g. string, integer or boolean value) + for key, val in d.items(): + if not isinstance(val, (BaseModel, str, int, bool)) and val is not None: + raise ValueError(f"Incorrect extension field value: {val}") - assert extension_depth(v) < 3 - return v + return d @property def bridge_info(self) -> Optional[BridgeInfo]: @@ -101,21 +115,6 @@ def decimals_must_be_uint8(cls, v: TokenDecimals): return v - @validator("extensions") - def extensions_must_contain_simple_types(cls, d: Optional[dict]) -> Optional[dict]: - if not d: - return d - - # `extensions` is `Dict[str, Union[str, int, bool, None]]`, but pydantic mutates entries - for key, val in d.items(): - if key in "bridgeInfo": - continue # don't parse valid extensions - - if not isinstance(val, (str, int, bool)) and val is not None: - raise ValueError(f"Incorrect extension field value: {val}") - - return d - class Tag(BaseModel): name: str From 9ededfede71472a726259bb386307ac8bd859ab6 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 17 Jul 2023 21:22:03 -0400 Subject: [PATCH 04/12] test: fix warning --- tests/functional/test_uniswap_examples.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/functional/test_uniswap_examples.py b/tests/functional/test_uniswap_examples.py index 5937d6d..5591434 100644 --- a/tests/functional/test_uniswap_examples.py +++ b/tests/functional/test_uniswap_examples.py @@ -8,7 +8,10 @@ from tokenlists import TokenList # NOTE: Must export GITHUB_ACCESS_TOKEN -UNISWAP_REPO = github.Github(os.environ["GITHUB_ACCESS_TOKEN"]).get_repo("Uniswap/token-lists") +UNISWAP_REPO = github.Github(auth=github.Auth.Token(os.environ["GITHUB_ACCESS_TOKEN"])).get_repo( + "Uniswap/token-lists" +) + UNISWAP_RAW_URL = "https://raw.githubusercontent.com/Uniswap/token-lists/master/test/schema/" From f8ddb80627faced28980835d81b4b61ea739e040 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 17 Jul 2023 21:22:41 -0400 Subject: [PATCH 05/12] refactor: don't serialize URLs to AnyUrl class --- tokenlists/typing.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tokenlists/typing.py b/tokenlists/typing.py index b85991d..8b7306c 100644 --- a/tokenlists/typing.py +++ b/tokenlists/typing.py @@ -51,10 +51,20 @@ class TokenInfo(BaseModel): name: TokenName decimals: TokenDecimals symbol: TokenSymbol - logoURI: Optional[AnyUrl] = None + logoURI: Optional[str] = None tags: Optional[List[TagId]] = None extensions: Optional[Dict[str, Any]] = None + @validator("logoURI") + def validate_uri(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + + if "://" not in v or not AnyUrl(v, scheme=v.split("://")[0]): + raise ValueError(f"'{v}' is not a valid URI") + + return v + @validator("extensions", pre=True) def parse_extensions(cls, v: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: # 1. Check extension depth first @@ -155,7 +165,7 @@ class TokenList(BaseModel): tokens: List[TokenInfo] keywords: Optional[List[str]] = None tags: Optional[Dict[TagId, Tag]] = None - logoURI: Optional[AnyUrl] = None + logoURI: Optional[str] = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -180,6 +190,16 @@ class Config: # NOTE: Not frozen as we may need to dynamically modify this froze = False + @validator("logoURI") + def validate_uri(cls, v: Optional[str]) -> Optional[str]: + if v is None: + return v + + if "://" not in v or not AnyUrl(v, scheme=v.split("://")[0]): + raise ValueError(f"'{v}' is not a valid URI") + + return v + def dict(self, *args, **kwargs) -> dict: data = super().dict(*args, **kwargs) # NOTE: This was the easiest way to make sure this property returns isoformat From e3a4ff1108e7c712df26cdf4d373e48b847d878a Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 17 Jul 2023 21:23:10 -0400 Subject: [PATCH 06/12] fix: also include unparsed dicts --- tokenlists/typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tokenlists/typing.py b/tokenlists/typing.py index 8b7306c..8570243 100644 --- a/tokenlists/typing.py +++ b/tokenlists/typing.py @@ -92,9 +92,9 @@ def extensions_must_contain_allowed_types( # NOTE: `extensions` is mapping from `str` to either: # - a parsed `dict` type (e.g. `BaseModel`) - # - a "simple" type (e.g. string, integer or boolean value) + # - a "simple" type (e.g. dict, string, integer or boolean value) for key, val in d.items(): - if not isinstance(val, (BaseModel, str, int, bool)) and val is not None: + if val is not None and not isinstance(val, (BaseModel, str, int, bool, dict)): raise ValueError(f"Incorrect extension field value: {val}") return d From dca903ad994fbf531e99cd558567096d21b22ceb Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 17 Jul 2023 21:30:50 -0400 Subject: [PATCH 07/12] test: remove test skips --- tests/functional/test_uniswap_examples.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/tests/functional/test_uniswap_examples.py b/tests/functional/test_uniswap_examples.py index 5591434..b86ada8 100644 --- a/tests/functional/test_uniswap_examples.py +++ b/tests/functional/test_uniswap_examples.py @@ -22,13 +22,6 @@ def test_uniswap_tokenlists(token_list_name): token_list = requests.get(UNISWAP_RAW_URL + token_list_name).json() - if token_list_name in ( - "example-crosschain.tokenlist.json", - "extensions-valid-object.tokenlist.json", - ): - # TODO: Unskip once can handle object extensions - pytest.skip("https://github.com/ApeWorX/py-tokenlists/issues/20") - if "invalid" not in token_list_name: assert TokenList.parse_obj(token_list).dict() == token_list From 0e4d6638119094214c12aa2934600f637746ccf1 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 17 Jul 2023 23:44:16 -0400 Subject: [PATCH 08/12] style: fix type extensions issue --- tests/functional/test_uniswap_examples.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_uniswap_examples.py b/tests/functional/test_uniswap_examples.py index b86ada8..828df0a 100644 --- a/tests/functional/test_uniswap_examples.py +++ b/tests/functional/test_uniswap_examples.py @@ -2,7 +2,7 @@ import github import pytest -import requests +import requests # type: ignore[import] from pydantic import ValidationError from tokenlists import TokenList From e0a053fd87d427467a8e17ac91929e1d42fa4d03 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 17 Jul 2023 23:58:58 -0400 Subject: [PATCH 09/12] fix: there was a update reference bug, so flatten structure a bit --- tokenlists/typing.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/tokenlists/typing.py b/tokenlists/typing.py index 8570243..6c0197b 100644 --- a/tokenlists/typing.py +++ b/tokenlists/typing.py @@ -28,23 +28,12 @@ class Config: froze = True -class BridgeInfoItem(BaseModel): +class BridgeInfo(BaseModel): tokenAddress: TokenAddress originBridgeAddress: Optional[TokenAddress] = None destBridgeAddress: Optional[TokenAddress] = None -class BridgeInfo(BaseModel): - __root__: Dict[str, BridgeInfoItem] - - @validator("__root__") - def validate_keys_are_chainIds(cls, v): - for chain_id in v: - int(chain_id) - - return v - - class TokenInfo(BaseModel): chainId: ChainId address: TokenAddress @@ -79,7 +68,8 @@ def extension_depth(obj: Optional[Dict[str, Any]]) -> int: # 2. Parse valid extensions if v and "bridgeInfo" in v: - v["bridgeInfo"] = BridgeInfo.parse_obj(v.pop("bridgeInfo")) + raw_bridge_info = v.pop("bridgeInfo") + v["bridgeInfo"] = {int(k): BridgeInfo.parse_obj(v) for k, v in raw_bridge_info.items()} return v From 50a7624f6a10cc80e9deb62d454cd34eed8f5527 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Mon, 17 Jul 2023 23:59:37 -0400 Subject: [PATCH 10/12] test: skip new key that we don't want to handle yet --- tests/functional/test_uniswap_examples.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/functional/test_uniswap_examples.py b/tests/functional/test_uniswap_examples.py index 828df0a..f733daa 100644 --- a/tests/functional/test_uniswap_examples.py +++ b/tests/functional/test_uniswap_examples.py @@ -22,6 +22,11 @@ def test_uniswap_tokenlists(token_list_name): token_list = requests.get(UNISWAP_RAW_URL + token_list_name).json() + if token_list_name == "example.tokenlist.json": + # NOTE: No idea why this breaking change was necessary + # https://github.com/Uniswap/token-lists/pull/420 + token_list.pop("tokenMap") + if "invalid" not in token_list_name: assert TokenList.parse_obj(token_list).dict() == token_list From a37aed13d9b63c5472fb070f8bf542e91b0cda38 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Tue, 18 Jul 2023 00:02:38 -0400 Subject: [PATCH 11/12] chore: deprecate Python 3.7 --- .github/workflows/test.yaml | 2 +- setup.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e02a780..890ebfb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -55,7 +55,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] # eventually add `windows-latest` - python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: [3.8, 3.9, "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/setup.py b/setup.py index 2fde40e..cd5cd40 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ long_description_content_type="text/markdown", url="https://github.com/ApeWorX/py-tokenlists", include_package_data=True, - python_requires=">=3.7.2,<4", + python_requires=">=3.8,<4", install_requires=[ "importlib-metadata ; python_version<'3.8'", "click>=8.1.3,<9", @@ -89,7 +89,6 @@ "Operating System :: MacOS", "Operating System :: POSIX", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", From f8eb236a001bb58cd405fe2c6a602da06ab60680 Mon Sep 17 00:00:00 2001 From: fubuloubu <3859395+fubuloubu@users.noreply.github.com> Date: Tue, 18 Jul 2023 00:10:31 -0400 Subject: [PATCH 12/12] style: ignore typing error for the moment --- tokenlists/_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tokenlists/_cli.py b/tokenlists/_cli.py index 2ca4b3e..a642c02 100644 --- a/tokenlists/_cli.py +++ b/tokenlists/_cli.py @@ -1,3 +1,5 @@ +# TODO: Seems like Click 8.1.5 introduced this +# mypy: disable-error-code=attr-defined import re import click