From bdd962d52f6b37f0262b0038803895341dfe5f7b Mon Sep 17 00:00:00 2001 From: Mikko Ohtamaa Date: Wed, 20 Sep 2023 13:53:00 +0200 Subject: [PATCH] Workaound LlamaNodes.com issues (#152) * Deal with custom LllamaNode replies --- CHANGELOG.md | 4 +++ eth_defi/chain.py | 33 +++++++++++++------ .../event_reader/reorganisation_monitor.py | 5 +-- eth_defi/provider/ankr.py | 7 +++- eth_defi/provider/llamanodes.py | 23 +++++++++++++ tests/provider/test_llamarpc.py | 19 +++++++++++ 6 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 eth_defi/provider/llamanodes.py create mode 100644 tests/provider/test_llamarpc.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b43f205c..f6121a60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 0.22.11 + +- Add `eth_defi.provider.llamanodes` and work around issues with LlamaNodes.com + # 0.22.10 - Move Ankr specific functionality to its own `eth_defi.provider.ankr` module diff --git a/eth_defi/chain.py b/eth_defi/chain.py index 66e3d328..78239812 100644 --- a/eth_defi/chain.py +++ b/eth_defi/chain.py @@ -19,6 +19,7 @@ from eth_defi.event_reader.conversion import convert_jsonrpc_value_to_int from eth_defi.middleware import http_retry_request_with_sleep_middleware +from eth_defi.provider.llamanodes import is_llama_bad_grapql_reply from eth_defi.provider.named import NamedProvider #: List of chain ids that need to have proof-of-authority middleweare installed @@ -161,6 +162,26 @@ def middleware(method: RPCEndpoint, params: Any) -> Optional[RPCResponse]: return api_counter +def get_graphql_url(provider: BaseProvider) -> str: + """Resolve potential GraphQL endpoint API for a JSON-RPC provider. + + See :py:func:`has_graphql_support`. + """ + + # See BaseNamedProvider + if hasattr(provider, "call_endpoint_uri"): + base_url = provider.call_endpoint_uri + elif hasattr(provider, "endpoint_uri"): + # HTTPProvider + base_url = provider.endpoint_uri + else: + raise AssertionError(f"Do not know how to extract endpoint URI: {provider}") + + graphql_url = urljoin(base_url, "graphql") + + return graphql_url + + def has_graphql_support(provider: BaseProvider) -> bool: """Check if a node has GoEthereum GraphQL API turned on. @@ -179,19 +200,11 @@ def has_graphql_support(provider: BaseProvider) -> bool: {"data":{"block":{"number":16328259}}} """ - # See BaseNamedProvider - if hasattr(provider, "call_endpoint_uri"): - base_url = provider.call_endpoint_uri - elif hasattr(provider, "endpoint_uri"): - # HTTPProvider - base_url = provider.endpoint_uri - else: - raise AssertionError(f"Do not know how to extract endpoint URI: {provider}") + graphql_url = get_graphql_url(provider) - graphql_url = urljoin(base_url, "graphql") try: resp = requests.get(graphql_url) - return resp.status_code == 400 + return resp.status_code == 400 and not is_llama_bad_grapql_reply(resp) except Exception as e: # ConnectionError, etc. return False diff --git a/eth_defi/event_reader/reorganisation_monitor.py b/eth_defi/event_reader/reorganisation_monitor.py index 78df85c8..e8c4fd4c 100644 --- a/eth_defi/event_reader/reorganisation_monitor.py +++ b/eth_defi/event_reader/reorganisation_monitor.py @@ -16,7 +16,7 @@ from tqdm import tqdm from web3 import Web3, HTTPProvider -from eth_defi.chain import has_graphql_support +from eth_defi.chain import has_graphql_support, get_graphql_url from eth_defi.event_reader.block_header import BlockHeader, Timestamp from eth_defi.provider.fallback import FallbackProvider from eth_defi.provider.mev_blocker import MEVBlockerProvider @@ -737,7 +737,8 @@ def create_reorganisation_monitor(web3: Web3, check_depth=250) -> Reorganisation if has_graphql_support(provider): # 10x faster /graphql implementation, # not provided by public nodes - reorg_mon = GraphQLReorganisationMonitor(graphql_url=urljoin(json_rpc_url, "/graphql"), check_depth=check_depth) + graphql_url = get_graphql_url(provider) + reorg_mon = GraphQLReorganisationMonitor(graphql_url=graphql_url, check_depth=check_depth) else: # Default slow implementation logger.warning("The node does not support /graphql interface. " "Downloading block headers and timestamps will be extremely slow." "Check documentation how to configure your node or choose a smaller timeframe for the buffer of trades.") diff --git a/eth_defi/provider/ankr.py b/eth_defi/provider/ankr.py index 0ec4dd90..68c067e2 100644 --- a/eth_defi/provider/ankr.py +++ b/eth_defi/provider/ankr.py @@ -1,8 +1,13 @@ """Ankr specific Web3.py functionality. - Ankr has issues with some JSON-RPC access patterns. - See also :py:mod:`eth_defi.provider.broken_provider`. +.. warning :: + + We do not recommend using Ankr as it randomly returns empty responses to eth_call RPC method + and this is not fixable at the client side. + +See also :py:mod:`eth_defi.provider.broken_provider`. """ from web3 import Web3 diff --git a/eth_defi/provider/llamanodes.py b/eth_defi/provider/llamanodes.py new file mode 100644 index 00000000..d74b0c5e --- /dev/null +++ b/eth_defi/provider/llamanodes.py @@ -0,0 +1,23 @@ +"""Fixes for quirks and features on LlamaNodes. + +- `LlamaNodes `__ runs RPC services at ``llamarpc.com`` + +- Their RPC nodes have some compatibility issues we address in this module + +See also :py:mod:`eth_defi.provider.broken_provider`. +""" +from requests import Response + + +def is_llama_bad_grapql_reply(resp: Response): + """Is the web server response fake 404 response from llamarpc.com + + llamarpc.com web server does not know how to use HTTP 404 status code. + + See :py:func:`eth_defi.chain.has_graphql_support`. + """ + try: + content = resp.json() + return content.get("error").get("message") == "UserKey was not a ULID or UUID" + except Exception: + return False diff --git a/tests/provider/test_llamarpc.py b/tests/provider/test_llamarpc.py new file mode 100644 index 00000000..ccc3b06e --- /dev/null +++ b/tests/provider/test_llamarpc.py @@ -0,0 +1,19 @@ +"""LlamaNodes specific tests""" + +import os + +import pytest +from web3 import Web3, HTTPProvider + +from eth_defi.chain import has_graphql_support + +JSON_RPC_LLAMA = os.environ.get("JSON_RPC_LLAMA") +pytestmark = pytest.mark.skipif(not JSON_RPC_LLAMA, reason="This test needs LlamaNodes come node via JSON_RPC_LLAMA") + + +def test_llama_is_bad(): + """Work around fake 404 response from llamarpc.com""" + provider = HTTPProvider(JSON_RPC_LLAMA) + web3 = Web3(provider) + assert not has_graphql_support(provider) + assert web3.eth.block_number > 1