Fix FallbackProvider to work with certain error codes (#156)

- `eth_sendRawTransaction` was not retried correctly for `nonce too low error` with LlamaNodes due to inconsistencies in error handling  n web3.py and Ethereum nodes

CHANGELOG.md

@@ -1,3 +1,10 @@
+# Current
+
+- Fix [FallbackProvider](https://web3-ethereum-defi.readthedocs.io/api/provider/_autosummary_provider/eth_defi.provider.fallback.html) to work with [certain problematic error codes](https://twitter.com/moo9000/status/1707672647264346205)
+- Log non-retryable exceptions in fallback middleware, so 
+  there is better diagnostics why fallback fails
+- Add `HotWallet.fill_in_gas_estimations()`
+
 # 0.22.14
 
 - Add `{'code': -32043, 'message': 'Requested data is not available'}` to RPC exceptions where we assume it's

eth_defi/hotwallet.py

@@ -18,6 +18,7 @@
 from web3 import Web3
 from web3.contract.contract import ContractFunction
 
+from eth_defi.gas import estimate_gas_fees, apply_gas
 from eth_defi.tx import decode_signed_transaction
 
 
@@ -178,6 +179,25 @@ def get_native_currency_balance(self, web3: Web3) -> Decimal:
         balance = web3.eth.get_balance(self.address)
         return web3.from_wei(balance, "ether")
 
+    @staticmethod
+    def fill_in_gas_price(web3: Web3, tx: dict) -> dict:
+        """Fills in the gas value fields for a transaction.
+
+        - Estimates raw transaction gas usage
+
+        - Uses web3 methods to get the gas value fields for the dict
+
+        - web3 offers different backends for this
+
+        - likely queries the values from the node
+
+        :return:
+            Transaction data (mutated) with gas values filled in.
+        """
+        price_data = estimate_gas_fees(web3)
+        apply_gas(tx, price_data)
+        return tx
+
     @staticmethod
     def from_private_key(key: str) -> "HotWallet":
         """Create a hot wallet from a private key that is passed in as a hex string.

eth_defi/middleware.py

@@ -77,14 +77,18 @@
 #: See GoEthereum error codes https://github.com/ethereum/go-ethereum/blob/master/rpc/errors.go
 #:
 DEFAULT_RETRYABLE_RPC_ERROR_CODES = (
+    # The node provider has corrupted database or something GoEthereum
+    # cannot handle gracefully.
     # ValueError: {'message': 'Internal JSON-RPC error.', 'code': -32603}
     -32603,
     # ValueError: {'code': -32000, 'message': 'nonce too low'}.
     # Might happen when we are broadcasting multiple transactions through multiple RPC providers
     # using eth_sendRawTransaction
     # One provide has not yet seeing a transaction broadcast through the other provider.
     -32000,
-
+    # ValueError: {'code': -32003, 'message': 'nonce too low'}.
+    # Anvil variant for nonce too low, same as above
+    -32003,
     # Some error we are getting from LlamaNodes eth_getLogs RPC that we do not know what it is all about
     # {'code': -32043, 'message': 'Requested data is not available'}
     -32043,

eth_defi/provider/fallback.py

@@ -105,6 +105,10 @@ def __init__(
         #: provider number -> API name -> call count mappings.
         # This tracks completed API requests.
         self.api_call_counts = defaultdict(Counter)
+
+        #: provider number-> api method name -> retry counts dict
+        self.api_retry_counts = defaultdict(Counter)
+
         self.retry_count = 0
         self.switchover_noisiness = switchover_noisiness
 
@@ -139,36 +143,49 @@ def make_request(self, method: RPCEndpoint, params: Any) -> RPCResponse:
         - If there are errors try cycle through providers and sleep
           between cycles until one provider works
         """
-
         current_sleep = self.sleep
-        for i in range(self.retries):
+        for i in range(self.retries + 1):
             provider = self.get_active_provider()
             try:
                 # Call the underlying provider
-                val = provider.make_request(method, params)
+                resp_data = provider.make_request(method, params)
+
+                # We need to manually raise the exception here,
+                # likely was raised by Web3.py itself in pre-6.0 versions.
+                # If this behavior is some legacy Web3.py behavior and not present anymore,
+                # we should replace this with a custom exception.
+                # Might be also related to EthereumTester only code paths.
+                if "error" in resp_data:
+                    # {'jsonrpc': '2.0', 'id': 23, 'error': {'code': -32003, 'message': 'nonce too low'}}
+                    # This will trigger exception that will be handled by is_retryable_http_exception()
+                    raise ValueError(resp_data["error"])
 
                 # Track API counts
                 self.api_call_counts[self.currently_active_provider][method] += 1
 
-                return val
+                return resp_data
 
             except Exception as e:
+                old_provider_name = get_provider_name(provider)
                 if is_retryable_http_exception(
                     e,
                     retryable_rpc_error_codes=self.retryable_rpc_error_codes,
                     retryable_status_codes=self.retryable_status_codes,
                     retryable_exceptions=self.retryable_exceptions,
                 ):
-                    old_provider_name = get_provider_name(provider)
                     self.switch_provider()
                     new_provider_name = get_provider_name(self.get_active_provider())
 
-                    if i < self.retries - 1:
-                        logger.log(self.switchover_noisiness, "Encountered JSON-RPC retryable error %s when calling method %s.\n" "Switching providers %s -> %s\n" "Retrying in %f seconds, retry #%d", e, method, old_provider_name, new_provider_name, current_sleep, i)
+                    if i < self.retries:
+                        logger.log(self.switchover_noisiness, "Encountered JSON-RPC retryable error %s when calling method %s.\n" "Switching providers %s -> %s\n" "Retrying in %f seconds, retry #%d / %d", e, method, old_provider_name, new_provider_name, current_sleep, i, self.retries)
                         time.sleep(current_sleep)
                         current_sleep *= self.backoff
                         self.retry_count += 1
+                        self.api_retry_counts[self.currently_active_provider][method] += 1
                         continue
                     else:
                         raise  # Out of retries
+                logger.info("Will not retry on %s, method %s, as not a retryable exception %s: %s", old_provider_name, method, e.__class__, e)
                 raise  # Not retryable exception
+
+        raise AssertionError("Should never be reached")

tests/rpc/test_fallback_provider.py

@@ -3,10 +3,14 @@
 
 import pytest
 import requests
+from eth_account import Account
 from web3 import HTTPProvider, Web3
 
 from eth_defi.anvil import launch_anvil, AnvilLaunch
+from eth_defi.gas import node_default_gas_price_strategy
+from eth_defi.hotwallet import HotWallet
 from eth_defi.provider.fallback import FallbackProvider
+from eth_defi.trace import assert_transaction_success_with_explanation
 
 
 @pytest.fixture(scope="module")
@@ -39,6 +43,18 @@ def fallback_provider(provider_1, provider_2) -> FallbackProvider:
     return provider
 
 
+@pytest.fixture()
+def web3(fallback_provider) -> Web3:
+    """Test account with built-in balance"""
+    return Web3(fallback_provider)
+
+
+@pytest.fixture()
+def deployer(web3) -> str:
+    """Test account with built-in balance"""
+    return web3.eth.accounts[0]
+
+
 def test_fallback_no_issue(anvil: AnvilLaunch, fallback_provider: FallbackProvider):
     """Callback goes through the first provider"""
     web3 = Web3(fallback_provider)
@@ -74,7 +90,7 @@ def test_fallback_double_fault(fallback_provider: FallbackProvider, provider_1,
         with pytest.raises(requests.exceptions.ConnectionError):
             web3.eth.block_number
 
-    assert fallback_provider.retry_count == 5
+    assert fallback_provider.retry_count == 6
 
 
 def test_fallback_double_fault_recovery(fallback_provider: FallbackProvider, provider_1, provider_2):
@@ -108,3 +124,47 @@ def test_fallback_unhandled_exception(fallback_provider: FallbackProvider, provi
     with patch.object(provider_1, "make_request", side_effect=RuntimeError):
         with pytest.raises(RuntimeError):
             web3.eth.block_number
+
+
+def test_fallback_nonce_too_low(web3, deployer: str):
+    """Retry nonce too low errors with eth_sendRawTransaction,
+
+    See if we can retry LlamanNodes nonce too low errors when sending multiple transactions.
+    """
+
+    web3.eth.set_gas_price_strategy(node_default_gas_price_strategy)
+
+    user = Account.create()
+    hot_wallet = HotWallet(user)
+
+    tx1_hash = web3.eth.send_transaction({"from": deployer, "to": user.address, "value": 5 * 10**18})
+    assert_transaction_success_with_explanation(web3, tx1_hash)
+
+    hot_wallet.sync_nonce(web3)
+
+    # First send a transaction with a correct nonce
+    tx2 = {"chainId": web3.eth.chain_id, "from": user.address, "to": deployer, "value": 1 * 10**18, "gas": 30_000}
+    HotWallet.fill_in_gas_price(web3, tx2)
+    signed_tx2 = hot_wallet.sign_transaction_with_new_nonce(tx2)
+    assert signed_tx2.nonce == 0
+    tx2_hash = web3.eth.send_raw_transaction(signed_tx2.rawTransaction)
+    assert_transaction_success_with_explanation(web3, tx2_hash)
+
+    fallback_provider = web3.provider
+    assert fallback_provider.api_call_counts[0]["eth_sendRawTransaction"] == 1
+    assert fallback_provider.api_retry_counts[0]["eth_sendRawTransaction"] == 0
+
+    # Then send a transaction with too low nonce.
+    # We are not interested that the transaction goes thru, only
+    # that it is retried.
+    tx3 = {"chainId": web3.eth.chain_id, "from": user.address, "to": deployer, "value": 1 * 10**18, "gas": 30_000}
+    HotWallet.fill_in_gas_price(web3, tx3)
+    hot_wallet.current_nonce = 0  # Spoof nonce
+    signed_tx3 = hot_wallet.sign_transaction_with_new_nonce(tx3)
+    assert signed_tx3.nonce == 0
+
+    with pytest.raises(ValueError):
+        # nonce too low happens during RPC call
+        tx3_hash = web3.eth.send_raw_transaction(signed_tx3.rawTransaction)
+
+    assert fallback_provider.api_retry_counts[0]["eth_sendRawTransaction"] == 3  # 5 attempts, 3 retries, the last retry does not count

tests/test_decode_tx.py

@@ -19,7 +19,7 @@
 from eth_typing import HexAddress, HexStr
 from web3 import HTTPProvider, Web3
 
-from eth_defi.anvil import fork_network_anvil
+from eth_defi.provider.anvil import fork_network_anvil
 from eth_defi.chain import install_chain_middleware
 from eth_defi.gas import node_default_gas_price_strategy
 from eth_defi.hotwallet import HotWallet