LpSugar: optimize pagination.

contracts/LpSugar.vy

@@ -7,14 +7,15 @@
 
 # Structs
 
-MAX_FACTORIES: constant(uint256) = 10
-MAX_POOLS: constant(uint256) = 2000
-MAX_TOKENS: constant(uint256) = 2000
-MAX_LPS: constant(uint256) = 500
-MAX_EPOCHS: constant(uint256) = 200
-MAX_REWARDS: constant(uint256) = 50
-MAX_POSITIONS: constant(uint256) = 200
-WEEK: constant(uint256) = 7 * 24 * 60 * 60
+MAX_FACTORIES: public(constant(uint256)) = 10
+MAX_POOLS: public(constant(uint256)) = 2000
+MAX_ITERATIONS: public(constant(uint256)) = 8000
+MAX_TOKENS: public(constant(uint256)) = 2000
+MAX_LPS: public(constant(uint256)) = 500
+MAX_EPOCHS: public(constant(uint256)) = 200
+MAX_REWARDS: public(constant(uint256)) = 50
+MAX_POSITIONS: public(constant(uint256)) = 200
+WEEK: public(constant(uint256)) = 7 * 24 * 60 * 60
 
 # Slot0 from CLPool.sol
 struct Slot:
@@ -290,8 +291,8 @@ def _pools(_limit: uint256, _offset: uint256)\
   factories: DynArray[address, MAX_FACTORIES] = self.registry.poolFactories()
   factories_count: uint256 = len(factories)
 
-  placeholder: address[4] = empty(address[4])
   to_skip: uint256 = _offset
+  visited: uint256 = 0
 
   pools: DynArray[address[4], MAX_POOLS] = \
     empty(DynArray[address[4], MAX_POOLS])
@@ -304,18 +305,19 @@ def _pools(_limit: uint256, _offset: uint256)\
     pools_count: uint256 = factory.allPoolsLength()
     nfpm: address = self._fetch_nfpm(factory.address)
 
-    for pindex in range(0, MAX_POOLS):
-      if pindex >= pools_count or len(pools) >= _limit + _offset:
+    for pindex in range(0, MAX_ITERATIONS):
+      if pindex >= pools_count or visited >= _limit + _offset or len(pools) >= MAX_POOLS:
         break
 
       # Since the convertor pool, first pool on one of the factories...
       if pindex == 0 and factory.allPools(0) == self.convertor:
         continue
 
+      visited += 1
+
       # Basically skip calls for offset records...
       if to_skip > 0:
         to_skip -= 1
-        pools.append(placeholder)
         continue
 
       pool_addr: address = factory.allPools(pindex)
@@ -349,8 +351,8 @@ def forSwaps(_limit: uint256, _offset: uint256) -> DynArray[SwapLp, MAX_POOLS]:
     nfpm: address = self._fetch_nfpm(factory.address)
     pools_count: uint256 = factory.allPoolsLength()
 
-    for pindex in range(0, MAX_POOLS):
-      if pindex >= pools_count:
+    for pindex in range(0, MAX_ITERATIONS):
+      if pindex >= pools_count or len(pools) >= MAX_POOLS:
         break
 
       # If no pools to process are left...
@@ -419,7 +421,7 @@ def tokens(_limit: uint256, _offset: uint256, _account: address, \
     col.append(self._token(_addresses[index], _account))
     seen.append(_addresses[index])
 
-  for index in range(_offset, _offset + MAX_TOKENS):
+  for index in range(0, MAX_POOLS):
     if len(col) >= _limit or index >= pools_count:
       break
 
@@ -469,7 +471,7 @@ def all(_limit: uint256, _offset: uint256) -> DynArray[Lp, MAX_LPS]:
   pools: DynArray[address[4], MAX_POOLS] = self._pools(_limit, _offset)
   pools_count: uint256 = len(pools)
 
-  for index in range(_offset, _offset + MAX_POOLS):
+  for index in range(0, MAX_POOLS):
     if len(col) == _limit or index >= pools_count:
       break
 
@@ -496,7 +498,7 @@ def byIndex(_index: uint256) -> Lp:
   """
   # Basically index is the limit and the offset is always one...
   # This will fire if _index is out of bounds
-  pool_data: address[4] = self._pools(1, _index)[_index]
+  pool_data: address[4] = self._pools(1, _index)[0]
   pool: IPool = IPool(pool_data[1])
   token0: address = pool.token0()
   token1: address = pool.token1()
@@ -668,7 +670,7 @@ def _positions(
     if nfpm.address == empty(address):
       pools_count: uint256 = factory.allPoolsLength()
 
-      for pindex in range(0, MAX_POOLS):
+      for pindex in range(0, MAX_ITERATIONS):
         if pindex >= pools_count or pools_done >= _limit:
           break
 
@@ -785,7 +787,7 @@ def _positions(
         positions.append(pos)
 
       # fetch staked CL positions
-      for pindex in range(0, MAX_POOLS):
+      for pindex in range(0, MAX_ITERATIONS):
         if pindex >= pools_count or pools_done >= _limit:
           break
 
@@ -1055,7 +1057,7 @@ def epochsLatest(_limit: uint256, _offset: uint256) \
 
   col: DynArray[LpEpoch, MAX_POOLS] = empty(DynArray[LpEpoch, MAX_POOLS])
 
-  for index in range(_offset, _offset + MAX_POOLS):
+  for index in range(0, MAX_ITERATIONS):
     if counted == _limit or index >= pools_count:
       break
 
@@ -1218,7 +1220,7 @@ def rewards(_limit: uint256, _offset: uint256, _venft_id: uint256) \
 
   col: DynArray[Reward, MAX_POOLS] = empty(DynArray[Reward, MAX_POOLS])
 
-  for pindex in range(_offset, _offset + MAX_POOLS):
+  for pindex in range(0, MAX_POOLS):
     if counted == _limit or pindex >= pools_count:
       break
 

tests/test_lp_sugar.py

@@ -115,6 +115,21 @@ def test_tokens(sugar_contract, TokenStruct, LpStruct):
     assert token2.token_address == second_lp.token0
 
 
+def test_tokens_large_limit(sugar_contract, TokenStruct):
+    many_tokens = list(map(
+        lambda _p: TokenStruct(*_p),
+        sugar_contract.tokens(2000, 1000, ADDRESS_ZERO, [])
+    ))
+
+    assert many_tokens is not None
+    assert len(many_tokens) > 100
+
+    token0 = many_tokens[0]
+
+    assert token0.symbol is not None
+    assert token0.decimals > 0
+
+
 def test_all(sugar_contract, LpStruct):
     first_lp = LpStruct(*sugar_contract.byIndex(0))
     second_lp = LpStruct(*sugar_contract.byIndex(1))
@@ -135,6 +150,16 @@ def test_all(sugar_contract, LpStruct):
     assert lp2.gauge == second_lp.gauge
 
 
+def test_all_pagination(sugar_contract, LpStruct):
+    max_lps = sugar_contract.MAX_LPS()
+
+    for i in range(0, max_lps, max_lps):
+        lps = sugar_contract.all(max_lps, 0)
+
+        assert lps is not None
+        assert len(lps) > max_lps - 1
+
+
 def test_all_limit_offset(sugar_contract, LpStruct):
     second_lp = LpStruct(*sugar_contract.byIndex(1))
     lps = list(map(