set_cache_size_limit()

HISTORY.rst

@@ -2,6 +2,11 @@
 History
 =======
 
+0.7.0 (2023-09-18)
+------------------
+
+* let user specify max cache size via `lakeapi.set_cache_size_limit()`
+
 0.6.4 (2023-08-05)
 ------------------
 

lakeapi/__init__.py

@@ -4,4 +4,4 @@
 __email__ = "[email protected]"
 __version__ = "__version__ = '0.6.4'"
 
-from .main import load_data, list_data, available_symbols, set_default_bucket, use_sample_data, cache # noqa
+from .main import load_data, list_data, available_symbols, set_default_bucket, use_sample_data, cache, set_cache_size_limit # noqa

lakeapi/_cache.py

@@ -5,19 +5,17 @@
 '''
 from typing import Any, Callable
 import joblib
-from pathlib import Path
 
-bytes_limit: int = 10_000_000_000
+default_bytes_limit: int = 10_000_000_000
 verbose_cache = 0
 
 _store: joblib.Memory = joblib.Memory(
 	'.lake_cache',
 	compress = 0,
-	bytes_limit = bytes_limit,
+	bytes_limit = default_bytes_limit,
 	verbose = verbose_cache,
 )
 cached: Callable[..., Callable[..., Any]] = _store.cache
-_store.reduce_size()
 
 if __name__ == '__main__':
 	import time

lakeapi/main.py

@@ -15,10 +15,11 @@
 from botocache.botocache import botocache_context
 
 import lakeapi._read_parquet
+import lakeapi._cache
 
 DataType = Literal["book", "book_delta", "trades", "trades_mpid", "candles", "level_1", "funding", "open_interest", "liquiditions"]
 
-cache = FSLRUCache(ttl=8 * 60 * 60, path=".lake_cache/boto", maxsize=1000)
+cache = FSLRUCache(ttl=8 * 60 * 60, path=".lake_cache/boto", maxsize=10_000)
 default_bucket = 'qnt.data/market-data/cryptofeed'
 is_anonymous_access = False
 
@@ -27,6 +28,14 @@ def set_default_bucket(bucket: str) -> None:
     global default_bucket
     default_bucket = bucket
 
+def set_cache_size_limit(limit_bytes: int) -> None:
+    '''
+    Set cache size limit in bytes.
+
+    :param limit_bytes: Cache size limit in bytes.
+    '''
+    lakeapi._cache._store.bytes_limit = limit_bytes
+
 def use_sample_data(anonymous_access: bool) -> None:
     '''
     Use sample data lake configuration, which is free for testing Lake.
@@ -150,6 +159,8 @@ def partition_filter(partition: Dict[str, str]) -> bool:
         df["next_funding_time"] = pd.to_datetime(df["next_funding_time"], unit="s", cache=True)
     if table == "trades":
         df.rename(columns={"id": "trade_id"}, inplace=True)
+
+    lakeapi._cache._store.reduce_size()
     return df
 
 def list_data(
@@ -252,7 +263,7 @@ def available_symbols(
         table="book",
         start=None, #datetime.datetime.now() - datetime.timedelta(days=2),
         end=None,
-        # symbols=["FRONT-BUSD"],
+        symbols=["FTRB-USDT"],
         exchanges=None,
     )
     pd.set_option("display.width", 1000)

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+filterwarnings =
+    ignore:.*cgi.* is deprecated.*:DeprecationWarning:botocore.utils