QUERIER: Configuring env variables to utilize latest table query in t…

…ests + logging.
planetlabs · Jul 26, 2024 · 327514d · 327514d
1 parent b06d770
commit 327514d
Show file tree

Hide file tree

Showing 4 changed files with 29 additions and 5 deletions.
diff --git a/api/datalake_api/querier.py b/api/datalake_api/querier.py
@@ -20,6 +20,11 @@
 import time
 import os
 
+import logging
+logging.basicConfig(level=logging.INFO)
+log = logging.getLogger(__name__)
+log.setLevel(logging.INFO)
+
 
 '''the maximum number of results to return to the user
 
@@ -179,7 +184,7 @@ def __init__(self, table_name, latest_table_name=None, dynamodb=None):
         self.latest_table_name = latest_table_name
         self.dynamodb = dynamodb
         self.use_latest_table = os.environ.get("DATALAKE_USE_LATEST_TABLE",
-                                               False)
+                                               "false").lower() == "true"
 
     def query_by_work_id(self, work_id, what, where=None, cursor=None):
         kwargs = self._prepare_work_id_kwargs(work_id, what)
@@ -341,7 +346,9 @@ def _latest_table(self):
         return self.dynamodb.Table(self.latest_table_name)
 
     def query_latest(self, what, where, lookback_days=DEFAULT_LOOKBACK_DAYS):
+        log.info('Inside query_latest method')
         if self.use_latest_table:
+            log.info('inside use_latest_table=TRUE')
             response = self._latest_table.query(
                 KeyConditionExpression=Key('what_where_key').eq(f'{what}:{where}')
             )

diff --git a/api/datalake_api/settings.py b/api/datalake_api/settings.py
@@ -14,6 +14,7 @@
 
 # default settings
 DYNAMODB_TABLE = 'test'
+DYNAMODB_LATEST_TABLE = 'test_latest'
 AWS_REGION = 'us-west-2'
 AWS_ACCESS_KEY_ID = None
 AWS_SECRET_ACCESS_KEY = None

diff --git a/api/datalake_api/v0.py b/api/datalake_api/v0.py
@@ -50,7 +50,9 @@ def get_dynamodb():
 def get_archive_querier():
     if not hasattr(app, 'archive_querier'):
         table_name = app.config.get('DYNAMODB_TABLE')
+        latest_table_name = app.config.get('DYNAMODB_LATEST_TABLE')
         app.archive_querier = ArchiveQuerier(table_name,
+                                             latest_table_name,
                                              dynamodb=get_dynamodb())
     return app.archive_querier
 

diff --git a/api/tests/test_archive_querier.py b/api/tests/test_archive_querier.py
@@ -123,10 +123,24 @@ def query_latest(self, what, where):
         return HttpRecord(**record)
 
 
-@pytest.fixture(params=[ArchiveQuerier, HttpQuerier],
-                ids=['archive_querier', 'http'])
-def querier(request, dynamodb):
-        return request.param('test', 'test_latest', dynamodb=dynamodb)
+@pytest.fixture(params=[
+    ('archive', 'use_latest'),
+    ('archive', 'use_default'),
+    ('http', 'use_latest'),
+    ('http', 'use_default')
+], ids=['archive_latest', 'archive-default', 'http-latest', 'http-default'])
+def querier(monkeypatch, request, dynamodb):
+        querier_type, table_usage = request.param
+
+        if table_usage == 'use_latest':
+            monkeypatch.setenv('DATALAKE_USE_LATEST_TABLE', 'true')
+        else:
+            monkeypatch.setenv('DATALAKE_USE_LATEST_TABLE', 'false')
+
+        if querier_type == 'http':
+            return HttpQuerier('test', 'test_latest', dynamodb=dynamodb)
+        else:
+            return ArchiveQuerier('test', 'test_latest', dynamodb=dynamodb)
 
 def in_url(result, part):
     url = result['url']