From f3cdb7d447f1f883bfb9d81a168569c2a7b10c2e Mon Sep 17 00:00:00 2001 From: Alex Bednarek Date: Thu, 27 Jun 2024 19:26:38 -0400 Subject: [PATCH] Ingester: Fixing imports, calling store_latest after default store method, adding --latest-table to cli. --- ingester/datalake_ingester/storage.py | 2 +- ingester/tests/test_ingester.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ingester/datalake_ingester/storage.py b/ingester/datalake_ingester/storage.py index 27e5dc5..ab49e6a 100644 --- a/ingester/datalake_ingester/storage.py +++ b/ingester/datalake_ingester/storage.py @@ -80,7 +80,7 @@ def store_latest(self, record): } if record['metadata']['work_id'] is None: - work_id_value = {'BOOL': False} + work_id_value = {'NULL': True} else: work_id_value = {'S': str(record['metadata']['work_id'])} diff --git a/ingester/tests/test_ingester.py b/ingester/tests/test_ingester.py index 82e6090..3b7ccfe 100644 --- a/ingester/tests/test_ingester.py +++ b/ingester/tests/test_ingester.py @@ -39,7 +39,6 @@ def test_ingest_random_latest(storage, dynamodb_latest_table, random_s3_file_mak ingester = Ingester(storage) ingester.ingest(url) records = [dict(r) for r in dynamodb_latest_table.scan()] - def convert_records(records): return {k: (decimal.Decimal(str(v)) if isinstance(v, (int, float)) else v) for k, v in records[0].items()}