allow scraper to handle a single filename on AWS

threnjen · Nov 24, 2024 · b4d509e · b4d509e
1 parent 6560cff
commit b4d509e
Showing 1 changed file with 15 additions and 4 deletions.
diff --git a/modules/lambda_functions/bgg_scraper_fargate_trigger.py b/modules/lambda_functions/bgg_scraper_fargate_trigger.py
@@ -47,9 +47,16 @@ def lambda_handler(event, context):
 
     print(terraform_state_file["outputs"])
 
-    file_prefixes = get_s3_keys_based_on_env(
-        directory=f'{CONFIGS[scraper_type]["raw_urls_directory"]}'
-    )
+    if not event.get("file_name"):
+        file_prefixes = get_s3_keys_based_on_env(
+            directory=f'{CONFIGS[scraper_type]["raw_urls_directory"]}'
+        )
+
+    else:
+        file_name = event.get("file_name")
+        file_prefixes = [
+            f"{WORKING_DIR}{CONFIGS[scraper_type]['raw_urls_directory']}{file_name}"
+        ]
 
     task_definition = (
         f"dev_{SCRAPER_TASK_DEFINITION}"
@@ -108,6 +115,10 @@ def lambda_handler(event, context):
 if __name__ == "__main__":
     scraper_type = sys.argv[1]
 
-    event = {"scraper_type": scraper_type}
+    try:
+        file_name = sys.argv[2]
+        event = {"scraper_type": scraper_type, "file_name": file_name}
+    except IndexError:
+        event = {"scraper_type": scraper_type, "file_name": None}
 
     lambda_handler(event, None)