redo to 29 URLS instead of 30

threnjen · Nov 28, 2024 · c425398 · c425398
1 parent a1e2425
commit c425398
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 10 deletions.
diff --git a/aws_dagster_bgg/assets/assets.py b/aws_dagster_bgg/assets/assets.py
@@ -130,7 +130,7 @@ def game_dfs_clean(
 
     raw_game_files = s3_resource.list_file_keys(bucket=bucket, key=key)
 
-    assert len(raw_game_files) == 30 if ENVIRONMENT == "prod" else 1
+    assert len(raw_game_files) == 29 if ENVIRONMENT == "prod" else 1
 
     task_definition = (
         "bgg_game_data_cleaner"
@@ -245,7 +245,7 @@ def ratings_dfs_dirty(
 
     raw_ratings_files = s3_resource.list_file_keys(bucket=bucket, key=key)
 
-    assert len(raw_ratings_files) == 30 if ENVIRONMENT == "prod" else 1
+    assert len(raw_ratings_files) == 29 if ENVIRONMENT == "prod" else 1
 
     task_definition = (
         "bgg_ratings_data_cleaner"
@@ -359,7 +359,7 @@ def user_dfs_dirty(
 
     raw_ratings_files = s3_resource.list_file_keys(bucket=bucket, key=key)
 
-    assert len(raw_ratings_files) == 30 if ENVIRONMENT == "prod" else 1
+    assert len(raw_ratings_files) == 29 if ENVIRONMENT == "prod" else 1
 
     task_definition = (
         "bgg_users_data_cleaner"
@@ -532,7 +532,7 @@ def scrape_data(
             ]
         }
         ecs_resource.launch_ecs_task(task_definition, overrides)
-        time.sleep(3)
+        time.sleep(10)
         logger.info(f"Launched ECS for filename: {filename}")
 
     compare_timestamps_for_refresh(

diff --git a/modules/lambda_functions/generate_game_urls_lambda.py b/modules/lambda_functions/generate_game_urls_lambda.py
@@ -9,7 +9,7 @@
 GAME_CONFIGS = CONFIGS["games"]
 
 url_block_size = 20
-number_url_files = 30
+number_url_files = 29
 
 
 def generate_raw_urls(game_ids: list[str]):

diff --git a/modules/lambda_functions/generate_ratings_urls_lambda.py b/modules/lambda_functions/generate_ratings_urls_lambda.py
@@ -10,8 +10,8 @@
 S3_SCRAPER_BUCKET = CONFIGS["s3_scraper_bucket"]
 RATING_CONFIGS = CONFIGS["ratings"]
 url_block_size = 20
-number_url_files = 30
-NUMBER_PROCESSES = 30
+number_url_files = 29
+NUMBER_PROCESSES = 29
 
 
 def generate_ratings_urls(game_entries):

diff --git a/modules/lambda_functions/generate_user_urls_lambda.py b/modules/lambda_functions/generate_user_urls_lambda.py
@@ -10,8 +10,8 @@
 S3_SCRAPER_BUCKET = CONFIGS["s3_scraper_bucket"]
 USER_CONFIGS = CONFIGS["users"]
 url_block_size = 20
-number_url_files = 30
-NUMBER_PROCESSES = 30
+number_url_files = 29
+NUMBER_PROCESSES = 29
 
 
 def generate_user_urls(user_ids):
@@ -42,7 +42,7 @@ def lambda_handler(event, context):
     print(type(user_ids))
     print(f"\nNumber of user ids: {len(user_ids)}\n")
 
-    total_url_files = 30
+    total_url_files = 29
     total_users = len(user_ids)
     urls_per_file = total_users // total_url_files