diff --git a/aws_dagster_bgg/assets/assets.py b/aws_dagster_bgg/assets/assets.py index 03c5bd3..4430138 100644 --- a/aws_dagster_bgg/assets/assets.py +++ b/aws_dagster_bgg/assets/assets.py @@ -130,7 +130,7 @@ def game_dfs_clean( raw_game_files = s3_resource.list_file_keys(bucket=bucket, key=key) - assert len(raw_game_files) == 30 if ENVIRONMENT == "prod" else 1 + assert len(raw_game_files) == 29 if ENVIRONMENT == "prod" else 1 task_definition = ( "bgg_game_data_cleaner" @@ -245,7 +245,7 @@ def ratings_dfs_dirty( raw_ratings_files = s3_resource.list_file_keys(bucket=bucket, key=key) - assert len(raw_ratings_files) == 30 if ENVIRONMENT == "prod" else 1 + assert len(raw_ratings_files) == 29 if ENVIRONMENT == "prod" else 1 task_definition = ( "bgg_ratings_data_cleaner" @@ -359,7 +359,7 @@ def user_dfs_dirty( raw_ratings_files = s3_resource.list_file_keys(bucket=bucket, key=key) - assert len(raw_ratings_files) == 30 if ENVIRONMENT == "prod" else 1 + assert len(raw_ratings_files) == 29 if ENVIRONMENT == "prod" else 1 task_definition = ( "bgg_users_data_cleaner" @@ -532,7 +532,7 @@ def scrape_data( ] } ecs_resource.launch_ecs_task(task_definition, overrides) - time.sleep(3) + time.sleep(10) logger.info(f"Launched ECS for filename: {filename}") compare_timestamps_for_refresh( diff --git a/modules/lambda_functions/generate_game_urls_lambda.py b/modules/lambda_functions/generate_game_urls_lambda.py index d06c5cd..eecdf6a 100644 --- a/modules/lambda_functions/generate_game_urls_lambda.py +++ b/modules/lambda_functions/generate_game_urls_lambda.py @@ -9,7 +9,7 @@ GAME_CONFIGS = CONFIGS["games"] url_block_size = 20 -number_url_files = 30 +number_url_files = 29 def generate_raw_urls(game_ids: list[str]): diff --git a/modules/lambda_functions/generate_ratings_urls_lambda.py b/modules/lambda_functions/generate_ratings_urls_lambda.py index 1b170a3..69f6000 100644 --- a/modules/lambda_functions/generate_ratings_urls_lambda.py +++ b/modules/lambda_functions/generate_ratings_urls_lambda.py @@ -10,8 +10,8 @@ S3_SCRAPER_BUCKET = CONFIGS["s3_scraper_bucket"] RATING_CONFIGS = CONFIGS["ratings"] url_block_size = 20 -number_url_files = 30 -NUMBER_PROCESSES = 30 +number_url_files = 29 +NUMBER_PROCESSES = 29 def generate_ratings_urls(game_entries): diff --git a/modules/lambda_functions/generate_user_urls_lambda.py b/modules/lambda_functions/generate_user_urls_lambda.py index bf724b3..c707578 100644 --- a/modules/lambda_functions/generate_user_urls_lambda.py +++ b/modules/lambda_functions/generate_user_urls_lambda.py @@ -10,8 +10,8 @@ S3_SCRAPER_BUCKET = CONFIGS["s3_scraper_bucket"] USER_CONFIGS = CONFIGS["users"] url_block_size = 20 -number_url_files = 30 -NUMBER_PROCESSES = 30 +number_url_files = 29 +NUMBER_PROCESSES = 29 def generate_user_urls(user_ids): @@ -42,7 +42,7 @@ def lambda_handler(event, context): print(type(user_ids)) print(f"\nNumber of user ids: {len(user_ids)}\n") - total_url_files = 30 + total_url_files = 29 total_users = len(user_ids) urls_per_file = total_users // total_url_files