attempts to reduce memory usage

threnjen · Dec 17, 2024 · 13ef7ee · 13ef7ee
1 parent 4ee5b57
commit 13ef7ee
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/aws_terraform_bgg/fargate_task_defs.tf b/aws_terraform_bgg/fargate_task_defs.tf
@@ -18,7 +18,7 @@ module "rag_description_generation_ecs" {
   execution_role_arn     = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.rag_description_generation}_FargateExecutionRole"
   image                  = "${data.aws_caller_identity.current.account_id}.dkr.ecr.${var.REGION}.amazonaws.com/${var.rag_description_generation}:latest"
   cpu                    = "1024"
-  memory                 = "4096"
+  memory                 = "8192"
   region                 = var.REGION
 }
 
@@ -33,7 +33,7 @@ module "dev_rag_description_generation_ecs" {
   execution_role_arn     = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/${var.rag_description_generation}_FargateExecutionRole"
   image                  = "${data.aws_caller_identity.current.account_id}.dkr.ecr.${var.REGION}.amazonaws.com/dev_${var.rag_description_generation}:latest"
   cpu                    = "1024"
-  memory                 = "4096"
+  memory                 = "8192"
   region                 = var.REGION
 }
 

diff --git a/modules/rag_description_generation/main.py b/modules/rag_description_generation/main.py
@@ -79,6 +79,7 @@ def load_reduced_game_df(self):
 
         del game_df
         gc.collect()
+        print("Loaded and refined games data")
 
         return game_df_reduced
 
@@ -88,9 +89,10 @@ def merge_game_df_with_ratings_df(self, game_df_reduced):
             path=RATINGS_CONFIGS["dirty_dfs_directory"],
             file_name="ratings_data.pkl",
         )
-        ratings_df["BGGId"] = ratings_df["BGGId"].astype("string")
         ratings_df = ratings_df[["username", "BGGId", "rating"]]
 
+        print("Loaded user ratings data")
+
         print(
             f"Reducing user ratings to only include games in the reduced game dataframe\n"
         )
@@ -100,6 +102,8 @@ def merge_game_df_with_ratings_df(self, game_df_reduced):
             how="inner",
         )
 
+        all_games_df["BGGId"] = all_games_df["BGGId"].astype(str)
+
         del game_df_reduced
         del ratings_df
         gc.collect()