rename files

.github/workflows/dev_deployment_ecs_scraper.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Build, tag, push image to AWS ECR game_data_scraper
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: dev_boardgamegeek_scraper
+          ECR_REPOSITORY: dev_bgg_scraper
         run: |
           DOCKER_BUILDKIT=1 docker build -f Dockerfiles/Dockerfile.game-data-scraper --build-arg GROUP="group1" -t $ECR_REGISTRY/$ECR_REPOSITORY .
           docker push $ECR_REGISTRY/$ECR_REPOSITORY

.github/workflows/dev_eployment_ecs_cleaner.yml

@@ -31,10 +31,10 @@ jobs:
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
 
-      - name: Build, tag, push image to AWS ECR boardgamegeek_cleaner
+      - name: Build, tag, push image to AWS ECR bgg_cleaner
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: dev_boardgamegeek_cleaner
+          ECR_REPOSITORY: dev_bgg_cleaner
         run: |
           DOCKER_BUILDKIT=1 docker build -f Dockerfiles/Dockerfile.game-data-cleaner --build-arg GROUP="group1" -t $ECR_REGISTRY/$ECR_REPOSITORY .
           docker push $ECR_REGISTRY/$ECR_REPOSITORY

.github/workflows/prod_deployment_ecs_cleaner.yml

@@ -23,10 +23,10 @@ jobs:
       - name: Login to Amazon ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
-      - name: Build, tag, push image to AWS ECR boardgamegeek_cleaner
+      - name: Build, tag, push image to AWS ECR bgg_cleaner
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: boardgamegeek_cleaner
+          ECR_REPOSITORY: bgg_cleaner
         run: |
           DOCKER_BUILDKIT=1 docker build -f Dockerfiles/Dockerfile.game-data-cleaner --build-arg GROUP="group1" -t $ECR_REGISTRY/$ECR_REPOSITORY .
           docker push $ECR_REGISTRY/$ECR_REPOSITORY

.github/workflows/prod_deployment_ecs_orchestrator.yml

@@ -26,7 +26,7 @@ jobs:
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
 
-      - name: Build, tag, push image to AWS ECR boardgamegeek_cleaner
+      - name: Build, tag, push image to AWS ECR bgg_cleaner
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           ECR_REPOSITORY: bgg_orchestrator

.github/workflows/prod_deployment_ecs_scraper.yml

@@ -26,10 +26,10 @@ jobs:
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
 
-      - name: Build, tag, push image to AWS ECR boardgamegeek_scraper
+      - name: Build, tag, push image to AWS ECR bgg_scraper
         env:
           ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: boardgamegeek_scraper
+          ECR_REPOSITORY: bgg_scraper
         run: |
           DOCKER_BUILDKIT=1 docker build -f Dockerfiles/Dockerfile.game-data-scraper --build-arg GROUP="group1" -t $ECR_REGISTRY/$ECR_REPOSITORY .
           docker push $ECR_REGISTRY/$ECR_REPOSITORY

Pipfile

@@ -17,6 +17,7 @@ awswrangler = "*"
 xmltodict = "*"
 dagster = "*"
 dagster-webserver = "*"
+tzlocal = "*"
 
 [dev-packages]
 pytest = "*"

aws_dagster_bgg/assets/assets.py

@@ -117,7 +117,7 @@ def game_dfs_dirty(
 
     assert len(raw_game_files) == 30
 
-    ecs_resource.launch_ecs_task(task_definition="boardgamegeek_cleaner")
+    ecs_resource.launch_ecs_task(task_definition="bgg_cleaner")
 
     data_set_file_names = [
         f"{configs['games']['dirty_dfs_directory']}/{x}_dirty.pkl" for x in data_sets

aws_terraform_bgg/elastic_container_registry.tf

@@ -5,10 +5,10 @@ locals {
   ecr_repositories = [
     module.bgg_boardgame_file_retrieval_ecr.ecr_repository_name,
     module.bgg_orchestrator_ecr.ecr_repository_name,
-    module.boardgamegeek_cleaner_ecr.ecr_repository_name,
-    module.dev_boardgamegeek_cleaner_ecr.ecr_repository_name,
-    module.boardgamegeek_scraper_ecr.ecr_repository_name,
-    module.dev_boardgamegeek_scraper_ecr.ecr_repository_name
+    module.bgg_cleaner_ecr.ecr_repository_name,
+    module.dev_bgg_cleaner_ecr.ecr_repository_name,
+    module.bgg_scraper_ecr.ecr_repository_name,
+    module.dev_bgg_scraper_ecr.ecr_repository_name
   ]
 }
 
@@ -22,22 +22,22 @@ module "bgg_orchestrator_ecr" {
   ecr_repository_name = "bgg_orchestrator"
 }
 
-module "boardgamegeek_cleaner_ecr" {
+module "bgg_cleaner_ecr" {
   source              = "./modules/ecr"
-  ecr_repository_name = "boardgamegeek_cleaner"
+  ecr_repository_name = "bgg_cleaner"
 }
 
-module "dev_boardgamegeek_cleaner_ecr" {
+module "dev_bgg_cleaner_ecr" {
   source              = "./modules/ecr"
-  ecr_repository_name = "dev_boardgamegeek_cleaner"
+  ecr_repository_name = "dev_bgg_cleaner"
 }
 
-module "boardgamegeek_scraper_ecr" {
+module "bgg_scraper_ecr" {
   source              = "./modules/ecr"
-  ecr_repository_name = "boardgamegeek_scraper"
+  ecr_repository_name = "bgg_scraper"
 }
 
-module "dev_boardgamegeek_scraper_ecr" {
+module "dev_bgg_scraper_ecr" {
   source              = "./modules/ecr"
-  ecr_repository_name = "dev_boardgamegeek_scraper"
+  ecr_repository_name = "dev_bgg_scraper"
 }

aws_terraform_bgg/iam_ecs_role.tf

@@ -11,7 +11,7 @@ module "bgg_orchestrator_FargateExecutionRole_role" {
 
 resource "aws_iam_role_policy_attachment" "S3_Access_bgg_orchestrator_FargateExecutionRole_attach" {
   role       = module.bgg_orchestrator_FargateExecutionRole_role.name
-  policy_arn = aws_iam_policy.S3_Access_boardgamegeek_scraper_policy.arn
+  policy_arn = aws_iam_policy.S3_Access_bgg_scraper_policy.arn
 }
 
 module "bgg_orchestrator_FargateTaskRole_role" {
@@ -21,59 +21,59 @@ module "bgg_orchestrator_FargateTaskRole_role" {
 
 resource "aws_iam_role_policy_attachment" "S3_Access_bgg_orchestrator_FargateTaskRole_attach" {
   role       = module.bgg_orchestrator_FargateTaskRole_role.name
-  policy_arn = aws_iam_policy.S3_Access_boardgamegeek_scraper_policy.arn
+  policy_arn = aws_iam_policy.S3_Access_bgg_scraper_policy.arn
 }
 
 resource "aws_iam_role_policy_attachment" "Cloudwatch_Put_Metrics_bgg_orchestrator_FargateTaskRole_roleattach" {
   role       = module.bgg_orchestrator_FargateTaskRole_role.name
   policy_arn = aws_iam_policy.Cloudwatch_Put_Metrics_policy.arn
 }
 
-module "boardgamegeek_cleaner_FargateExecutionRole_role" {
+module "bgg_cleaner_FargateExecutionRole_role" {
   source          = "./modules/iam_ecs_roles"
-  task_definition = "boardgamegeek_cleaner_FargateExecutionRole"
+  task_definition = "bgg_cleaner_FargateExecutionRole"
 }
 
-resource "aws_iam_role_policy_attachment" "S3_Access_boardgamegeek_cleaner_FargateExecutionRole_attach" {
-  role       = module.boardgamegeek_cleaner_FargateExecutionRole_role.name
-  policy_arn = aws_iam_policy.S3_Access_boardgamegeek_scraper_policy.arn
+resource "aws_iam_role_policy_attachment" "S3_Access_bgg_cleaner_FargateExecutionRole_attach" {
+  role       = module.bgg_cleaner_FargateExecutionRole_role.name
+  policy_arn = aws_iam_policy.S3_Access_bgg_scraper_policy.arn
 }
 
-module "boardgamegeek_cleaner_FargateTaskRole_role" {
+module "bgg_cleaner_FargateTaskRole_role" {
   source          = "./modules/iam_ecs_roles"
-  task_definition = "boardgamegeek_cleaner_FargateTaskRole"
+  task_definition = "bgg_cleaner_FargateTaskRole"
 }
 
-resource "aws_iam_role_policy_attachment" "S3_Access_boardgamegeekboardgamegeek_cleaner_FargateTaskRoleattach" {
-  role       = module.boardgamegeek_cleaner_FargateTaskRole_role.name
-  policy_arn = aws_iam_policy.S3_Access_boardgamegeek_scraper_policy.arn
+resource "aws_iam_role_policy_attachment" "S3_Access_boardgamegeekbgg_cleaner_FargateTaskRoleattach" {
+  role       = module.bgg_cleaner_FargateTaskRole_role.name
+  policy_arn = aws_iam_policy.S3_Access_bgg_scraper_policy.arn
 }
 
-resource "aws_iam_role_policy_attachment" "Cloudwatch_Put_Metricsboardgamegeek_cleaner_FargateTaskRoleattach" {
-  role       = module.boardgamegeek_cleaner_FargateTaskRole_role.name
+resource "aws_iam_role_policy_attachment" "Cloudwatch_Put_Metricsbgg_cleaner_FargateTaskRoleattach" {
+  role       = module.bgg_cleaner_FargateTaskRole_role.name
   policy_arn = aws_iam_policy.Cloudwatch_Put_Metrics_policy.arn
 }
 
-module "boardgamegeek_scraper_FargateExecutionRole_role" {
+module "bgg_scraper_FargateExecutionRole_role" {
   source          = "./modules/iam_ecs_roles"
-  task_definition = "boardgamegeek_scraper_FargateExecutionRole"
+  task_definition = "bgg_scraper_FargateExecutionRole"
 }
-resource "aws_iam_role_policy_attachment" "S3_Access_boardgamegeek_scraperboardgamegeek_scraper_FargateExecutionRole" {
-  role       = module.boardgamegeek_scraper_FargateExecutionRole_role.name
-  policy_arn = aws_iam_policy.S3_Access_boardgamegeek_scraper_policy.arn
+resource "aws_iam_role_policy_attachment" "S3_Access_bgg_scraperbgg_scraper_FargateExecutionRole" {
+  role       = module.bgg_scraper_FargateExecutionRole_role.name
+  policy_arn = aws_iam_policy.S3_Access_bgg_scraper_policy.arn
 }
 
-module "boardgamegeek_scraper_FargateTaskRole_role" {
+module "bgg_scraper_FargateTaskRole_role" {
   source          = "./modules/iam_ecs_roles"
-  task_definition = "boardgamegeek_scraper_FargateTaskRole"
+  task_definition = "bgg_scraper_FargateTaskRole"
 }
-resource "aws_iam_role_policy_attachment" "S3_Access_boardgamegeekboardgamegeek_scraper_FargateTaskRoleattach" {
-  role       = module.boardgamegeek_scraper_FargateTaskRole_role.name
-  policy_arn = aws_iam_policy.S3_Access_boardgamegeek_scraper_policy.arn
+resource "aws_iam_role_policy_attachment" "S3_Access_boardgamegeekbgg_scraper_FargateTaskRoleattach" {
+  role       = module.bgg_scraper_FargateTaskRole_role.name
+  policy_arn = aws_iam_policy.S3_Access_bgg_scraper_policy.arn
 }
 
-resource "aws_iam_role_policy_attachment" "Cloudwatch_Put_Metric_boardgamegeek_scraper_FargateTaskRoleattach" {
-  role       = module.boardgamegeek_scraper_FargateTaskRole_role.name
+resource "aws_iam_role_policy_attachment" "Cloudwatch_Put_Metric_bgg_scraper_FargateTaskRoleattach" {
+  role       = module.bgg_scraper_FargateTaskRole_role.name
   policy_arn = aws_iam_policy.Cloudwatch_Put_Metrics_policy.arn
 }
 

aws_terraform_bgg/iam_lambda_role.tf

@@ -5,7 +5,7 @@ module "bgg_boardgame_file_retrieval_role" {
 
 resource "aws_iam_role_policy_attachment" "bgg_boardgame_file_retrieval_attach" {
   role       = module.bgg_boardgame_file_retrieval_role.role_name
-  policy_arn = aws_iam_policy.S3_Access_boardgamegeek_scraper_policy.arn
+  policy_arn = aws_iam_policy.S3_Access_bgg_scraper_policy.arn
 }
 
 module "bgg_generate_game_urls_lambda_role" {
@@ -14,7 +14,7 @@ module "bgg_generate_game_urls_lambda_role" {
 }
 resource "aws_iam_role_policy_attachment" "bgg_generate_game_urls_lambda_role" {
   role       = module.bgg_generate_game_urls_lambda_role.role_name
-  policy_arn = aws_iam_policy.S3_Access_boardgamegeek_scraper_policy.arn
+  policy_arn = aws_iam_policy.S3_Access_bgg_scraper_policy.arn
 }
 
 module "bgg_generate_user_urls_lambda_role" {
@@ -24,37 +24,37 @@ module "bgg_generate_user_urls_lambda_role" {
 
 resource "aws_iam_role_policy_attachment" "bgg_generate_user_urls_lambda_attach" {
   role       = module.bgg_generate_user_urls_lambda_role.role_name
-  policy_arn = aws_iam_policy.S3_Access_boardgamegeek_scraper_policy.arn
+  policy_arn = aws_iam_policy.S3_Access_bgg_scraper_policy.arn
 }
 
-module "boardgamegeek_scraper_fargate_trigger_role" {
+module "bgg_scraper_fargate_trigger_role" {
   source    = "./modules/iam_lambda_roles"
-  role_name = "boardgamegeek_scraper_fargate_trigger_role"
+  role_name = "bgg_scraper_fargate_trigger_role"
 }
 
-resource "aws_iam_role_policy_attachment" "boardgamegeek_scraper_describe_attach" {
-  role       = module.boardgamegeek_scraper_fargate_trigger_role.role_name
+resource "aws_iam_role_policy_attachment" "bgg_scraper_describe_attach" {
+  role       = module.bgg_scraper_fargate_trigger_role.role_name
   policy_arn = module.bgg_scraper_describe_task_def_policy.lambda_ecs_trigger_arn
 }
 
-resource "aws_iam_role_policy_attachment" "boardgamegeek_scraper_S3_attach" {
-  role       = module.boardgamegeek_scraper_fargate_trigger_role.role_name
-  policy_arn = aws_iam_policy.S3_Access_boardgamegeek_scraper_policy.arn
+resource "aws_iam_role_policy_attachment" "bgg_scraper_S3_attach" {
+  role       = module.bgg_scraper_fargate_trigger_role.role_name
+  policy_arn = aws_iam_policy.S3_Access_bgg_scraper_policy.arn
 }
 
 module "bgg_cleaner_fargate_trigger_role" {
   source    = "./modules/iam_lambda_roles"
   role_name = "bgg_cleaner_fargate_trigger_role"
 }
 
-resource "aws_iam_role_policy_attachment" "boardgamegeek_cleaner_describe_attach" {
+resource "aws_iam_role_policy_attachment" "bgg_cleaner_describe_attach" {
   role       = module.bgg_cleaner_fargate_trigger_role.role_name
   policy_arn = module.bgg_cleaner_describe_task_def_policy.lambda_ecs_trigger_arn
 }
 
-resource "aws_iam_role_policy_attachment" "boardgamegeek_cleaner_s3_attach" {
+resource "aws_iam_role_policy_attachment" "bgg_cleaner_s3_attach" {
   role       = module.bgg_cleaner_fargate_trigger_role.role_name
-  policy_arn = aws_iam_policy.S3_Access_boardgamegeek_scraper_policy.arn
+  policy_arn = aws_iam_policy.S3_Access_bgg_scraper_policy.arn
 }
 
 module "bgg_orchestrator_fargate_trigger_role" {
@@ -69,6 +69,6 @@ resource "aws_iam_role_policy_attachment" "bgg_orchestrator_describe_attach" {
 
 resource "aws_iam_role_policy_attachment" "bgg_orchestrator_s3_attach" {
   role       = module.bgg_orchestrator_fargate_trigger_role.role_name
-  policy_arn = aws_iam_policy.S3_Access_boardgamegeek_scraper_policy.arn
+  policy_arn = aws_iam_policy.S3_Access_bgg_scraper_policy.arn
 }