update terraform makefile to clean env file for AWS

.github/workflows/dev_deployment.yml → .github/temp_off/dev_deployment_ecs_scraper.yml

@@ -38,12 +38,3 @@ jobs:
         run: |
           DOCKER_BUILDKIT=1 docker build -f Dockerfiles/Dockerfile.game-data-scraper --build-arg GROUP="group1" -t $ECR_REGISTRY/$ECR_REPOSITORY .
           docker push $ECR_REGISTRY/$ECR_REPOSITORY
-          
-      - name: Build, tag, push image to AWS ECR boardgamegeek_cleaner
-        env:
-          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
-          ECR_REPOSITORY: boardgamegeek_cleaner_dev
-        run: |
-          DOCKER_BUILDKIT=1 docker build -f Dockerfiles/Dockerfile.game-data-cleaner --build-arg GROUP="group1" -t $ECR_REGISTRY/$ECR_REPOSITORY .
-          docker push $ECR_REGISTRY/$ECR_REPOSITORY
-      

.github/temp_off/dev_eployment_ecs_cleaner.yml

@@ -0,0 +1,41 @@
+name: DEV deployment to boardgamegeek cleaner
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+permissions:
+  id-token: write # This is required for requesting the JWT
+  contents: read  # This is required for actions/checkout
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_GITHUB_ROLE }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build, tag, push image to AWS ECR boardgamegeek_cleaner
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: boardgamegeek_cleaner_dev
+        run: |
+          DOCKER_BUILDKIT=1 docker build -f Dockerfiles/Dockerfile.game-data-cleaner --build-arg GROUP="group1" -t $ECR_REGISTRY/$ECR_REPOSITORY .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY
+      

.github/temp_off/prod_deployment_ecs_bgg_file_retrieval.yml

@@ -0,0 +1,35 @@
+name: PROD deployment to bgg file retrieval
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  id-token: write # This is required for requesting the JWT
+  contents: read  # This is required for actions/checkout
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_GITHUB_ROLE }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build, tag, push image to AWS ECR bgg_boardgame_file_retrieval
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: bgg_boardgame_file_retrieval
+        run: |
+          DOCKER_BUILDKIT=1 docker build -f Dockerfiles/Dockerfile.bgg_boardgame_file_retrieval -t $ECR_REGISTRY/$ECR_REPOSITORY .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY

.github/temp_off/prod_deployment_ecs_cleaner.yml

@@ -0,0 +1,32 @@
+name: PROD deployment to boardgamegeek cleaner
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  id-token: write # This is required for requesting the JWT
+  contents: read  # This is required for actions/checkout
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_GITHUB_ROLE }}
+          aws-region: ${{ secrets.AWS_REGION }}
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+      - name: Build, tag, push image to AWS ECR boardgamegeek_cleaner
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: boardgamegeek_cleaner
+        run: |
+          DOCKER_BUILDKIT=1 docker build -f Dockerfiles/Dockerfile.game-data-cleaner --build-arg GROUP="group1" -t $ECR_REGISTRY/$ECR_REPOSITORY .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY

.github/temp_off/prod_deployment_ecs_scraper.yml

@@ -0,0 +1,35 @@
+name: PROD deployment to boardgamegeek scraper
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  id-token: write # This is required for requesting the JWT
+  contents: read  # This is required for actions/checkout
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_GITHUB_ROLE }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build, tag, push image to AWS ECR boardgamegeek_scraper
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: boardgamegeek_scraper
+        run: |
+          DOCKER_BUILDKIT=1 docker build -f Dockerfiles/Dockerfile.game-data-scraper --build-arg GROUP="group1" -t $ECR_REGISTRY/$ECR_REPOSITORY .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY

.github/workflows/prod_deployment_ecs_orchestrator.yml

@@ -0,0 +1,36 @@
+name: PROD deployment to boardgamegeek orchestrator
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  id-token: write # This is required for requesting the JWT
+  contents: read  # This is required for actions/checkout
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.AWS_GITHUB_ROLE }}
+          aws-region: ${{ secrets.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build, tag, push image to AWS ECR boardgamegeek_cleaner
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: bgg_orchestrator
+        run: |
+          DOCKER_BUILDKIT=1 docker build -f Dockerfiles/Dockerfile.bgg_orchestrator --build-arg ENV="prod" --build-arg ASSET="all" -t $ECR_REGISTRY/$ECR_REPOSITORY .
+          docker push $ECR_REGISTRY/$ECR_REPOSITORY
+          

Dockerfiles/Dockerfile.bgg_orchestrator

@@ -10,23 +10,23 @@ RUN apt-get update && apt-get install -y \
 RUN pip3 install pipenv
 
 # Copy the source code into the container
-# COPY data game_data_scraper/data
 COPY aws_dagster_bgg aws_dagster_bgg
-COPY utils utils
 COPY aws_dagster_bgg/Pipfile* .
+COPY utils utils
 COPY config.py .
-COPY dagster_cloud.yaml .
-COPY aws_dagster_bgg/pyproject.toml .
+COPY pyproject.toml .
 
 # Install dependencies with pipenv
 RUN pipenv sync
 
+# Set environment variables with ARG for build-time and ENV for runtime
 ARG ENV
 ENV ENV=$ENV
 
 ARG ASSET
 ENV ASSET=$ASSET
 
+# Expose the port (if needed for the service)
 EXPOSE 3000
 
 # Set the entry point and command

aws_dagster_bgg/__init__.py

@@ -29,9 +29,9 @@
         "s3_resource": S3Resource(
             region_name=REGION,
         ),
-        "dynamodb_resource": DynamoDBResource(
-            region_name=REGION, table_name="boardgamegeek"
-        ),
+        # "dynamodb_resource": DynamoDBResource(
+        #     region_name=REGION, table_name="boardgamegeek"
+        # ),
         "lambda_resource": LambdaHandlerResource(region_name=REGION),
         "ecs_resource": ECSResource(region_name=REGION),
         "config_resource": ConfigResource(