Josef final

.dockerignore

@@ -0,0 +1,4 @@
+Volumes
+*.egg-info
+.venv
+__pycache__

.env

@@ -0,0 +1,33 @@
+# Green Light Super User
+ADMIN_USERNAME=admin
+ADMIN_PASSWORD=adm1n 
+
+# ML Model API
+MODEL_PATH=/models
+MODEL_FILENAME="trained_model.joblib"
+MODEL_API_PORTS="8001:8000"
+
+# Road Accidents DB (Postgresql)
+ROAD_ACCIDENTS_POSTGRES_DB=RoadAccidents
+ROAD_ACCIDENTS_POSTGRES_PORT=5432
+ROAD_ACCIDENTS_POSTGRES_HOST=postgres_road_accidents
+# Raw Road Accidents CSV files upload path in Docker
+ROAD_ACCIDENTS_DATA_DIRECTORIES=/road_accidents_data_directories/
+# The name of the Airflow variable used between DAGs to indicate the time new Road Accidents data added to the db.
+AIRFLOW_NEW_DATA_IN_ROAD_ACCIDENTS_DB_VARNAME="new_road_accident_data_in_db_timestamp"
+
+# Road Accidents DB PgAdmin
+[email protected]
+PGADMIN_DEFAULT_PASSWORD=admin
+
+# Airflow
+AIRFLOW_UID=501
+AIRFLOW_GID=0
+AIRFLOW_PROJ_DIR=./Volumes/airflow
+
+# for Testing purposes 
+
+DOCKERIZED=True
+
+# TODO 
+# Add volume dir names

.github/workflows/build-and-push-docker-images.yml

@@ -0,0 +1,74 @@
+name: Continuous Delivery when a PR is merged to master
+
+on:
+    pull_request:
+        types:
+        - closed
+
+permissions:
+  contents: read
+
+jobs:
+    RoadAccidentsDatabaseIngestion:
+      if: github.event.pull_request.merged == true
+      runs-on: ubuntu-latest
+
+      steps:
+        - uses: actions/checkout@v3
+
+        - name: Login to the Docker Hub
+          uses: docker/login-action@v1
+          with:
+            username: roadaccidentsmlops24
+            password: ${{ secrets.DOCKER_HUB_PASS }}
+
+        - name: Build and push the road_accidents_database_ingestion Docker image to the roadaccidentsmlops24 DockerHub
+          uses: docker/build-push-action@v2
+          with:
+            context: ./python-packages/
+            file: ./python-packages/airflowdb.Dockerfile
+            push: true
+            tags: roadaccidentsmlops24/airflowdb:latest, roadaccidentsmlops24/airflowdb:${{ github.run_number }}
+
+    ModelApi:
+      if: github.event.pull_request.merged == true
+      runs-on: ubuntu-latest
+
+      steps:
+        - uses: actions/checkout@v3
+
+        - name: Login to the Docker Hub
+          uses: docker/login-action@v1
+          with:
+            username: roadaccidentsmlops24
+            password: ${{ secrets.DOCKER_HUB_PASS }}
+
+        - name: Build and push the model_api Docker image to the roadaccidentsmlops24 DockerHub
+          uses: docker/build-push-action@v2
+          with:
+            context: ./python-packages/model_api/
+            push: true
+            tags: roadaccidentsmlops24/model_api:latest, roadaccidentsmlops24/model_api:${{ github.run_number }}
+            dockerfile: ./python-packages/model_api/Dockerfile
+
+
+    GreenLightUi:
+      if: github.event.pull_request.merged == true
+      runs-on: ubuntu-latest
+
+      steps:
+        - uses: actions/checkout@v3
+
+        - name: Login to the Docker Hub
+          uses: docker/login-action@v1
+          with:
+            username: roadaccidentsmlops24
+            password: ${{ secrets.DOCKER_HUB_PASS }}
+
+        - name: Build and push the green_light_ui Docker image to the roadaccidentsmlops24 DockerHub
+          uses: docker/build-push-action@v2
+          with:
+            context: ./python-packages/green_light_ui/
+            push: true
+            tags: roadaccidentsmlops24/accidents_ui:latest, roadaccidentsmlops24/accidents_ui:${{ github.run_number }}
+            dockerfile: ./python-packages/green_light_ui/Dockerfile

.github/workflows/deploy-docker-images-on-new-tag.yml

@@ -0,0 +1,84 @@
+name: Build and deploy Docker images to Hub on new Tag creation
+
+on:
+  push:
+    tags:
+      - 'v*.*.*'
+jobs:
+  DeployRoadAccidentsDatabaseIngestionDockerImage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set env
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+
+      - name: GetTagVersion
+        run: |
+          echo $RELEASE_VERSION
+          echo ${{ env.RELEASE_VERSION }}
+
+      - name: Login to the Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: roadaccidentsmlops24
+          password: ${{ secrets.DOCKER_HUB_PASS }}
+
+      - name: Build and push the road_accidents_database_ingestion Docker image to the roadaccidentsmlops24 DockerHub
+        uses: docker/build-push-action@v2
+        with:
+          context: ./python-packages/
+          file: ./python-packages/airflowdb.Dockerfile
+          push: true
+          tags: roadaccidentsmlops24/airflowdb:latest, roadaccidentsmlops24/airflowdb:${{ env.RELEASE_VERSION }}
+
+  DeployModelApiDockerImage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set env
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+
+      - name: GetTagVersion
+        run: |
+          echo $RELEASE_VERSION
+          echo ${{ env.RELEASE_VERSION }}
+
+      - name: Login to the Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: roadaccidentsmlops24
+          password: ${{ secrets.DOCKER_HUB_PASS }}
+
+      - name: Build and push the model_api Docker image to the roadaccidentsmlops24 DockerHub
+        uses: docker/build-push-action@v2
+        with:
+          context: ./python-packages/model_api/
+          push: true
+          tags: roadaccidentsmlops24/model_api:latest, roadaccidentsmlops24/model_api:${{ env.RELEASE_VERSION }}
+          dockerfile: ./python-packages/model_api/Dockerfile
+
+  DeployGreenLightUiImage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set env
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+
+      - name: GetTagVersion
+        run: |
+          echo $RELEASE_VERSION
+          echo ${{ env.RELEASE_VERSION }}
+
+      - name: Login to the Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: roadaccidentsmlops24
+          password: ${{ secrets.DOCKER_HUB_PASS }}
+
+      - name: Build and push the green_light_ui Docker image to the roadaccidentsmlops24 DockerHub
+        uses: docker/build-push-action@v2
+        with:
+          context: ./python-packages/green_light_ui/
+          push: true
+          tags: roadaccidentsmlops24/accidents_ui:latest, roadaccidentsmlops24/accidents_ui:${{ env.RELEASE_VERSION }}
+          dockerfile: ./python-packages/green_light_ui/Dockerfile      

.github/workflows/python-app.yml

@@ -1,40 +1,129 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
-'''
-name: Python application
+
+# https://www.youtube.com/watch?v=R8_veQiYBjI&ab_channel=TechWorldwithNana
+
+name: Continuous Integration
 
 on:
   push:
     branches: [ "master" ]
   pull_request:
     branches: [ "master" ]
 
+# DOCKER_HUB_PASS
+
 permissions:
   contents: read
 
 jobs:
-  build:
+  RoadAccidentsDatabaseIngestion:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.12"
+
+      - name: Update pip
+        run: |
+          python -m pip install --upgrade pip
+
+      - name: Install road_accidents_database_ingestion package 
+        run: |
+          python -m pip install -e python-packages/road_accidents_database_ingestion
+
+      - name: Test road_accidents_database_ingestion package
+        run: |
+          pytest python-packages/road_accidents_database_ingestion/tests
+
+      - name: Login to the Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: roadaccidentsmlops24
+          password: ${{ secrets.DOCKER_HUB_PASS }}
 
+      - name: Build the road_accidents_database_ingestion Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./python-packages/
+          file: ./python-packages/airflowdb.Dockerfile
+          push: false
+          tags: roadaccidentsmlops24/airflowdb:latest, roadaccidentsmlops24/airflowdb:${{ github.run_number }}
+
+  ModelApi:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v3
-      with:
-        python-version: "3.10"
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test with pytest
-      run: |
-        pytest
-'''
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.12"
+
+      - name: Update pip
+        run: |
+          python -m pip install --upgrade pip
+
+      - name: Install model_api package 
+        run: |
+          python -m pip install -e python-packages/model_api
+
+      - name: Test model_api package
+        run: |
+          pytest python-packages/model_api/tests
+
+      - name: Login to the Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: roadaccidentsmlops24
+          password: ${{ secrets.DOCKER_HUB_PASS }}
+
+      - name: Build the model_api Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./python-packages/model_api/
+          push: false
+          tags: roadaccidentsmlops24/model_api:latest, roadaccidentsmlops24/model_api:${{ github.run_number }}
+          dockerfile: ./python-packages/model_api/Dockerfile
+
+  GreenLightUi:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.12"
+
+      - name: Update pip
+        run: |
+          python -m pip install --upgrade pip
+
+      - name: Install green_light_ui package 
+        run: |
+          python -m pip install -e python-packages/green_light_ui
+
+      - name: Test green_light_ui package
+        run: |
+          pytest python-packages/green_light_ui/tests
+
+      - name: Login to the Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: roadaccidentsmlops24
+          password: ${{ secrets.DOCKER_HUB_PASS }}
+
+      - name: Build the green_light_ui Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: ./python-packages/green_light_ui/
+          push: false
+          tags: roadaccidentsmlops24/accidents_ui:latest, roadaccidentsmlops24/accidents_ui:${{ github.run_number }}
+          dockerfile: ./python-packages/green_light_ui/Dockerfile