Merge pull request #3 from DataScientest-Studio/mike0.9

Fully integrated local dev env and actions build-push workflow

.env.example

@@ -1,7 +1,160 @@
 
-# get a token from https://www.themoviedb.org/settings/api
-TMDB_API_TOKEN=
+TMDB_API_TOKEN=get_a_key_from_https://www.themoviedb.org/settings/api
 
-SUPABASE_URL="http://localhost:8000"
-SUPABASE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE
+SUPABASE_URL="http://kong:8000"
+MLFLOW_TRACKING_URI="http://tracking_server:5000"
+
+##############################################################################################
+###################################### SUPABASE - BEGIN ######################################
+##############################################################################################
+
+############
+# Secrets
+# YOU MUST CHANGE THESE BEFORE GOING INTO PRODUCTION
+############
+
+POSTGRES_PASSWORD=your-super-secret-and-long-postgres-password
+JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long
+ANON_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJhbm9uIiwKICAgICJpc3MiOiAic3VwYWJhc2UtZGVtbyIsCiAgICAiaWF0IjogMTY0MTc2OTIwMCwKICAgICJleHAiOiAxNzk5NTM1NjAwCn0.dc_X5iR_VP_qT0zsiyj_I_OZ2T9FtRU2BBNWN8Bu4GE
 SERVICE_ROLE_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyAgCiAgICAicm9sZSI6ICJzZXJ2aWNlX3JvbGUiLAogICAgImlzcyI6ICJzdXBhYmFzZS1kZW1vIiwKICAgICJpYXQiOiAxNjQxNzY5MjAwLAogICAgImV4cCI6IDE3OTk1MzU2MDAKfQ.DaYlNEoUrrEn2Ig7tqibS-PHK5vgusbcbo7X36XVt4Q
+DASHBOARD_USERNAME=supabase
+DASHBOARD_PASSWORD=this_password_is_insecure_and_should_be_updated
+
+############
+# Database - You can change these to any PostgreSQL database that has logical replication enabled.
+############
+
+POSTGRES_HOST=db
+POSTGRES_DB=postgres
+POSTGRES_PORT=5432
+# default user is postgres
+
+############
+# Supavisor -- Database pooler
+############
+POOLER_PROXY_PORT_TRANSACTION=6543
+POOLER_DEFAULT_POOL_SIZE=20
+POOLER_MAX_CLIENT_CONN=100
+POOLER_TENANT_ID=your-tenant-id
+
+
+############
+# API Proxy - Configuration for the Kong Reverse proxy.
+############
+
+KONG_HTTP_PORT=8000
+KONG_HTTPS_PORT=8443
+
+
+############
+# API - Configuration for PostgREST.
+############
+
+PGRST_DB_SCHEMAS=public,storage,graphql_public
+
+
+############
+# Auth - Configuration for the GoTrue authentication server.
+############
+
+## General
+SITE_URL=http://localhost:3000
+ADDITIONAL_REDIRECT_URLS=
+JWT_EXPIRY=3600
+DISABLE_SIGNUP=false
+API_EXTERNAL_URL=http://localhost:8000
+
+## Mailer Config
+MAILER_URLPATHS_CONFIRMATION="/auth/v1/verify"
+MAILER_URLPATHS_INVITE="/auth/v1/verify"
+MAILER_URLPATHS_RECOVERY="/auth/v1/verify"
+MAILER_URLPATHS_EMAIL_CHANGE="/auth/v1/verify"
+
+## Email auth
+ENABLE_EMAIL_SIGNUP=true
+ENABLE_EMAIL_AUTOCONFIRM=true
+SMTP_ADMIN_EMAIL=[email protected]
+SMTP_HOST=supabase-mail
+SMTP_PORT=2500
+SMTP_USER=fake_mail_user
+SMTP_PASS=fake_mail_password
+SMTP_SENDER_NAME=fake_sender
+ENABLE_ANONYMOUS_USERS=false
+
+## Phone auth
+ENABLE_PHONE_SIGNUP=true
+ENABLE_PHONE_AUTOCONFIRM=true
+
+
+############
+# Studio - Configuration for the Dashboard
+############
+
+STUDIO_DEFAULT_ORGANIZATION=Default Organization
+STUDIO_DEFAULT_PROJECT=Default Project
+
+STUDIO_PORT=3000
+# replace if you intend to use Studio outside of localhost
+SUPABASE_PUBLIC_URL=http://localhost:8000
+
+# Enable webp support
+IMGPROXY_ENABLE_WEBP_DETECTION=true
+
+############
+# Functions - Configuration for Functions
+############
+# NOTE: VERIFY_JWT applies to all functions. Per-function VERIFY_JWT is not supported yet.
+FUNCTIONS_VERIFY_JWT=false
+
+############
+# Logs - Configuration for Logflare
+# Please refer to https://supabase.com/docs/reference/self-hosting-analytics/introduction
+############
+
+LOGFLARE_LOGGER_BACKEND_API_KEY=your-super-secret-and-long-logflare-key
+
+# Change vector.toml sinks to reflect this change
+LOGFLARE_API_KEY=your-super-secret-and-long-logflare-key
+
+# Docker socket location - this value will differ depending on your OS
+DOCKER_SOCKET_LOCATION=/var/run/docker.sock
+
+# Google Cloud Project details
+GOOGLE_PROJECT_ID=GOOGLE_PROJECT_ID
+GOOGLE_PROJECT_NUMBER=GOOGLE_PROJECT_NUMBER
+
+###################################### SUPABASE - END ######################################
+
+############################################################################################
+###################################### MLFLOW - BEGIN ######################################
+############################################################################################
+
+# Postgres configuration
+PG_USER=mlflow
+PG_PASSWORD=mlflow
+PG_DATABASE=mlflow
+PG_PORT=5432
+
+# MLflow configuration
+MLFLOW_PORT=5001
+MLFLOW_BUCKET_NAME=mlflow
+
+# MinIO access keys - these are needed by MLflow
+MINIO_ACCESS_KEY=oD63bXddzg54JmLq7G5e
+MINIO_SECRET_ACCESS_KEY=LoWHkKcwnnUdyTrP4RruAHEycPXFqi3Hc76S6mqT
+
+# MinIO configuration
+MINIO_ROOT_USER=minio_user
+MINIO_ROOT_PASSWORD=minio_pwd
+MINIO_ADDRESS=:9000
+MINIO_STORAGE_USE_HTTPS=false
+MINIO_CONSOLE_ADDRESS=:9001
+MINIO_PORT=9000
+MINIO_CONSOLE_PORT=9001
+
+###################################### MLFLOW - END ######################################
+
+############################################################################################
+###################################### AIRFLOW - BEGIN ######################################
+############################################################################################
+

.github/workflows/build-and-push.yml

@@ -0,0 +1,47 @@
+name: Build and Push Docker Images
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build and push API image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./app/Dockerfile.apipredict
+          push: true
+          tags: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/api-predict:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/api-predict:${{ github.sha }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/api-predict:${{ github.ref_name }}
+
+      - name: Build and push Streamlit image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./app
+          file: ./app/Dockerfile.streamlit
+          push: true
+          tags: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/streamlit:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/streamlit:${{ github.sha }}
+            ${{ secrets.DOCKERHUB_USERNAME }}/streamlit:${{ github.ref_name }}

.github/workflows/test-api.yml

@@ -0,0 +1,49 @@
+name: Test API Predict
+
+on:
+  push:
+    branches: ["master", "mike0.9"]
+  pull_request:
+    branches: ["master"]
+
+jobs:
+  test-api:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.12"
+
+      - name: Install test dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest requests
+
+      - name: Create Docker network
+        run: docker network create backend
+
+      - name: Build and start API container
+        run: |
+          docker compose -f app/docker-compose.yml up -d api-predict --build
+
+      - name: Wait for API to be ready
+        run: |
+          timeout 60s bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:8002/model_info)" != "200" ]]; do sleep 1; done' || false
+
+      - name: Run API tests
+        run: |
+          pytest tests/test_api_predict.py -v
+
+      - name: Docker logs on failure
+        if: failure()
+        run: docker compose -f app/docker-compose.yml logs api-predict
+
+      - name: Stop containers
+        if: always()
+        run: |
+          docker compose -f app/docker-compose.yml down
+          docker network rm backend

.gitignore

@@ -104,3 +104,14 @@ trash/
 
 # pytest cache
 .pytest_cache/
+
+# MinIO
+minio_data/
+
+# mlflow db
+db_data/
+
+notebooks/
+*.ipynb
+
+todo.md