🔨 Update Pvs

Makefile

@@ -1,219 +1,114 @@
+# Define namespaces for Kubernetes
 NAMESPACE1 = api
 NAMESPACE2 = airflow
 NAMESPACE3 = mlflow
 
-.PHONY: help setup1 setup2 start stop down restart logs-supabase logs-airflow logs-api logs-fastapi clean network all namespace pv secrets configmaps deployments services ingress clean-kube-reco clean-kube-airflow apply-configmap start-minikube start-airflow pv-airflow reco start-mlflow
+# Declare phony targets that do not correspond to files
+.PHONY: help start-all start-minikube install-helm start-airflow start-mlflow start-api delete-pv-airflow check-kube change-namespace-api change-namespace-airflow change-namespace-mlflow clean-kube-api clean-kube-airflow clean-kube-mlflow clean-kube-all
 
 # Help command to list all available targets
 help:
 	@echo "Usage: make [target]"
 	@echo "Targets:"
-	@echo "  setup1      	- Setup environment, load initial data and set .env files based on .env.example"
-	@echo "  setup2      	- Build all services and load data"
-	@echo "  start      	- Start all services"
-	@echo "  stop       	- Stop all services"
-	@echo "  restart    	- Restart all services"
-	@echo "  logs-supabase  - Show logs for supabase"
-	@echo "  logs-airflow   - Show logs for airflow"
-	@echo "  logs-api       - Show logs for api"
-	@echo "  logs-fastapi   - Show logs for FastAPI"
-	@echo "  clean          - Remove all containers and networks"
-	@echo "  clean-db       - Delete all data in the database and reload the schema and data"
-	@echo "  network        - apply the Docker network 'backend'"
-
-# Setup: Setup environment, load initial data and set env files based on .env.example
-# TODO: gérer le fait que l'on ait pas les posterUrl a ce stade pour le build des features
-setup1:
-	@echo "###### SETUP ENV #########"
-	# python3 -m venv .venv
-	# source .venv/bin/activate
-	pip install -r requirements-dev.txt
-	@echo "###### DATA & MODEL ######"
-	@echo 'Chargement des données'
-	python ml/src/data/import_raw_data.py
-	@echo 'Création des features'
-	python ml/src/features/build_features.py
-	@echo 'Entrainement des modèles SVD & KNN'
-	python ml/src/models/train_model.py
-	@echo "###### ENV VARIABLES #####"
-	cd postgres && cp .env.example .env
-	cd airflow && cp .env.example .env
-	cp .env.example .env
-	@echo "##########################"
-	@echo "Set the desired env variables in the .env files (postgres/.env, airflow/.env and .env) then run 'make setup2'"
-
-# Setup: Build all services and load data
-setup2: network
-	cd postgres && docker compose up --build --no-start
-	docker compose up --build --no-start
-	cd airflow && echo "AIRFLOW_UID=$(shell id -u)" >> .env
-	cd airflow && docker compose up --build --no-start
-	@echo "##########################"
-	@echo "Run 'make start' to start the services"
-
-# Start: start all services
-start: network
-	cd postgres && docker compose up -d
-	./wait-for-it.sh 0.0.0.0:5432 --timeout=60 --strict -- echo "Database is up"
-	docker compose up -d
-	cd airflow && docker compose up -d
-	@echo "##########################"
-	@echo "Pg Admin: http://localhost:5050"
-	@echo "Airflow: http://127.0.0.1:8081"
-	@echo "Streamlit: http://localhost:8501"
-	@echo "FastAPI: http://localhost:8002/docs"
-	@echo "Grafana: http://localhost:3000"
-	@echo "MlFlow: http://localhost:5000"
-
-# Stop: stop all services
-stop:
-	docker compose stop
-	docker compose -f airflow/docker-compose.yaml stop
-	docker compose -f postgres/docker-compose.yml stop
-
-down:
-	docker compose down --volumes --remove-orphans
-	docker compose -f airflow/docker-compose.yaml down --volumes --remove-orphans
-	docker compose -f postgres/docker-compose.yml down --volumes --remove-orphans
-
-# Restart: restart all services
-restart: stop start
-
-
-logs-airflow:
-	docker compose -f airflow/docker-compose.yaml logs -f
-
-logs-api:
-	docker compose logs -f
-
-logs-fastapi:
-	docker compose logs -f fastapi
-
-# Clean: stop and remove all containers, networks, and volumes for all services
-clean:
-	cd supabase/docker && docker compose down -v
-	cd airflow && docker compose down -v
-	docker compose down -v
-	docker network rm backend || true
-	bash clean_docker.sh
-
-# Clean-db: delete all data in the database and reload the schema and data
-clean-db: network
-	cd supabase/docker && docker compose down -v
-	rm -rf supabase/docker/volumes/db/data/
-	cd supabase/docker && docker compose up -d
-	sleep 10 && python ml/src/data/load_data_in_db.py
-	@echo "##########################"
-	@echo "Run 'make start' to start all the services"
-
-# Network: apply the Docker network 'backend'
-network:
-	docker network apply backend || true
-
-
-
-###### MAKEFILE KUBERNETES
-all: namespace install-airflow pv-airflow pv secrets configmaps deployments services ingress
-
+	@echo "  start-all     - Start all services"
+	@echo "  start-minikube - Start Minikube with specified resources"
+	@echo "  install-helm   - Install Helm package manager"
+	@echo "  start-airflow  - Deploy Airflow using Helm"
+	@echo "  start-mlflow   - Deploy MLflow using Helm"
+	@echo "  start-api      - Deploy API services"
+	@echo "  delete-pv-airflow - Delete persistent volumes for Airflow"
+	@echo "  check-kube     - Verify kubectl is connected to a cluster"
+	@echo "  change-namespace-* - Change current namespace context for Kubernetes"
+	@echo "  clean-kube-*   - Clean up specific namespaces"
+
+###### MAKEFILE KUBERNETES ######
+
+# Start all services
+start-all: start-airflow start-mlflow start-api
+
+# Start Minikube with specified resources
 start-minikube:
-	minikube start --driver=docker --memory=8192 --cpus=4 --mount --mount-string= "/home/antoine/jul24_cmlops_reco_film/:/host"
+	minikube start --driver=docker --memory=8192 --cpus=4 --mount --mount-string="/home/antoine/jul24_cmlops_reco_film/:/host"
 
+# Install Helm package manager
 install-helm:
 	curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
-	chmod 700 get_helm.sh
+	chmod +x get_helm.sh
 	./get_helm.sh
 
-# Installation de helm Airflow
+# Deploy Airflow using Helm
 start-airflow:
 	sudo apt-get update
 	helm repo add apache-airflow https://airflow.apache.org
-	helm upgrade --install airflow apache-airflow/airflow --namespace airflow --create-namespace -f kubernetes/airflow/my_airflow_values.yml
+	helm upgrade --install airflow apache-airflow/airflow --namespace $(NAMESPACE2) --create-namespace -f kubernetes/airflow/my_airflow_values.yml
+
+	# Apply persistent volumes and claims for Airflow
 	kubectl apply -f kubernetes/persistent-volumes/airflow-local-dags-folder-pv.yml
-	kubectl apply -f kubernetes/persistent-volumes/airflow-local-dags-folder-pvc.yml
-	kubectl apply -f kubernetes/persistent-volumes/airflow-local-logs-folder-pv.yml
-	kubectl apply -f kubernetes/persistent-volumes/airflow-local-logs-folder-pvc.yml
 	kubectl apply -f kubernetes/persistent-volumes/mlfow-storage-pv.yml
+	kubectl apply -f kubernetes/persistent-volumes/airflow-local-logs-folder-pv.yml
+	kubectl apply -f kubernetes/persistent-volumes/airflow-local-dags-folder-pvc.yml
 	kubectl apply -f kubernetes/persistent-volumes/mlfow-storage-pvc.yml
+	kubectl apply -f kubernetes/persistent-volumes/airflow-local-logs-folder-pvc.yml
+
+	# Apply secrets and configmaps for Airflow
 	kubectl apply -f kubernetes/secrets/airflow-secrets.yaml
 	kubectl apply -f kubernetes/configmaps/airflow-configmaps.yml
+
+	# Deploy pgAdmin service for managing PostgreSQL databases
 	kubectl apply -f kubernetes/deployments/pgadmin-deployment.yml
 	kubectl apply -f kubernetes/services/pgadmin-service.yml
 
+# Deploy MLflow using Helm
 start-mlflow:
 	helm repo add bitnami https://charts.bitnami.com/bitnami
 	helm repo update
-	helm install mlf-ts bitnami/mlflow --namespace mlflow --create-namespace
+	helm install mlf-ts bitnami/mlflow --namespace $(NAMESPACE3) --create-namespace
+
 	kubectl apply -f kubernetes/services/mlflow-service.yml
 
+# Deploy API services (FastAPI and Streamlit)
 start-api:
-	kubectl create namespace api
+	kubectl create namespace $(NAMESPACE1) || true # Avoid error if namespace already exists
 	kubectl apply -f kubernetes/deployments/fastapi-deployment.yml
 	kubectl apply -f kubernetes/deployments/streamlit-deployment.yml
 	kubectl apply -f kubernetes/secrets/api-secrets.yaml
 	kubectl apply -f kubernetes/services/api-service.yml
 
+# Delete persistent volumes for Airflow (if they exist)
 delete-pv-airflow:
-	kubectl delete pv airflow-local-dags-folder || true
-	kubectl delete pv airflow-local-logs-folder || true
-
+	kubectl delete pv airflow-local-dags-folder-pv || true
+	kubectl delete pv airflow-local-logs-folder-pv || true
 
-# Vérifie si kubectl est connecté à un cluster
+# Check if kubectl is connected to a Kubernetes cluster
 check-kube:
-	@kubectl cluster-info > /dev/null 2>&1 || { echo "kubectl n'est pas connecté à un cluster"; exit 1; }
-
-reco: namespace pv secrets configmaps deployments services ingress
-
-
-namespace: check-kube
-	kubectl apply -f kubernetes/namespace/namespace.yml --validate=false
+	@kubectl cluster-info > /dev/null 2>&1 || { echo "kubectl is not connected to a cluster"; exit 1; }
 
+# Change the current namespace context for Kubernetes commands (API)
 change-namespace-api:
 	kubectl config set-context --current --namespace=$(NAMESPACE1)
 
+# Change the current namespace context for Kubernetes commands (Airflow)
 change-namespace-airflow:
 	kubectl config set-context --current --namespace=$(NAMESPACE2)
 
+# Change the current namespace context for Kubernetes commands (MLFlow)
 change-namespace-mlflow:
 	kubectl config set-context --current --namespace=$(NAMESPACE3)
 
-pv: check-kube
-	kubectl apply -f kubernetes/persistent-volumes/fastapi-persistent-volume.yml --validate=false
-	kubectl apply -f kubernetes/persistent-volumes/grafana-persistent-volume.yml --validate=false
-	kubectl apply -f kubernetes/persistent-volumes/minio-persistent-volumes.yml --validate=false
-	kubectl apply -f kubernetes/persistent-volumes/postgres-api-persistent-volumes.yml --validate=false
-	kubectl apply -f kubernetes/persistent-volumes/prometheus-persistent-volume.yml --validate=false
-	kubectl apply -f kubernetes/persistent-volumes/pgadmin-persistent-volume.yml --validate=false
-
-secrets: check-kube
-	kubectl apply -f kubernetes/secrets/secrets.yml --validate=false
-
-configmaps: check-kube
-	kubectl apply -f kubernetes/configmaps/configmaps.yml --validate=false
-
-deployments: check-kube
-	kubectl apply -f kubernetes/deployments/postgres-api-deployment.yml --validate=false
-	kubectl apply -f kubernetes/deployments/postgres-mlflow-deployment.yml --validate=false
-	kubectl apply -f kubernetes/deployments/mlflow-deployment.yml --validate=false
-	kubectl apply -f kubernetes/deployments/fastapi-deployment.yml --validate=false
-	kubectl apply -f kubernetes/deployments/streamlit-deployment.yml --validate=false
-	kubectl apply -f kubernetes/deployments/prometheus-deployment.yml --validate=false
-	kubectl apply -f kubernetes/deployments/grafana-deployment.yml --validate=false
-	kubectl apply -f kubernetes/deployments/node-exporter-deployment.yml --validate=false
-	kubectl apply -f kubernetes/deployments/minio-deployment.yml --validate=false
-	kubectl apply -f kubernetes/deployments/postgres-exporter-deployment.yml --validate=false
-	kubectl apply -f kubernetes/deployments/pgadmin-deployment.yml --validate=false
-
-services: check-kube
-	kubectl apply -f kubernetes/services/services.yml
-
-ingress: check-kube
-	kubectl apply -f kubernetes/ingress/ingress.yml
-
-clean-kube-reco: check-kube
-	kubectl delete namespace $(NAMESPACE1)
+# Clean up specific namespaces in Kubernetes (API)
+clean-kube-api: check-kube
+	kubectl delete namespace $(NAMESPACE1) || true
 
+# Clean up specific namespaces in Kubernetes (Airflow)
 clean-kube-airflow: check-kube
-	kubectl delete namespace $(NAMESPACE2)
+	kubectl delete namespace $(NAMESPACE2) || true
 
+# Clean up specific namespaces in Kubernetes (MLFlow)
 clean-kube-mlflow: check-kube
-	kubectl delete namespace $(NAMESPACE3)
+	kubectl delete namespace $(NAMESPACE3) || true
+
+# Clean up all specified namespaces in Kubernetes
+clean-kube-all: check-kube
+	kubectl delete namespace $(NAMESPACE1) || true
+	kubectl delete namespace $(NAMESPACE2) || true
+	kubectl delete namespace $(NAMESPACE3) || true

docker/fastapi/requirements.txt

@@ -1,4 +1,4 @@
-fastapi==0.95.2  # ou la dernière version stable
+fastapi
 uvicorn==0.22.0
 pydantic==1.10.4
 passlib==1.7.4

docker/streamlit/Dockerfile

@@ -13,9 +13,6 @@ RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
 # Copier les fichiers de l'application dans le conteneur
 COPY ./app /app
 
-# Exposer le port utilisé par Streamlit
-EXPOSE 8501
-
 # Définir le point d'entrée pour exécuter Streamlit
 ENTRYPOINT ["streamlit", "run"]
 

kubernetes/airflow/dags/dag_train_models.py

@@ -33,7 +33,7 @@
     secrets= [secret_password],
     volumes=[
         k8s.V1Volume(
-            name="model-storage",
+            name="model-storage-pv",
             persistent_volume_claim=k8s.V1PersistentVolumeClaimVolumeSource(claim_name="model-storage-pvc")
         )
     ],

kubernetes/airflow/logs/scheduler/latest

@@ -1 +1 @@
-2024-12-03
+2024-12-05

kubernetes/airflow/my_airflow_values.yml

@@ -1,12 +1,12 @@
 dags:
   persistence:
     enabled: true
-    existingClaim: airflow-local-dags-folder-pvc # PVC existant pour les DAGs
+    existingClaim: airflow-local-dags-folder
 
 logs:
   persistence:
     enabled: true
-    existingClaim: airflow-local-logs-folder-pvc # PVC existant pour les logs
+    existingClaim: airflow-local-logs-folder
 
 uid: 1000
 gid: 1000

kubernetes/deployments/streamlit-deployment.yml

@@ -3,7 +3,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: streamlit
-  namespace: reco-movies
+  namespace: api
 spec:
   replicas: 1
   selector:

kubernetes/persistent-volumes/airflow-local-dags-folder-pv.yml

@@ -1,15 +1,15 @@
 apiVersion: v1
 kind: PersistentVolume
 metadata:
-  name: airflow-local-dags-folder-pv
+  name: airflow-local-dags-folder
 spec:
+  storageClassName: local-path
   capacity:
     storage: 5Gi
   accessModes:
     - ReadWriteOnce
-  storageClassName: local-path
   claimRef:
     namespace: airflow
-    name: airflow-local-dags-folder-pvc
+    name: airflow-local-dags-folder
   hostPath:
     path: "/host/kubernetes/airflow/dags"

kubernetes/persistent-volumes/airflow-local-dags-folder-pvc.yml

@@ -1,12 +1,12 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: airflow-local-dags-folder-pvc
+  name: airflow-local-dags-folder
   namespace: airflow
 spec:
+  storageClassName: local-path
   accessModes:
     - ReadWriteOnce
   resources:
     requests:
       storage: 5Gi
-  storageClassName: local-path

kubernetes/persistent-volumes/airflow-local-logs-folder-pv.yml

@@ -1,15 +1,15 @@
 apiVersion: v1
 kind: PersistentVolume
 metadata:
-  name: airflow-local-logs-folder-pv
+  name: airflow-local-logs-folder
 spec:
+  storageClassName: local-path
   capacity:
-    storage: 2Gi # Capacité du PV
+    storage: 10Gi # Capacité du PV
   accessModes:
     - ReadWriteOnce # Mode d'accès
-  storageClassName: local-path
   claimRef:
     namespace: airflow # Namespace pour le PVC qui utilise ce PV
-    name: airflow-local-logs-folder-pvc
+    name: airflow-local-logs-folder
   hostPath:
     path: "/host/kubernetes/airflow/logs"