🚧 Update kubernetes _ airflow Ok

Makefile

@@ -1,6 +1,7 @@
-NAMESPACE = reco-movies
+NAMESPACE1 = reco-movies
+NAMESPACE2 = airflow
 
-.PHONY: help setup1 setup2 start stop down restart logs-supabase logs-airflow logs-api clean network all namespace pv secrets configmaps deployments services ingress clean-kube create-configmap load-data-minikube
+.PHONY: help setup1 setup2 start stop down restart logs-supabase logs-airflow logs-api clean network all namespace pv secrets configmaps deployments services ingress clean-kube-reco clean-kube-airflow create-configmap load-data-minikube install-airflow pv-airflow
 
 # Help command to list all available targets
 help:
@@ -104,15 +105,36 @@ clean-db: network
 network:
 	docker network create backend || true
 
+
+
 ###### MAKEFILE KUBERNETES
-all:  namespace pv secrets configmaps deployments services ingress
+all: load-data-minikube namespace install-airflow pv-airflow pv secrets configmaps deployments services ingress
+
+# Installation de helm Airflow
+install-airflow:
+	helm repo add apache-airflow https://airflow.apache.org
+	helm repo update
+	helm -n airflow upgrade --install airflow apache-airflow/airflow -f kubernetes/airflow/my_values.yml
+
+delete-airflow-statefulsets:
+	kubectl delete statefulset -n airflow airflow-triggerer || true
+	kubectl delete statefulset -n airflow airflow-worker || true
+
+pv-airflow:
+	kubectl apply -f kubernetes/airflow/airflow-local-dags-folder-pv.yml -n airflow --validate=false
+	kubectl apply -f kubernetes/airflow/airflow-local-dags-folder-pvc.yml -n airflow  --validate=false
+	kubectl apply -f kubernetes/airflow/airflow-local-logs-folder-pv.yml -n airflow --validate=false
+	kubectl apply -f kubernetes/airflow/airflow-local-logs-folder-pvc.yml -n airflow --validate=false
 
 # Chargement des données dans minikube : https://minikube.sigs.k8s.io/docs/handbook/filesync/
 load-data-minikube:
 	mkdir -p ~/.minikube/files/processed_raw
+	mkdir -p ~/.minikube/files/dags
+	mkdir -p ~/.minikube/files/logs
 	cp -r ml/data/processed/* ~/.minikube/files/processed_raw
 	cp -r postgres/init.sql ~/.minikube/files/init.sql
 	cp -r prometheus/prometheus.yml ~/.minikube/files/prometheus.yml
+	cp -r airflow/dags/* ~/.minikube/files/dags
 	minikube start
 
 # Vérifie si kubectl est connecté à un cluster
@@ -122,8 +144,6 @@ check-kube:
 
 namespace: check-kube
 	kubectl apply -f kubernetes/namespace/namespace.yml --validate=false
-	kubectl config set-context --current --namespace=reco-movies
-
 
 pv: check-kube
 	kubectl apply -f kubernetes/persistent-volumes/fastapi-persistent-volume.yml --validate=false
@@ -148,7 +168,6 @@ deployments: check-kube
 	kubectl apply -f kubernetes/deployments/prometheus-deployment.yml --validate=false
 	kubectl apply -f kubernetes/deployments/grafana-deployment.yml --validate=false
 	kubectl apply -f kubernetes/deployments/node-exporter-deployment.yml --validate=false
-	kubectl apply -f kubernetes/deployments/airflow-deployment.yml --validate=false
 	kubectl apply -f kubernetes/deployments/minio-deployment.yml --validate=false
 	kubectl apply -f kubernetes/deployments/postgres-exporter-deployment.yml --validate=false
 	kubectl apply -f kubernetes/deployments/pgadmin-deployment.yml --validate=false
@@ -159,5 +178,8 @@ services: check-kube
 ingress: check-kube
 	kubectl apply -f kubernetes/ingress/ingress.yml
 
-clean-kube: check-kube
-	kubectl delete namespace $(NAMESPACE)
+clean-kube-reco: check-kube
+	kubectl delete namespace $(NAMESPACE1)
+
+clean-kube-airflow: check-kube
+	kubectl delete namespace $(NAMESPACE2)

kubernetes/airflow/airflow-local-dags-folder-pv.yml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: airflow-local-dags-folder
+spec:
+  storageClassName: local-path
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  claimRef:
+    namespace: airflow
+    name: airflow-local-dags-folder
+  hostPath:
+    path: "/dags"

kubernetes/airflow/airflow-local-dags-folder-pvc.yml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: airflow-local-dags-folder
+spec:
+  storageClassName: local-path
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi

kubernetes/airflow/airflow-logs-folder-pvc.yml

@@ -0,0 +1,12 @@
+# Persistent Volume Claims
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: airflow-logs-folder
+  namespace: airflow
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi

kubernetes/airflow/my_values.yml

@@ -0,0 +1,19 @@
+dags:
+  persistence:
+    enabled: true
+    existingClaim: airflow-local-dags-folder
+
+logs:
+  persistence:
+    enabled: true
+    existingClaim: airflow-local-logs-folder
+
+uid: 1000
+gid: 1000
+
+scheduler:
+  env:
+    - name: AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL
+      value: "30"
+
+executor: CeleryKubernetesExecutor

kubernetes/deployments/mlflow-deployment.yml

@@ -1,4 +1,3 @@
-# Déploiement MLflow Webserver
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -23,9 +22,6 @@ spec:
           envFrom:
             - configMapRef:
                 name: postgres-mlflow-config
-            - secretKeyRef:
-                name: postgres-mlflow-secrets
-                key: POSTGRES_PASSWORD
           env:
             - name: MLFLOW_S3_ENDPOINT_URL
               value: http://minio-service:9000
@@ -39,3 +35,8 @@ spec:
                 secretKeyRef:
                   name: postgres-mlflow-secrets
                   key: AWS_SECRET_ACCESS_KEY
+            - name: POSTGRES_PASSWORD # Ajoutez cette ligne pour inclure le mot de passe PostgreSQL.
+              valueFrom:
+                secretKeyRef:
+                  name: postgres-mlflow-secrets
+                  key: POSTGRES_PASSWORD

kubernetes/namespace/namespace.yml

@@ -2,3 +2,9 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: reco-movies
+
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: airflow