🚧 Update volumes

Makefile

@@ -1,5 +1,6 @@
 NAMESPACE1 = reco-movies
 NAMESPACE2 = airflow
+NAMESPACE3 = mlflow
 
 .PHONY: help setup1 setup2 start stop down restart logs-supabase logs-airflow logs-api logs-fastapi clean network all namespace pv secrets configmaps deployments services ingress clean-kube-reco clean-kube-airflow apply-configmap start-minikube start-airflow pv-airflow reco start-mlflow
 
@@ -127,20 +128,22 @@ start-airflow:
 	sudo apt-get update
 	helm repo add apache-airflow https://airflow.apache.org
 	helm upgrade --install airflow apache-airflow/airflow --namespace airflow --create-namespace -f kubernetes/airflow/my_airflow_values.yml
-	kubectl apply -f kubernetes/storageclass/storageclass.yaml -n airflow
-	kubectl apply -f kubernetes/persistent-volumes/airflow-local-dags-folder.yml -n airflow
-	kubectl apply -f kubernetes/persistent-volumes/airflow-local-logs-folder.yml -n airflow
-	kubectl apply -f kubernetes/persistent-volumes/mlfow_storage.yml
-	kubectl apply -f kubernetes/secrets/airflow-secrets.yaml -n airflow
-	kubectl apply -f kubernetes/configmaps/airflow_configmaps.yml -n airflow
-	kubectl apply -f kubernetes/deployments/pgadmin-deployment.yml -n airflow
-	kubectl apply -f kubernetes/services/pgadmin_service.yml -n airflow
+	kubectl apply -f kubernetes/persistent-volumes/airflow-local-dags-folder-pv.yml
+	kubectl apply -f kubernetes/persistent-volumes/airflow-local-dags-folder-pvc.yml
+	kubectl apply -f kubernetes/persistent-volumes/airflow-local-logs-folder-pv.yml
+	kubectl apply -f kubernetes/persistent-volumes/airflow-local-logs-folder-pvc.yml
+	kubectl apply -f kubernetes/persistent-volumes/mlfow-storage-pv.yml
+	kubectl apply -f kubernetes/persistent-volumes/mlfow-storage-pvc.yml
+	kubectl apply -f kubernetes/secrets/airflow-secrets.yaml
+	kubectl apply -f kubernetes/configmaps/airflow-configmaps.yml
+	kubectl apply -f kubernetes/deployments/pgadmin-deployment.yml
+	kubectl apply -f kubernetes/services/pgadmin-service.yml
 
 start-mlflow:
 	helm repo add bitnami https://charts.bitnami.com/bitnami
 	helm repo update
 	helm install mlf-ts bitnami/mlflow --namespace mlflow --create-namespace
-	kubectl apply -f kubernetes/services/mlflow_service.yml -n mlflow
+	kubectl apply -f kubernetes/services/mlflow-service.yml
 
 
 delete-pv-airflow:
@@ -194,6 +197,9 @@ ingress: check-kube
 clean-kube-reco: check-kube
 	kubectl delete namespace $(NAMESPACE1)
 
-
 clean-kube-airflow: check-kube
-	kubectl delete namespace $(NAMESPACE2)
+	kubectl delete namespace $(NAMESPACE2)
+
+
+clean-kube-mlflow: check-kube
+	kubectl delete namespace $(NAMESPACE3)

kubernetes/airflow/my_airflow_values.yml

@@ -1,12 +1,12 @@
 dags:
   persistence:
     enabled: true
-    existingClaim: airflow-local-dags-folder
+    existingClaim: airflow-local-dags-folder-pvc # PVC existant pour les DAGs
 
 logs:
   persistence:
     enabled: true
-    existingClaim: airflow-local-logs-folder
+    existingClaim: airflow-local-logs-folder-pvc # PVC existant pour les logs
 
 uid: 1000
 gid: 1000
@@ -20,4 +20,4 @@ executor: CeleryKubernetesExecutor
 
 webserverSecretKey: 370562eacc440d7d9a9a1ad86eef576c
 
-AIRFLOW_VAR_MLFLOW_SERVER: "http://mlflow-server.airflow.svc.cluster.local:5000"
+AIRFLOW_VAR_MLFLOW_SERVER: "mlf-ts-mlflow-tracking.mlflow.svc.cluster.local:80"

kubernetes/persistent-volumes/airflow-local-dags-folder-pv.yml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: airflow-local-dags-folder-pv
+spec:
+  capacity:
+    storage: 5Gi
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: local-path
+  claimRef:
+    namespace: airflow
+    name: airflow-local-dags-folder-pvc
+  hostPath:
+    path: "/host/kubernetes/airflow/dags"

kubernetes/persistent-volumes/airflow-local-dags-folder-pvc.yml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: airflow-local-dags-folder-pvc
+  namespace: airflow
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+  storageClassName: local-path

kubernetes/persistent-volumes/airflow-local-logs-folder-pv.yml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: airflow-local-logs-folder-pv
+spec:
+  capacity:
+    storage: 2Gi # Capacité du PV
+  accessModes:
+    - ReadWriteOnce # Mode d'accès
+  storageClassName: local-path
+  claimRef:
+    namespace: airflow # Namespace pour le PVC qui utilise ce PV
+    name: airflow-local-logs-folder-pvc
+  hostPath:
+    path: "/host/kubernetes/airflow/logs"

kubernetes/persistent-volumes/airflow-local-logs-folder-pvc.yml

@@ -0,0 +1,13 @@
+# Persistent Volume Claims
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: airflow-local-logs-folder-pvc
+  namespace: airflow # Namespace où le PVC sera créé
+spec:
+  accessModes:
+    - ReadWriteOnce # Mode d'accès requis pour le PVC
+  storageClassName: local-path
+  resources:
+    requests:
+      storage: 2Gi # Capacité demandée par le PVC

kubernetes/persistent-volumes/mlfow-storage-pv.yml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: models-pv
+spec:
+  capacity:
+    storage: 5Gi
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: local-path
+  claimRef:
+    namespace: airflow
+    name: models-pvc
+  hostPath:
+    path: "/host/kubernetes/mlflow/models"

kubernetes/persistent-volumes/mlfow_storage.yml → kubernetes/persistent-volumes/mlfow-storage-pvc.yml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: mlflow-models-pvc
+  name: models-pvc
   namespace: airflow
 spec:
   accessModes:

kubernetes/services/mlflow_service.yml → kubernetes/services/mlflow-service.yml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: mlflow-server
-  namespace: airflow
+  namespace: mlflow
 spec:
   type: ClusterIP # Changez en NodePort si vous voulez accéder depuis l'extérieur du cluster
   ports:

ml/notebooks/mlruns/0/meta.yaml

@@ -0,0 +1,6 @@
+artifact_location: file:///home/antoine/jul24_cmlops_reco_film/ml/notebooks/mlruns/0
+creation_time: 1733157425809
+experiment_id: '0'
+last_update_time: 1733157425809
+lifecycle_stage: active
+name: Default

ml/notebooks/mlruns/357506974039482680/065fd872829c4333a024fc21923bf380/meta.yaml

@@ -0,0 +1,15 @@
+artifact_uri: file:///home/antoine/jul24_cmlops_reco_film/ml/notebooks/mlruns/357506974039482680/065fd872829c4333a024fc21923bf380/artifacts
+end_time: 1733158213697
+entry_point_name: ''
+experiment_id: '357506974039482680'
+lifecycle_stage: active
+run_id: 065fd872829c4333a024fc21923bf380
+run_name: default-params
+run_uuid: 065fd872829c4333a024fc21923bf380
+source_name: ''
+source_type: 4
+source_version: ''
+start_time: 1733157833329
+status: 3
+tags: []
+user_id: antoine

ml/notebooks/mlruns/357506974039482680/065fd872829c4333a024fc21923bf380/tags/mlflow.runName

@@ -0,0 +1 @@
+default-params

ml/notebooks/mlruns/357506974039482680/065fd872829c4333a024fc21923bf380/tags/mlflow.source.name

@@ -0,0 +1 @@
+/home/antoine/miniconda3/envs/reco_movies/lib/python3.8/site-packages/ipykernel_launcher.py

ml/notebooks/mlruns/357506974039482680/065fd872829c4333a024fc21923bf380/tags/mlflow.source.type

@@ -0,0 +1 @@
+LOCAL

ml/notebooks/mlruns/357506974039482680/065fd872829c4333a024fc21923bf380/tags/mlflow.user

@@ -0,0 +1 @@
+antoine

ml/notebooks/mlruns/357506974039482680/meta.yaml

@@ -0,0 +1,6 @@
+artifact_location: file:///home/antoine/jul24_cmlops_reco_film/ml/notebooks/mlruns/357506974039482680
+creation_time: 1733157425828
+experiment_id: '357506974039482680'
+last_update_time: 1733157425828
+lifecycle_stage: active
+name: reco_movies