🚧 Update kubernetes

Makefile

@@ -1,7 +1,7 @@
 NAMESPACE1 = reco-movies
 NAMESPACE2 = airflow
 
-.PHONY: help setup1 setup2 start stop down restart logs-supabase logs-airflow logs-api clean network all namespace pv secrets configmaps deployments services ingress clean-kube-reco clean-kube-airflow create-configmap load-data-minikube install-airflow pv-airflow
+.PHONY: help setup1 setup2 start stop down restart logs-supabase logs-airflow logs-api logs-fastapi clean network all namespace pv secrets configmaps deployments services ingress clean-kube-reco clean-kube-airflow create-configmap load-data-minikube install-airflow pv-airflow airflow reco
 
 # Help command to list all available targets
 help:
@@ -15,6 +15,7 @@ help:
 	@echo "  logs-supabase  - Show logs for supabase"
 	@echo "  logs-airflow   - Show logs for airflow"
 	@echo "  logs-api       - Show logs for api"
+	@echo "  logs-fastapi   - Show logs for FastAPI"
 	@echo "  clean          - Remove all containers and networks"
 	@echo "  clean-db       - Delete all data in the database and reload the schema and data"
 	@echo "  network        - Create the Docker network 'backend'"
@@ -84,6 +85,9 @@ logs-airflow:
 logs-api:
 	docker compose logs -f
 
+logs-fastapi:
+	docker compose logs -f fastapi
+
 # Clean: stop and remove all containers, networks, and volumes for all services
 clean:
 	cd supabase/docker && docker compose down -v
@@ -112,6 +116,10 @@ all: load-data-minikube namespace install-airflow pv-airflow pv secrets configma
 
 # Installation de helm Airflow
 install-airflow:
+	sudo apt-get update
+	curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+	chmod 700 get_helm.sh
+	./get_helm.sh
 	helm repo add apache-airflow https://airflow.apache.org
 	helm repo update
 	helm -n airflow upgrade --install airflow apache-airflow/airflow -f kubernetes/airflow/my_values.yml
@@ -126,6 +134,9 @@ pv-airflow:
 	kubectl apply -f kubernetes/airflow/airflow-local-logs-folder-pv.yml -n airflow --validate=false
 	kubectl apply -f kubernetes/airflow/airflow-local-logs-folder-pvc.yml -n airflow --validate=false
 
+airflow: namespace pv-airflow
+	helm -n airflow upgrade --install airflow apache-airflow/airflow -f kubernetes/airflow/my_values.yml
+
 # Chargement des données dans minikube : https://minikube.sigs.k8s.io/docs/handbook/filesync/
 load-data-minikube:
 	mkdir -p ~/.minikube/files/processed_raw
@@ -141,6 +152,8 @@ load-data-minikube:
 check-kube:
 	@kubectl cluster-info > /dev/null 2>&1 || { echo "kubectl n'est pas connecté à un cluster"; exit 1; }
 
+reco: namespace pv secrets configmaps deployments services ingress
+
 
 namespace: check-kube
 	kubectl apply -f kubernetes/namespace/namespace.yml --validate=false

kubernetes/airflow/airflow-local-logs-folder-pv.yml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: airflow-local-logs-folder
+spec:
+  storageClassName: local-path
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  claimRef:
+    namespace: airflow
+    name: airflow-local-logs-folder
+  hostPath:
+    path: "/logs"

kubernetes/airflow/airflow-logs-folder-pvc.yml → kubernetes/airflow/airflow-local-logs-folder-pvc.yml

@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: airflow-logs-folder
+  name: airflow-local-logs-folder
   namespace: airflow
 spec:
   accessModes:

kubernetes/configmaps/configmaps.yml

@@ -22,6 +22,7 @@ metadata:
 data:
   POSTGRES_DB: "postgres-api-service"
   POSTGRES_USER: "antoine"
+  POSTGRES_HOST: "postgres-api-service"
 
 ---
 # ConfigMap pour Prometheus (Configuration)
@@ -49,3 +50,7 @@ metadata:
 data:
   MODEL_PATH: "/app/model/model.pkl"
   DATA_PATH: "/app/raw/movies.csv"
+  POSTGRES_HOST: "postgres-api-service"
+  POSTGRES_PORT: "5432"
+  POSTGRES_DB: "postgres-api-service"
+  POSTGRES_USER: "antoine"

kubernetes/deployments/fastapi-deployment.yml

@@ -35,6 +35,10 @@ spec:
                 secretKeyRef:
                   name: postgres-api-secrets
                   key: POSTGRES_PASSWORD
+            - name: POSTGRES_HOST
+              value: "postgres-api-service"
+            - name: UVICORN_RELOAD
+              value: "false"
           ports:
             - containerPort: 8000
           volumeMounts:

kubernetes/deployments/postgres-api-deployment.yml

@@ -38,6 +38,8 @@ spec:
                 secretKeyRef:
                   name: postgres-api-secrets
                   key: POSTGRES_PASSWORD
+            - name: POSTGRES_HOST
+              value: "postgres-api-service"
           volumeMounts:
             - name: pg-data
               mountPath: /var/lib/postgresql/data
@@ -54,19 +56,19 @@ spec:
       volumes:
         - name: init-sql
           hostPath:
-            path: /init.sql
+            path: /home/antoine/jul24_cmlops_reco_film/postgres/init.sql
         - name: processed-movies-volume
           hostPath:
-            path: /processed_raw/processed_movies.csv
+            path: /home/antoine/jul24_cmlops_reco_film/ml/data/processed/processed_movies.csv
         - name: processed-ratings-volume
           hostPath:
-            path: /processed_raw/processed_ratings.csv # Chemin sur l'hôte.
+            path: /home/antoine/jul24_cmlops_reco_film/ml/data/processed/processed_ratings.csv
         - name: processed-links-volume
           hostPath:
-            path: /processed_raw/processed_links.csv # Chemin sur l'hôte.
+            path: /home/antoine/jul24_cmlops_reco_film/ml/data/processed/processed_links.csv
         - name: users-volume
           hostPath:
-            path: /processed_raw/users.csv # Chemin sur l'hôte.
+            path: /home/antoine/jul24_cmlops_reco_film/ml/data/processed/users.csv
         - name: pg-data
           persistentVolumeClaim:
             claimName: pvc-postgres-api-volume

kubernetes/services/services.yml

@@ -15,7 +15,7 @@ spec:
   type: ClusterIP
 
 ---
-## Service FastAPI (LoadBalancer)
+## Service FastAPI (NodePort)
 apiVersion: v1
 kind: Service
 metadata:
@@ -27,9 +27,10 @@ spec:
   ports:
     - port: 8000
       targetPort: 8000
-  type: LoadBalancer
+      nodePort: 30001
+  type: NodePort
 ---
-## Service Streamlit (LoadBalancer)
+## Service Streamlit (NodePort)
 apiVersion: v1
 kind: Service
 metadata:
@@ -40,9 +41,11 @@ spec:
     app: streamlit
   ports:
     - port: 8501
-  type: LoadBalancer
+      targetPort: 8501
+      nodePort: 30002
+  type: NodePort
 ---
-## Service Prometheus (ClusterIP)
+## Service Prometheus (NodePort)
 apiVersion: v1
 kind: Service
 metadata:
@@ -53,9 +56,11 @@ spec:
     app: prometheus
   ports:
     - port: 9090
-  type: ClusterIP
+      targetPort: 9090
+      nodePort: 30003
+  type: NodePort
 ---
-## Service Grafana (LoadBalancer)
+## Service Grafana (NodePort)
 apiVersion: v1
 kind: Service
 metadata:
@@ -66,7 +71,9 @@ spec:
     app: grafana
   ports:
     - port: 3000
-  type: LoadBalancer
+      targetPort: 3000
+      nodePort: 30004
+  type: NodePort
 
 ---
 ## Service Postgres-API (ClusterIP)
@@ -84,7 +91,7 @@ spec:
   type: ClusterIP
 
 ---
-## Service MinIO (ClusterIP)
+## Service MinIO (NodePort)
 apiVersion: v1
 kind: Service
 metadata:
@@ -97,10 +104,12 @@ spec:
     - name: minio
       port: 9000
       targetPort: 9000
+      nodePort: 30005
     - name: console
       port: 9001
       targetPort: 9001
-  type: ClusterIP
+      nodePort: 30006
+  type: NodePort
 
 ---
 apiVersion: v1
@@ -115,4 +124,21 @@ spec:
     - protocol: TCP
       port: 80
       targetPort: 80
-  type: ClusterIP
+      nodePort: 30007
+  type: NodePort
+
+---
+## Service Airflow Webserver (NodePort)
+apiVersion: v1
+kind: Service
+metadata:
+  name: airflow-webserver-service
+  namespace: airflow
+spec:
+  selector:
+    app: airflow-webserver
+  ports:
+    - port: 8080
+      targetPort: 8080
+      nodePort: 30008
+  type: NodePort