From a36b0fd49c3391affc6b6e527687f26dfff85fc6 Mon Sep 17 00:00:00 2001 From: AntoinePELAMOURGUES Date: Thu, 28 Nov 2024 17:56:54 +0400 Subject: [PATCH] :construction: Update kubernetes --- Makefile | 15 +++++- .../airflow/airflow-local-logs-folder-pv.yml | 15 ++++++ ....yml => airflow-local-logs-folder-pvc.yml} | 2 +- kubernetes/configmaps/configmaps.yml | 5 ++ kubernetes/deployments/fastapi-deployment.yml | 4 ++ .../deployments/postgres-api-deployment.yml | 12 +++-- kubernetes/services/services.yml | 48 ++++++++++++++----- 7 files changed, 83 insertions(+), 18 deletions(-) create mode 100644 kubernetes/airflow/airflow-local-logs-folder-pv.yml rename kubernetes/airflow/{airflow-logs-folder-pvc.yml => airflow-local-logs-folder-pvc.yml} (84%) diff --git a/Makefile b/Makefile index acb78d2..af047d4 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ NAMESPACE1 = reco-movies NAMESPACE2 = airflow -.PHONY: help setup1 setup2 start stop down restart logs-supabase logs-airflow logs-api clean network all namespace pv secrets configmaps deployments services ingress clean-kube-reco clean-kube-airflow create-configmap load-data-minikube install-airflow pv-airflow +.PHONY: help setup1 setup2 start stop down restart logs-supabase logs-airflow logs-api logs-fastapi clean network all namespace pv secrets configmaps deployments services ingress clean-kube-reco clean-kube-airflow create-configmap load-data-minikube install-airflow pv-airflow airflow reco # Help command to list all available targets help: @@ -15,6 +15,7 @@ help: @echo " logs-supabase - Show logs for supabase" @echo " logs-airflow - Show logs for airflow" @echo " logs-api - Show logs for api" + @echo " logs-fastapi - Show logs for FastAPI" @echo " clean - Remove all containers and networks" @echo " clean-db - Delete all data in the database and reload the schema and data" @echo " network - Create the Docker network 'backend'" @@ -84,6 +85,9 @@ logs-airflow: logs-api: docker compose logs -f +logs-fastapi: + docker compose logs -f fastapi + # Clean: stop and remove all containers, networks, and volumes for all services clean: cd supabase/docker && docker compose down -v @@ -112,6 +116,10 @@ all: load-data-minikube namespace install-airflow pv-airflow pv secrets configma # Installation de helm Airflow install-airflow: + sudo apt-get update + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh helm repo add apache-airflow https://airflow.apache.org helm repo update helm -n airflow upgrade --install airflow apache-airflow/airflow -f kubernetes/airflow/my_values.yml @@ -126,6 +134,9 @@ pv-airflow: kubectl apply -f kubernetes/airflow/airflow-local-logs-folder-pv.yml -n airflow --validate=false kubectl apply -f kubernetes/airflow/airflow-local-logs-folder-pvc.yml -n airflow --validate=false +airflow: namespace pv-airflow + helm -n airflow upgrade --install airflow apache-airflow/airflow -f kubernetes/airflow/my_values.yml + # Chargement des données dans minikube : https://minikube.sigs.k8s.io/docs/handbook/filesync/ load-data-minikube: mkdir -p ~/.minikube/files/processed_raw @@ -141,6 +152,8 @@ load-data-minikube: check-kube: @kubectl cluster-info > /dev/null 2>&1 || { echo "kubectl n'est pas connecté à un cluster"; exit 1; } +reco: namespace pv secrets configmaps deployments services ingress + namespace: check-kube kubectl apply -f kubernetes/namespace/namespace.yml --validate=false diff --git a/kubernetes/airflow/airflow-local-logs-folder-pv.yml b/kubernetes/airflow/airflow-local-logs-folder-pv.yml new file mode 100644 index 0000000..af077d3 --- /dev/null +++ b/kubernetes/airflow/airflow-local-logs-folder-pv.yml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: airflow-local-logs-folder +spec: + storageClassName: local-path + capacity: + storage: 2Gi + accessModes: + - ReadWriteOnce + claimRef: + namespace: airflow + name: airflow-local-logs-folder + hostPath: + path: "/logs" diff --git a/kubernetes/airflow/airflow-logs-folder-pvc.yml b/kubernetes/airflow/airflow-local-logs-folder-pvc.yml similarity index 84% rename from kubernetes/airflow/airflow-logs-folder-pvc.yml rename to kubernetes/airflow/airflow-local-logs-folder-pvc.yml index e004c0e..0b04fec 100644 --- a/kubernetes/airflow/airflow-logs-folder-pvc.yml +++ b/kubernetes/airflow/airflow-local-logs-folder-pvc.yml @@ -2,7 +2,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: airflow-logs-folder + name: airflow-local-logs-folder namespace: airflow spec: accessModes: diff --git a/kubernetes/configmaps/configmaps.yml b/kubernetes/configmaps/configmaps.yml index 092d4be..bba9e9b 100644 --- a/kubernetes/configmaps/configmaps.yml +++ b/kubernetes/configmaps/configmaps.yml @@ -22,6 +22,7 @@ metadata: data: POSTGRES_DB: "postgres-api-service" POSTGRES_USER: "antoine" + POSTGRES_HOST: "postgres-api-service" --- # ConfigMap pour Prometheus (Configuration) @@ -49,3 +50,7 @@ metadata: data: MODEL_PATH: "/app/model/model.pkl" DATA_PATH: "/app/raw/movies.csv" + POSTGRES_HOST: "postgres-api-service" + POSTGRES_PORT: "5432" + POSTGRES_DB: "postgres-api-service" + POSTGRES_USER: "antoine" diff --git a/kubernetes/deployments/fastapi-deployment.yml b/kubernetes/deployments/fastapi-deployment.yml index 13a54ff..0b441a2 100644 --- a/kubernetes/deployments/fastapi-deployment.yml +++ b/kubernetes/deployments/fastapi-deployment.yml @@ -35,6 +35,10 @@ spec: secretKeyRef: name: postgres-api-secrets key: POSTGRES_PASSWORD + - name: POSTGRES_HOST + value: "postgres-api-service" + - name: UVICORN_RELOAD + value: "false" ports: - containerPort: 8000 volumeMounts: diff --git a/kubernetes/deployments/postgres-api-deployment.yml b/kubernetes/deployments/postgres-api-deployment.yml index 89d0aee..73034ca 100644 --- a/kubernetes/deployments/postgres-api-deployment.yml +++ b/kubernetes/deployments/postgres-api-deployment.yml @@ -38,6 +38,8 @@ spec: secretKeyRef: name: postgres-api-secrets key: POSTGRES_PASSWORD + - name: POSTGRES_HOST + value: "postgres-api-service" volumeMounts: - name: pg-data mountPath: /var/lib/postgresql/data @@ -54,19 +56,19 @@ spec: volumes: - name: init-sql hostPath: - path: /init.sql + path: /home/antoine/jul24_cmlops_reco_film/postgres/init.sql - name: processed-movies-volume hostPath: - path: /processed_raw/processed_movies.csv + path: /home/antoine/jul24_cmlops_reco_film/ml/data/processed/processed_movies.csv - name: processed-ratings-volume hostPath: - path: /processed_raw/processed_ratings.csv # Chemin sur l'hôte. + path: /home/antoine/jul24_cmlops_reco_film/ml/data/processed/processed_ratings.csv - name: processed-links-volume hostPath: - path: /processed_raw/processed_links.csv # Chemin sur l'hôte. + path: /home/antoine/jul24_cmlops_reco_film/ml/data/processed/processed_links.csv - name: users-volume hostPath: - path: /processed_raw/users.csv # Chemin sur l'hôte. + path: /home/antoine/jul24_cmlops_reco_film/ml/data/processed/users.csv - name: pg-data persistentVolumeClaim: claimName: pvc-postgres-api-volume diff --git a/kubernetes/services/services.yml b/kubernetes/services/services.yml index 955ee4e..95e315e 100644 --- a/kubernetes/services/services.yml +++ b/kubernetes/services/services.yml @@ -15,7 +15,7 @@ spec: type: ClusterIP --- -## Service FastAPI (LoadBalancer) +## Service FastAPI (NodePort) apiVersion: v1 kind: Service metadata: @@ -27,9 +27,10 @@ spec: ports: - port: 8000 targetPort: 8000 - type: LoadBalancer + nodePort: 30001 + type: NodePort --- -## Service Streamlit (LoadBalancer) +## Service Streamlit (NodePort) apiVersion: v1 kind: Service metadata: @@ -40,9 +41,11 @@ spec: app: streamlit ports: - port: 8501 - type: LoadBalancer + targetPort: 8501 + nodePort: 30002 + type: NodePort --- -## Service Prometheus (ClusterIP) +## Service Prometheus (NodePort) apiVersion: v1 kind: Service metadata: @@ -53,9 +56,11 @@ spec: app: prometheus ports: - port: 9090 - type: ClusterIP + targetPort: 9090 + nodePort: 30003 + type: NodePort --- -## Service Grafana (LoadBalancer) +## Service Grafana (NodePort) apiVersion: v1 kind: Service metadata: @@ -66,7 +71,9 @@ spec: app: grafana ports: - port: 3000 - type: LoadBalancer + targetPort: 3000 + nodePort: 30004 + type: NodePort --- ## Service Postgres-API (ClusterIP) @@ -84,7 +91,7 @@ spec: type: ClusterIP --- -## Service MinIO (ClusterIP) +## Service MinIO (NodePort) apiVersion: v1 kind: Service metadata: @@ -97,10 +104,12 @@ spec: - name: minio port: 9000 targetPort: 9000 + nodePort: 30005 - name: console port: 9001 targetPort: 9001 - type: ClusterIP + nodePort: 30006 + type: NodePort --- apiVersion: v1 @@ -115,4 +124,21 @@ spec: - protocol: TCP port: 80 targetPort: 80 - type: ClusterIP + nodePort: 30007 + type: NodePort + +--- +## Service Airflow Webserver (NodePort) +apiVersion: v1 +kind: Service +metadata: + name: airflow-webserver-service + namespace: airflow +spec: + selector: + app: airflow-webserver + ports: + - port: 8080 + targetPort: 8080 + nodePort: 30008 + type: NodePort