diff --git a/kubernetes/configmaps/configmaps.yml b/kubernetes/configmaps/configmaps.yml new file mode 100644 index 0000000..6a285a7 --- /dev/null +++ b/kubernetes/configmaps/configmaps.yml @@ -0,0 +1,50 @@ +# ConfigMap pour les services MLflow et API +apiVersion: v1 +kind: ConfigMap +metadata: + name: posgres-mlfow-config + namespace: reco-movies +data: + POSTGRES_DB: "mlflow" + POSTGRES_USER: "mlflow" + MLFLOW_S3_ENDPOINT_URL: "http://s3-artifact-storage:9000" + MLFLOW_TRACKING_URI: "http://mlflow-service:5000" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgres-api-config + namespace: reco-movies +data: + POSTGRES_DB: "reco_movies" + POSTGRES_USER: "antoine" + +--- +# ConfigMap pour Prometheus (Configuration) +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + namespace: reco-movies +data: + prometheus.yml: | + global: + scrape_interval: '15s' + scrape_configs: + - job_name: 'fastapi' + static_configs: + - targets: ['fastapi-service:8000'] + +--- +# Configmap pour fastapi +apiVersion: v1 +kind: ConfigMap +metadata: + name: fastapi-config + namespace: reco-movies +data: + MODEL_PATH: "/app/model/model.pkl" + DATA_PATH: "/app/raw/movies.csv" + HOST: "0.0.0.0" + PORT: "8000" diff --git a/kubernetes/deployment.yml b/kubernetes/deployment.yml deleted file mode 100644 index 0982785..0000000 --- a/kubernetes/deployment.yml +++ /dev/null @@ -1,396 +0,0 @@ ---- -apiVersion: v1 -kind: Namespace -metadata: - name: reco-movies - ---- -# Persistent Volumes Claims -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: postgres-pvc - namespace: reco-movies -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - ---- -# CONFIGMAP POUR NOS SERVICES -apiVersion: v1 -kind: ConfigMap -metadata: - name: posgres-mlfow-config - namespace: reco-movies -data: - POSTGRES_DB: "mlflow" - POSTGRES_USER: "mlflow" - POSTGRES_PASSWORD: "mlflow" - MLFLOW_S3_ENDPOINT_URL: "http://s3-artifact-storage:9000" - MLFLOW_TRACKING_URI: "http://mlflow-service:5000" - AWS_ACCESS_KEY_ID: "mlflow_access" - AWS_SECRET_ACCESS_KEY: "mlflow_secret" - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: postgres-api-config - namespace: reco-movies -data: - POSTGRES_DB: "reco_movies" - POSTGRES_USER: "antoine" - ---- -# SECRETS - -apiVersion: v1 -kind: Secret -metadata: - name: postgres-api-secrets - namespace: reco-movies -type: Opaque -data: - POSTGRES_PASSWORD: ZGF0YXNjaWVudGVzdA== # base64 encoded - ---- -apiVersion: v1 -kind: Secret -metadata: - name: postgres-mlflow-secrets - namespace: reco-movies -type: Opaque -data: - POSTGRES_PASSWORD: bWxmbG93 # base64 encoded - AWS_ACCESS_KEY_ID: bWxmbG93X2FjY2Vzcw== # base64 encoded 'mlflow_access' - AWS_SECRET_ACCESS_KEY: bWxmbG93X3NlY3JldA== # base64 encoded 'mlflow_secret' - ---- -# Postgres API Deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: api-postgres - namespace: reco-movies -spec: - replicas: 2 - strategy: - type: RollingUpdate - selector: - matchLabels: - app: api-postgres - template: - metadata: - labels: - app: api-postgres - spec: - containers: - - name: db - image: postgres:16-alpine - envFrom: - - configMapRef: - name: postgres-api-config - - secretRef: - name: postgres-api-secrets - ports: - - containerPort: 5432 - volumeMounts: - - name: pg-data - mountPath: /var/lib/postgresql/data - volumes: - - name: pg-data - persistentVolumeClaim: - claimName: postgres-pvc - ---- -# Postgres MLFLOW Deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mlflow-postgres - namespace: reco-movies -spec: - replicas: 1 - selector: - matchLabels: - app: postgres-mlflow - template: - metadata: - labels: - app: postgres-mlflow - spec: - containers: - - name: postgres-mlflow - image: postgres:13 - envFrom: - - configMapRef: - name: posgres-mlfow-config - - secretRef: - name: postgres-mlflow-secrets - ports: - - containerPort: 5429 - volumeMounts: - - name: postgres-db-volume-mlflow - mountPath: /var/lib/postgresql/data - volumes: - - name: postgres-db-volume-mlflow - persistentVolumeClaim: - claimName: postgres-pvc ---- -# MLflow Deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: mlflow - namespace: reco-movies -spec: - replicas: 1 - selector: - matchLabels: - app: mlflow - template: - metadata: - labels: - app: mlflow - spec: - containers: - - name: mlflow-webserver - image: python:3.9-slim-buster - ports: - - containerPort: 5000 - envFrom: - - configMapRef: - name: posgres-mlfow-config - - secretRef: - name: posgres-mlfow-secrets - ---- -# FastAPI Deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: fastapi - namespace: reco-movies -spec: - replicas: 2 - selector: - matchLabels: - app: fastapi - template: - metadata: - labels: - app: fastapi - spec: - containers: - - name: fastapi - image: python:3.12 - ports: - - containerPort: 8002 - volumeMounts: - - name: ml-data - mountPath: /app/raw - - name: ml-models - mountPath: /app/model - volumes: - - name: ml-data - emptyDir: {} - - name: ml-models - emptyDir: {} - ---- -# Streamlit Deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: streamlit - namespace: reco-movies -spec: - replicas: 1 - selector: - matchLabels: - app: streamlit - template: - metadata: - labels: - app: streamlit - spec: - containers: - - name: streamlit - image: python:3.12 - ports: - - containerPort: 8501 - ---- -# Prometheus Deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: prometheus - namespace: reco-movies -spec: - replicas: 1 - selector: - matchLabels: - app: prometheus - template: - metadata: - labels: - app: prometheus - spec: - containers: - - name: prometheus - image: prom/prometheus:latest - ports: - - containerPort: 9090 - volumeMounts: - - name: prometheus-config - mountPath: /etc/prometheus/ - - name: shared-prometheus - mountPath: /var/log/prometheus - volumes: - - name: prometheus-config - configMap: - name: prometheus-config - - name: shared-prometheus - emptyDir: {} - ---- -# Grafana Deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: grafana - namespace: reco-movies -spec: - replicas: 1 - selector: - matchLabels: - app: grafana - template: - metadata: - labels: - app: grafana - spec: - containers: - - name: grafana - image: grafana/grafana-enterprise - ports: - - containerPort: 3000 - volumeMounts: - - name: grafana-data - mountPath: /var/lib/grafana - volumes: - - name: grafana-data - emptyDir: {} - ---- -# Services -apiVersion: v1 -kind: Service -metadata: - name: mlflow-service - namespace: reco-movies -spec: - selector: - app: mlflow - ports: - - port: 5000 - type: ClusterIP - ---- -apiVersion: v1 -kind: Service -metadata: - name: fastapi-service - namespace: reco-movies -spec: - selector: - app: fastapi - ports: - - port: 8000 - type: LoadBalancer - ---- -apiVersion: v1 -kind: Service -metadata: - name: streamlit-service - namespace: reco-movies -spec: - selector: - app: streamlit - ports: - - port: 8501 - type: LoadBalancer - ---- -apiVersion: v1 -kind: Service -metadata: - name: prometheus-service - namespace: reco-movies -spec: - selector: - app: prometheus - ports: - - port: 9090 - type: ClusterIP - ---- -apiVersion: v1 -kind: Service -metadata: - name: grafana-service - namespace: reco-movies -spec: - selector: - app: grafana - ports: - - port: 3000 - type: LoadBalancer - ---- -# ConfigMap pour Prometheus -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus-config - namespace: reco-movies -data: - prometheus.yml: | - global: - scrape_interval: 15s - scrape_configs: - - job_name: 'fastapi' - static_configs: - - targets: ['fastapi-service:8000'] - ---- -# Ingress -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: app-ingress - namespace: reco-movies -spec: - rules: - - host: reco-movies.local - http: - paths: - - path: /api - pathType: Prefix - backend: - service: - name: fastapi-service - port: - number: 8000 - - path: / - pathType: Prefix - backend: - service: - name: streamlit-service - port: - number: 8501 diff --git a/kubernetes/deployments/airflow-deployment.yml b/kubernetes/deployments/airflow-deployment.yml new file mode 100644 index 0000000..444fc82 --- /dev/null +++ b/kubernetes/deployments/airflow-deployment.yml @@ -0,0 +1,100 @@ +# Airflow Scheduler Deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: airflow-scheduler + namespace: reco-movies +spec: + replicas: 1 # Un seul scheduler est nécessaire dans une configuration standard. + selector: + matchLabels: + app: airflow-scheduler # Label utilisé par le service. + template: + metadata: + labels: + app: airflow-scheduler # Labels du pod. + spec: + containers: + - name: scheduler # Nom du conteneur. + image: antoinepela/projet_reco_movies:airflow-scheduler-latest + env: + - name: AIRFLOW__CORE__SQL_ALCHEMY_CONN + value: postgresql+psycopg2://antoine@api-postgres:5432/reco_movies + - name: AIRFLOW__CORE__EXECUTOR + value: CeleryExecutor + - name: AIRFLOW__CORE__LOAD_EXAMPLES + value: "False" + - name: AIRFLOW__CORE__FERNET_KEY + value: "YXJlbnRvaW5lcGVsYQ==" + - name: AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION + value: "False" + - name: AIRFLOW__CORE__DAG_DISCOVERY_SAFE_MODE + value: "False" + - name: AIRFLOW__CORE__DAGS_FOLDER + value: /usr/local/airflow/dags + +--- +# Airflow Worker Deployment + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: airflow-worker + namespace: reco-movies +spec: + replicas: 2 # Plusieurs workers peuvent être nécessaires. + selector: + matchLabels: + app: airflow-worker + template: + metadata: + labels: + app: airflow-worker + spec: + containers: + - name: airflow-worker + image: antoinepela/projet_reco_movies:airflow-worker-latest + +--- +# Airflow Webserver Deployment + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: airflow-webserver + namespace: reco-movies +spec: + replicas: 1 + selector: + matchLabels: + app: airflow-webserver + template: + metadata: + labels: + app: airflow-webserver + spec: + containers: + - name: airflow-webserver + image: antoinepela/projet_reco_movies:airflow-webserver-latest + +--- +# Airflow Flower Deployment + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: airflow-flower + namespace: reco-movies +spec: + replicas: 1 + selector: + matchLabels: + app: airflow-flower + template: + metadata: + labels: + app: airflow-flower + spec: + containers: + - name: airflow-flower + image: antoinepela/projet_reco_movies:airflow-flower-latest diff --git a/kubernetes/deployments/fastapi-deployment.yml b/kubernetes/deployments/fastapi-deployment.yml new file mode 100644 index 0000000..4a35b2e --- /dev/null +++ b/kubernetes/deployments/fastapi-deployment.yml @@ -0,0 +1,38 @@ +# Déploiement Postgres pour l'API FastAPI +apiVersion: apps/v1 +kind: Deployment +metadata: + name: fastapi + namespace: reco-movies +spec: + replicas: 2 + selector: + matchLabels: + app: fastapi + template: + metadata: + labels: + app: fastapi + spec: + containers: + - name: fastapi + image: antoinepela/projet_reco_movies:fastapi-latest + envFrom: + - configMapRef: + name: fastapi-config + - secretRef: + name: fastapi-secrets + ports: + - containerPort: 8000 + volumeMounts: + - name: processed-data-volume + mountPath: /app/raw + - name: models-volume + mountPath: /app/model + volumes: + - name: processed-data-volume + persistentVolumeClaim: + claimName: pvc-processed-data + - name: models-volume + persistentVolumeClaim: + claimName: pvc-models diff --git a/kubernetes/deployments/grafana-deployment.yml b/kubernetes/deployments/grafana-deployment.yml new file mode 100644 index 0000000..2762e47 --- /dev/null +++ b/kubernetes/deployments/grafana-deployment.yml @@ -0,0 +1,28 @@ +# Déploiement Grafana +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: reco-movies +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + containers: + - name: grafana + image: antoinepela/projet_reco_movies:grafana-latest + ports: + - containerPort: 3000 + volumeMounts: + - name: grafana-logs + mountPath: /var/lib/grafana + volumes: + - name: grafana-logs + persistentVolumeClaim: + claimName: grafana-logs diff --git a/kubernetes/deployments/minio-deployment.yml b/kubernetes/deployments/minio-deployment.yml new file mode 100644 index 0000000..dc1b907 --- /dev/null +++ b/kubernetes/deployments/minio-deployment.yml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: s3-artifact-storage + namespace: reco-movies +spec: + replicas: 1 + selector: + matchLabels: + app: s3-artifact-storage + template: + metadata: + labels: + app: s3-artifact-storage + spec: + containers: + - name: s3-storage + image: minio/minio + ports: + - containerPort: 9000 # Port pour MinIO. + - containerPort: 9001 # Port pour l'interface web de MinIO. + env: + - name: MINIO_ROOT_USER + value: "mlflow_access" + - name: MINIO_ROOT_PASSWORD + value: "mlflow_secret" + command: ["server", "/data", "--console-address", ":9001"] # Commande pour démarrer MinIO. + volumeMounts: + - name: minio-data-volume + mountPath: /data # Chemin où les données seront stockées. + volumes: + - name: minio-data-volume + persistentVolumeClaim: + claimName: pvc-minio-data # Référence au PVC pour le stockage des données. diff --git a/kubernetes/deployments/mlflow-deployment.yml b/kubernetes/deployments/mlflow-deployment.yml new file mode 100644 index 0000000..211887d --- /dev/null +++ b/kubernetes/deployments/mlflow-deployment.yml @@ -0,0 +1,26 @@ +# Déploiement MLflow Webserver +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mlflow + namespace: reco-movies +spec: + replicas: 1 + selector: + matchLabels: + app: mlflow + template: + metadata: + labels: + app: mlflow + spec: + containers: + - name: mlflow-webserver + image: antoinepela/projet_reco_movies:mlflow-webserver-latest + ports: + - containerPort: 5000 + envFrom: + - configMapRef: + name: posgres-mlfow-config + - secretRef: + name: postgres-mlflow-secrets diff --git a/kubernetes/deployments/node-exporter-deployment.yml b/kubernetes/deployments/node-exporter-deployment.yml new file mode 100644 index 0000000..974665b --- /dev/null +++ b/kubernetes/deployments/node-exporter-deployment.yml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: node-exporter + namespace: reco-movies +spec: + replicas: 1 + selector: + matchLabels: + app: node-exporter + template: + metadata: + labels: + app: node-exporter + spec: + containers: + - name: node-exporter + image: antoinepela/projet_reco_movies:node-exporter-latest + ports: + - containerPort: 9100 # Port exposé pour Node Exporter. diff --git a/kubernetes/deployments/postgres-api-deployment.yml b/kubernetes/deployments/postgres-api-deployment.yml new file mode 100644 index 0000000..0ba9199 --- /dev/null +++ b/kubernetes/deployments/postgres-api-deployment.yml @@ -0,0 +1,56 @@ +# Déploiement Postgres pour l'API FastAPI +apiVersion: apps/v1 +kind: Deployment +metadata: + name: api-postgres + namespace: reco-movies +spec: + replicas: 2 + strategy: + type: RollingUpdate + selector: + matchLabels: + app: api-postgres + template: + metadata: + labels: + app: api-postgres + spec: + containers: + - name: postgres-api + image: antoinepela/projet_reco_movies:postgres-api-latest + envFrom: + - configMapRef: + name: postgres-api-config + - secretRef: + name: postgres-api-secrets + ports: + - containerPort: 5432 # Port par défaut de Postgres + volumeMounts: + - name: init-sql-volume + mountPath: /docker-entrypoint-initdb.d/init.sql # Chemin où PostgreSQL cherche les scripts d'initialisation. + - name: processed-movies-volume + mountPath: /docker-entrypoint-initdb.d/processed_movies.csv # Chemin pour le fichier CSV. + - name: processed-ratings-volume + mountPath: /docker-entrypoint-initdb.d/processed_ratings.csv # Chemin pour le fichier CSV. + - name: processed-links-volume + mountPath: /docker-entrypoint-initdb.d/processed_links.csv # Chemin pour le fichier CSV. + - name: posgres-api-volume + mountPath: /var/lib/postgresql/data # Chemin où PostgreSQL stocke ses données. + volumes: + - name: init-sql-volume + persistentVolumeClaim: + claimName: pvc-init-sql # Référence au PVC pour init.sql. + - name: processed-movies-volume + persistentVolumeClaim: + claimName: pvc-processed-movies # Référence au PVC pour processed_movies.csv. + - name: processed-ratings-volume + persistentVolumeClaim: + claimName: pvc-processed-ratings # Référence au PVC pour processed_ratings.csv. + - name: processed-links-volume + persistentVolumeClaim: + claimName: pvc-processed-links # Référence au PVC pour processed_links.csv. + - name: posgres-api-volume + persistentVolumeClaim: + claimName: posgres-api-volume # Référence au PVC pour processed_links.csv. + diff --git a/kubernetes/deployments/postgres-exporter-deployment.yml b/kubernetes/deployments/postgres-exporter-deployment.yml new file mode 100644 index 0000000..2937ab0 --- /dev/null +++ b/kubernetes/deployments/postgres-exporter-deployment.yml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres-exporter + namespace: reco-movies +spec: + replicas: 1 + selector: + matchLabels: + app: postgres-exporter + template: + metadata: + labels: + app: postgres-exporter + spec: + containers: + - name: postgres-exporter + image: prometheuscommunity/postgres-exporter # Utilisez l'image officielle de Postgres Exporter. + ports: + - containerPort: 9187 # Port exposé pour Postgres Exporter. + env: + - name: DATA_SOURCE_NAME + value: "postgresql://$(POSTGRES_USER):$(POSTGRES_PASSWORD)@db:5432/$(POSTGRES_DB)?sslmode=disable" # Informations de connexion à la base de données. + envFrom: + - configMapRef: + name: postgres-api-config + - secretRef: + name: postgres-api-secrets # Référence au secret contenant les informations d'authentification. + diff --git a/kubernetes/deployments/postgres-mlflow-deployment.yml b/kubernetes/deployments/postgres-mlflow-deployment.yml new file mode 100644 index 0000000..09c1685 --- /dev/null +++ b/kubernetes/deployments/postgres-mlflow-deployment.yml @@ -0,0 +1,26 @@ +# Déploiement Postgres pour MLflow +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mlflow-postgres + namespace: reco-movies +spec: + replicas: 1 + selector: + matchLabels: + app: postgres-mlflow + template: + metadata: + labels: + app: postgres-mlflow + spec: + containers: + - name: postgres-mlflow + image: antoinepela/projet_reco_movies:postgres-mlflow-latest + envFrom: + - configMapRef: + name: posgres-mlfow-config + - secretRef: + name: postgres-mlflow-secrets + ports: + - containerPort: 5429 # Port personnalisé pour MLflow DB diff --git a/kubernetes/deployments/prometheus-deployment.yml b/kubernetes/deployments/prometheus-deployment.yml new file mode 100644 index 0000000..a863765 --- /dev/null +++ b/kubernetes/deployments/prometheus-deployment.yml @@ -0,0 +1,36 @@ +# Déploiement Prometheus +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: reco-movies +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + containers: + - name: prometheus + image: antoinepela/projet_reco_movies:prometheus-latest + ports: + - containerPort: 9090 + envFrom: + - configMapRef: + name: prometheus-config + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus/prometheus.yml # Chemin où le fichier de configuration sera monté. + - name: prometheus-logs + mountPath: /var/log/prometheus # Répertoire pour les logs. + volumes: + - name: prometheus-config-volume + persistentVolumeClaim: + claimName: pvc-prometheus-config # Référence au PVC pour le fichier de configuration. + - name: prometheus-logs + persistentVolumeClaim: + claimName: prometheus-logs diff --git a/kubernetes/deployments/streamlit-deployment.yml b/kubernetes/deployments/streamlit-deployment.yml new file mode 100644 index 0000000..b8ff8c1 --- /dev/null +++ b/kubernetes/deployments/streamlit-deployment.yml @@ -0,0 +1,21 @@ +# Déploiement Streamlit +apiVersion: apps/v1 +kind: Deployment +metadata: + name: streamlit + namespace: reco-movies +spec: + replicas: 1 + selector: + matchLabels: + app: streamlit + template: + metadata: + labels: + app: streamlit + spec: + containers: + - name: streamlit + image: antoinepela/projet_reco_movies:streamlit-latest + ports: + - containerPort: 8501 diff --git a/kubernetes/ingress/ingress.yml b/kubernetes/ingress/ingress.yml new file mode 100644 index 0000000..c830388 --- /dev/null +++ b/kubernetes/ingress/ingress.yml @@ -0,0 +1,25 @@ +# Ingress Configuration (pour accéder aux services externes) +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: app-ingress + namespace: reco-movies +spec: + rules: + - host: "reco-movies.local" + http: + paths: + - path: "/api" + pathType: Prefix + backend: + service: + name: fastapi-service + port: + number: 8000 + - path: "/" + pathType: Prefix + backend: + service: + name: streamlit-service + port: + number: 8501 diff --git a/kubernetes/namespace/namespace.yml b/kubernetes/namespace/namespace.yml new file mode 100644 index 0000000..3b4d690 --- /dev/null +++ b/kubernetes/namespace/namespace.yml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: reco-movies diff --git a/kubernetes/persistent-volumes/fastapi-persistent-volume.yml b/kubernetes/persistent-volumes/fastapi-persistent-volume.yml new file mode 100644 index 0000000..6e7fc37 --- /dev/null +++ b/kubernetes/persistent-volumes/fastapi-persistent-volume.yml @@ -0,0 +1,54 @@ +# pv-processed-data.yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-processed-data +spec: + capacity: + storage: 1Gi # Ajustez la taille selon vos besoins. + accessModes: + - ReadWriteMany # Permettre plusieurs montages en lecture-écriture. + hostPath: + path: /home/antoine/jul24_cmlops_reco_film/ml/data/processed + +--- +# pv-models.yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-models +spec: + capacity: + storage: 1Gi # Ajustez la taille selon vos besoins. + accessModes: + - ReadWriteMany # Permettre plusieurs montages en lecture-écriture. + hostPath: + path: /home/antoine/jul24_cmlops_reco_film/ml/models + +--- +# pvc-processed-data.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-processed-data + namespace: reco-movies +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + +--- +# pvc-models.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-models + namespace: reco-movies +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi diff --git a/kubernetes/persistent-volumes/grafana-persistent-volume.yml b/kubernetes/persistent-volumes/grafana-persistent-volume.yml new file mode 100644 index 0000000..6f87dd6 --- /dev/null +++ b/kubernetes/persistent-volumes/grafana-persistent-volume.yml @@ -0,0 +1,12 @@ +# Persistent Volume Claims +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: grafana-logs + namespace: reco-movies +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi diff --git a/kubernetes/persistent-volumes/minio-persistent-volumes.yml b/kubernetes/persistent-volumes/minio-persistent-volumes.yml new file mode 100644 index 0000000..338ef61 --- /dev/null +++ b/kubernetes/persistent-volumes/minio-persistent-volumes.yml @@ -0,0 +1,12 @@ +# pvc-minio-data.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-minio-data + namespace: reco-movies +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi # Doit correspondre à la taille du PV. diff --git a/kubernetes/persistent-volumes/models-pvc.yml b/kubernetes/persistent-volumes/models-pvc.yml new file mode 100644 index 0000000..2b4f660 --- /dev/null +++ b/kubernetes/persistent-volumes/models-pvc.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-models + namespace: reco-movies +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 500Mi + storageClassName: standard diff --git a/kubernetes/persistent-volumes/postgres-api-persistent-volumes.yml b/kubernetes/persistent-volumes/postgres-api-persistent-volumes.yml new file mode 100644 index 0000000..7e5e081 --- /dev/null +++ b/kubernetes/persistent-volumes/postgres-api-persistent-volumes.yml @@ -0,0 +1,128 @@ +### CREATION DES PV + +# pv-init-sql.yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-init-sql +spec: + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /home/antoine/jul24_cmlops_reco_film/postgres/init.sql + +--- +# pv-processed-movies.yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-processed-movies +spec: + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /home/antoine/jul24_cmlops_reco_film/ml/data/processed/processed_movies.csv + +--- +# pv-processed-ratings.yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-processed-ratings +spec: + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /home/antoine/jul24_cmlops_reco_film/ml/data/processed/processed_ratings.csv + +--- +# pv-processed-links.yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-processed-links +spec: + capacity: + storage: 1Gi + accessModes: + - ReadWriteOnce + hostPath: + path: /home/antoine/jul24_cmlops_reco_film/ml/data/processed/processed_links.csv + +--- +### CREATION DES PVC + +# pvc-init-sql.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-init-sql + namespace: reco-movies +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + +--- +# pvc-processed-movies.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-processed-movies + namespace: reco-movies +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + +--- +# pvc-processed-ratings.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-processed-ratings + namespace: reco-movies +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + +--- +# pvc-processed-links.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-processed-links + namespace: reco-movies +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + +--- +# pvc-processed-links.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: posgres-api-volume + namespace: reco-movies +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi diff --git a/kubernetes/persistent-volumes/processed-data-pvc.yml b/kubernetes/persistent-volumes/processed-data-pvc.yml new file mode 100644 index 0000000..dd25a60 --- /dev/null +++ b/kubernetes/persistent-volumes/processed-data-pvc.yml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-processed-data + namespace: reco-movies +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 1Gi + storageClassName: standard diff --git a/kubernetes/persistent-volumes/prometheus-persistent-volume.yml b/kubernetes/persistent-volumes/prometheus-persistent-volume.yml new file mode 100644 index 0000000..1951ad9 --- /dev/null +++ b/kubernetes/persistent-volumes/prometheus-persistent-volume.yml @@ -0,0 +1,38 @@ +# pv-prometheus-config.yaml +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pv-prometheus-config +spec: + capacity: + storage: 1Gi # Ajustez la taille selon vos besoins. + accessModes: + - ReadWriteOnce + hostPath: + path: /home/antoine/jul24_cmlops_reco_film/prometheus/prometheus.yml +--- +# pvc-prometheus-config.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-prometheus-config + namespace: reco-movies +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +# Persistent Volume Claims +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: prometheus-logs + namespace: reco-movies +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi diff --git a/kubernetes/secrets/secrets.yml b/kubernetes/secrets/secrets.yml new file mode 100644 index 0000000..c3de117 --- /dev/null +++ b/kubernetes/secrets/secrets.yml @@ -0,0 +1,33 @@ +# Secrets pour les mots de passe de la base de données +apiVersion: v1 +kind: Secret +metadata: + name: postgres-api-secrets + namespace: reco-movies +type: Opaque +data: + POSTGRES_PASSWORD: ZGF0YXNjaWVudGVzdA== # base64 encoded password + +--- +apiVersion: v1 +kind: Secret +metadata: + name: postgres-mlflow-secrets + namespace: reco-movies +type: Opaque +data: + POSTGRES_PASSWORD: bWxmbG93 # base64 encoded password for MLflow DB + AWS_ACCESS_KEY_ID: bWxmbG93X2FjY2Vzcw== # base64 encoded 'mlflow_access' + AWS_SECRET_ACCESS_KEY: bWxmbG93X3NlY3JldA== # base64 encoded 'mlflow_secret' + +# Secrets Fastapi +--- +apiVersion: v1 +kind: Secret +metadata: + name: fastapi-secrets + namespace: reco-movies +type: Opaque +data: + # Ces valeurs doivent être encodées en base64 + API_KEY: "dm90cmVfY2xlX2FwaV9pY2k=" # Exemple encodé en base64 diff --git a/kubernetes/services/services.yml b/kubernetes/services/services.yml new file mode 100644 index 0000000..e03bb2d --- /dev/null +++ b/kubernetes/services/services.yml @@ -0,0 +1,70 @@ +# Services pour chaque application + +## Service MLflow (ClusterIP) +apiVersion: v1 +kind: Service +metadata: + name: mlflow-service + namespace: reco-movies +spec: + selector: + app: mlflow + ports: + - port: 5000 + type: ClusterIP + +--- +## Service FastAPI (LoadBalancer) +apiVersion: v1 +kind: Service +metadata: + name: fastapi-service + namespace: reco-movies +spec: + selector: + app: fastapi + ports: + - port: 8000 + type: LoadBalancer +--- +## Service Streamlit (LoadBalancer) +apiVersion: v1 +kind: Service +metadata: + name: streamlit-service + namespace: reco-movies +spec: + selector: + app: streamlit + ports: + - port: 8501 + type: LoadBalancer +--- +## Service Prometheus (ClusterIP) +apiVersion: v1 +kind: Service +metadata: + name: prometheus-service + namespace: reco-movies +spec: + selector: + app: prometheus + ports: + - port: 9090 + type: ClusterIP +--- +## Service Grafana (LoadBalancer) +apiVersion: v1 +kind: Service +metadata: + name: grafana-service + namespace: reco-movies +spec: + selector: + app: grafana + ports: + - port: 3000 + type: LoadBalancer + +--- +