refactor: Refactor garafana and prometheus deployment stuff (#646)

<!-- ELLIPSIS_HIDDEN -->



> [!IMPORTANT]
> Refactor Grafana and Prometheus deployment by consolidating
configurations into a single `monitoring` directory and updating related
`docker-compose` setups.
> 
>   - **Refactoring**:
> - Consolidate Grafana and Prometheus configurations into `monitoring`
directory.
> - Remove `grafana/docker-compose.yml` and
`prometheus/docker-compose.yml`.
> - Update `docker-compose.yml` to include
`monitoring/docker-compose.yml` instead of separate Grafana and
Prometheus files.
>   - **File Moves**:
> - Move `grafana/provisioning/datasources/datasource.yml` to
`monitoring/grafana/provisioning/datasources/datasource.yml`.
> - Move `prometheus/config/prometheus.yml` to
`monitoring/prometheus/config/prometheus.yml`.
>   - **Configuration Changes**:
> - Update `monitoring/docker-compose.yml` to include both Grafana and
Prometheus services with shared volume configurations.
> - Add `depends_on` condition for Prometheus service in
`monitoring/docker-compose.yml`.
> 
> <sup>This description was created by </sup>[<img alt="Ellipsis"
src="https://img.shields.io/badge/Ellipsis-blue?color=175173">](https://www.ellipsis.dev?ref=julep-ai%2Fjulep&utm_source=github&utm_medium=referral)<sup>
for cb36d21. It will automatically
update as commits are pushed.</sup>

<!-- ELLIPSIS_HIDDEN -->

docker-compose.yml

@@ -12,8 +12,7 @@ include:
   - ./scheduler/docker-compose.yml
   - ./llm-proxy/docker-compose.yml
   - ./integrations-service/docker-compose.yml
-  - ./prometheus/docker-compose.yml
-  - ./grafana/docker-compose.yml
+  - ./monitoring/docker-compose.yml
   - ./blob-store/docker-compose.yml
 
   # TODO: Enable after testing

monitoring/README.md

@@ -1,41 +1,28 @@
-# TODO: Fix monitoring services
-# SCRUM-25
-
 # Compose sample
 ### Prometheus & Grafana
 
 Project structure:
 ```
-.
-├── compose.yaml
+├── README.md
+├── docker-compose.yml
 ├── grafana
-│   └── datasource.yml
-├── prometheus
-│   └── prometheus.yml
-└── README.md
+│   └── provisioning
+│       └── datasources
+│           └── datasource.yml
+└── prometheus
+    └── config
+        └── prometheus.yml
 ```
 
-[_compose.yaml_](compose.yaml)
-```
-services:
-  prometheus:
-    image: prom/prometheus
-    ...
-    ports:
-      - 9090:9090
-  grafana:
-    image: grafana/grafana
-    ...
-    ports:
-      - 3000:3000
-```
-The compose file defines a stack with two services `prometheus` and `grafana`.
+The docker compose file defines a stack with two services `prometheus` and `grafana`.
 When deploying the stack, docker compose maps port the default ports for each service to the equivalent ports on the host in order to inspect easier the web interface of each service.
 Make sure the ports 9090 and 3000 on the host are not already in use.
 
 ## Deploy with docker compose
 
 ```
+$ docker volume create grafana_data
+$ docker volume create prometheus_data
 $ docker compose up -d
 Creating network "prometheus-grafana_default" with the default driver
 Creating volume "prometheus-grafana_prom_data" with default driver

monitoring/docker-compose.yml

@@ -1,39 +1,39 @@
 name: julep-monitoring
 
 services:
-  portainer:
-    image: portainer/portainer-ce:latest
-    container_name: portainer
-    restart: unless-stopped
-    ports:
-      - "9000:8000"
-      - "9443:9443"
-    volumes:
-      - portainer_data:/data
-      - /var/run/docker.sock:/var/run/docker.sock
-
   prometheus:
     image: prom/prometheus
     container_name: prometheus
-    command:
-      - "--config.file=/etc/prometheus/prometheus.yml"
-    restart: unless-stopped
+    profiles:
+      - multi-tenant
+
     volumes:
-      - ./prometheus:/etc/prometheus
-      - prom_data:/prometheus
+      - ./config/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+
+    depends_on:
+      agents-api-multi-tenant:
+        condition: service_started
+
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
 
   grafana:
     image: grafana/grafana
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD}
+      - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER}
     container_name: grafana
     ports:
       - 3000:3000
-    restart: unless-stopped
-    environment:
-      - GF_SECURITY_ADMIN_USER=admin
-      - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD}
     volumes:
-      - ./grafana:/etc/grafana/provisioning/datasources
+      - grafana_data:/var/lib/grafana
+      - ./provisioning:/etc/grafana/provisioning
+    profiles:
+      - multi-tenant
 
 volumes:
-  prom_data:
-  portainer_data:
+  prometheus_data:
+    external: true
+  grafana_data:
+    external: true