fixing hbk prometheus

Signed-off-by: Sanskar Bhushan <[email protected]>
Obmondo · Oct 28, 2024 · 733f95d · 733f95d
1 parent 5d4b98b
commit 733f95d
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 0 deletions.
diff --git a/argocd-helm-charts/prometheus-linuxaid/templates/prometheus.yaml b/argocd-helm-charts/prometheus-linuxaid/templates/prometheus.yaml
@@ -47,5 +47,13 @@ spec:
           requests:
             storage: {{ .Values.prometheus.storage.size }}
         storageClassName: {{ .Values.prometheus.storage.className }}
+  containers:
+  - name: prometheus
+    startupProbe:
+      failureThreshold: {{ .Values.prometheus.startupProbe.failureThreshold }}
+    livenessProbe:
+      failureThreshold: {{ .Values.prometheus.livenessProbe.failureThreshold }}
+    readinessProbe:
+      failureThreshold: {{ .Values.prometheus.readinessProbe.failureThreshold }}
   version: {{ .Values.prometheus.version }}
   enableRemoteWriteReceiver: true
diff --git a/argocd-helm-charts/prometheus-linuxaid/values.yaml b/argocd-helm-charts/prometheus-linuxaid/values.yaml
@@ -12,6 +12,14 @@ prometheus:
     requests:
       cpu: 600m
       memory: 2Gi
+  # in prometheus pods that takes some recovery time due to large number of WALs we need to increase the probe time.
+  # ref. - https://github.com/prometheus-operator/prometheus-operator/blob/378d36df448366414de53a66a64020cd053002b7/pkg/prometheus/statefulset.go#L641-L643
+  startupProbe:
+    failureThreshold: 240
+  livenessProbe:
+    failureThreshold: 240
+  readinessProbe:
+    failureThreshold: 240
 grafana:
   server: graphs.obmondo.com
   version: 11.1.4