Add PROBE_MONITOR_RETRY_LIMIT environment variable and update monitor…

… retry logic
OneUptime · Dec 9, 2024 · 9e117f3 · 9e117f3
1 parent 4a6edfa
commit 9e117f3
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 8 deletions.
diff --git a/HelmChart/Public/oneuptime/templates/probe.yaml b/HelmChart/Public/oneuptime/templates/probe.yaml
@@ -64,6 +64,8 @@ spec:
               value: {{ $val.description }}
             - name: PROBE_MONITORING_WORKERS
               value: {{ $val.monitoringWorkers | squote }}
+            - name: PROBE_MONITOR_RETRY_LIMIT
+              value: {{ $val.monitorRetryLimit | squote }}
             # syntheticMonitorScriptTimeoutInMs
             - name: PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS
               value: {{ $val.syntheticMonitorScriptTimeoutInMs | squote }}

diff --git a/Probe/Config.ts b/Probe/Config.ts
@@ -73,3 +73,9 @@ export const PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS: number = process
       process.env["PROBE_CUSTOM_CODE_MONITOR_SCRIPT_TIMEOUT_IN_MS"].toString(),
     )
   : 60000;
+
+export const PROBE_MONITOR_RETRY_LIMIT: number = process.env[
+  "PROBE_MONITOR_RETRY_LIMIT"
+]
+  ? parseInt(process.env["PROBE_MONITOR_RETRY_LIMIT"].toString())
+  : 3;
diff --git a/Probe/Index.ts b/Probe/Index.ts
@@ -1,4 +1,4 @@
-import { PROBE_MONITORING_WORKERS } from "./Config";
+import { PROBE_MONITOR_RETRY_LIMIT, PROBE_MONITORING_WORKERS } from "./Config";
 import "./Jobs/Alive";
 import FetchListAndProbe from "./Jobs/Monitor/FetchList";
 import FetchMonitorTest from "./Jobs/Monitor/FetchMonitorTest";
@@ -35,6 +35,10 @@ const init: PromiseVoidFunction = async (): Promise<void> => {
     await App.addDefaultRoutes();
 
     try {
+      logger.debug(
+        `This probe will retyr monitor for: ${PROBE_MONITOR_RETRY_LIMIT} times`,
+      );
+
       // Register this probe.
       await Register.registerProbe();
 

diff --git a/Probe/Utils/Monitors/Monitor.ts b/Probe/Utils/Monitors/Monitor.ts
@@ -1,4 +1,4 @@
-import { PROBE_INGEST_URL } from "../../Config";
+import { PROBE_INGEST_URL, PROBE_MONITOR_RETRY_LIMIT } from "../../Config";
 import ProbeUtil from "../Probe";
 import ProbeAPIRequest from "../ProbeAPIRequest";
 import ApiMonitor, { APIResponse } from "./MonitorTypes/ApiMonitor";
@@ -203,7 +203,7 @@ export default class MonitorUtil {
           monitorStep.data?.monitorDestination,
           new Port(80), // use port 80 by default.
           {
-            retry: 10,
+            retry: PROBE_MONITOR_RETRY_LIMIT,
             monitorId: monitorId,
             timeout: new PositiveNumber(60000), // 60 seconds
           },
@@ -220,7 +220,7 @@ export default class MonitorUtil {
         const response: PingResponse | null = await PingMonitor.ping(
           monitorStep.data?.monitorDestination,
           {
-            retry: 10,
+            retry: PROBE_MONITOR_RETRY_LIMIT,
             monitorId: monitorId,
             timeout: new PositiveNumber(60000), // 60 seconds
           },
@@ -257,7 +257,7 @@ export default class MonitorUtil {
         monitorStep.data?.monitorDestination,
         monitorStep.data.monitorDestinationPort,
         {
-          retry: 10,
+          retry: PROBE_MONITOR_RETRY_LIMIT,
           monitorId: monitorId,
           timeout: new PositiveNumber(60000), // 60 seconds
         },
@@ -333,7 +333,7 @@ export default class MonitorUtil {
       const response: SslResponse | null = await SSLMonitor.ping(
         monitorStep.data?.monitorDestination as URL,
         {
-          retry: 10,
+          retry: PROBE_MONITOR_RETRY_LIMIT,
           monitorId: monitorId,
           timeout: new PositiveNumber(60000), // 60 seconds
         },
@@ -362,7 +362,7 @@ export default class MonitorUtil {
         {
           isHeadRequest: MonitorUtil.isHeadRequest(monitorStep),
           monitorId: monitorId,
-          retry: 10,
+          retry: PROBE_MONITOR_RETRY_LIMIT,
           timeout: new PositiveNumber(60000), // 60 seconds
           doNotFollowRedirects: monitorStep.data?.doNotFollowRedirects || false,
         },
@@ -404,7 +404,7 @@ export default class MonitorUtil {
           requestBody: requestBody || undefined,
           monitorId: monitorId,
           requestType: monitorStep.data?.requestType || HTTPMethod.GET,
-          retry: 10,
+          retry: PROBE_MONITOR_RETRY_LIMIT,
           timeout: new PositiveNumber(60000), // 60 seconds
           doNotFollowRedirects: monitorStep.data?.doNotFollowRedirects || false,
         },