[Webcrawler] Adding a Temporal heartbeat in the crawler error handler (…

…#3978)
dust-tt · Feb 27, 2024 · d6d7b28 · d6d7b28
1 parent d2e1a68
commit d6d7b28
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/connectors/src/connectors/webcrawler/temporal/activities.ts b/connectors/src/connectors/webcrawler/temporal/activities.ts
@@ -241,6 +241,12 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) {
         );
         crawlingError++;
       },
+      errorHandler: () => {
+        // Errors are already logged by the crawler, so we are not re-logging them here.
+        Context.current().heartbeat({
+          type: "error_handler",
+        });
+      },
     },
     new Configuration({
       purgeOnStart: true,