[connector] fix: Add heartbeat in webcrawler (#5871)

dust-tt · Jun 26, 2024 · 58d890c · 58d890c
1 parent bdf6b42
commit 58d890c
Showing 1 changed file with 7 additions and 0 deletions.
diff --git a/connectors/src/connectors/webcrawler/temporal/activities.ts b/connectors/src/connectors/webcrawler/temporal/activities.ts
@@ -93,6 +93,10 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) {
       navigationTimeoutSecs: 10,
       preNavigationHooks: [
         async (crawlingContext) => {
+          Context.current().heartbeat({
+            type: "pre_navigation",
+          });
+
           const { address, family } = await getIpAddressForUrl(
             crawlingContext.request.url
           );
@@ -318,6 +322,9 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) {
         await reportInitialSyncProgress(connector.id, `${pageCount} pages`);
       },
       failedRequestHandler: async (context, error) => {
+        Context.current().heartbeat({
+          type: "failed_request",
+        });
         childLogger.error(
           {
             url: context.request.url,