Adding heartbeat for the webcrawler workflow (#3161)

dust-tt · Jan 11, 2024 · 4ee043b · 4ee043b
1 parent 96ed9b7
commit 4ee043b
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/connectors/src/connectors/webcrawler/temporal/activities.ts b/connectors/src/connectors/webcrawler/temporal/activities.ts
@@ -1,4 +1,5 @@
 import { ModelId } from "@dust-tt/types";
+import { Context } from "@temporalio/activity";
 import { CheerioCrawler, Configuration } from "crawlee";
 import turndown from "turndown";
 
@@ -45,6 +46,9 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) {
       maxConcurrency: CONCURRENCY,
 
       async requestHandler({ $, request, enqueueLinks }) {
+        Context.current().heartbeat({
+          type: "http_request",
+        });
         const extracted = new turndown()
           .remove(["style", "script", "iframe"])
           .turndown($.html());