move to Section for upsertToDatasource (#2732)

dust-tt · Dec 1, 2023 · 2aa97a7 · 2aa97a7
1 parent 61f1926
commit 2aa97a7
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 12 deletions.
diff --git a/connectors/src/connectors/github/temporal/activities.ts b/connectors/src/connectors/github/temporal/activities.ts
@@ -157,7 +157,11 @@ export async function githubUpsertIssueActivity(
   await upsertToDatasource({
     dataSourceConfig,
     documentId,
-    documentText: renderedIssue,
+    documentContent: {
+      prefix: null,
+      content: renderedIssue,
+      sections: [],
+    },
     documentUrl: issue.url,
     timestampMs: lastUpdateTimestamp,
     tags: tags,
@@ -287,7 +291,11 @@ export async function githubUpsertDiscussionActivity(
   await upsertToDatasource({
     dataSourceConfig,
     documentId,
-    documentText: renderedDiscussion,
+    documentContent: {
+      prefix: null,
+      content: renderedDiscussion,
+      sections: [],
+    },
     documentUrl: discussion.url,
     timestampMs: new Date(discussion.createdAt).getTime(),
     tags,

diff --git a/connectors/src/connectors/google_drive/temporal/activities.ts b/connectors/src/connectors/google_drive/temporal/activities.ts
@@ -484,7 +484,11 @@ async function syncOneFile(
     await upsertToDatasource({
       dataSourceConfig,
       documentId,
-      documentText: documentContent,
+      documentContent: {
+        prefix: null,
+        content: documentContent,
+        sections: [],
+      },
       documentUrl: file.webViewLink,
       timestampMs: file.updatedAtMs,
       tags,

diff --git a/connectors/src/connectors/notion/temporal/activities.ts b/connectors/src/connectors/notion/temporal/activities.ts
@@ -1740,7 +1740,11 @@ export async function renderAndUpsertPageFromCache({
         workspaceAPIKey: connector.workspaceAPIKey,
       },
       documentId,
-      documentText: renderedPage,
+      documentContent: {
+        prefix: null,
+        content: renderedPage,
+        sections: [],
+      },
       documentUrl: pageCacheEntry.url,
       timestampMs: updatedTime,
       tags: getTagsForPage({

diff --git a/connectors/src/connectors/slack/temporal/activities.ts b/connectors/src/connectors/slack/temporal/activities.ts
@@ -444,7 +444,11 @@ export async function syncNonThreaded(
   await upsertToDatasource({
     dataSourceConfig,
     documentId,
-    documentText: text,
+    documentContent: {
+      prefix: null,
+      content: text,
+      sections: [],
+    },
     documentUrl: sourceUrl,
     timestampMs: createdAt,
     tags,
@@ -614,7 +618,11 @@ export async function syncThread(
   await upsertToDatasource({
     dataSourceConfig,
     documentId,
-    documentText: text,
+    documentContent: {
+      prefix: null,
+      content: text,
+      sections: [],
+    },
     documentUrl: sourceUrl,
     timestampMs: createdAt,
     tags,

diff --git a/connectors/src/lib/data_sources.ts b/connectors/src/lib/data_sources.ts
@@ -1,4 +1,8 @@
-import { PostDataSourceDocumentRequestBody } from "@dust-tt/types";
+import {
+  CoreAPIDataSourceDocumentSection,
+  PostDataSourceDocumentRequestBody,
+  sectionFullText,
+} from "@dust-tt/types";
 import axios, { AxiosRequestConfig, AxiosResponse } from "axios";
 
 import logger from "@connectors/logger/logger";
@@ -23,7 +27,7 @@ type UpsertContext = {
 type UpsertToDataSourceParams = {
   dataSourceConfig: DataSourceConfig;
   documentId: string;
-  documentText: string;
+  documentContent: CoreAPIDataSourceDocumentSection;
   documentUrl?: string;
   timestampMs?: number;
   tags?: string[];
@@ -37,7 +41,7 @@ export const upsertToDatasource = withRetries(_upsertToDatasource);
 async function _upsertToDatasource({
   dataSourceConfig,
   documentId,
-  documentText,
+  documentContent,
   documentUrl,
   timestampMs,
   tags,
@@ -49,7 +53,7 @@ async function _upsertToDatasource({
     ...loggerArgs,
     documentId,
     documentUrl,
-    documentLength: documentText.length,
+    documentLength: sectionFullText(documentContent).length,
     workspaceId: dataSourceConfig.workspaceId,
     dataSourceName: dataSourceConfig.dataSourceName,
   });
@@ -66,8 +70,8 @@ async function _upsertToDatasource({
   const urlSafeName = encodeURIComponent(dataSourceConfig.dataSourceName);
   const endpoint = `${DUST_FRONT_API}/api/v1/w/${dataSourceConfig.workspaceId}/data_sources/${urlSafeName}/documents/${documentId}`;
   const dustRequestPayload: PostDataSourceDocumentRequestBody = {
-    text: documentText,
-    section: null,
+    text: null,
+    section: documentContent,
     source_url: documentUrl,
     timestamp: timestampMs,
     tags: tags?.map((tag) => tag.substring(0, 512)),