feat(kw-search): pass title & mime when upserting tables from connect…

…ors (#8962) * feat(kw-search): pass title and mimetype when upserting tables from connectors * improve mimetype for notion DB --------- Co-authored-by: Henry Fontanier <[email protected]>
dust-tt · Nov 27, 2024 · 04d68c9 · 04d68c9
1 parent 551d655
commit 04d68c9
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 0 deletions.
diff --git a/connectors/src/connectors/google_drive/temporal/spreadsheets.ts b/connectors/src/connectors/google_drive/temporal/spreadsheets.ts
@@ -83,6 +83,8 @@ async function upsertGdriveTable(
     truncate: true,
     parents: [tableId, ...parents],
     useAppForHeaderDetection: true,
+    title: `${spreadsheet.title} - ${title}`,
+    mimeType: "application/vnd.google-apps.spreadsheet",
   });
 
   logger.info(loggerArgs, "[Spreadsheet] Table upserted.");

diff --git a/connectors/src/connectors/microsoft/temporal/spreadsheets.ts b/connectors/src/connectors/microsoft/temporal/spreadsheets.ts
@@ -97,6 +97,9 @@ async function upsertMSTable(
     truncate: true,
     parents,
     useAppForHeaderDetection: true,
+    title: `${spreadsheet.name} - ${worksheet.name}`,
+    mimeType:
+      "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
   });
 
   logger.info(loggerArgs, "[Spreadsheet] Table upserted.");

diff --git a/connectors/src/connectors/notion/temporal/activities.ts b/connectors/src/connectors/notion/temporal/activities.ts
@@ -1812,6 +1812,8 @@ export async function renderAndUpsertPageFromCache({
               // We only update the rowId of for the page without truncating the rest of the table (incremental sync).
               truncate: false,
               parents,
+              title: parentDb.title ?? "Untitled Notion Database",
+              mimeType: "application/vnd.notion.database",
             }),
           localLogger
         );
@@ -2522,6 +2524,8 @@ export async function upsertDatabaseStructuredDataFromCache({
         // We overwrite the whole table since we just fetched all child pages.
         truncate: true,
         parents,
+        title: dbModel.title ?? "Untitled Notion Database",
+        mimeType: "notion/database",
       }),
     localLogger
   );

diff --git a/connectors/src/connectors/shared/file.ts b/connectors/src/connectors/shared/file.ts
@@ -78,6 +78,8 @@ export async function handleCsvFile({
       },
       truncate: true,
       parents,
+      title: fileName,
+      mimeType: "text/csv",
     });
   } catch (err) {
     localLogger.warn({ error: err }, "Error while parsing or upserting table");

diff --git a/connectors/src/lib/data_sources.ts b/connectors/src/lib/data_sources.ts
@@ -709,6 +709,8 @@ export async function upsertTableFromCsv({
   truncate,
   parents,
   useAppForHeaderDetection,
+  title,
+  mimeType,
 }: {
   dataSourceConfig: DataSourceConfig;
   tableId: string;
@@ -719,6 +721,8 @@ export async function upsertTableFromCsv({
   truncate: boolean;
   parents: string[];
   useAppForHeaderDetection?: boolean;
+  title: string;
+  mimeType: string;
 }) {
   const localLogger = logger.child({ ...loggerArgs, tableId, tableName });
   const statsDTags = [
@@ -754,6 +758,8 @@ export async function upsertTableFromCsv({
     truncate,
     async: true,
     useAppForHeaderDetection,
+    title,
+    mimeType,
   };
   const dustRequestConfig: AxiosRequestConfig = {
     headers: {