Continued move to dustAPIDataSourceId (#7082)

* getDataSourceDocumentVersions * update document methods * dataSourceTokenize * deleteTable, upssertTable, upsertTableRows * getTable * getTable * updateTableParents * getTableRow * getTableRows * deleteTableRow
dust-tt · Sep 3, 2024 · 879b4af · 879b4af
1 parent f119b69
commit 879b4af
Show file tree

Hide file tree

Showing 20 changed files with 106 additions and 104 deletions.
diff --git a/front/admin/cli.ts b/front/admin/cli.ts
@@ -423,7 +423,7 @@ const dataSource = async (command: string, args: parseArgs.ParsedArgs) => {
       }
       const delRes = await coreAPI.deleteDataSourceDocument({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         documentId: args.documentId,
       });
       if (delRes.isErr()) {

diff --git a/front/components/assistant_builder/server_side_props_helpers.ts b/front/components/assistant_builder/server_side_props_helpers.ts
@@ -200,7 +200,7 @@ export async function buildInitialActions({
 
           const coreAPITable = await coreAPI.getTable({
             projectId: dataSourceView.dataSource.dustAPIProjectId,
-            dataSourceName: dataSourceView.dataSource.name,
+            dataSourceId: dataSourceView.dataSource.dustAPIDataSourceId,
             tableId: t.tableId,
           });
 

diff --git a/front/lib/api/data_source_view.ts b/front/lib/api/data_source_view.ts
@@ -160,7 +160,7 @@ export async function getContentNodesForStaticDataSourceView(
     return new Ok(documentsAsContentNodes);
   } else {
     const tablesRes = await coreAPI.getTables({
-      dataSourceName: dataSource.name,
+      dataSourceId: dataSource.dustAPIDataSourceId,
       projectId: dataSource.dustAPIProjectId,
       viewFilter: dataSourceView.toViewFilter(),
     });

diff --git a/front/lib/api/tables.ts b/front/lib/api/tables.ts
@@ -3,6 +3,7 @@ import type {
   CoreAPIRow,
   CoreAPIRowValue,
   CoreAPITable,
+  DataSourceType,
   Result,
   WorkspaceType,
 } from "@dust-tt/types";
@@ -58,26 +59,26 @@ export type TableOperationError =
 
 export async function deleteTable({
   owner,
-  projectId,
-  dataSourceName,
+  dataSource,
   tableId,
 }: {
   owner: WorkspaceType;
-  projectId: string;
-  dataSourceName: string;
+  dataSource: DataSourceType;
   tableId: string;
 }): Promise<Result<null, TableOperationError>> {
   const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger);
 
   const deleteRes = await coreAPI.deleteTable({
-    projectId,
-    dataSourceName,
+    projectId: dataSource.dustAPIProjectId,
+    dataSourceId: dataSource.dustAPIDataSourceId,
     tableId,
   });
   if (deleteRes.isErr()) {
     logger.error(
       {
-        dataSourceName,
+        projectId: dataSource.dustAPIProjectId,
+        dataSourceId: dataSource.dustAPIDataSourceId,
+        dataSourceName: dataSource.name,
         workspaceId: owner.id,
         error: deleteRes.error,
       },
@@ -101,7 +102,8 @@ export async function deleteTable({
   await AgentTablesQueryConfigurationTable.destroy({
     where: {
       dataSourceWorkspaceId: owner.sId,
-      dataSourceId: dataSourceName,
+      // TODO(DATASOURCE_SID); state storing the datasource name
+      dataSourceId: dataSource.name,
       tableId,
     },
   });
@@ -111,8 +113,7 @@ export async function deleteTable({
 
 export async function upsertTableFromCsv({
   auth,
-  projectId,
-  dataSourceName,
+  dataSource,
   tableName,
   tableDescription,
   tableId,
@@ -123,8 +124,7 @@ export async function upsertTableFromCsv({
   truncate,
 }: {
   auth: Authenticator;
-  projectId: string;
-  dataSourceName: string;
+  dataSource: DataSourceType;
   tableName: string;
   tableDescription: string;
   tableId: string;
@@ -166,10 +166,11 @@ export async function upsertTableFromCsv({
       logger.error(
         {
           ...errorDetails,
-          dataSourceName,
+          projectId: dataSource.dustAPIProjectId,
+          dataSourceId: dataSource.dustAPIDataSourceId,
+          dataSourceName: dataSource.name,
           tableName,
           tableId,
-          projectId,
         },
         "CSV parsing error."
       );
@@ -190,10 +191,11 @@ export async function upsertTableFromCsv({
     logger.error(
       {
         ...errorDetails,
-        dataSourceName,
+        projectId: dataSource.dustAPIProjectId,
+        dataSourceId: dataSource.dustAPIDataSourceId,
+        dataSourceName: dataSource.name,
         tableName,
         tableId,
-        projectId,
       },
       "CSV input validation error: too many rows."
     );
@@ -202,8 +204,8 @@ export async function upsertTableFromCsv({
 
   const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger);
   const tableRes = await coreAPI.upsertTable({
-    projectId,
-    dataSourceName,
+    projectId: dataSource.dustAPIProjectId,
+    dataSourceId: dataSource.dustAPIDataSourceId,
     tableId,
     name: tableName,
     description: tableDescription,
@@ -221,11 +223,12 @@ export async function upsertTableFromCsv({
     logger.error(
       {
         ...errorDetails,
-        dataSourceName,
+        projectId: dataSource.dustAPIProjectId,
+        dataSourceId: dataSource.dustAPIDataSourceId,
+        dataSourceName: dataSource.name,
         workspaceId: owner.id,
         tableId,
         tableName,
-        projectId,
       },
       "Error upserting table in CoreAPI."
     );
@@ -234,8 +237,8 @@ export async function upsertTableFromCsv({
 
   if (csvRows) {
     const rowsRes = await coreAPI.upsertTableRows({
-      projectId,
-      dataSourceName,
+      projectId: dataSource.dustAPIProjectId,
+      dataSourceId: dataSource.dustAPIDataSourceId,
       tableId,
       rows: csvRows,
       truncate,
@@ -250,18 +253,19 @@ export async function upsertTableFromCsv({
       logger.error(
         {
           ...errorDetails,
-          dataSourceName,
+          projectId: dataSource.dustAPIProjectId,
+          dataSourceId: dataSource.dustAPIDataSourceId,
+          dataSourceName: dataSource.name,
           workspaceId: owner.id,
           tableId,
           tableName,
-          projectId,
         },
         "Error upserting rows in CoreAPI."
       );
 
       const delRes = await coreAPI.deleteTable({
-        projectId,
-        dataSourceName,
+        projectId: dataSource.dustAPIProjectId,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         tableId,
       });
 
@@ -270,11 +274,12 @@ export async function upsertTableFromCsv({
           {
             type: "internal_server_error",
             coreAPIError: delRes.error,
-            dataSourceName,
+            projectId: dataSource.dustAPIProjectId,
+            dataSourceId: dataSource.dustAPIDataSourceId,
+            dataSourceName: dataSource.name,
             workspaceId: owner.id,
             tableId,
             tableName,
-            projectId,
           },
           "Failed to delete table after failed row upsert."
         );

diff --git a/front/lib/document_tracker.ts b/front/lib/document_tracker.ts
@@ -176,14 +176,14 @@ export async function updateTrackedDocuments(
   if (hasExistingTrackedDocs && !hasRemainingTrackedDocs) {
     await coreAPI.updateDataSourceDocumentTags({
       projectId: dataSource.dustAPIProjectId,
-      dataSourceName: dataSource.name,
+      dataSourceId: dataSource.dustAPIDataSourceId,
       removeTags: ["__DUST_TRACKED"],
       documentId,
     });
   } else if (!hasExistingTrackedDocs && hasRemainingTrackedDocs) {
     await coreAPI.updateDataSourceDocumentTags({
       projectId: dataSource.dustAPIProjectId,
-      dataSourceName: dataSource.name,
+      dataSourceId: dataSource.dustAPIDataSourceId,
       addTags: ["__DUST_TRACKED"],
       documentId,
     });

diff --git a/front/lib/documents_post_process_hooks/hooks/data_source_helpers.ts b/front/lib/documents_post_process_hooks/hooks/data_source_helpers.ts
@@ -26,7 +26,7 @@ export async function getPreviousDocumentVersion({
   const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger);
   const versions = await coreAPI.getDataSourceDocumentVersions({
     projectId: dataSource.dustAPIProjectId,
-    dataSourceName: dataSource.name,
+    dataSourceId: dataSource.dustAPIDataSourceId,
     documentId: documentId,
     limit: 1,
     offset: 1,

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/documents/[documentId]/index.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/documents/[documentId]/index.ts
@@ -452,7 +452,7 @@ async function handler(
         // Create document with the Dust internal API.
         const upsertRes = await coreAPI.upsertDataSourceDocument({
           projectId: dataSource.dustAPIProjectId,
-          dataSourceName: dataSource.name,
+          dataSourceId: dataSource.dustAPIDataSourceId,
           documentId: req.query.documentId as string,
           tags: bodyValidation.right.tags || [],
           parents: bodyValidation.right.parents || [],
@@ -516,7 +516,7 @@ async function handler(
 
       const delRes = await coreAPI.deleteDataSourceDocument({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         documentId: req.query.documentId as string,
       });
 

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/documents/[documentId]/parents.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/documents/[documentId]/parents.ts
@@ -125,7 +125,7 @@ async function handler(
       const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger);
       const updateRes = await coreAPI.updateDataSourceDocumentParents({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         documentId: req.query.documentId as string,
         parents: req.body.parents,
       });

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/index.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/index.ts
@@ -152,7 +152,7 @@ async function handler(
       const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger);
       const tableRes = await coreAPI.getTable({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         tableId,
       });
       if (tableRes.isErr()) {

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/parents.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/parents.ts
@@ -82,7 +82,7 @@ async function handler(
       const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger);
       const updateRes = await coreAPI.updateTableParents({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         tableId: req.query.tId as string,
         parents,
       });

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/rows/[rId].ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/rows/[rId].ts
@@ -169,7 +169,7 @@ async function handler(
     case "GET":
       const rowRes = await coreAPI.getTableRow({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         tableId,
         rowId,
       });
@@ -201,7 +201,7 @@ async function handler(
     case "DELETE":
       const deleteRes = await coreAPI.deleteTableRow({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         tableId,
         rowId,
       });

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/rows/index.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/[tId]/rows/index.ts
@@ -249,7 +249,7 @@ async function handler(
 
       const listRes = await coreAPI.getTableRows({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         tableId,
         offset,
         limit,
@@ -317,7 +317,7 @@ async function handler(
       });
       const upsertRes = await coreAPI.upsertTableRows({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         tableId: tableId,
         rows: rowsToUpsert,
         truncate,
@@ -346,7 +346,7 @@ async function handler(
       // Upsert is succesful, retrieve the updated table.
       const tableRes = await coreAPI.getTable({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         tableId,
       });
       if (tableRes.isErr()) {

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/index.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/tables/index.ts
@@ -168,7 +168,7 @@ async function handler(
     case "GET":
       const tablesRes = await coreAPI.getTables({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
       });
 
       if (tablesRes.isErr()) {
@@ -230,7 +230,7 @@ async function handler(
 
       const tRes = await coreAPI.getTables({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
       });
 
       if (tRes.isErr()) {
@@ -265,7 +265,7 @@ async function handler(
 
       const upsertRes = await coreAPI.upsertTable({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         tableId,
         name,
         description,

diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/tokenize.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/tokenize.ts
@@ -84,7 +84,7 @@ async function handler(
       const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger);
       const coreTokenizeRes = await coreAPI.dataSourceTokenize({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         text,
       });
       if (coreTokenizeRes.isErr()) {

diff --git a/front/pages/api/w/[wId]/data_sources/[name]/documents/[documentId]/index.ts b/front/pages/api/w/[wId]/data_sources/[name]/documents/[documentId]/index.ts
@@ -207,7 +207,7 @@ async function handler(
       // Create document with the Dust internal API.
       const upsertRes = await coreAPI.upsertDataSourceDocument({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         documentId: req.query.documentId as string,
         tags,
         parents: bodyValidation.right.parents || [],
@@ -283,7 +283,7 @@ async function handler(
 
       const deleteRes = await coreAPI.deleteDataSourceDocument({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         documentId: req.query.documentId as string,
       });
 

diff --git a/front/pages/api/w/[wId]/data_sources/[name]/tables/[tId]/index.ts b/front/pages/api/w/[wId]/data_sources/[name]/tables/[tId]/index.ts
@@ -63,7 +63,7 @@ async function handler(
       const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger);
       const tableRes = await coreAPI.getTable({
         projectId: dataSource.dustAPIProjectId,
-        dataSourceName: dataSource.name,
+        dataSourceId: dataSource.dustAPIDataSourceId,
         tableId,
       });
       if (tableRes.isErr()) {
@@ -134,8 +134,7 @@ export async function handleDeleteTableByIdRequest(
 ) {
   const delRes = await deleteTable({
     owner,
-    projectId: dataSource.dustAPIProjectId,
-    dataSourceName: dataSource.name,
+    dataSource,
     tableId,
   });