feat: table CSV upload / delete from folders page (#3293)

* Csv upload in folders

* remove tables from lab

* fix nav post rebase

* improve whitelisting

---------

Co-authored-by: Henry Fontanier <[email protected]>

core/bin/dust_api.rs

@@ -1779,7 +1779,11 @@ async fn data_sources_delete(
                 None,
             ),
             Some(ds) => match ds
-                .delete(state.store.clone(), state.qdrant_clients.clone())
+                .delete(
+                    state.store.clone(),
+                    state.databases_store.clone(),
+                    state.qdrant_clients.clone(),
+                )
                 .await
             {
                 Err(e) => error_response(
@@ -1915,6 +1919,52 @@ async fn tables_list(
     }
 }
 
+async fn tables_delete(
+    extract::Path((project_id, data_source_id, table_id)): extract::Path<(i64, String, String)>,
+    extract::Extension(state): extract::Extension<Arc<APIState>>,
+) -> (StatusCode, Json<APIResponse>) {
+    let project = project::Project::new_from_id(project_id);
+
+    match state
+        .store
+        .load_table(&project, &data_source_id, &table_id)
+        .await
+    {
+        Err(e) => error_response(
+            StatusCode::INTERNAL_SERVER_ERROR,
+            "internal_server_error",
+            "Failed to load table",
+            Some(e),
+        ),
+        Ok(None) => error_response(
+            StatusCode::NOT_FOUND,
+            "table_not_found",
+            &format!("No table found for id `{}`", table_id),
+            None,
+        ),
+        Ok(Some(table)) => match table
+            .delete(state.store.clone(), state.databases_store.clone())
+            .await
+        {
+            Err(e) => error_response(
+                StatusCode::INTERNAL_SERVER_ERROR,
+                "internal_server_error",
+                "Failed to delete table",
+                Some(e),
+            ),
+            Ok(_) => (
+                StatusCode::OK,
+                Json(APIResponse {
+                    error: None,
+                    response: Some(json!({
+                        "success": true,
+                    })),
+                }),
+            ),
+        },
+    }
+}
+
 #[derive(serde::Deserialize)]
 struct TablesRowsUpsertPayload {
     rows: Vec<Row>,
@@ -2443,6 +2493,10 @@ fn main() {
             "/projects/:project_id/data_sources/:data_source_id/tables",
             get(tables_list),
         )
+        .route(
+            "/projects/:project_id/data_sources/:data_source_id/tables/:table_id",
+            delete(tables_delete),
+        )
         .route(
             "/projects/:project_id/data_sources/:data_source_id/tables/:table_id/rows",
             post(tables_rows_upsert),

core/src/data_sources/data_source.rs

@@ -1,6 +1,7 @@
 use crate::consts::DATA_SOURCE_DOCUMENT_SYSTEM_TAG_PREFIX;
 use crate::data_sources::qdrant::{QdrantClients, QdrantDataSourceConfig};
 use crate::data_sources::splitter::{splitter, SplitterID};
+use crate::databases_store::store::DatabasesStore;
 use crate::project::Project;
 use crate::providers::embedder::{EmbedderRequest, EmbedderVector};
 use crate::providers::provider::{provider, ProviderID};
@@ -1803,6 +1804,7 @@ impl DataSource {
     pub async fn delete(
         &self,
         store: Box<dyn Store + Sync + Send>,
+        databases_store: Box<dyn DatabasesStore + Sync + Send>,
         qdrant_clients: QdrantClients,
     ) -> Result<()> {
         if qdrant_clients
@@ -1831,7 +1833,12 @@ impl DataSource {
         let (tables, total) = store
             .list_tables(&self.project, &self.data_source_id, None)
             .await?;
-        try_join_all(tables.iter().map(|t| t.delete(store.clone()))).await?;
+        try_join_all(
+            tables
+                .iter()
+                .map(|t| t.delete(store.clone(), databases_store.clone())),
+        )
+        .await?;
 
         utils::done(&format!(
             "Deleted tables: data_source_id={} table_count={}",

core/src/databases/database.rs

@@ -187,10 +187,34 @@ impl Table {
         }
     }
 
-    pub async fn delete(&self, store: Box<dyn Store + Sync + Send>) -> Result<()> {
+    pub async fn delete(
+        &self,
+        store: Box<dyn Store + Sync + Send>,
+        databases_store: Box<dyn DatabasesStore + Sync + Send>,
+    ) -> Result<()> {
+        // Invalidate the databases that use the table.
+        try_join_all(
+            (store
+                .find_databases_using_table(
+                    &self.project,
+                    &self.data_source_id,
+                    &self.table_id,
+                    HEARTBEAT_INTERVAL_MS,
+                )
+                .await?)
+                .into_iter()
+                .map(|db| invalidate_database(db, store.clone())),
+        )
+        .await?;
+
+        // Delete the table rows.
+        databases_store.delete_table_rows(&self.unique_id()).await?;
+
         store
             .delete_table(&self.project, &self.data_source_id, &self.table_id)
-            .await
+            .await?;
+
+        Ok(())
     }
 
     pub async fn upsert_rows(

front/components/sparkle/navigation.tsx

@@ -10,7 +10,6 @@ import {
   PlanetIcon,
   PuzzleIcon,
   RobotSharedIcon,
-  ServerIcon,
   ShapesIcon,
 } from "@dust-tt/sparkle";
 import { GlobeAltIcon } from "@dust-tt/sparkle";
@@ -41,17 +40,17 @@ export type SubNavigationAssistantsId =
   | "personal_assistants"
   | "data_sources_url"
   | "developers"
-  | "extract"
-  | "tables";
+  | "extract";
 
 export type SubNavigationAdminId = "subscription" | "workspace" | "members";
+
 export type SubNavigationAppId =
   | "specification"
   | "datasets"
   | "execute"
   | "runs"
   | "settings";
-export type SubNavigationLabId = "extract" | "databases";
+export type SubNavigationLabId = "extract";
 
 export type SparkleAppLayoutNavigation = {
   id:
@@ -221,13 +220,6 @@ export const subNavigationBuild = ({
           href: `/w/${owner.sId}/u/extract`,
           current: current === "extract",
         },
-        {
-          id: "tables",
-          label: "Tables",
-          icon: ServerIcon,
-          href: `/w/${owner.sId}/tables`,
-          current: current === "tables",
-        },
       ],
     });
   }

front/lib/development.ts

@@ -18,7 +18,13 @@ export function isDevelopmentOrDustWorkspace(owner: WorkspaceType) {
 
 export function isActivatedStructuredDB(owner: WorkspaceType) {
   // We will manually add workspace ids here.
-  return isDevelopmentOrDustWorkspace(owner) || owner.sId === "bd133dacaa"; // Daph's workspace;
+  return (
+    isDevelopmentOrDustWorkspace(owner) ||
+    [
+      "47cc56f99e", // Henry's workspace;
+      "bd133dacaa", // Daph's workspace;
+    ].includes(owner.sId)
+  );
 }
 
 export function isActivatedPublicURLs(owner: WorkspaceType) {

front/lib/swr.ts

@@ -592,12 +592,12 @@ export function useTable({
 }: {
   workspaceId: string;
   dataSourceName: string;
-  tableId: string;
+  tableId: string | null;
 }) {
   const tableFetcher: Fetcher<GetTableResponseBody> = fetcher;
 
   const { data, error, mutate } = useSWR(
-    dataSourceName
+    tableId
       ? `/api/w/${workspaceId}/data_sources/${dataSourceName}/tables/${tableId}`
       : null,
     tableFetcher

front/pages/api/w/[wId]/data_sources/[name]/tables/[tId]/index.ts

@@ -81,9 +81,10 @@ async function handler(
     });
   }
 
+  const coreAPI = new CoreAPI(logger);
+
   switch (req.method) {
     case "GET":
-      const coreAPI = new CoreAPI(logger);
       const tableRes = await coreAPI.getTable({
         projectId: dataSource.dustAPIProjectId,
         dataSourceName: dataSource.name,
@@ -111,12 +112,40 @@ async function handler(
 
       return res.status(200).json({ table });
 
+    case "DELETE":
+      const deleteRes = await coreAPI.deleteTable({
+        projectId: dataSource.dustAPIProjectId,
+        dataSourceName: dataSource.name,
+        tableId,
+      });
+      if (deleteRes.isErr()) {
+        logger.error(
+          {
+            dataSourcename: dataSource.name,
+            workspaceId: owner.id,
+            error: deleteRes.error,
+          },
+          "Failed to delete table."
+        );
+        return apiError(req, res, {
+          status_code: 500,
+          api_error: {
+            type: "internal_server_error",
+            message: "Failed to delete table.",
+          },
+        });
+      }
+
+      res.status(200).end();
+      return;
+
     default:
       return apiError(req, res, {
         status_code: 405,
         api_error: {
           type: "method_not_supported_error",
-          message: "The method passed is not supported, GET is expected.",
+          message:
+            "The method passed is not supported, GET or DELETE is expected.",
         },
       });
   }

front/pages/api/w/[wId]/data_sources/[name]/tables/csv.ts

@@ -22,6 +22,10 @@ const CreateTableFromCsvSchema = t.type({
   csv: t.string,
 });
 
+export type CreateTableFromCsvRequestBody = t.TypeOf<
+  typeof CreateTableFromCsvSchema
+>;
+
 type RowValue =
   | number
   | boolean