Can upload multiple documents with fileId

front/components/DataSourceViewDocumentModal.tsx

@@ -25,7 +25,6 @@ export default function DataSourceViewDocumentModal({
       dataSourceView,
       owner,
     });
-  console.log(isDocumentError);
 
   const { title, text } = useMemo(() => {
     if (!document) {

front/components/data_source/MultipleDocumentsUpload.tsx

@@ -3,16 +3,19 @@ import type {
   DataSourceViewType,
   LightWorkspaceType,
   PlanType,
-  PostDataSourceWithNameDocumentRequestBody,
 } from "@dust-tt/types";
-import { Err, Ok } from "@dust-tt/types";
+import { Err, supportedPlainTextExtensions } from "@dust-tt/types";
 import type { ChangeEvent } from "react";
 import { useCallback, useEffect, useRef, useState } from "react";
 
 import { DocumentLimitPopup } from "@app/components/data_source/DocumentLimitPopup";
-import { handleFileUploadToText } from "@app/lib/client/handle_file_upload";
-
-const UPLOAD_ACCEPT = [".txt", ".pdf", ".md", ".csv"];
+import type {
+  FileBlob,
+  FileBlobWithFileId,
+} from "@app/hooks/useFileUploaderService";
+import { useFileUploaderService } from "@app/hooks/useFileUploaderService";
+import { useCreateDataSourceViewDocument } from "@app/lib/swr/data_source_view_documents";
+import { getFileProcessedUrl } from "@app/lib/swr/file";
 
 type MultipleDocumentsUploadProps = {
   dataSourceView: DataSourceViewType;
@@ -31,15 +34,11 @@ export const MultipleDocumentsUpload = ({
   totalNodesCount,
   plan,
 }: MultipleDocumentsUploadProps) => {
+  const sendNotification = useSendNotification();
   const fileInputRef = useRef<HTMLInputElement>(null);
   const [isLimitPopupOpen, setIsLimitPopupOpen] = useState(false);
   const [wasOpened, setWasOpened] = useState(isOpen);
 
-  const [isBulkFilesUploading, setIsBulkFilesUploading] = useState<null | {
-    total: number;
-    completed: number;
-  }>(null);
-
   const close = useCallback(
     (save: boolean) => {
       // Clear the values of the file input
@@ -51,117 +50,149 @@ export const MultipleDocumentsUpload = ({
     [onClose]
   );
 
-  const sendNotification = useSendNotification();
-
-  const handleUpsert = useCallback(
-    async (text: string, documentId: string) => {
-      const body: PostDataSourceWithNameDocumentRequestBody = {
-        name: documentId,
-        timestamp: null,
-        parents: null,
-        section: {
-          prefix: null,
-          content: text,
-          sections: [],
-        },
-        text: null,
-        source_url: undefined,
-        tags: [],
-        light_document_output: true,
-        upsert_context: null,
-        async: false,
-      };
-
-      try {
-        const res = await fetch(
-          `/api/w/${owner.sId}/spaces/${dataSourceView.spaceId}/data_sources/${
-            dataSourceView.dataSource.sId
-          }/documents`,
-          {
-            method: "POST",
-            headers: {
-              "Content-Type": "application/json",
-            },
-            body: JSON.stringify(body),
-          }
-        );
-
-        if (!res.ok) {
-          let errMsg = "";
-          try {
-            const data = await res.json();
-            errMsg = data.error.message;
-          } catch (e) {
-            errMsg = "An error occurred while uploading your document.";
-          }
-          return new Err(errMsg);
-        }
-      } catch (e) {
-        return new Err("An error occurred while uploading your document.");
+  const getFileProcessedContent = useCallback(
+    async (fileId: string) => {
+      const url = getFileProcessedUrl(owner, fileId);
+      const res = await fetch(url);
+      if (!res.ok) {
+        return new Err(`Error reading the file content: ${res.status}`);
       }
-
-      return new Ok(null);
+      const content = await res.text();
+      if (content === null || content === "") {
+        return new Err("Empty file content");
+      }
+      return content;
     },
-    [dataSourceView.dataSource.sId, dataSourceView.spaceId, owner.sId]
+    [owner]
   );
 
+  // Used for creating files, with text extraction post-processing
+  const fileUploaderService = useFileUploaderService({
+    owner,
+    useCase: "folder",
+  });
+
+  // Mutation for creating documents, throw error on partial failure
+  const createDocumentMutation = useCreateDataSourceViewDocument(
+    owner,
+    dataSourceView
+  );
+
+  const [isBulkFilesUploading, setIsBulkFilesUploading] = useState<null | {
+    total: number;
+    completed: number;
+  }>(null);
+
   const handleFileChange = useCallback(
     async (
       e: ChangeEvent<HTMLInputElement> & { target: { files: File[] } }
     ) => {
-      if (e.target.files && e.target.files.length > 0) {
-        if (
-          plan.limits.dataSources.documents.count != -1 &&
-          e.target.files.length + totalNodesCount >
-            plan.limits.dataSources.documents.count
-        ) {
-          setIsLimitPopupOpen(true);
-          return;
-        }
-        const files = e.target.files;
-        let i = 0;
-        for (const file of files) {
-          setIsBulkFilesUploading({
-            total: files.length,
-            completed: i++,
+      // Empty file input
+      if (!e.target.files || e.target.files.length === 0) {
+        close(false);
+        return;
+      }
+
+      // Open plan popup if limit is reached
+      if (
+        plan.limits.dataSources.documents.count != -1 &&
+        e.target.files.length + totalNodesCount >
+          plan.limits.dataSources.documents.count
+      ) {
+        setIsLimitPopupOpen(true);
+        return;
+      }
+
+      setIsBulkFilesUploading({
+        total: e.target.files.length,
+        completed: 0,
+      });
+
+      // upload Files and get FileBlobs (only keep successfull uploads)
+      // Each individual error triggers a notification
+      const fileBlobs = (await fileUploaderService.handleFileChange(e))?.filter(
+        (fileBlob: FileBlob): fileBlob is FileBlobWithFileId =>
+          !!fileBlob.fileId
+      );
+      if (!fileBlobs || fileBlobs.length === 0) {
+        sendNotification({
+          type: "error",
+          title: "Error uploading files",
+          description: "An error occurred while uploading your files.",
+        });
+        setIsBulkFilesUploading(null);
+        close(false);
+        fileUploaderService.resetUpload();
+        return;
+      }
+
+      // upsert the file as Data Source Documents
+      // Done 1 by 1 for simplicity
+      let i = 0;
+      for (const blob of fileBlobs) {
+        setIsBulkFilesUploading({
+          total: fileBlobs.length,
+          completed: i++,
+        });
+
+        // get processed text
+        const content = await getFileProcessedContent(blob.fileId);
+        if (content instanceof Err) {
+          sendNotification({
+            type: "error",
+            title: `Error processing document ${blob.filename}`,
+            description: content.error,
           });
-          try {
-            const uploadRes = await handleFileUploadToText(file);
-            if (uploadRes.isErr()) {
+          continue;
+        }
+
+        // Create the document
+        const documentRequestBody = {
+          name: blob.filename,
+          timestamp: null,
+          parents: null,
+          section: {
+            prefix: null,
+            content: content,
+            sections: [],
+          },
+          text: null,
+          source_url: undefined,
+          tags: [],
+          light_document_output: true,
+          upsert_context: null,
+          async: false,
+        };
+        await createDocumentMutation.trigger(
+          { documentBody: documentRequestBody },
+          {
+            onError: (error: Error) => {
               sendNotification({
                 type: "error",
-                title: `Error uploading document ${file.name}`,
-                description: uploadRes.error.message,
+                title: `Error uploading document ${blob.filename}`,
+                description: error.message,
               });
-            } else {
-              const upsertRes = await handleUpsert(
-                uploadRes.value.content,
-                file.name
-              );
-              if (upsertRes.isErr()) {
-                sendNotification({
-                  type: "error",
-                  title: `Error uploading document ${file.name}`,
-                  description: upsertRes.error,
-                });
-              }
-            }
-          } catch (e) {
-            sendNotification({
-              type: "error",
-              title: "Error uploading document",
-              description: `An error occurred while uploading your documents.`,
-            });
+              console.error(error);
+            },
           }
-        }
-        setIsBulkFilesUploading(null);
-        close(true);
-      } else {
-        close(false);
+        );
       }
+
+      sendNotification({
+        type: "success",
+        title: "Files uploaded",
+        description: "Done uploading your files.",
+      });
+
+      // Reset the upload state
+      setIsBulkFilesUploading(null);
+      fileUploaderService.resetUpload();
+      close(true);
     },
     [
-      handleUpsert,
+      createDocumentMutation,
+      fileUploaderService,
+      getFileProcessedContent,
       close,
       plan.limits.dataSources.documents.count,
       sendNotification,
@@ -173,6 +204,7 @@ export const MultipleDocumentsUpload = ({
     close(false);
   }, [close]);
 
+  // Effect: add event listener to file input
   useEffect(() => {
     const ref = fileInputRef.current;
     ref?.addEventListener("cancel", handleFileInputBlur);
@@ -181,6 +213,7 @@ export const MultipleDocumentsUpload = ({
     };
   }, [handleFileInputBlur]);
 
+  // Effect: open file input when the dialog is opened
   useEffect(() => {
     if (isOpen && !wasOpened) {
       const ref = fileInputRef.current;
@@ -220,7 +253,7 @@ export const MultipleDocumentsUpload = ({
       <input
         className="hidden"
         type="file"
-        accept={UPLOAD_ACCEPT.join(",")}
+        accept={supportedPlainTextExtensions.join(", ")}
         ref={fileInputRef}
         multiple={true}
         onChange={handleFileChange}

front/hooks/useFileUploaderService.ts

@@ -20,7 +20,7 @@ import { useState } from "react";
 
 import { getMimeTypeFromFile } from "@app/lib/file";
 
-interface FileBlob {
+export interface FileBlob {
   contentType: SupportedFileContentType;
   file: File;
   filename: string;
@@ -31,7 +31,7 @@ interface FileBlob {
   size: number;
   publicUrl?: string;
 }
-
+export type FileBlobWithFileId = FileBlob & { fileId: string };
 type FileBlobUploadErrorCode =
   | "failed_to_upload_file"
   | "file_type_not_supported";
@@ -126,7 +126,7 @@ export function useFileUploaderService({
         if (fileBlobs.some((f) => f.id === file.name)) {
           sendNotification({
             type: "error",
-            title: "File already exists.",
+            title: `Failed to upload file ${file.name}`,
             description: `File "${file.name}" is already uploaded.`,
           });
 
@@ -315,7 +315,6 @@ export function useFileUploaderService({
     setFileBlobs([]);
   };
 
-  type FileBlobWithFileId = FileBlob & { fileId: string };
   function fileBlobHasFileId(
     fileBlob: FileBlob
   ): fileBlob is FileBlobWithFileId {

front/lib/swr/file.ts

@@ -3,6 +3,11 @@ import type { SWRConfiguration } from "swr";
 
 import { useSWRWithDefaults } from "@app/lib/swr/swr";
 
+export const getFileProcessedUrl = (
+  owner: LightWorkspaceType,
+  fileId: string
+) => `/api/w/${owner.sId}/files/${fileId}?action=view&version=processed`;
+
 export function useFileProcessedContent(
   owner: LightWorkspaceType,
   fileId: string | null,
@@ -17,9 +22,7 @@ export function useFileProcessedContent(
     error,
     mutate,
   } = useSWRWithDefaults(
-    isDisabled
-      ? null
-      : `/api/w/${owner.sId}/files/${fileId}?action=view&version=processed`,
+    isDisabled ? null : getFileProcessedUrl(owner, fileId),
     // Stream fetcher -> don't try to parse the stream
     // Wait for initial response to trigger swr error handling
     async (...args) => {