wip

front/lib/api/data_sources.ts

@@ -13,6 +13,7 @@ import type {
   DataSourceType,
   DataSourceWithConnectorDetailsType,
   FrontDataSourceDocumentSectionType,
+  ModelId,
   PlanType,
   Result,
   UpsertTableFromCsvRequestType,
@@ -699,12 +700,14 @@ export async function createDataSourceWithoutProvider(
     space,
     name,
     description,
+    conversationId,
   }: {
     plan: PlanType;
     owner: WorkspaceType;
     space: SpaceResource;
     name: string;
     description: string | null;
+    conversationId?: ModelId;
   }
 ): Promise<
   Result<
@@ -797,6 +800,7 @@ export async function createDataSourceWithoutProvider(
         dustAPIDataSourceId: dustDataSource.value.data_source.data_source_id,
         workspaceId: owner.id,
         assistantDefaultSelected: false,
+        conversationId,
       },
       space
     );

front/lib/api/files/upload.ts

@@ -7,6 +7,7 @@ import type {
 } from "@dust-tt/types";
 import {
   Err,
+  isSupportedPlainTextContentType,
   isTextExtractionSupportedContentType,
   Ok,
   pagePrefixesPerMimeType,
@@ -19,13 +20,21 @@ import sharp from "sharp";
 import type { TransformCallback } from "stream";
 import { PassThrough, Readable, Transform } from "stream";
 import { pipeline } from "stream/promises";
+import { v4 as uuidv4 } from "uuid";
 
+import { isJITActionsEnabled } from "@app/lib/api/assistant/jit_actions";
 import config from "@app/lib/api/config";
 import type { CSVRow } from "@app/lib/api/csv";
 import { analyzeCSVColumns } from "@app/lib/api/csv";
+import {
+  createDataSourceWithoutProvider,
+  upsertDocument,
+} from "@app/lib/api/data_sources";
 import type { Authenticator } from "@app/lib/auth";
 import type { DustError } from "@app/lib/error";
+import { DataSourceResource } from "@app/lib/resources/data_source_resource";
 import type { FileResource } from "@app/lib/resources/file_resource";
+import { SpaceResource } from "@app/lib/resources/space_resource";
 import logger from "@app/logger/logger";
 
 const UPLOAD_DELAY_AFTER_CREATION_MS = 1000 * 60 * 1; // 1 minute.
@@ -509,5 +518,77 @@ export async function uploadToCloudStorage(
     });
   }
 
+  // Upsert to the conversation datasource & generate a snippet.
+  // TODO(JIT) the tool output flow do not go through this path.
+  const jitEnabled = await isJITActionsEnabled(auth);
+  if (
+    jitEnabled &&
+    (file.useCase === "conversation" || file.useCase === "tool_output") &&
+    isSupportedPlainTextContentType(file.contentType)
+  ) {
+    // Upsert the file to the conversation datasource.
+    const conversationsSpace =
+      await SpaceResource.fetchWorkspaceConversationsSpace(auth);
+
+    // TODO(JIT) we need to retrieve the conversation linked to the file somehow.
+    const conversationId = 345;
+
+    // Fetch the datasource linked to the conversation...
+    let dataSource = await DataSourceResource.fetchByConversationId(
+      auth,
+      conversationId
+    );
+
+    if (!dataSource) {
+      // ...or create a new one.
+
+      // IMPORTANT: never use the conversation sID in the name or description, as conversation sIDs are used as secrets to share the conversation within the workspace users.
+      const name = `Conversation ${uuidv4()}`;
+      const r = await createDataSourceWithoutProvider(auth, {
+        plan: auth.getNonNullablePlan(),
+        owner: auth.getNonNullableWorkspace(),
+        space: conversationsSpace,
+        name: name,
+        description: "Files uploaded to conversation",
+        conversationId,
+      });
+
+      if (r.isErr()) {
+        return new Err({
+          name: "dust_error",
+          code: "internal_server_error",
+          message: `Failed to create datasource : ${r.error}`,
+        });
+      }
+
+      dataSource = r.value.dataSource;
+    }
+
+    const documentId = `${file.id}`; // Use the file id as the document id to make it easy to track the document back to the file.
+    const sourceUrl = file.getPublicUrl(auth);
+
+    // TODO(JIT) we should extract sections from the processing of the file done above.
+    // TODO(JIT) note, upsertDocument do not call runPostUpsertHooks (seems used for document tracker)
+    const upsertRes = await upsertDocument({
+      name: documentId,
+      source_url: sourceUrl,
+      text: "fake content",
+      parents: [documentId],
+      tags: [`title:${file.fileName}`],
+      light_document_output: true,
+      dataSource,
+      auth,
+    });
+
+    if (upsertRes.isErr()) {
+      return new Err({
+        name: "dust_error",
+        code: "internal_server_error",
+        message: "There was an error upserting the document.",
+        data_source_error: upsertRes.error,
+      });
+    }
+  }
+
   return new Ok(file);
 }

front/lib/resources/data_source_resource.ts

@@ -263,6 +263,21 @@ export class DataSourceResource extends ResourceWithSpace<DataSourceModel> {
     return dataSource ?? null;
   }
 
+  static async fetchByConversationId(
+    auth: Authenticator,
+    conversationId: number,
+    options?: FetchDataSourceOptions
+  ): Promise<DataSourceResource | null> {
+    const [dataSource] = await this.baseFetch(auth, options, {
+      where: {
+        conversationId,
+        workspaceId: auth.getNonNullableWorkspace().id,
+      },
+    });
+
+    return dataSource ?? null;
+  }
+
   // TODO(DATASOURCE_SID): remove
   static async fetchByNames(
     auth: Authenticator,

front/lib/resources/space_resource.ts

@@ -260,6 +260,20 @@ export class SpaceResource extends BaseResource<SpaceModel> {
     return space;
   }
 
+  static async fetchWorkspaceConversationsSpace(
+    auth: Authenticator
+  ): Promise<SpaceResource> {
+    const [space] = await this.baseFetch(auth, {
+      where: { kind: "conversations" },
+    });
+
+    if (!space) {
+      throw new Error("Conversations space not found.");
+    }
+
+    return space;
+  }
+
   static async fetchById(
     auth: Authenticator,
     sId: string,