diff --git a/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts b/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts index 41ac9919eacd..6cf4b0e07c95 100644 --- a/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts +++ b/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts @@ -1,4 +1,4 @@ -import { INTERCOM_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import { makeScript } from "scripts/helpers"; import { @@ -36,7 +36,7 @@ async function createFolderNodes(execute: boolean) { parents: [getTeamsInternalId(connector.id)], parentId: null, title: "Conversations", - mimeType: INTERCOM_MIME_TYPES.CONVERSATIONS, + mimeType: MIME_TYPES.INTERCOM.TEAMS_FOLDER, }); } @@ -60,7 +60,7 @@ async function createFolderNodes(execute: boolean) { parents: [teamInternalId, getTeamsInternalId(connector.id)], parentId: getTeamsInternalId(connector.id), title: team.name, - mimeType: INTERCOM_MIME_TYPES.TEAM, + mimeType: MIME_TYPES.INTERCOM.TEAM, }); } }, @@ -99,7 +99,7 @@ async function createFolderNodes(execute: boolean) { parents: [helpCenterInternalId], parentId: null, title: helpCenter.name, - mimeType: INTERCOM_MIME_TYPES.HELP_CENTER, + mimeType: MIME_TYPES.INTERCOM.HELP_CENTER, }); } @@ -133,7 +133,7 @@ async function createFolderNodes(execute: boolean) { parents: collectionParents, parentId: collectionParents[1] || null, title: collection.name, - mimeType: INTERCOM_MIME_TYPES.COLLECTION, + mimeType: MIME_TYPES.INTERCOM.COLLECTION, }); } }, diff --git a/connectors/migrations/20250110_investigate_zendesk_hc.ts b/connectors/migrations/20250110_investigate_zendesk_hc.ts new file mode 100644 index 000000000000..a83f7c348380 --- /dev/null +++ b/connectors/migrations/20250110_investigate_zendesk_hc.ts @@ -0,0 +1,100 @@ +import { makeScript } from "scripts/helpers"; + +import { isZendeskNotFoundError } from "@connectors/connectors/zendesk/lib/errors"; +import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token"; +import { + fetchZendeskBrand, + fetchZendeskCategoriesInBrand, + fetchZendeskCurrentUser, + getZendeskBrandSubdomain, +} from "@connectors/connectors/zendesk/lib/zendesk_api"; +import { ZENDESK_BATCH_SIZE } from "@connectors/connectors/zendesk/temporal/config"; +import { ConnectorResource } from "@connectors/resources/connector_resource"; +import { ZendeskBrandResource } from "@connectors/resources/zendesk_resources"; + +makeScript({}, async ({ execute }, logger) => { + const connectors = await ConnectorResource.listByType("zendesk", {}); + + for (const connector of connectors) { + const connectorId = connector.id; + if (connector.isPaused()) { + continue; + } + const { accessToken, subdomain } = await getZendeskSubdomainAndAccessToken( + connector.connectionId + ); + const user = await fetchZendeskCurrentUser({ accessToken, subdomain }); + const brandsOnDb = await ZendeskBrandResource.fetchByConnector(connector); + for (const brandOnDb of brandsOnDb) { + const { brandId } = brandOnDb; + if (!execute) { + logger.info({ brandId }, "DRY: Fetching brand"); + continue; + } + + const fetchedBrand = await fetchZendeskBrand({ + brandId, + accessToken, + subdomain, + }); + const brandSubdomain = await getZendeskBrandSubdomain({ + brandId, + connectorId, + accessToken, + subdomain, + }); + + let couldFetchCategories; + try { + await fetchZendeskCategoriesInBrand(accessToken, { + brandSubdomain, + pageSize: ZENDESK_BATCH_SIZE, + }); + couldFetchCategories = true; + } catch (e) { + if (isZendeskNotFoundError(e)) { + couldFetchCategories = false; + // if (fetchedBrand?.has_help_center) { + // const url = `https://${brandSubdomain}.zendesk.com/api/v2/help_center/articles.json`; + // const res = await fetch(url, { + // method: "GET", + // headers: { + // Authorization: `Bearer ${accessToken}`, + // "Content-Type": "application/json", + // }, + // }); + // const text = await res.text(); + // logger.error( + // { + // res, + // brandSubdomain, + // status: res.status, + // statusText: res.statusText, + // headers: res.headers, + // text, + // body: res.body, + // }, + // "Failed to fetch categories" + // ); + // await res.json(); + // } + } else { + throw e; + } + } + logger.info( + { + connectorId, + brandId, + couldFetchCategories, + brandSubdomain, + hasHelpCenter: fetchedBrand?.has_help_center, + helpCenterState: fetchedBrand?.help_center_state, + userRole: user.role, + userActive: user.active, + }, + `FETCH` + ); + } + } +}); diff --git a/connectors/src/api/get_content_node_parents.ts b/connectors/src/api/get_content_node_parents.ts deleted file mode 100644 index 26a12ebc4062..000000000000 --- a/connectors/src/api/get_content_node_parents.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { WithConnectorsAPIErrorReponse } from "@dust-tt/types"; -import type { Request, Response } from "express"; -import { isLeft } from "fp-ts/lib/Either"; -import * as t from "io-ts"; -import * as reporter from "io-ts-reporters"; - -import type { ContentNodeParentIdsBlob } from "@connectors/lib/api/content_nodes"; -import { getParentIdsForContentNodes } from "@connectors/lib/api/content_nodes"; -import logger from "@connectors/logger/logger"; -import { apiError, withLogging } from "@connectors/logger/withlogging"; -import { ConnectorResource } from "@connectors/resources/connector_resource"; - -const GetContentNodesParentsRequestBodySchema = t.type({ - internalIds: t.array(t.string), -}); - -export type GetContentNodesParentsRequestBody = t.TypeOf< - typeof GetContentNodesParentsRequestBodySchema ->; - -type GetContentNodesResponseBody = WithConnectorsAPIErrorReponse<{ - nodes: ContentNodeParentIdsBlob[]; -}>; - -const _getContentNodesParents = async ( - req: Request< - { connector_id: string }, - GetContentNodesResponseBody, - GetContentNodesParentsRequestBody - >, - res: Response -) => { - const connector = await ConnectorResource.fetchById(req.params.connector_id); - if (!connector) { - return apiError(req, res, { - status_code: 404, - api_error: { - type: "connector_not_found", - message: "Connector not found", - }, - }); - } - - const bodyValidation = GetContentNodesParentsRequestBodySchema.decode( - req.body - ); - if (isLeft(bodyValidation)) { - const pathError = reporter.formatValidationErrors(bodyValidation.left); - return apiError(req, res, { - status_code: 400, - api_error: { - type: "invalid_request_error", - message: `Invalid request body: ${pathError}`, - }, - }); - } - - const { internalIds } = bodyValidation.right; - - const parentsRes = await getParentIdsForContentNodes(connector, internalIds); - if (parentsRes.isErr()) { - logger.error(parentsRes.error, "Failed to get content node parents."); - return apiError(req, res, { - status_code: 500, - api_error: { - type: "internal_server_error", - message: parentsRes.error.message, - }, - }); - } - - return res.status(200).json({ nodes: parentsRes.value }); -}; - -export const getContentNodesParentsAPIHandler = withLogging( - _getContentNodesParents -); diff --git a/connectors/src/api_server.ts b/connectors/src/api_server.ts index 11490c655e99..ff0b3ebb9efd 100644 --- a/connectors/src/api_server.ts +++ b/connectors/src/api_server.ts @@ -13,7 +13,6 @@ import { getConnectorsAPIHandler, } from "@connectors/api/get_connector"; import { getConnectorPermissionsAPIHandler } from "@connectors/api/get_connector_permissions"; -import { getContentNodesParentsAPIHandler } from "@connectors/api/get_content_node_parents"; import { getContentNodesAPIHandler } from "@connectors/api/get_content_nodes"; import { pauseConnectorAPIHandler } from "@connectors/api/pause_connector"; import { resumeConnectorAPIHandler } from "@connectors/api/resume_connector"; @@ -112,11 +111,6 @@ export function startServer(port: number) { "/connectors/:connector_id/permissions", getConnectorPermissionsAPIHandler ); - app.post( - // must be POST because of body - "/connectors/:connector_id/content_nodes/parents", - getContentNodesParentsAPIHandler - ); app.post( // must be POST because of body "/connectors/:connector_id/content_nodes", diff --git a/connectors/src/connectors/confluence/lib/permissions.ts b/connectors/src/connectors/confluence/lib/permissions.ts index b286026d12b4..48ac475a6873 100644 --- a/connectors/src/connectors/confluence/lib/permissions.ts +++ b/connectors/src/connectors/confluence/lib/permissions.ts @@ -46,7 +46,6 @@ export function createContentNodeFromSpace( const spaceId = isConfluenceSpaceModel(space) ? space.spaceId : space.id; return { - provider: "confluence", internalId: makeSpaceInternalId(spaceId), parentInternalId: null, type: "folder", @@ -54,7 +53,6 @@ export function createContentNodeFromSpace( sourceUrl: `${baseUrl}/wiki${urlSuffix}`, expandable: isExpandable, permission, - dustDocumentId: null, lastUpdatedAt: null, }; } @@ -66,7 +64,6 @@ export function createContentNodeFromPage( isExpandable = false ): ContentNode { return { - provider: "confluence", internalId: makePageInternalId(page.pageId), parentInternalId: parent.type === "space" @@ -77,7 +74,6 @@ export function createContentNodeFromPage( sourceUrl: `${baseUrl}/wiki${page.externalUrl}`, expandable: isExpandable, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }; } diff --git a/connectors/src/connectors/confluence/temporal/activities.ts b/connectors/src/connectors/confluence/temporal/activities.ts index c42bda84ed5d..56dffa86281d 100644 --- a/connectors/src/connectors/confluence/temporal/activities.ts +++ b/connectors/src/connectors/confluence/temporal/activities.ts @@ -1,8 +1,8 @@ import type { ModelId } from "@dust-tt/types"; import { - CONFLUENCE_MIME_TYPES, ConfluenceClientError, isConfluenceNotFoundError, + MIME_TYPES, } from "@dust-tt/types"; import { Op } from "sequelize"; import TurndownService from "turndown"; @@ -222,7 +222,7 @@ export async function confluenceUpsertSpaceFolderActivity({ parents: [makeSpaceInternalId(spaceId)], parentId: null, title: spaceName, - mimeType: CONFLUENCE_MIME_TYPES.SPACE, + mimeType: MIME_TYPES.CONFLUENCE.SPACE, }); } @@ -329,7 +329,7 @@ async function upsertConfluencePageToDataSource({ timestampMs: lastPageVersionCreatedAt.getTime(), upsertContext: { sync_type: syncType }, title: page.title, - mimeType: CONFLUENCE_MIME_TYPES.PAGE, + mimeType: MIME_TYPES.CONFLUENCE.PAGE, async: true, }); } diff --git a/connectors/src/connectors/github/index.ts b/connectors/src/connectors/github/index.ts index 4ebfa01d7b52..19ffd791ab8f 100644 --- a/connectors/src/connectors/github/index.ts +++ b/connectors/src/connectors/github/index.ts @@ -286,7 +286,6 @@ export class GithubConnectorManager extends BaseConnectorManager { nodes = nodes.concat( page.map((repo) => ({ - provider: c.type, internalId: getRepositoryInternalId(repo.id), parentInternalId: null, type: "folder", @@ -294,7 +293,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, })) ); @@ -358,7 +356,6 @@ export class GithubConnectorManager extends BaseConnectorManager { if (latestIssue) { nodes.push({ - provider: c.type, internalId: getIssuesInternalId(repoId), parentInternalId, type: "database", @@ -366,14 +363,12 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + "/issues", expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: latestIssue.updatedAt.getTime(), }); } if (latestDiscussion) { nodes.push({ - provider: c.type, internalId: getDiscussionsInternalId(repoId), parentInternalId, type: "channel", @@ -381,14 +376,12 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + "/discussions", expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: latestDiscussion.updatedAt.getTime(), }); } if (codeRepo) { nodes.push({ - provider: c.type, internalId: getCodeRootInternalId(repoId), parentInternalId, type: "folder", @@ -396,7 +389,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: codeRepo.codeUpdatedAt.getTime(), }); } @@ -431,7 +423,6 @@ export class GithubConnectorManager extends BaseConnectorManager { directories.forEach((directory) => { nodes.push({ - provider: c.type, internalId: directory.internalId, parentInternalId, type: "folder", @@ -439,14 +430,12 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: directory.sourceUrl, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: directory.codeUpdatedAt.getTime(), }); }); files.forEach((file) => { nodes.push({ - provider: c.type, internalId: file.documentId, parentInternalId, type: "file", @@ -454,7 +443,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: file.sourceUrl, expandable: false, permission: "read", - dustDocumentId: file.documentId, lastUpdatedAt: file.codeUpdatedAt.getTime(), }); }); @@ -609,16 +597,13 @@ export class GithubConnectorManager extends BaseConnectorManager { return; } nodes.push({ - provider: c.type, internalId: getRepositoryInternalId(repoId), parentInternalId: null, type: "folder", title: repo.name, - titleWithParentsContext: `[${repo.name}] - Full repository`, sourceUrl: repo.url, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); }); @@ -630,16 +615,13 @@ export class GithubConnectorManager extends BaseConnectorManager { return; } nodes.push({ - provider: c.type, internalId: getIssuesInternalId(repoId), parentInternalId: getRepositoryInternalId(repoId), type: "database", title: "Issues", - titleWithParentsContext: `[${repo.name}] Issues`, sourceUrl: repo.url + "/issues", expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); }); @@ -649,16 +631,13 @@ export class GithubConnectorManager extends BaseConnectorManager { return; } nodes.push({ - provider: c.type, internalId: getDiscussionsInternalId(repoId), parentInternalId: getRepositoryInternalId(repoId), type: "channel", title: "Discussions", - titleWithParentsContext: `[${repo.name}] Discussions`, sourceUrl: repo.url + "/discussions", expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); }); @@ -670,7 +649,6 @@ export class GithubConnectorManager extends BaseConnectorManager { return; } nodes.push({ - provider: c.type, internalId: getIssueInternalId(repoId, issueNumber), parentInternalId: getIssuesInternalId(repoId), type: "file", @@ -678,7 +656,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + `/issues/${issueNumber}`, expandable: false, permission: "read", - dustDocumentId: getIssueInternalId(repoId, issueNumber), lastUpdatedAt: issue.updatedAt.getTime(), }); }); @@ -690,7 +667,6 @@ export class GithubConnectorManager extends BaseConnectorManager { return; } nodes.push({ - provider: c.type, internalId: getDiscussionInternalId(repoId, discussionNumber), parentInternalId: getDiscussionsInternalId(repoId), type: "file", @@ -698,65 +674,48 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + `/discussions/${discussionNumber}`, expandable: false, permission: "read", - dustDocumentId: getDiscussionInternalId(repoId, discussionNumber), lastUpdatedAt: discussion.updatedAt.getTime(), }); }); // Constructing Nodes for Code fullCodeInRepos.forEach((codeRepo) => { - const repo = uniqueRepos[parseInt(codeRepo.repoId)]; nodes.push({ - provider: c.type, internalId: getCodeRootInternalId(codeRepo.repoId), parentInternalId: getRepositoryInternalId(codeRepo.repoId), type: "folder", title: "Code", - titleWithParentsContext: repo ? `[${repo.name}] Code` : "Code", sourceUrl: codeRepo.sourceUrl, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: codeRepo.codeUpdatedAt.getTime(), }); }); // Constructing Nodes for Code Directories codeDirectories.forEach((directory) => { - const repo = uniqueRepos[parseInt(directory.repoId)]; nodes.push({ - provider: c.type, internalId: directory.internalId, parentInternalId: directory.parentInternalId, type: "folder", title: directory.dirName, - titleWithParentsContext: repo - ? `[${repo.name}] ${directory.dirName} (code)` - : directory.dirName, sourceUrl: directory.sourceUrl, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: directory.codeUpdatedAt.getTime(), }); }); // Constructing Nodes for Code Files codeFiles.forEach((file) => { - const repo = uniqueRepos[parseInt(file.repoId)]; nodes.push({ - provider: c.type, internalId: file.documentId, parentInternalId: file.parentInternalId, type: "file", title: file.fileName, - titleWithParentsContext: repo - ? `[${repo.name}] ${file.fileName} (code)` - : file.fileName, sourceUrl: file.sourceUrl, expandable: false, permission: "read", - dustDocumentId: file.documentId, lastUpdatedAt: file.codeUpdatedAt.getTime(), }); }); diff --git a/connectors/src/connectors/github/temporal/activities.ts b/connectors/src/connectors/github/temporal/activities.ts index 57910729485c..11920a79999f 100644 --- a/connectors/src/connectors/github/temporal/activities.ts +++ b/connectors/src/connectors/github/temporal/activities.ts @@ -1,5 +1,5 @@ import type { CoreAPIDataSourceDocumentSection, ModelId } from "@dust-tt/types"; -import { assertNever, GITHUB_MIME_TYPES } from "@dust-tt/types"; +import { assertNever, MIME_TYPES } from "@dust-tt/types"; import { Context } from "@temporalio/activity"; import { hash as blake3 } from "blake3"; import { promises as fs } from "fs"; @@ -308,7 +308,7 @@ export async function githubUpsertIssueActivity( sync_type: isBatchSync ? "batch" : "incremental", }, title: issue.title, - mimeType: GITHUB_MIME_TYPES.ISSUE, + mimeType: MIME_TYPES.GITHUB.ISSUE, async: true, }); @@ -494,7 +494,7 @@ export async function githubUpsertDiscussionActivity( sync_type: isBatchSync ? "batch" : "incremental", }, title: discussion.title, - mimeType: GITHUB_MIME_TYPES.DISCUSSION, + mimeType: MIME_TYPES.GITHUB.DISCUSSION, async: true, }); @@ -960,7 +960,7 @@ export async function githubCodeSyncActivity({ title: "Code", parents: [getCodeRootInternalId(repoId), getRepositoryInternalId(repoId)], parentId: getRepositoryInternalId(repoId), - mimeType: GITHUB_MIME_TYPES.CODE_ROOT, + mimeType: MIME_TYPES.GITHUB.CODE_ROOT, }); let githubCodeRepository = await GithubCodeRepository.findOne({ @@ -1172,7 +1172,7 @@ export async function githubCodeSyncActivity({ sync_type: isBatchSync ? "batch" : "incremental", }, title: f.fileName, - mimeType: GITHUB_MIME_TYPES.CODE_FILE, + mimeType: MIME_TYPES.GITHUB.CODE_FILE, async: true, }); @@ -1216,7 +1216,7 @@ export async function githubCodeSyncActivity({ parents, parentId: parents[1], title: d.dirName, - mimeType: GITHUB_MIME_TYPES.CODE_DIRECTORY, + mimeType: MIME_TYPES.GITHUB.CODE_DIRECTORY, }); // Find directory or create it. @@ -1356,7 +1356,7 @@ export async function githubUpsertRepositoryFolderActivity({ title: repoName, parents: [getRepositoryInternalId(repoId)], parentId: null, - mimeType: GITHUB_MIME_TYPES.REPOSITORY, + mimeType: MIME_TYPES.GITHUB.REPOSITORY, }); } @@ -1377,7 +1377,7 @@ export async function githubUpsertIssuesFolderActivity({ title: "Issues", parents: [getIssuesInternalId(repoId), getRepositoryInternalId(repoId)], parentId: getRepositoryInternalId(repoId), - mimeType: GITHUB_MIME_TYPES.ISSUES, + mimeType: MIME_TYPES.GITHUB.ISSUES, }); } @@ -1401,7 +1401,7 @@ export async function githubUpsertDiscussionsFolderActivity({ getRepositoryInternalId(repoId), ], parentId: getRepositoryInternalId(repoId), - mimeType: GITHUB_MIME_TYPES.DISCUSSIONS, + mimeType: MIME_TYPES.GITHUB.DISCUSSIONS, }); } @@ -1422,6 +1422,6 @@ export async function githubUpsertCodeRootFolderActivity({ title: "Code", parents: [getCodeRootInternalId(repoId), getRepositoryInternalId(repoId)], parentId: getRepositoryInternalId(repoId), - mimeType: GITHUB_MIME_TYPES.CODE_ROOT, + mimeType: MIME_TYPES.GITHUB.CODE_ROOT, }); } diff --git a/connectors/src/connectors/google_drive/index.ts b/connectors/src/connectors/google_drive/index.ts index b504633b1d49..4987259ccb79 100644 --- a/connectors/src/connectors/google_drive/index.ts +++ b/connectors/src/connectors/google_drive/index.ts @@ -8,7 +8,6 @@ import { Err, getGoogleIdsFromSheetContentNodeInternalId, getGoogleSheetContentNodeInternalId, - getGoogleSheetTableId, isGoogleSheetContentNodeInternalId, Ok, removeNulls, @@ -25,10 +24,7 @@ import { } from "@connectors/connectors/google_drive/lib"; import { GOOGLE_DRIVE_SHARED_WITH_ME_VIRTUAL_ID } from "@connectors/connectors/google_drive/lib/consts"; import { getGoogleDriveObject } from "@connectors/connectors/google_drive/lib/google_drive_api"; -import { - getGoogleDriveEntityDocumentId, - getPermissionViewType, -} from "@connectors/connectors/google_drive/lib/permissions"; +import { getPermissionViewType } from "@connectors/connectors/google_drive/lib/permissions"; import { folderHasChildren, getDrives, @@ -321,12 +317,10 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { const type = getPermissionViewType(f); return { - provider: c.type, internalId: getInternalId(f.driveFileId), parentInternalId: null, type, title: f.name || "", - dustDocumentId: getGoogleDriveEntityDocumentId(f), lastUpdatedAt: f.lastUpsertedTs?.getTime() || null, sourceUrl: getSourceUrlForGoogleDriveFiles(f), expandable: await isDriveObjectExpandable({ @@ -345,7 +339,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { nodes = nodes.concat( sheets.map((s) => { return { - provider: c.type, internalId: getGoogleSheetContentNodeInternalId( s.driveFileId, s.driveSheetId @@ -353,15 +346,10 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { parentInternalId: getInternalId(s.driveFileId), type: "database" as const, title: s.name || "", - dustDocumentId: null, lastUpdatedAt: s.updatedAt.getTime() || null, sourceUrl: null, expandable: false, permission: "read", - dustTableId: getGoogleSheetTableId( - s.driveFileId, - s.driveSheetId - ), }; }) ); @@ -394,7 +382,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { ); } return { - provider: c.type, internalId: getInternalId(driveObject.id), parentInternalId: // note: if the parent is null, the drive object falls at top-level @@ -402,7 +389,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { type: "folder" as const, title: driveObject.name, sourceUrl: driveObject.webViewLink || null, - dustDocumentId: null, lastUpdatedAt: driveObject.updatedAtMs || null, expandable: await folderHasChildren( this.connectorId, @@ -422,14 +408,12 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { // Adding a fake "Shared with me" node, to allow the user to see their shared files // that are not living in a shared drive. nodes.push({ - provider: c.type, internalId: getInternalId(GOOGLE_DRIVE_SHARED_WITH_ME_VIRTUAL_ID), parentInternalId: null, type: "folder" as const, preventSelection: true, title: "Shared with me", sourceUrl: null, - dustDocumentId: null, lastUpdatedAt: null, expandable: true, permission: "none", @@ -491,7 +475,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { ); return { - provider: c.type, internalId: getInternalId(driveObject.id), parentInternalId: driveObject.parent && getInternalId(driveObject.parent), @@ -502,7 +485,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { this.connectorId, driveObject.id ), - dustDocumentId: null, lastUpdatedAt: driveObject.updatedAtMs || null, permission: (await GoogleDriveFolders.findOne({ where: { @@ -678,12 +660,10 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { const sourceUrl = getSourceUrlForGoogleDriveFiles(f); return { - provider: "google_drive", internalId: getInternalId(f.driveFileId), parentInternalId: null, type, title: f.name || "", - dustDocumentId: getGoogleDriveEntityDocumentId(f), lastUpdatedAt: f.lastUpsertedTs?.getTime() || null, sourceUrl, expandable: await isDriveObjectExpandable({ @@ -718,7 +698,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { })(); const sheetNodes: ContentNode[] = sheets.map((s) => ({ - provider: "google_drive", internalId: getGoogleSheetContentNodeInternalId( s.driveFileId, s.driveSheetId @@ -726,7 +705,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { parentInternalId: getInternalId(s.driveFileId), type: "database", title: s.name || "", - dustDocumentId: null, lastUpdatedAt: s.updatedAt.getTime() || null, sourceUrl: `https://docs.google.com/spreadsheets/d/${s.driveFileId}/edit#gid=${s.driveSheetId}`, expandable: false, @@ -979,13 +957,11 @@ async function getFoldersAsContentNodes({ } const sourceUrl = `https://drive.google.com/drive/folders/${f.folderId}`; return { - provider: "google_drive", internalId: getInternalId(f.folderId), parentInternalId: null, type: "folder", title: fd.name || "", sourceUrl, - dustDocumentId: null, lastUpdatedAt: fd.updatedAtMs || null, expandable: await isDriveObjectExpandable({ objectId: f.folderId, diff --git a/connectors/src/connectors/google_drive/lib/permissions.ts b/connectors/src/connectors/google_drive/lib/permissions.ts index 8983e8b3f722..226cd96dcea0 100644 --- a/connectors/src/connectors/google_drive/lib/permissions.ts +++ b/connectors/src/connectors/google_drive/lib/permissions.ts @@ -1,8 +1,4 @@ -import { - isGoogleDriveFolder, - isGoogleDriveSpreadSheetFile, -} from "@connectors/connectors/google_drive/temporal/mime_types"; -import { getInternalId } from "@connectors/connectors/google_drive/temporal/utils"; +import { isGoogleDriveFolder } from "@connectors/connectors/google_drive/temporal/mime_types"; import type { GoogleDriveFiles } from "@connectors/lib/models/google_drive"; export function getPermissionViewType(file: GoogleDriveFiles) { @@ -12,11 +8,3 @@ export function getPermissionViewType(file: GoogleDriveFiles) { return "file"; } - -export function getGoogleDriveEntityDocumentId(file: GoogleDriveFiles) { - if (isGoogleDriveSpreadSheetFile(file) || isGoogleDriveFolder(file)) { - return null; - } - - return getInternalId(file.driveFileId); -} diff --git a/connectors/src/connectors/google_drive/temporal/activities.ts b/connectors/src/connectors/google_drive/temporal/activities.ts index d9ee917aa548..ab15cd7c5729 100644 --- a/connectors/src/connectors/google_drive/temporal/activities.ts +++ b/connectors/src/connectors/google_drive/temporal/activities.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { GOOGLE_DRIVE_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import { uuid4 } from "@temporalio/workflow"; import type { drive_v3 } from "googleapis"; import type { GaxiosResponse, OAuth2Client } from "googleapis-common"; @@ -74,7 +74,7 @@ export async function upsertSharedWithMeFolder(connectorId: ModelId) { parents: [folderId], parentId: null, title: "Shared with me", - mimeType: GOOGLE_DRIVE_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.GOOGLE_DRIVE.FOLDER, }); } @@ -515,7 +515,7 @@ export async function incrementalSync( parents, parentId: parents[1] || null, title: driveFile.name ?? "", - mimeType: GOOGLE_DRIVE_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.GOOGLE_DRIVE.FOLDER, }); await GoogleDriveFiles.upsert({ @@ -860,7 +860,7 @@ export async function markFolderAsVisited( parents, parentId: parents[1] || null, title: file.name ?? "", - mimeType: GOOGLE_DRIVE_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.GOOGLE_DRIVE.FOLDER, }); await GoogleDriveFiles.upsert({ diff --git a/connectors/src/connectors/intercom/index.ts b/connectors/src/connectors/intercom/index.ts index 93ca8cbab6d5..ff13edc25a57 100644 --- a/connectors/src/connectors/intercom/index.ts +++ b/connectors/src/connectors/intercom/index.ts @@ -1,5 +1,9 @@ -import type { ConnectorPermission, ContentNode, Result } from "@dust-tt/types"; -import type { ContentNodesViewType } from "@dust-tt/types"; +import type { + ConnectorPermission, + ContentNode, + ContentNodesViewType, + Result, +} from "@dust-tt/types"; import { Err, Ok } from "@dust-tt/types"; import { Op } from "sequelize"; @@ -39,8 +43,10 @@ import type { RetrievePermissionsErrorCode, UpdateConnectorErrorCode, } from "@connectors/connectors/interface"; -import { ConnectorManagerError } from "@connectors/connectors/interface"; -import { BaseConnectorManager } from "@connectors/connectors/interface"; +import { + BaseConnectorManager, + ConnectorManagerError, +} from "@connectors/connectors/interface"; import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config"; import { ExternalOAuthTokenError } from "@connectors/lib/error"; import { @@ -613,7 +619,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { const nodes: ContentNode[] = []; for (const helpCenter of helpCenters) { nodes.push({ - provider: "intercom", internalId: getHelpCenterInternalId( this.connectorId, helpCenter.helpCenterId @@ -624,13 +629,11 @@ export class IntercomConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: true, permission: helpCenter.permission, - dustDocumentId: null, lastUpdatedAt: null, }); } for (const collection of collections) { nodes.push({ - provider: "intercom", internalId: getHelpCenterCollectionInternalId( this.connectorId, collection.collectionId @@ -646,13 +649,11 @@ export class IntercomConnectorManager extends BaseConnectorManager { sourceUrl: collection.url, expandable: true, permission: collection.permission, - dustDocumentId: null, lastUpdatedAt: collection.lastUpsertedTs?.getTime() || null, }); } for (const article of articles) { nodes.push({ - provider: "intercom", internalId: getHelpCenterArticleInternalId( this.connectorId, article.articleId @@ -668,13 +669,11 @@ export class IntercomConnectorManager extends BaseConnectorManager { sourceUrl: article.url, expandable: false, permission: article.permission, - dustDocumentId: null, lastUpdatedAt: article.lastUpsertedTs?.getTime() || null, }); } if (isAllConversations) { nodes.push({ - provider: "intercom", internalId: getTeamsInternalId(this.connectorId), parentInternalId: null, type: "channel", @@ -685,13 +684,11 @@ export class IntercomConnectorManager extends BaseConnectorManager { intercomWorkspace.syncAllConversations === "activated" ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: null, }); } for (const team of teams) { nodes.push({ - provider: "intercom", internalId: getTeamInternalId(this.connectorId, team.teamId), parentInternalId: getTeamsInternalId(this.connectorId), type: "channel", @@ -699,7 +696,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: false, permission: team.permission, - dustDocumentId: null, lastUpdatedAt: null, }); } diff --git a/connectors/src/connectors/intercom/lib/conversation_permissions.ts b/connectors/src/connectors/intercom/lib/conversation_permissions.ts index a9b771705235..622e70ae7992 100644 --- a/connectors/src/connectors/intercom/lib/conversation_permissions.ts +++ b/connectors/src/connectors/intercom/lib/conversation_permissions.ts @@ -134,7 +134,6 @@ export async function retrieveIntercomConversationsPermissions({ if (isReadPermissionsOnly) { if (isRootLevel && isAllConversationsSynced) { nodes.push({ - provider: "intercom", internalId: allTeamsInternalId, parentInternalId: null, type: "channel", @@ -143,12 +142,10 @@ export async function retrieveIntercomConversationsPermissions({ expandable: false, preventSelection: false, permission: isAllConversationsSynced ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: null, }); } else if (isRootLevel && hasTeamsWithReadPermission) { nodes.push({ - provider: "intercom", internalId: allTeamsInternalId, parentInternalId: null, type: "channel", @@ -157,7 +154,6 @@ export async function retrieveIntercomConversationsPermissions({ expandable: true, preventSelection: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -165,7 +161,6 @@ export async function retrieveIntercomConversationsPermissions({ if (parentInternalId === allTeamsInternalId) { teamsWithReadPermission.forEach((team) => { nodes.push({ - provider: connector.type, internalId: getTeamInternalId(connectorId, team.teamId), parentInternalId: allTeamsInternalId, type: "folder", @@ -173,7 +168,6 @@ export async function retrieveIntercomConversationsPermissions({ sourceUrl: null, expandable: false, permission: team.permission, - dustDocumentId: null, lastUpdatedAt: null, }); }); @@ -183,7 +177,6 @@ export async function retrieveIntercomConversationsPermissions({ const teams = await fetchIntercomTeams({ accessToken }); if (isRootLevel) { nodes.push({ - provider: "intercom", internalId: allTeamsInternalId, parentInternalId: null, type: "channel", @@ -192,7 +185,6 @@ export async function retrieveIntercomConversationsPermissions({ expandable: true, preventSelection: false, permission: isAllConversationsSynced ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -202,7 +194,6 @@ export async function retrieveIntercomConversationsPermissions({ return teamFromDb.teamId === team.id; }); nodes.push({ - provider: connector.type, internalId: getTeamInternalId(connectorId, team.id), parentInternalId: allTeamsInternalId, type: "folder", @@ -210,7 +201,6 @@ export async function retrieveIntercomConversationsPermissions({ sourceUrl: null, expandable: false, permission: isTeamInDb ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: null, }); }); diff --git a/connectors/src/connectors/intercom/lib/help_center_permissions.ts b/connectors/src/connectors/intercom/lib/help_center_permissions.ts index 4898c9f37ab5..995ca834e38e 100644 --- a/connectors/src/connectors/intercom/lib/help_center_permissions.ts +++ b/connectors/src/connectors/intercom/lib/help_center_permissions.ts @@ -364,7 +364,6 @@ export async function retrieveIntercomHelpCentersPermissions({ }, }); nodes = helpCentersFromDb.map((helpCenter) => ({ - provider: connector.type, internalId: getHelpCenterInternalId( connectorId, helpCenter.helpCenterId @@ -375,13 +374,11 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: null, expandable: true, permission: helpCenter.permission, - dustDocumentId: null, lastUpdatedAt: helpCenter.updatedAt.getTime(), })); } else { const helpCenters = await fetchIntercomHelpCenters({ accessToken }); nodes = helpCenters.map((helpCenter) => ({ - provider: connector.type, internalId: getHelpCenterInternalId(connectorId, helpCenter.id), parentInternalId: null, type: "database", @@ -390,7 +387,6 @@ export async function retrieveIntercomHelpCentersPermissions({ expandable: true, preventSelection: true, permission: "none", - dustDocumentId: null, lastUpdatedAt: null, })); } @@ -425,7 +421,6 @@ export async function retrieveIntercomHelpCentersPermissions({ }); if (isReadPermissionsOnly) { nodes = collectionsInDb.map((collection) => ({ - provider: connector.type, internalId: getHelpCenterCollectionInternalId( connectorId, collection.collectionId @@ -438,7 +433,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: collection.url, expandable: true, permission: collection.permission, - dustDocumentId: null, lastUpdatedAt: collection.updatedAt.getTime() || null, })); } else { @@ -452,7 +446,6 @@ export async function retrieveIntercomHelpCentersPermissions({ (c) => c.collectionId === collection.id ); return { - provider: connector.type, internalId: getHelpCenterCollectionInternalId( connectorId, collection.id @@ -468,7 +461,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: collection.url, expandable: false, // WE DO NOT LET EXPAND BELOW LEVEL 1 WHEN SELECTING NODES permission: matchingCollectionInDb ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: matchingCollectionInDb?.updatedAt.getTime() || null, }; }); @@ -493,7 +485,6 @@ export async function retrieveIntercomHelpCentersPermissions({ }); const collectionNodes: ContentNode[] = collectionsInDb.map( (collection) => ({ - provider: connector.type, internalId: getHelpCenterCollectionInternalId( connectorId, collection.collectionId @@ -509,7 +500,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: collection.url, expandable: true, permission: collection.permission, - dustDocumentId: null, lastUpdatedAt: collection.lastUpsertedTs?.getTime() || null, }) ); @@ -522,7 +512,6 @@ export async function retrieveIntercomHelpCentersPermissions({ }, }); const articleNodes: ContentNode[] = articlesInDb.map((article) => ({ - provider: connector.type, internalId: getHelpCenterArticleInternalId( connectorId, article.articleId @@ -535,7 +524,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: article.url, expandable: false, permission: article.permission, - dustDocumentId: null, lastUpdatedAt: article.updatedAt.getTime(), })); diff --git a/connectors/src/connectors/intercom/lib/permissions.ts b/connectors/src/connectors/intercom/lib/permissions.ts index 30c27f54b6bd..c4a5734c58ca 100644 --- a/connectors/src/connectors/intercom/lib/permissions.ts +++ b/connectors/src/connectors/intercom/lib/permissions.ts @@ -50,7 +50,6 @@ export async function retrieveSelectedNodes({ ); collectionsNodes.push({ - provider: connector.type, internalId: getHelpCenterCollectionInternalId( connectorId, collection.collectionId @@ -64,7 +63,6 @@ export async function retrieveSelectedNodes({ sourceUrl: collection.url, expandable, permission: collection.permission, - dustDocumentId: null, lastUpdatedAt: collection.updatedAt.getTime() || null, }); }); @@ -79,7 +77,6 @@ export async function retrieveSelectedNodes({ intercomWorkspace?.syncAllConversations === "scheduled_activate" ) { teamsNodes.push({ - provider: connector.type, internalId: getTeamsInternalId(connectorId), parentInternalId: null, type: "channel", @@ -87,7 +84,6 @@ export async function retrieveSelectedNodes({ sourceUrl: null, expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -100,7 +96,6 @@ export async function retrieveSelectedNodes({ }); teams.forEach((team) => { teamsNodes.push({ - provider: connector.type, internalId: getTeamInternalId(connectorId, team.teamId), parentInternalId: getTeamsInternalId(connectorId), type: "folder", @@ -108,7 +103,6 @@ export async function retrieveSelectedNodes({ sourceUrl: null, expandable: false, permission: team.permission, - dustDocumentId: null, lastUpdatedAt: team.updatedAt.getTime() || null, }); }); diff --git a/connectors/src/connectors/intercom/temporal/activities.ts b/connectors/src/connectors/intercom/temporal/activities.ts index fd7196176193..3ee734db84e4 100644 --- a/connectors/src/connectors/intercom/temporal/activities.ts +++ b/connectors/src/connectors/intercom/temporal/activities.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { INTERCOM_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import { Op } from "sequelize"; import { getIntercomAccessToken } from "@connectors/connectors/intercom/lib/intercom_access_token"; @@ -176,7 +176,7 @@ export async function syncHelpCenterOnlyActivity({ title: helpCenterOnIntercom.display_name || "Help Center", parents: [helpCenterInternalId], parentId: null, - mimeType: INTERCOM_MIME_TYPES.HELP_CENTER, + mimeType: MIME_TYPES.INTERCOM.HELP_CENTER, }); // If all children collections are not allowed anymore we delete the Help Center data @@ -509,7 +509,7 @@ export async function syncTeamOnlyActivity({ title: teamOnIntercom.name, parents: [teamInternalId, getTeamsInternalId(connectorId)], parentId: getTeamsInternalId(connectorId), - mimeType: INTERCOM_MIME_TYPES.TEAM, + mimeType: MIME_TYPES.INTERCOM.TEAM, }); return true; @@ -744,6 +744,6 @@ export async function upsertIntercomTeamsFolderActivity({ title: "Conversations", parents: [getTeamsInternalId(connectorId)], parentId: null, - mimeType: INTERCOM_MIME_TYPES.CONVERSATIONS, + mimeType: MIME_TYPES.INTERCOM.TEAMS_FOLDER, }); } diff --git a/connectors/src/connectors/intercom/temporal/sync_conversation.ts b/connectors/src/connectors/intercom/temporal/sync_conversation.ts index 93c245ed4ae9..48622f7b4910 100644 --- a/connectors/src/connectors/intercom/temporal/sync_conversation.ts +++ b/connectors/src/connectors/intercom/temporal/sync_conversation.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { INTERCOM_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import TurndownService from "turndown"; import { getIntercomAccessToken } from "@connectors/connectors/intercom/lib/intercom_access_token"; @@ -332,7 +332,7 @@ export async function syncConversation({ sync_type: syncType, }, title: convoTitle, - mimeType: INTERCOM_MIME_TYPES.CONVERSATION, + mimeType: MIME_TYPES.INTERCOM.CONVERSATION, async: true, }); } diff --git a/connectors/src/connectors/intercom/temporal/sync_help_center.ts b/connectors/src/connectors/intercom/temporal/sync_help_center.ts index 8868f7163376..9655b5fee61b 100644 --- a/connectors/src/connectors/intercom/temporal/sync_help_center.ts +++ b/connectors/src/connectors/intercom/temporal/sync_help_center.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { INTERCOM_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import TurndownService from "turndown"; import { getIntercomAccessToken } from "@connectors/connectors/intercom/lib/intercom_access_token"; @@ -229,7 +229,7 @@ export async function upsertCollectionWithChildren({ title: collection.name, parents: collectionParents, parentId: collectionParents[1], - mimeType: INTERCOM_MIME_TYPES.COLLECTION, + mimeType: MIME_TYPES.INTERCOM.COLLECTION, }); // Then we call ourself recursively on the children collections @@ -429,7 +429,7 @@ export async function upsertArticle({ sync_type: "batch", }, title: article.title, - mimeType: INTERCOM_MIME_TYPES.ARTICLE, + mimeType: MIME_TYPES.INTERCOM.ARTICLE, async: true, }); await articleOnDb.update({ diff --git a/connectors/src/connectors/microsoft/lib/content_nodes.ts b/connectors/src/connectors/microsoft/lib/content_nodes.ts index f65e83166eae..fe665918991a 100644 --- a/connectors/src/connectors/microsoft/lib/content_nodes.ts +++ b/connectors/src/connectors/microsoft/lib/content_nodes.ts @@ -17,7 +17,6 @@ export function getRootNodes(): ContentNode[] { export function getSitesRootAsContentNode(): ContentNode { return { - provider: "microsoft", internalId: internalIdFromTypeAndPath({ itemAPIPath: "", nodeType: "sites-root", @@ -26,7 +25,6 @@ export function getSitesRootAsContentNode(): ContentNode { type: "folder", title: "Sites", sourceUrl: null, - dustDocumentId: null, lastUpdatedAt: null, preventSelection: true, expandable: true, @@ -36,7 +34,6 @@ export function getSitesRootAsContentNode(): ContentNode { export function getTeamsRootAsContentNode(): ContentNode { return { - provider: "microsoft", internalId: internalIdFromTypeAndPath({ itemAPIPath: "", nodeType: "teams-root", @@ -45,7 +42,6 @@ export function getTeamsRootAsContentNode(): ContentNode { type: "folder", title: "Teams", sourceUrl: null, - dustDocumentId: null, lastUpdatedAt: null, preventSelection: true, expandable: true, @@ -54,7 +50,6 @@ export function getTeamsRootAsContentNode(): ContentNode { } export function getTeamAsContentNode(team: microsoftgraph.Team): ContentNode { return { - provider: "microsoft", internalId: internalIdFromTypeAndPath({ itemAPIPath: `/teams/${team.id}`, nodeType: "team", @@ -63,7 +58,6 @@ export function getTeamAsContentNode(team: microsoftgraph.Team): ContentNode { type: "folder", title: team.displayName || "unnamed", sourceUrl: team.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, preventSelection: true, expandable: true, @@ -80,7 +74,6 @@ export function getSiteAsContentNode( throw new Error("Site id is required"); } return { - provider: "microsoft", internalId: internalIdFromTypeAndPath({ itemAPIPath: getSiteAPIPath(site), nodeType: "site", @@ -89,7 +82,6 @@ export function getSiteAsContentNode( type: "folder", title: site.displayName || site.name || "unnamed", sourceUrl: site.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, preventSelection: true, expandable: true, @@ -111,7 +103,6 @@ export function getChannelAsContentNode( } return { - provider: "microsoft", internalId: internalIdFromTypeAndPath({ itemAPIPath: `/teams/${parentInternalId}/channels/${channel.id}`, nodeType: "channel", @@ -120,7 +111,6 @@ export function getChannelAsContentNode( type: "channel", title: channel.displayName || "unnamed", sourceUrl: channel.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: false, permission: "none", @@ -136,13 +126,11 @@ export function getDriveAsContentNode( throw new Error("Drive id is required"); } return { - provider: "microsoft", internalId: getDriveInternalId(drive), parentInternalId, type: "folder", title: drive.name || "unnamed", sourceUrl: drive.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: true, permission: "none", @@ -153,13 +141,11 @@ export function getFolderAsContentNode( parentInternalId: string ): ContentNode { return { - provider: "microsoft", internalId: getDriveItemInternalId(folder), parentInternalId, type: "folder", title: folder.name || "unnamed", sourceUrl: folder.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: true, permission: "none", @@ -171,13 +157,11 @@ export function getFileAsContentNode( parentInternalId: string ): ContentNode { return { - provider: "microsoft", internalId: getDriveItemInternalId(file), parentInternalId, type: "file", title: file.name || "unnamed", sourceUrl: file.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: false, permission: "none", @@ -207,13 +191,11 @@ export function getMicrosoftNodeAsContentNode( } return { - provider: "microsoft", internalId: node.internalId, parentInternalId: node.parentInternalId, type, title: node.name || "unnamed", sourceUrl: node.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: isExpandable, permission: "none", diff --git a/connectors/src/connectors/microsoft/temporal/activities.ts b/connectors/src/connectors/microsoft/temporal/activities.ts index a64f72b3ac59..66fe7a5030c8 100644 --- a/connectors/src/connectors/microsoft/temporal/activities.ts +++ b/connectors/src/connectors/microsoft/temporal/activities.ts @@ -1,9 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { - cacheWithRedis, - MICROSOFT_MIME_TYPES, - removeNulls, -} from "@dust-tt/types"; +import { cacheWithRedis, MIME_TYPES, removeNulls } from "@dust-tt/types"; import type { Client } from "@microsoft/microsoft-graph-client"; import { GraphError } from "@microsoft/microsoft-graph-client"; import type { DriveItem } from "@microsoft/microsoft-graph-types"; @@ -213,7 +209,7 @@ export async function getRootNodesToSyncFromResources( parents: [createdOrUpdatedResource.internalId], parentId: null, title: createdOrUpdatedResource.name ?? "", - mimeType: MICROSOFT_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.MICROSOFT.FOLDER, }), { concurrency: 5 } ); @@ -485,7 +481,7 @@ export async function syncFiles({ parents: [createdOrUpdatedResource.internalId, ...parentsOfParent], parentId: parentsOfParent[0], title: createdOrUpdatedResource.name ?? "", - mimeType: MICROSOFT_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.MICROSOFT.FOLDER, }), { concurrency: 5 } ); @@ -659,7 +655,7 @@ export async function syncDeltaForRootNodesInDrive({ parents: [blob.internalId], parentId: null, title: blob.name ?? "", - mimeType: MICROSOFT_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.MICROSOFT.FOLDER, }); // add parent information to new node resource. for the toplevel folder, @@ -875,7 +871,7 @@ async function updateDescendantsParentsInCore({ parents, parentId: parents[1] || null, title: folder.name ?? "", - mimeType: MICROSOFT_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.MICROSOFT.FOLDER, }); await concurrentExecutor( diff --git a/connectors/src/connectors/notion/index.ts b/connectors/src/connectors/notion/index.ts index fea202ad3086..e21c77ce5520 100644 --- a/connectors/src/connectors/notion/index.ts +++ b/connectors/src/connectors/notion/index.ts @@ -1,9 +1,8 @@ import type { ContentNode, ContentNodesViewType, Result } from "@dust-tt/types"; import { Err, - getNotionDatabaseTableId, getOAuthConnectionAccessToken, - NOTION_MIME_TYPES, + MIME_TYPES, Ok, } from "@dust-tt/types"; import _ from "lodash"; @@ -105,7 +104,7 @@ export class NotionConnectorManager extends BaseConnectorManager { parents: [folderId], parentId: null, title: "Orphaned Resources", - mimeType: NOTION_MIME_TYPES.UNKNOWN_FOLDER, + mimeType: MIME_TYPES.NOTION.UNKNOWN_FOLDER, }); try { @@ -455,7 +454,6 @@ export class NotionConnectorManager extends BaseConnectorManager { const expandable = Boolean(hasChildrenByPageId[page.notionPageId]); return { - provider: c.type, internalId: nodeIdFromNotionId(page.notionPageId), parentInternalId: !page.parentId || page.parentId === "workspace" @@ -466,7 +464,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: page.notionUrl || null, expandable, permission: "read", - dustDocumentId: nodeIdFromNotionId(page.notionPageId), lastUpdatedAt: page.lastUpsertedTs?.getTime() || null, }; }; @@ -479,7 +476,6 @@ export class NotionConnectorManager extends BaseConnectorManager { const getDbNodes = async (db: NotionDatabase): Promise => { return { - provider: c.type, internalId: nodeIdFromNotionId(db.notionDatabaseId), parentInternalId: !db.parentId || db.parentId === "workspace" @@ -490,7 +486,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: db.notionUrl || null, expandable: true, permission: "read", - dustDocumentId: nodeIdFromNotionId(`database-${db.notionDatabaseId}`), lastUpdatedAt: db.structuredDataUpsertedTs?.getTime() ?? null, }; }; @@ -504,7 +499,6 @@ export class NotionConnectorManager extends BaseConnectorManager { // We also need to return a "fake" top-level folder call "Orphaned" to include resources // we haven't been able to find a parent for. folderNodes.push({ - provider: c.type, // Orphaned resources in the database will have "unknown" as their parentId. internalId: nodeIdFromNotionId("unknown"), parentInternalId: null, @@ -513,7 +507,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -553,7 +546,6 @@ export class NotionConnectorManager extends BaseConnectorManager { const hasChildrenByPageId = await hasChildren(pages, this.connectorId); const pageNodes: ContentNode[] = await Promise.all( pages.map(async (page) => ({ - provider: "notion", internalId: nodeIdFromNotionId(page.notionPageId), parentInternalId: !page.parentId || page.parentId === "workspace" @@ -564,14 +556,11 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: page.notionUrl || null, expandable: Boolean(hasChildrenByPageId[page.notionPageId]), permission: "read", - dustDocumentId: nodeIdFromNotionId(page.notionPageId), lastUpdatedAt: page.lastUpsertedTs?.getTime() || null, - dustTableId: null, })) ); const dbNodes: ContentNode[] = dbs.map((db) => ({ - provider: "notion", internalId: nodeIdFromNotionId(db.notionDatabaseId), parentInternalId: !db.parentId || db.parentId === "workspace" @@ -582,9 +571,7 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: db.notionUrl || null, expandable: true, permission: "read", - dustDocumentId: nodeIdFromNotionId(`database-${db.notionDatabaseId}`), lastUpdatedAt: null, - dustTableId: getNotionDatabaseTableId(db.notionDatabaseId), })); const contentNodes = pageNodes.concat(dbNodes); @@ -593,7 +580,6 @@ export class NotionConnectorManager extends BaseConnectorManager { const orphanedCount = await getOrphanedCount(this.connectorId); if (orphanedCount > 0) { contentNodes.push({ - provider: "notion", // Orphaned resources in the database will have "unknown" as their parentId. internalId: nodeIdFromNotionId("unknown"), parentInternalId: null, @@ -602,7 +588,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); } diff --git a/connectors/src/connectors/notion/temporal/activities.ts b/connectors/src/connectors/notion/temporal/activities.ts index 864fcd25842f..719f3dd0829f 100644 --- a/connectors/src/connectors/notion/temporal/activities.ts +++ b/connectors/src/connectors/notion/temporal/activities.ts @@ -7,7 +7,7 @@ import type { import { assertNever, getNotionDatabaseTableId, - NOTION_MIME_TYPES, + MIME_TYPES, slugify, } from "@dust-tt/types"; import { isFullBlock, isFullPage, isNotionClientError } from "@notionhq/client"; @@ -1829,7 +1829,7 @@ export async function renderAndUpsertPageFromCache({ parents: parents, parentId: parents[1] || null, title: parentDb.title ?? "Untitled Notion Database", - mimeType: NOTION_MIME_TYPES.DATABASE, + mimeType: MIME_TYPES.NOTION.DATABASE, }), localLogger ); @@ -2053,7 +2053,7 @@ export async function renderAndUpsertPageFromCache({ sync_type: isFullSync ? "batch" : "incremental", }, title: title ?? "", - mimeType: NOTION_MIME_TYPES.PAGE, + mimeType: MIME_TYPES.NOTION.PAGE, async: true, }); } @@ -2550,7 +2550,7 @@ export async function upsertDatabaseStructuredDataFromCache({ parents: parentIds, parentId: parentIds[1] || null, title: dbModel.title ?? "Untitled Notion Database", - mimeType: NOTION_MIME_TYPES.DATABASE, + mimeType: MIME_TYPES.NOTION.DATABASE, }), localLogger ); @@ -2611,7 +2611,7 @@ export async function upsertDatabaseStructuredDataFromCache({ sync_type: "batch", }, title: databaseName, - mimeType: NOTION_MIME_TYPES.DATABASE, + mimeType: MIME_TYPES.NOTION.DATABASE, async: true, }); } else { diff --git a/connectors/src/connectors/slack/index.ts b/connectors/src/connectors/slack/index.ts index 9843b63e7686..f839596ad1b9 100644 --- a/connectors/src/connectors/slack/index.ts +++ b/connectors/src/connectors/slack/index.ts @@ -1,11 +1,11 @@ import type { ConnectorPermission, ContentNode, + ContentNodesViewType, ModelId, Result, SlackConfigurationType, } from "@dust-tt/types"; -import type { ContentNodesViewType } from "@dust-tt/types"; import { Err, isSlackAutoReadPatterns, @@ -20,8 +20,10 @@ import type { RetrievePermissionsErrorCode, UpdateConnectorErrorCode, } from "@connectors/connectors/interface"; -import { ConnectorManagerError } from "@connectors/connectors/interface"; -import { BaseConnectorManager } from "@connectors/connectors/interface"; +import { + BaseConnectorManager, + ConnectorManagerError, +} from "@connectors/connectors/interface"; import { getChannels } from "@connectors/connectors/slack//temporal/activities"; import { getBotEnabled } from "@connectors/connectors/slack/bot"; import { joinChannel } from "@connectors/connectors/slack/lib/channels"; @@ -397,7 +399,6 @@ export class SlackConnectorManager extends BaseConnectorManager ({ - provider: "slack", internalId: slackChannelInternalIdFromSlackChannelId(ch.slackChannelId), parentInternalId: null, type: "channel", @@ -405,9 +406,7 @@ export class SlackConnectorManager extends BaseConnectorManager ({ - provider: "slack", internalId: slackChannelInternalIdFromSlackChannelId(ch.slackChannelId), parentInternalId: null, type: "channel", @@ -629,9 +627,7 @@ export class SlackConnectorManager extends BaseConnectorManager t.startsWith("title:"))?.split(":")[1] ?? "", - mimeType: SLACK_MIME_TYPES.THREAD, + mimeType: MIME_TYPES.SLACK.MESSAGES, async: true, }); } @@ -850,7 +846,7 @@ export async function syncThread( sync_type: isBatchSync ? "batch" : "incremental", }, title: tags.find((t) => t.startsWith("title:"))?.split(":")[1] ?? "", - mimeType: SLACK_MIME_TYPES.THREAD, + mimeType: MIME_TYPES.SLACK.THREAD, async: true, }); } diff --git a/connectors/src/connectors/snowflake/lib/content_nodes.ts b/connectors/src/connectors/snowflake/lib/content_nodes.ts index f94659e8c03d..7f3101ff3fe6 100644 --- a/connectors/src/connectors/snowflake/lib/content_nodes.ts +++ b/connectors/src/connectors/snowflake/lib/content_nodes.ts @@ -34,7 +34,6 @@ export const getContentNodeFromInternalId = ( if (type === "database") { return { - provider: "snowflake", internalId: databaseName as string, parentInternalId: null, type: "folder", @@ -43,13 +42,11 @@ export const getContentNodeFromInternalId = ( expandable: true, preventSelection: false, permission, - dustDocumentId: null, lastUpdatedAt: null, }; } if (type === "schema") { return { - provider: "snowflake", internalId: `${databaseName}.${schemaName}`, parentInternalId: databaseName as string, type: "folder", @@ -58,13 +55,11 @@ export const getContentNodeFromInternalId = ( expandable: true, preventSelection: false, permission, - dustDocumentId: null, lastUpdatedAt: null, }; } if (type === "table") { return { - provider: "snowflake", internalId: `${databaseName}.${schemaName}.${tableName}`, parentInternalId: `${databaseName}.${schemaName}`, type: "database", @@ -73,7 +68,6 @@ export const getContentNodeFromInternalId = ( expandable: false, preventSelection: false, permission, - dustDocumentId: null, lastUpdatedAt: null, }; } diff --git a/connectors/src/connectors/snowflake/temporal/activities.ts b/connectors/src/connectors/snowflake/temporal/activities.ts index d7947bc4527e..2a6e8a3a4a51 100644 --- a/connectors/src/connectors/snowflake/temporal/activities.ts +++ b/connectors/src/connectors/snowflake/temporal/activities.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { isSnowflakeCredentials, SNOWFLAKE_MIME_TYPES } from "@dust-tt/types"; +import { isSnowflakeCredentials, MIME_TYPES } from "@dust-tt/types"; import { connectToSnowflake, @@ -168,7 +168,7 @@ export async function syncSnowflakeConnection(connectorId: ModelId) { title: table.databaseName, parents: [table.databaseName], parentId: null, - mimeType: SNOWFLAKE_MIME_TYPES.DATABASE, + mimeType: MIME_TYPES.SNOWFLAKE.DATABASE, }); // upsert a folder for the schema (child of the database) @@ -179,7 +179,7 @@ export async function syncSnowflakeConnection(connectorId: ModelId) { title: table.schemaName, parents: [schemaId, table.databaseName], parentId: table.databaseName, - mimeType: SNOWFLAKE_MIME_TYPES.SCHEMA, + mimeType: MIME_TYPES.SNOWFLAKE.SCHEMA, }); await upsertDataSourceRemoteTable({ @@ -192,7 +192,7 @@ export async function syncSnowflakeConnection(connectorId: ModelId) { parents: [table.internalId, schemaId, table.databaseName], parentId: schemaId, title: table.name, - mimeType: SNOWFLAKE_MIME_TYPES.TABLE, + mimeType: MIME_TYPES.SNOWFLAKE.TABLE, }); await table.update({ lastUpsertedAt: new Date(), diff --git a/connectors/src/connectors/webcrawler/index.ts b/connectors/src/connectors/webcrawler/index.ts index 99e06b576ef6..f53373063ecb 100644 --- a/connectors/src/connectors/webcrawler/index.ts +++ b/connectors/src/connectors/webcrawler/index.ts @@ -19,8 +19,10 @@ import type { RetrievePermissionsErrorCode, UpdateConnectorErrorCode, } from "@connectors/connectors/interface"; -import { ConnectorManagerError } from "@connectors/connectors/interface"; -import { BaseConnectorManager } from "@connectors/connectors/interface"; +import { + BaseConnectorManager, + ConnectorManagerError, +} from "@connectors/connectors/interface"; import { getDisplayNameForFolder, getDisplayNameForPage, @@ -199,7 +201,6 @@ export class WebcrawlerConnectorManager extends BaseConnectorManager !excludedFoldersSet.has(f.url)) .map((folder): ContentNode => { return { - provider: "webcrawler", internalId: folder.internalId, parentInternalId: folder.parentUrl ? stableIdForUrl({ @@ -211,7 +212,6 @@ export class WebcrawlerConnectorManager extends BaseConnectorManager { nodes.push({ - provider: "webcrawler", internalId: folder.internalId, parentInternalId: folder.parentUrl, title: getDisplayNameForFolder(folder), sourceUrl: folder.url, expandable: true, permission: "read", - dustDocumentId: null, type: "folder", lastUpdatedAt: folder.updatedAt.getTime(), }); }); pages.forEach((page) => { nodes.push({ - provider: "webcrawler", internalId: page.documentId, parentInternalId: page.parentUrl, title: getDisplayNameForPage(page), sourceUrl: page.url, expandable: false, permission: "read", - dustDocumentId: page.documentId, type: "file", lastUpdatedAt: page.updatedAt.getTime(), }); diff --git a/connectors/src/connectors/webcrawler/temporal/activities.ts b/connectors/src/connectors/webcrawler/temporal/activities.ts index d7fb4b9d9420..83feabe76167 100644 --- a/connectors/src/connectors/webcrawler/temporal/activities.ts +++ b/connectors/src/connectors/webcrawler/temporal/activities.ts @@ -1,9 +1,9 @@ import type { CoreAPIDataSourceDocumentSection, ModelId } from "@dust-tt/types"; import { + MIME_TYPES, stripNullBytes, WEBCRAWLER_MAX_DEPTH, WEBCRAWLER_MAX_PAGES, - WEBCRAWLER_MIME_TYPES, } from "@dust-tt/types"; import { validateUrl } from "@dust-tt/types/src/shared/utils/url_utils"; import { Context } from "@temporalio/activity"; @@ -292,7 +292,7 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) { parents, parentId: parents[1] || null, title: folder, - mimeType: WEBCRAWLER_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.WEBCRAWLER.FOLDER, }); createdFolders.add(folder); diff --git a/connectors/src/connectors/zendesk/lib/brand_permissions.ts b/connectors/src/connectors/zendesk/lib/brand_permissions.ts index 078dffc13a3c..eae056f2f511 100644 --- a/connectors/src/connectors/zendesk/lib/brand_permissions.ts +++ b/connectors/src/connectors/zendesk/lib/brand_permissions.ts @@ -9,10 +9,7 @@ import { forbidSyncZendeskTickets, } from "@connectors/connectors/zendesk/lib/ticket_permissions"; import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token"; -import { - createZendeskClient, - isBrandHelpCenterEnabled, -} from "@connectors/connectors/zendesk/lib/zendesk_api"; +import { createZendeskClient } from "@connectors/connectors/zendesk/lib/zendesk_api"; import logger from "@connectors/logger/logger"; import { ZendeskBrandResource } from "@connectors/resources/zendesk_resources"; @@ -50,8 +47,6 @@ export async function allowSyncZendeskBrand({ return false; } - const helpCenterEnabled = isBrandHelpCenterEnabled(fetchedBrand); - // creating the brand if it does not exist yet in db if (!brand) { await ZendeskBrandResource.makeNew({ @@ -61,7 +56,7 @@ export async function allowSyncZendeskBrand({ brandId: fetchedBrand.id, name: fetchedBrand.name || "Brand", ticketsPermission: "read", - helpCenterPermission: helpCenterEnabled ? "read" : "none", + helpCenterPermission: fetchedBrand.has_help_center ? "read" : "none", url: fetchedBrand.url, }, }); @@ -69,7 +64,7 @@ export async function allowSyncZendeskBrand({ // setting the permissions for the brand: // can be redundant if already set when creating the brand but necessary because of the categories. - if (helpCenterEnabled) { + if (fetchedBrand.has_help_center) { // allow the categories await allowSyncZendeskHelpCenter({ connectorId, diff --git a/connectors/src/connectors/zendesk/lib/cli.ts b/connectors/src/connectors/zendesk/lib/cli.ts index bba88a3dcb28..371649d82cc0 100644 --- a/connectors/src/connectors/zendesk/lib/cli.ts +++ b/connectors/src/connectors/zendesk/lib/cli.ts @@ -2,12 +2,14 @@ import type { ZendeskCheckIsAdminResponseType, ZendeskCommandType, ZendeskCountTicketsResponseType, + ZendeskFetchBrandResponseType, ZendeskFetchTicketResponseType, ZendeskResyncTicketsResponseType, } from "@dust-tt/types"; import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendesk/lib/zendesk_access_token"; import { + fetchZendeskBrand, fetchZendeskCurrentUser, fetchZendeskTicket, fetchZendeskTicketCount, @@ -17,6 +19,7 @@ import { launchZendeskTicketReSyncWorkflow } from "@connectors/connectors/zendes import { default as topLogger } from "@connectors/logger/logger"; import { ConnectorResource } from "@connectors/resources/connector_resource"; import { + ZendeskBrandResource, ZendeskConfigurationResource, ZendeskTicketResource, } from "@connectors/resources/zendesk_resources"; @@ -29,6 +32,7 @@ export const zendesk = async ({ | ZendeskCountTicketsResponseType | ZendeskResyncTicketsResponseType | ZendeskFetchTicketResponseType + | ZendeskFetchBrandResponseType > => { const logger = topLogger.child({ majorCommand: "zendesk", command, args }); @@ -143,5 +147,27 @@ export const zendesk = async ({ isTicketOnDb: ticketOnDb !== null, }; } + case "fetch-brand": { + if (!connector) { + throw new Error(`Connector ${connectorId} not found`); + } + const brandId = args.brandId ? args.brandId : null; + if (!brandId) { + throw new Error(`Missing --brandId argument`); + } + + const brand = await fetchZendeskBrand({ + brandId, + ...(await getZendeskSubdomainAndAccessToken(connector.connectionId)), + }); + const brandOnDb = await ZendeskBrandResource.fetchByBrandId({ + connectorId: connector.id, + brandId, + }); + return { + brand: brand as { [key: string]: unknown } | null, + brandOnDb: brandOnDb as { [key: string]: unknown } | null, + }; + } } }; diff --git a/connectors/src/connectors/zendesk/lib/permissions.ts b/connectors/src/connectors/zendesk/lib/permissions.ts index b542bf58c544..0c4d05868380 100644 --- a/connectors/src/connectors/zendesk/lib/permissions.ts +++ b/connectors/src/connectors/zendesk/lib/permissions.ts @@ -18,7 +18,6 @@ import { getZendeskSubdomainAndAccessToken } from "@connectors/connectors/zendes import { changeZendeskClientSubdomain, createZendeskClient, - isBrandHelpCenterEnabled, } from "@connectors/connectors/zendesk/lib/zendesk_api"; import type { ConnectorResource } from "@connectors/resources/connector_resource"; import { @@ -77,7 +76,6 @@ async function getRootLevelContentNodes( brandsInDatabase .find((b) => b.brandId === brand.id) ?.toContentNode(connectorId) ?? { - provider: "zendesk", internalId: getBrandInternalId({ connectorId, brandId: brand.id }), parentInternalId: null, type: "folder", @@ -85,7 +83,6 @@ async function getRootLevelContentNodes( sourceUrl: brand.brand_url, expandable: true, permission: "none", - dustDocumentId: null, lastUpdatedAt: null, } ); @@ -118,7 +115,6 @@ async function getBrandChildren( const { result: { brand: fetchedBrand }, } = await zendeskApiClient.brand.show(brandId); - const helpCenterEnabled = isBrandHelpCenterEnabled(fetchedBrand); if (isReadPermissionsOnly) { if (brandInDb?.ticketsPermission === "read") { @@ -126,14 +122,16 @@ async function getBrandChildren( brandInDb.getTicketsContentNode(connector.id, { expandable: true }) ); } - if (helpCenterEnabled && brandInDb?.helpCenterPermission === "read") { + if ( + fetchedBrand.has_help_center && + brandInDb?.helpCenterPermission === "read" + ) { nodes.push(brandInDb.getHelpCenterContentNode(connector.id)); } } else { const ticketsNode: ContentNode = brandInDb?.getTicketsContentNode( connector.id ) ?? { - provider: "zendesk", internalId: getTicketsInternalId({ connectorId: connector.id, brandId }), parentInternalId: parentInternalId, type: "folder", @@ -141,17 +139,15 @@ async function getBrandChildren( sourceUrl: null, expandable: false, permission: "none", - dustDocumentId: null, lastUpdatedAt: null, }; nodes.push(ticketsNode); // only displaying the Help Center node if the brand has an enabled Help Center - if (helpCenterEnabled) { + if (fetchedBrand.has_help_center) { const helpCenterNode: ContentNode = brandInDb?.getHelpCenterContentNode( connector.id ) ?? { - provider: "zendesk", internalId: getHelpCenterInternalId({ connectorId: connector.id, brandId, @@ -162,7 +158,6 @@ async function getBrandChildren( sourceUrl: null, expandable: true, permission: "none", - dustDocumentId: null, lastUpdatedAt: null, }; nodes.push(helpCenterNode); @@ -212,7 +207,6 @@ async function getHelpCenterChildren( categoriesInDatabase .find((c) => c.categoryId === category.id) ?.toContentNode(connectorId) ?? { - provider: "zendesk", internalId: getCategoryInternalId({ connectorId, brandId, @@ -224,7 +218,6 @@ async function getHelpCenterChildren( sourceUrl: category.html_url, expandable: false, permission: "none", - dustDocumentId: null, lastUpdatedAt: null, } ); diff --git a/connectors/src/connectors/zendesk/lib/sync_article.ts b/connectors/src/connectors/zendesk/lib/sync_article.ts index 1c31c0dd3919..10fda3049149 100644 --- a/connectors/src/connectors/zendesk/lib/sync_article.ts +++ b/connectors/src/connectors/zendesk/lib/sync_article.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { ZENDESK_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import TurndownService from "turndown"; import type { @@ -176,7 +176,7 @@ export async function syncArticle({ loggerArgs: { ...loggerArgs, articleId: article.id }, upsertContext: { sync_type: "batch" }, title: article.title, - mimeType: ZENDESK_MIME_TYPES.ARTICLE, + mimeType: MIME_TYPES.ZENDESK.ARTICLE, async: true, }); await articleInDb.update({ lastUpsertedTs: new Date(currentSyncDateMs) }); diff --git a/connectors/src/connectors/zendesk/lib/sync_category.ts b/connectors/src/connectors/zendesk/lib/sync_category.ts index 1ae683817183..8e793151c0ae 100644 --- a/connectors/src/connectors/zendesk/lib/sync_category.ts +++ b/connectors/src/connectors/zendesk/lib/sync_category.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { ZENDESK_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import type { ZendeskFetchedCategory } from "@connectors/@types/node-zendesk"; import { @@ -118,6 +118,6 @@ export async function syncCategory({ parents, parentId: parents[1], title: categoryInDb.name, - mimeType: ZENDESK_MIME_TYPES.CATEGORY, + mimeType: MIME_TYPES.ZENDESK.CATEGORY, }); } diff --git a/connectors/src/connectors/zendesk/lib/sync_ticket.ts b/connectors/src/connectors/zendesk/lib/sync_ticket.ts index 814c32251e04..336e112dc5da 100644 --- a/connectors/src/connectors/zendesk/lib/sync_ticket.ts +++ b/connectors/src/connectors/zendesk/lib/sync_ticket.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { ZENDESK_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import TurndownService from "turndown"; import type { @@ -237,7 +237,7 @@ ${comments loggerArgs: { ...loggerArgs, ticketId: ticket.id }, upsertContext: { sync_type: "batch" }, title: ticket.subject, - mimeType: ZENDESK_MIME_TYPES.TICKET, + mimeType: MIME_TYPES.ZENDESK.TICKET, async: true, }); await ticketInDb.update({ lastUpsertedTs: new Date(currentSyncDateMs) }); diff --git a/connectors/src/connectors/zendesk/lib/zendesk_api.ts b/connectors/src/connectors/zendesk/lib/zendesk_api.ts index bcec94f0f497..1bcbad33a9af 100644 --- a/connectors/src/connectors/zendesk/lib/zendesk_api.ts +++ b/connectors/src/connectors/zendesk/lib/zendesk_api.ts @@ -206,15 +206,6 @@ export async function fetchZendeskBrand({ } } -/** - * Finds out whether a brand has a help center enabled. - * has_help_center can be true without having an enabled help center, which leads to 404 when retreving categories. - * @param brand A brand fetched from the Zendesk API. - */ -export function isBrandHelpCenterEnabled(brand: ZendeskFetchedBrand): boolean { - return brand.has_help_center && brand.help_center_state === "enabled"; -} - /** * Fetches a single article from the Zendesk API. */ @@ -256,17 +247,24 @@ export async function fetchZendeskCategoriesInBrand( hasMore: boolean; nextLink: string | null; }> { - const response = await fetchFromZendeskWithRetries({ - url: - url ?? // using the URL if we got one, reconstructing it otherwise - `https://${brandSubdomain}.zendesk.com/api/v2/help_center/categories?page[size]=${pageSize}`, - accessToken, - }); - return { - categories: response.categories, - hasMore: response.meta.has_more, - nextLink: response.links.next, - }; + try { + const response = await fetchFromZendeskWithRetries({ + url: + url ?? // using the URL if we got one, reconstructing it otherwise + `https://${brandSubdomain}.zendesk.com/api/v2/help_center/categories?page[size]=${pageSize}`, + accessToken, + }); + return { + categories: response.categories, + hasMore: response.meta.has_more, + nextLink: response.links.next, + }; + } catch (e) { + if (isZendeskNotFoundError(e)) { + return { categories: [], hasMore: false, nextLink: null }; + } + throw e; + } } /** diff --git a/connectors/src/connectors/zendesk/temporal/activities.ts b/connectors/src/connectors/zendesk/temporal/activities.ts index f36d4915183d..edc94327f86d 100644 --- a/connectors/src/connectors/zendesk/temporal/activities.ts +++ b/connectors/src/connectors/zendesk/temporal/activities.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { ZENDESK_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import _ from "lodash"; import { getBrandInternalId } from "@connectors/connectors/zendesk/lib/id_conversions"; @@ -11,6 +11,7 @@ import { changeZendeskClientSubdomain, createZendeskClient, fetchZendeskArticlesInCategory, + fetchZendeskBrand, fetchZendeskCategoriesInBrand, fetchZendeskManyUsers, fetchZendeskTicketComments, @@ -137,7 +138,7 @@ export async function syncZendeskBrandActivity({ parents: [brandInternalId], parentId: null, title: brandInDb.name, - mimeType: ZENDESK_MIME_TYPES.BRAND, + mimeType: MIME_TYPES.ZENDESK.BRAND, }); // using the content node to get one source of truth regarding the parent relationship @@ -148,7 +149,7 @@ export async function syncZendeskBrandActivity({ parents: [helpCenterNode.internalId, helpCenterNode.parentInternalId], parentId: helpCenterNode.parentInternalId, title: helpCenterNode.title, - mimeType: ZENDESK_MIME_TYPES.HELP_CENTER, + mimeType: MIME_TYPES.ZENDESK.HELP_CENTER, }); // using the content node to get one source of truth regarding the parent relationship @@ -159,7 +160,7 @@ export async function syncZendeskBrandActivity({ parents: [ticketsNode.internalId, ticketsNode.parentInternalId], parentId: ticketsNode.parentInternalId, title: ticketsNode.title, - mimeType: ZENDESK_MIME_TYPES.TICKETS, + mimeType: MIME_TYPES.ZENDESK.TICKETS, }); // updating the entry in db @@ -178,6 +179,7 @@ export async function syncZendeskBrandActivity({ /** * Retrieves the IDs of every brand in db that has read permissions on their Help Center or in one of their Categories. + * Removes the permissions beforehand for Help Center that have been deleted or disabled on Zendesk. * This activity will be used to retrieve the brands that need to be incrementally synced. * * Note: in this approach; if a single category has read permissions and not its Help Center, @@ -189,7 +191,36 @@ export async function getZendeskHelpCenterReadAllowedBrandIdsActivity( // fetching the brands that have a Help Center selected as a whole const brandsWithHelpCenter = await ZendeskBrandResource.fetchHelpCenterReadAllowedBrandIds(connectorId); - // fetching the brands that have at least one Category selected + + // cleaning up Brands (resp. Help Centers) that don't exist on Zendesk anymore (resp. have been deleted) + const connector = await ConnectorResource.fetchById(connectorId); + if (!connector) { + throw new Error("[Zendesk] Connector not found."); + } + const { subdomain, accessToken } = await getZendeskSubdomainAndAccessToken( + connector.connectionId + ); + for (const brandId of brandsWithHelpCenter) { + const fetchedBrand = await fetchZendeskBrand({ + accessToken, + subdomain, + brandId, + }); + const brandInDb = await ZendeskBrandResource.fetchByBrandId({ + connectorId, + brandId, + }); + if (!fetchedBrand) { + await brandInDb?.revokeTicketsPermissions(); + await brandInDb?.revokeHelpCenterPermissions(); + } else if (!fetchedBrand.has_help_center) { + await brandInDb?.revokeHelpCenterPermissions(); + } + } + + // fetching the brands that have at least one Category selected: + // we need to do that because we can only fetch diffs at the brand level. + // We will filter later on the categories allowed. const brandWithCategories = await ZendeskCategoryResource.fetchBrandIdsOfReadOnlyCategories( connectorId @@ -335,7 +366,7 @@ export async function syncZendeskCategoryActivity({ parents, parentId: parents[1], title: categoryInDb.name, - mimeType: ZENDESK_MIME_TYPES.CATEGORY, + mimeType: MIME_TYPES.ZENDESK.CATEGORY, }); // otherwise, we update the category name and lastUpsertedTs diff --git a/connectors/src/connectors/zendesk/temporal/incremental_activities.ts b/connectors/src/connectors/zendesk/temporal/incremental_activities.ts index 6990dce9189f..af12a1eb7b97 100644 --- a/connectors/src/connectors/zendesk/temporal/incremental_activities.ts +++ b/connectors/src/connectors/zendesk/temporal/incremental_activities.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { ZENDESK_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import { syncArticle } from "@connectors/connectors/zendesk/lib/sync_article"; import { @@ -154,7 +154,7 @@ export async function syncZendeskArticleUpdateBatchActivity({ parents, parentId: parents[1], title: category.name, - mimeType: ZENDESK_MIME_TYPES.CATEGORY, + mimeType: MIME_TYPES.ZENDESK.CATEGORY, }); } else { /// ignoring these to proceed with the other articles, but these might have to be checked at some point diff --git a/connectors/src/resources/zendesk_resources.ts b/connectors/src/resources/zendesk_resources.ts index de45cb15a23d..2a077a3ddb80 100644 --- a/connectors/src/resources/zendesk_resources.ts +++ b/connectors/src/resources/zendesk_resources.ts @@ -330,7 +330,6 @@ export class ZendeskBrandResource extends BaseResource { toContentNode(connectorId: number): ContentNode { const { brandId } = this; return { - provider: "zendesk", internalId: getBrandInternalId({ connectorId, brandId }), parentInternalId: null, type: "folder", @@ -342,7 +341,6 @@ export class ZendeskBrandResource extends BaseResource { this.ticketsPermission === "read" ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: this.updatedAt.getTime(), }; } @@ -353,7 +351,6 @@ export class ZendeskBrandResource extends BaseResource { ): ContentNode & { parentInternalId: string } { const { brandId } = this; return { - provider: "zendesk", internalId: getHelpCenterInternalId({ connectorId, brandId }), parentInternalId: getBrandInternalId({ connectorId, brandId }), type: "folder", @@ -361,7 +358,6 @@ export class ZendeskBrandResource extends BaseResource { sourceUrl: null, expandable: true, permission: this.helpCenterPermission, - dustDocumentId: null, lastUpdatedAt: null, }; } @@ -375,7 +371,6 @@ export class ZendeskBrandResource extends BaseResource { ): ContentNode & { parentInternalId: string } { const { brandId } = this; return { - provider: "zendesk", internalId: getTicketsInternalId({ connectorId, brandId }), parentInternalId: getBrandInternalId({ connectorId, brandId }), type: "folder", @@ -383,7 +378,6 @@ export class ZendeskBrandResource extends BaseResource { sourceUrl: null, expandable: expandable, permission: this.ticketsPermission, - dustDocumentId: null, lastUpdatedAt: null, }; } @@ -641,7 +635,6 @@ export class ZendeskCategoryResource extends BaseResource { ): ContentNode { const { brandId, categoryId, permission } = this; return { - provider: "zendesk", internalId: getCategoryInternalId({ connectorId, brandId, categoryId }), parentInternalId: getHelpCenterInternalId({ connectorId, brandId }), type: "folder", @@ -649,7 +642,6 @@ export class ZendeskCategoryResource extends BaseResource { sourceUrl: this.url, expandable: expandable, permission, - dustDocumentId: null, lastUpdatedAt: this.updatedAt.getTime(), }; } @@ -727,7 +719,6 @@ export class ZendeskTicketResource extends BaseResource { toContentNode(connectorId: number): ContentNode { const { brandId, ticketId } = this; return { - provider: "zendesk", internalId: getTicketInternalId({ connectorId, brandId, ticketId }), parentInternalId: getTicketsInternalId({ connectorId, brandId }), type: "file", @@ -735,7 +726,6 @@ export class ZendeskTicketResource extends BaseResource { sourceUrl: this.url, expandable: false, permission: this.permission, - dustDocumentId: null, lastUpdatedAt: this.updatedAt.getTime(), }; } @@ -939,7 +929,6 @@ export class ZendeskArticleResource extends BaseResource { toContentNode(connectorId: number): ContentNode { const { brandId, categoryId, articleId } = this; return { - provider: "zendesk", internalId: getArticleInternalId({ connectorId, brandId, articleId }), parentInternalId: getCategoryInternalId({ connectorId, @@ -951,7 +940,6 @@ export class ZendeskArticleResource extends BaseResource { sourceUrl: this.url, expandable: false, permission: this.permission, - dustDocumentId: null, lastUpdatedAt: this.updatedAt.getTime(), }; } diff --git a/core/admin/elasticsearch_run_command_from_file.sh b/core/admin/elasticsearch_run_command_from_file.sh new file mode 100755 index 000000000000..695ea2a4bfff --- /dev/null +++ b/core/admin/elasticsearch_run_command_from_file.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Runs a single command on the Elasticsearch cluster +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +if [ ! -f "$1" ]; then + echo "Error: File $1 not found" + exit 1 +fi + +# Extract method and path from first line +read -r ES_METHOD ES_PATH < "$1" +# Get JSON body (skip first line) +ES_BODY=$(sed '1d' "$1") + +# Validate JSON +if ! echo "$ES_BODY" | jq . >/dev/null 2>&1; then + echo "Error: Invalid JSON body" + exit 1 +fi + +read -p "Run command from file ${1} in region ${DUST_REGION}? [y/N] " response +if [[ ! $response =~ ^[Yy]$ ]]; then + echo "Operation cancelled" + exit 0 +fi + +curl -X "$ES_METHOD" "${ELASTICSEARCH_URL}/${ES_PATH}" \ + -H "Content-Type: application/json" \ + -u "${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -d "$ES_BODY" \ No newline at end of file diff --git a/core/bin/core_api.rs b/core/bin/core_api.rs index a8efe1d9f86c..486abdf3f8db 100644 --- a/core/bin/core_api.rs +++ b/core/bin/core_api.rs @@ -2187,6 +2187,7 @@ struct DatabasesTablesUpsertPayload { tags: Vec, parent_id: Option, parents: Vec, + source_url: Option, // Remote DB specifics remote_database_table_id: Option, @@ -2260,6 +2261,7 @@ async fn tables_upsert( timestamp: payload.timestamp.unwrap_or(utils::now()), tags: payload.tags, parents: payload.parents, + source_url: payload.source_url, remote_database_table_id: payload.remote_database_table_id, remote_database_secret_id: payload.remote_database_secret_id, title: payload.title, @@ -2928,6 +2930,7 @@ struct FoldersUpsertPayload { parents: Vec, title: String, mime_type: String, + source_url: Option, provider_visibility: Option, } @@ -2992,6 +2995,7 @@ async fn folders_upsert( parents: payload.parents, title: payload.title, mime_type: payload.mime_type, + source_url: payload.source_url, provider_visibility: payload.provider_visibility, }, ) diff --git a/core/src/data_sources/data_source.rs b/core/src/data_sources/data_source.rs index e3fd366c9140..fec0eb2e6633 100644 --- a/core/src/data_sources/data_source.rs +++ b/core/src/data_sources/data_source.rs @@ -235,6 +235,7 @@ impl From for Node { document.provider_visibility, document.parent_id, document.parents.clone(), + document.source_url, ) } } diff --git a/core/src/data_sources/folder.rs b/core/src/data_sources/folder.rs index e458fa9b2330..52bb49825e24 100644 --- a/core/src/data_sources/folder.rs +++ b/core/src/data_sources/folder.rs @@ -12,6 +12,7 @@ pub struct Folder { parent_id: Option, parents: Vec, mime_type: String, + source_url: Option, provider_visibility: Option, } @@ -25,6 +26,7 @@ impl Folder { parent_id: Option, parents: Vec, mime_type: String, + source_url: Option, provider_visibility: Option, ) -> Self { Folder { @@ -36,6 +38,7 @@ impl Folder { parent_id, parents, mime_type, + source_url, provider_visibility, } } @@ -61,6 +64,9 @@ impl Folder { pub fn parents(&self) -> &Vec { &self.parents } + pub fn source_url(&self) -> &Option { + &self.source_url + } pub fn mime_type(&self) -> &str { &self.mime_type } @@ -82,6 +88,7 @@ impl From for Node { folder.provider_visibility, folder.parent_id, folder.parents, + folder.source_url, ) } } diff --git a/core/src/data_sources/node.rs b/core/src/data_sources/node.rs index 74c8f37dfef4..4e89e3317369 100644 --- a/core/src/data_sources/node.rs +++ b/core/src/data_sources/node.rs @@ -32,6 +32,7 @@ pub struct Node { pub provider_visibility: Option, pub parent_id: Option, pub parents: Vec, + pub source_url: Option, } impl Node { @@ -46,6 +47,7 @@ impl Node { provider_visibility: Option, parent_id: Option, parents: Vec, + source_url: Option, ) -> Self { Node { data_source_id: data_source_id.to_string(), @@ -58,6 +60,7 @@ impl Node { provider_visibility: provider_visibility.clone(), parent_id: parent_id.clone(), parents, + source_url, } } @@ -97,6 +100,7 @@ impl Node { self.parent_id, self.parents, self.mime_type, + self.source_url, self.provider_visibility, ) } diff --git a/core/src/databases/table.rs b/core/src/databases/table.rs index 8c7751bd66bc..a2bec9483dbf 100644 --- a/core/src/databases/table.rs +++ b/core/src/databases/table.rs @@ -65,6 +65,7 @@ pub struct Table { provider_visibility: Option, parent_id: Option, parents: Vec, + source_url: Option, schema: Option, schema_stale_at: Option, @@ -89,6 +90,7 @@ impl Table { tags: Vec, parent_id: Option, parents: Vec, + source_url: Option, schema: Option, schema_stale_at: Option, remote_database_table_id: Option, @@ -109,6 +111,7 @@ impl Table { provider_visibility, parent_id, parents, + source_url, schema, schema_stale_at, remote_database_table_id, @@ -143,6 +146,9 @@ impl Table { pub fn parents(&self) -> &Vec { &self.parents } + pub fn source_url(&self) -> &Option { + &self.source_url + } pub fn name(&self) -> &str { &self.name } @@ -258,6 +264,7 @@ impl From for Node { table.provider_visibility, table.parents.get(1).cloned(), table.parents, + table.source_url, ) } } @@ -607,6 +614,7 @@ mod tests { vec![], None, vec![], + None, Some(schema), None, None, diff --git a/core/src/search_stores/indices/data_sources_nodes_2.mappings.json b/core/src/search_stores/indices/data_sources_nodes_2.mappings.json index a7ff9c56e81d..816ccbeecfab 100644 --- a/core/src/search_stores/indices/data_sources_nodes_2.mappings.json +++ b/core/src/search_stores/indices/data_sources_nodes_2.mappings.json @@ -34,6 +34,10 @@ }, "mime_type": { "type": "keyword" + }, + "source_url": { + "type": "keyword", + "index": false } } } diff --git a/core/src/search_stores/migrations/20250113_add_source_url.http b/core/src/search_stores/migrations/20250113_add_source_url.http new file mode 100644 index 000000000000..dbe94e7a37fe --- /dev/null +++ b/core/src/search_stores/migrations/20250113_add_source_url.http @@ -0,0 +1,9 @@ +PUT core.data_sources_nodes/_mapping +{ + "properties": { + "source_url": { + "type": "keyword", + "index": false + } + } +} diff --git a/core/src/stores/migrations/20250107_drop_parents_columns.sql b/core/src/stores/migrations/20250107_drop_parents_columns.sql index eba635cba8cd..b91d0cac41fc 100644 --- a/core/src/stores/migrations/20250107_drop_parents_columns.sql +++ b/core/src/stores/migrations/20250107_drop_parents_columns.sql @@ -1,2 +1,5 @@ -ALTER TABLE data_sources_documents DROP COLUMN parents; -ALTER TABLE tables DROP COLUMN parents; +DROP INDEX IF EXISTS idx_data_sources_documents_parents_array; +DROP INDEX IF EXISTS idx_tables_parents_array; + +ALTER TABLE data_sources_documents DROP COLUMN IF EXISTS parents; +ALTER TABLE tables DROP COLUMN IF EXISTS parents; diff --git a/core/src/stores/migrations/20250109_nodes_add_source_url.sql b/core/src/stores/migrations/20250109_nodes_add_source_url.sql new file mode 100644 index 000000000000..da4c286a3bf6 --- /dev/null +++ b/core/src/stores/migrations/20250109_nodes_add_source_url.sql @@ -0,0 +1 @@ +ALTER TABLE data_sources_nodes ADD COLUMN source_url TEXT; diff --git a/core/src/stores/postgres.rs b/core/src/stores/postgres.rs index 410d456e2fc9..b45c3f28ad41 100644 --- a/core/src/stores/postgres.rs +++ b/core/src/stores/postgres.rs @@ -11,7 +11,6 @@ use std::hash::Hasher; use std::str::FromStr; use tokio_postgres::types::ToSql; use tokio_postgres::{NoTls, Transaction}; -use tracing::info; use crate::data_sources::data_source::DocumentStatus; use crate::data_sources::node::{Node, NodeType}; @@ -55,6 +54,7 @@ pub struct UpsertNode<'a> { pub mime_type: &'a str, pub provider_visibility: &'a Option, pub parents: &'a Vec, + pub source_url: &'a Option, } impl PostgresStore { @@ -156,27 +156,6 @@ impl PostgresStore { row_id: i64, tx: &Transaction<'_>, ) -> Result<()> { - // Check that all parents exist in data_sources_nodes. - let stmt_check = tx - .prepare( - "SELECT COUNT(*) FROM data_sources_nodes - WHERE data_source = $1 AND node_id = ANY($2)", - ) - .await?; - let count: i64 = tx - .query_one(&stmt_check, &[&data_source_row_id, &upsert_params.parents]) - .await? - .get(0); - if count != upsert_params.parents.len() as i64 { - info!( - data_source_id = data_source_row_id, - node_id = upsert_params.node_id, - parents = ?upsert_params.parents, - operation = "upsert_node", - "[KWSEARCH] invariant_parents_missing_in_nodes" - ); - } - let created = utils::now(); let (document_row_id, table_row_id, folder_row_id) = match upsert_params.node_type { @@ -1479,24 +1458,6 @@ impl Store for PostgresStore { let tx = c.transaction().await?; - // Check that all parents exist in data_sources_nodes. - let count: u64 = tx - .execute( - "SELECT COUNT(*) FROM data_sources_nodes - WHERE data_source = $1 AND node_id = ANY($2)", - &[&data_source_row_id, &parents], - ) - .await?; - if count != parents.len() as u64 { - info!( - data_source_id = data_source_row_id, - node_id = document_id, - parents = ?parents, - operation = "update_document_parents", - "[KWSEARCH] invariant_parents_missing_in_nodes" - ); - } - // Update parents on nodes table. tx.execute( "UPDATE data_sources_nodes SET parents = $1 \ @@ -1966,6 +1927,7 @@ impl Store for PostgresStore { mime_type: &document.mime_type, provider_visibility: &document.provider_visibility, parents: &document.parents, + source_url: &document.source_url, }, data_source_row_id, document_row_id, @@ -2728,6 +2690,7 @@ impl Store for PostgresStore { upsert_params.tags, upsert_params.parents.get(1).cloned(), upsert_params.parents, + upsert_params.source_url, parsed_schema, table_schema_stale_at.map(|t| t as u64), upsert_params.remote_database_table_id, @@ -2743,6 +2706,7 @@ impl Store for PostgresStore { mime_type: table.mime_type(), provider_visibility: table.provider_visibility(), parents: table.parents(), + source_url: table.source_url(), }, data_source_row_id, table_row_id, @@ -2830,24 +2794,6 @@ impl Store for PostgresStore { let tx = c.transaction().await?; - // Check that all parents exist in data_sources_nodes. - let count: u64 = tx - .execute( - "SELECT COUNT(*) FROM data_sources_nodes - WHERE data_source = $1 AND node_id = ANY($2)", - &[&data_source_row_id, &parents], - ) - .await?; - if count != parents.len() as u64 { - info!( - data_source_id = data_source_row_id, - node_id = table_id, - parents = ?parents, - operation = "update_table_parents", - "[KWSEARCH] invariant_parents_missing_in_nodes" - ); - } - // Update parents on nodes table. let stmt = tx .prepare( @@ -2932,7 +2878,7 @@ impl Store for PostgresStore { let stmt = c .prepare( "SELECT t.created, t.table_id, t.name, t.description, \ - t.timestamp, t.tags_array, dsn.parents, \ + t.timestamp, t.tags_array, dsn.parents, dsn.source_url, \ t.schema, t.schema_stale_at, \ t.remote_database_table_id, t.remote_database_secret_id, \ dsn.title, dsn.mime_type, dsn.provider_visibility \ @@ -2951,6 +2897,7 @@ impl Store for PostgresStore { Vec, Vec, Option, + Option, Option, Option, Option, @@ -2974,6 +2921,7 @@ impl Store for PostgresStore { r[0].get(11), r[0].get(12), r[0].get(13), + r[0].get(14), )), _ => unreachable!(), }; @@ -2988,6 +2936,7 @@ impl Store for PostgresStore { timestamp, tags, parents, + source_url, schema, schema_stale_at, remote_database_table_id, @@ -3022,6 +2971,7 @@ impl Store for PostgresStore { tags, parents.get(1).cloned(), parents, + source_url, parsed_schema, schema_stale_at.map(|t| t as u64), remote_database_table_id, @@ -3096,7 +3046,7 @@ impl Store for PostgresStore { t.timestamp, t.tags_array, dsn.parents, \ t.schema, t.schema_stale_at, \ t.remote_database_table_id, t.remote_database_secret_id, \ - dsn.title, dsn.mime_type, dsn.provider_visibility \ + dsn.title, dsn.mime_type, dsn.source_url, dsn.provider_visibility \ FROM tables t INNER JOIN data_sources_nodes dsn ON dsn.table=t.id \ WHERE {} ORDER BY t.timestamp DESC", where_clauses.join(" AND "), @@ -3138,7 +3088,8 @@ impl Store for PostgresStore { let remote_database_secret_id: Option = r.get(10); let title: String = r.get(11); let mime_type: String = r.get(12); - let provider_visibility: Option = r.get(13); + let source_url: Option = r.get(13); + let provider_visibility: Option = r.get(14); let parsed_schema: Option = match schema { None => None, @@ -3166,6 +3117,7 @@ impl Store for PostgresStore { tags, parents.get(1).cloned(), parents, + source_url, parsed_schema, schema_stale_at.map(|t| t as u64), remote_database_table_id, @@ -3302,6 +3254,7 @@ impl Store for PostgresStore { upsert_params.parents.get(1).cloned(), upsert_params.parents, upsert_params.mime_type, + upsert_params.source_url, upsert_params.provider_visibility, ); @@ -3314,6 +3267,7 @@ impl Store for PostgresStore { title: folder.title(), mime_type: folder.mime_type(), parents: folder.parents(), + source_url: folder.source_url(), }, data_source_row_id, folder_row_id, @@ -3423,7 +3377,7 @@ impl Store for PostgresStore { } let sql = format!( - "SELECT dsn.node_id, dsn.title, dsn.timestamp, dsn.parents, dsn.mime_type, dsn.provider_visibility \ + "SELECT dsn.node_id, dsn.title, dsn.timestamp, dsn.parents, dsn.mime_type, dsn.source_url, dsn.provider_visibility \ FROM data_sources_nodes dsn \ WHERE dsn.folder IS NOT NULL AND {} ORDER BY dsn.timestamp DESC", where_clauses.join(" AND "), @@ -3472,7 +3426,8 @@ impl Store for PostgresStore { let timestamp: i64 = r.get(2); let parents: Vec = r.get(3); let mime_type: String = r.get(4); - let provider_visibility: Option = r.get(5); + let source_url: Option = r.get(5); + let provider_visibility: Option = r.get(6); Ok(Folder::new( data_source_id.clone(), @@ -3483,6 +3438,7 @@ impl Store for PostgresStore { parents.get(1).cloned(), parents, mime_type, + source_url, provider_visibility, )) }) @@ -3556,7 +3512,7 @@ impl Store for PostgresStore { let stmt = c .prepare( - "SELECT timestamp, title, mime_type, provider_visibility, parents, node_id, document, \"table\", folder \ + "SELECT timestamp, title, mime_type, provider_visibility, parents, node_id, document, \"table\", folder, source_url \ FROM data_sources_nodes \ WHERE data_source = $1 AND node_id = $2 LIMIT 1", ) @@ -3581,6 +3537,7 @@ impl Store for PostgresStore { (None, None, Some(id)) => (NodeType::Folder, id), _ => unreachable!(), }; + let source_url: Option = row[0].get::<_, Option>(9); Ok(Some(( Node::new( &data_source_id, @@ -3593,6 +3550,7 @@ impl Store for PostgresStore { provider_visibility, parents.get(1).cloned(), parents, + source_url, ), row_id, ))) @@ -3611,7 +3569,7 @@ impl Store for PostgresStore { let stmt = c .prepare( - "SELECT dsn.timestamp, dsn.title, dsn.mime_type, dsn.provider_visibility, dsn.parents, dsn.node_id, dsn.document, dsn.\"table\", dsn.folder, ds.data_source_id, ds.internal_id, dsn.id \ + "SELECT dsn.timestamp, dsn.title, dsn.mime_type, dsn.provider_visibility, dsn.parents, dsn.node_id, dsn.document, dsn.\"table\", dsn.folder, ds.data_source_id, ds.internal_id, dsn.source_url, dsn.id \ FROM data_sources_nodes dsn JOIN data_sources ds ON dsn.data_source = ds.id \ WHERE dsn.id > $1 ORDER BY dsn.id ASC LIMIT $2", ) @@ -3639,7 +3597,8 @@ impl Store for PostgresStore { (None, None, Some(id)) => (NodeType::Folder, id), _ => unreachable!(), }; - let row_id = row.get::<_, i64>(10); + let source_url: Option = row.get::<_, Option>(11); + let row_id = row.get::<_, i64>(12); ( Node::new( &data_source_id, @@ -3652,6 +3611,7 @@ impl Store for PostgresStore { provider_visibility, parents.get(1).cloned(), parents, + source_url, ), row_id, element_row_id, diff --git a/core/src/stores/store.rs b/core/src/stores/store.rs index 39f70e4b185e..6a5e64a735ca 100644 --- a/core/src/stores/store.rs +++ b/core/src/stores/store.rs @@ -67,6 +67,7 @@ pub struct TableUpsertParams { pub timestamp: u64, pub tags: Vec, pub parents: Vec, + pub source_url: Option, pub remote_database_table_id: Option, pub remote_database_secret_id: Option, pub title: String, @@ -80,6 +81,7 @@ pub struct FolderUpsertParams { pub title: String, pub parents: Vec, pub mime_type: String, + pub source_url: Option, pub provider_visibility: Option, } @@ -602,8 +604,9 @@ pub const POSTGRES_TABLES: [&'static str; 16] = [ mime_type TEXT NOT NULL, provider_visibility TEXT, parents TEXT[] NOT NULL, + source_url TEXT, document BIGINT, - \"table\" BIGINT, + \"table\" BIGINT, folder BIGINT, FOREIGN KEY(data_source) REFERENCES data_sources(id), FOREIGN KEY(document) REFERENCES data_sources_documents(id), @@ -617,7 +620,7 @@ pub const POSTGRES_TABLES: [&'static str; 16] = [ );", ]; -pub const SQL_INDEXES: [&'static str; 35] = [ +pub const SQL_INDEXES: [&'static str; 33] = [ "CREATE INDEX IF NOT EXISTS idx_specifications_project_created ON specifications (project, created);", "CREATE INDEX IF NOT EXISTS @@ -664,16 +667,12 @@ pub const SQL_INDEXES: [&'static str; 35] = [ ON data_sources_documents (data_source, document_id, created DESC);", "CREATE INDEX IF NOT EXISTS idx_data_sources_documents_tags_array ON data_sources_documents USING GIN (tags_array);", - "CREATE INDEX IF NOT EXISTS - idx_data_sources_documents_parents_array ON data_sources_documents USING GIN (parents);", "CREATE UNIQUE INDEX IF NOT EXISTS idx_databases_table_ids_hash ON databases (table_ids_hash);", "CREATE UNIQUE INDEX IF NOT EXISTS idx_tables_data_source_table_id ON tables (data_source, table_id);", "CREATE INDEX IF NOT EXISTS idx_tables_tags_array ON tables USING GIN (tags_array);", - "CREATE INDEX IF NOT EXISTS - idx_tables_parents_array ON tables USING GIN (parents);", "CREATE UNIQUE INDEX IF NOT EXISTS idx_sqlite_workers_url ON sqlite_workers (url);", "CREATE INDEX IF NOT EXISTS diff --git a/front/components/ConnectorPermissionsModal.tsx b/front/components/ConnectorPermissionsModal.tsx index e75e19edc248..7be2be126627 100644 --- a/front/components/ConnectorPermissionsModal.tsx +++ b/front/components/ConnectorPermissionsModal.tsx @@ -28,10 +28,10 @@ import { } from "@dust-tt/sparkle"; import type { APIError, - BaseContentNode, ConnectorPermission, ConnectorProvider, ConnectorType, + ContentNode, DataSourceType, LightWorkspaceType, UpdateConnectorRequestBody, @@ -946,7 +946,7 @@ export async function confirmPrivateNodesSync({ selectedNodes, confirm, }: { - selectedNodes: BaseContentNode[]; + selectedNodes: ContentNode[]; confirm: (n: ConfirmDataType) => Promise; }): Promise { // confirmation in case there are private nodes diff --git a/front/components/ContentNodeTree.tsx b/front/components/ContentNodeTree.tsx index 81b05bb24b2f..e595d5e6f19c 100644 --- a/front/components/ContentNodeTree.tsx +++ b/front/components/ContentNodeTree.tsx @@ -8,7 +8,7 @@ import { Tooltip, Tree, } from "@dust-tt/sparkle"; -import type { APIError, BaseContentNode } from "@dust-tt/types"; +import type { APIError, ContentNode } from "@dust-tt/types"; import type { ReactNode } from "react"; import React, { useCallback, useContext, useState } from "react"; @@ -17,7 +17,7 @@ import { classNames, timeAgoFrom } from "@app/lib/utils"; const unselectedChildren = ( selection: Record, - node: BaseContentNode + node: ContentNode ) => Object.entries(selection).reduce((acc, [k, v]) => { const shouldUnselect = v.parents.includes(node.internalId); @@ -32,7 +32,7 @@ const unselectedChildren = ( }, {}); export type UseResourcesHook = (parentId: string | null) => { - resources: BaseContentNode[]; + resources: ContentNode[]; isResourcesLoading: boolean; isResourcesError: boolean; resourcesError?: APIError | null; @@ -40,7 +40,7 @@ export type UseResourcesHook = (parentId: string | null) => { export type ContentNodeTreeItemStatus = { isSelected: boolean; - node: BaseContentNode; + node: ContentNode; parents: string[]; }; @@ -121,7 +121,7 @@ function ContentNodeTreeChildren({ ); const getCheckedState = useCallback( - (node: BaseContentNode) => { + (node: ContentNode) => { if (!selectedNodes) { return false; } @@ -249,15 +249,15 @@ function ContentNodeTreeChildren({ size="xs" icon={BracesIcon} onClick={() => { - if (n.dustDocumentId) { - onDocumentViewClick(n.dustDocumentId); + if (n.type === "file") { + onDocumentViewClick(n.internalId); } }} className={classNames( - n.dustDocumentId ? "" : "pointer-events-none opacity-0" + n.type === "file" ? "" : "pointer-events-none opacity-0" )} - disabled={!n.dustDocumentId} - variant="ghost" + disabled={n.type !== "file"} + variant="outline" /> )} diff --git a/front/components/actions/dust_app_run/DustAppRunActionDetails.tsx b/front/components/actions/dust_app_run/DustAppRunActionDetails.tsx index 37075dbebfbc..73f0d240d38c 100644 --- a/front/components/actions/dust_app_run/DustAppRunActionDetails.tsx +++ b/front/components/actions/dust_app_run/DustAppRunActionDetails.tsx @@ -11,6 +11,7 @@ import { useMemo } from "react"; import { ActionDetailsWrapper } from "@app/components/actions/ActionDetailsWrapper"; import type { ActionDetailsComponentBaseProps } from "@app/components/actions/types"; +import { DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY } from "@app/lib/api/assistant/actions/constants"; export function DustAppRunActionDetails({ action, @@ -51,12 +52,14 @@ function DustAppRunParamsDetails({ action }: { action: DustAppRunActionType }) { return (
- {Object.entries(params).map(([k, v], idx) => ( -

- {capitalize(k)}: - {` ${v}`} -

- ))} + {Object.entries(params) + .filter(([k]) => k !== DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY) + .map(([k, v], idx) => ( +

+ {capitalize(k)}: + {` ${v}`} +

+ ))}
); } diff --git a/front/components/app/blocks/Database.tsx b/front/components/app/blocks/Database.tsx index c77a23483116..c23613eb4d93 100644 --- a/front/components/app/blocks/Database.tsx +++ b/front/components/app/blocks/Database.tsx @@ -141,7 +141,7 @@ export function TablesManager({ currentTableId={table.table_id} onTableUpdate={(selectedTable) => { updateTableConfig(index, { - table_id: selectedTable.dustDocumentId!, + table_id: selectedTable.internalId, }); }} excludeTables={getSelectedTables()} diff --git a/front/components/assistant/AssistantActions.tsx b/front/components/assistant/AssistantActions.tsx deleted file mode 100644 index a184a4636c25..000000000000 --- a/front/components/assistant/AssistantActions.tsx +++ /dev/null @@ -1,133 +0,0 @@ -import { Dialog, useSendNotification } from "@dust-tt/sparkle"; -import type { - LightAgentConfigurationType, - PostOrPatchAgentConfigurationRequestBody, - WorkspaceType, -} from "@dust-tt/types"; - -import { - useAgentConfiguration, - useUpdateUserFavorite, -} from "@app/lib/swr/assistants"; - -export function RemoveAssistantFromFavoritesDialog({ - owner, - agentConfiguration, - show, - onClose, -}: { - owner: WorkspaceType; - agentConfiguration: LightAgentConfigurationType; - show: boolean; - onClose: () => void; -}) { - const { updateUserFavorite } = useUpdateUserFavorite({ - owner, - agentConfigurationId: agentConfiguration.sId, - }); - - return ( - { - void updateUserFavorite(false); - onClose(); - }} - > -
- This will remove the assistant from favorites. You can add it back at - any time from the Chat homepage. -
- - ); -} - -export function RemoveAssistantFromWorkspaceDialog({ - owner, - agentConfiguration, - show, - onClose, - onRemove, -}: { - owner: WorkspaceType; - agentConfiguration: LightAgentConfigurationType; - show: boolean; - onClose: () => void; - onRemove: () => void; -}) { - const sendNotification = useSendNotification(); - - const { agentConfiguration: detailedConfiguration } = useAgentConfiguration({ - workspaceId: owner.sId, - agentConfigurationId: agentConfiguration.sId, - }); - - return ( - { - if (!detailedConfiguration) { - throw new Error("Agent configuration not found"); - } - const body: PostOrPatchAgentConfigurationRequestBody = { - assistant: { - name: agentConfiguration.name, - description: agentConfiguration.description, - instructions: agentConfiguration.instructions, - pictureUrl: agentConfiguration.pictureUrl, - status: "active", - scope: "published", - model: agentConfiguration.model, - actions: detailedConfiguration.actions, - templateId: agentConfiguration.templateId, - maxStepsPerRun: agentConfiguration.maxStepsPerRun, - visualizationEnabled: agentConfiguration.visualizationEnabled, - }, - }; - - const res = await fetch( - `/api/w/${owner.sId}/assistant/agent_configurations/${agentConfiguration.sId}`, - { - method: "PATCH", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(body), - } - ); - if (!res.ok) { - const data = await res.json(); - sendNotification({ - title: `Error removing from Company assistants`, - description: data.error.message, - type: "error", - }); - } else { - sendNotification({ - title: `Assistant removed from Company assistants`, - type: "success", - }); - onRemove(); - } - - onClose(); - }} - > -
-
- Removing the assistant from the Company assistants means it won't be - automatically active for members anymore. -
-
Any workspace member will be able to modify the assistant.
-
- - ); -} diff --git a/front/components/assistant/AssistantDetails.tsx b/front/components/assistant/AssistantDetails.tsx index 4946b3ed9380..914e0e087dc2 100644 --- a/front/components/assistant/AssistantDetails.tsx +++ b/front/components/assistant/AssistantDetails.tsx @@ -2,7 +2,6 @@ import { Avatar, BarChartIcon, Button, - Card, CardGrid, ChatBubbleLeftRightIcon, ChatBubbleThoughtIcon, @@ -14,6 +13,7 @@ import { HandThumbDownIcon, HandThumbUpIcon, InformationCircleIcon, + LockIcon, Page, Sheet, SheetContainer, @@ -25,6 +25,7 @@ import { TabsList, TabsTrigger, Tooltip, + ValueCard, } from "@dust-tt/sparkle"; import type { AgentConfigurationScope, @@ -142,48 +143,49 @@ function AssistantDetailsPerformance({ ) : ( - -
-
-
Active Users
-
-
- {agentAnalytics?.users ? ( - <> -
- {agentAnalytics.users.length} -
+ +
+ {agentAnalytics?.users ? ( + <> +
+ {agentAnalytics.users.length} +
- - {removeNulls(agentAnalytics.users.map((top) => top.user)) - .slice(0, 5) - .map((user) => ( - - } - label={user.fullName} - /> - ))} - - - ) : ( - "-" - )} + + {removeNulls( + agentAnalytics.users.map((top) => top.user) + ) + .slice(0, 5) + .map((user) => ( + + } + label={user.fullName} + /> + ))} + + + ) : ( + "-" + )} +
-
- + } + className="h-32" + /> - -
-
-
Reactions
-
+ {agentConfiguration.scope !== "global" && agentAnalytics?.feedbacks ? ( @@ -205,13 +207,12 @@ function AssistantDetailsPerformance({ "-" )}
- - - -
-
-
Conversations
-
+ } + className="h-32" + /> +
@@ -224,13 +225,12 @@ function AssistantDetailsPerformance({
- - - -
-
-
Messages
-
+ } + className="h-32" + /> +
@@ -243,18 +243,19 @@ function AssistantDetailsPerformance({
- - + } + className="h-32" + /> )} {agentConfiguration.scope !== "global" && ( - <> - +
+ - +
)} ); @@ -267,11 +268,14 @@ export function AssistantDetails({ }: AssistantDetailsProps) { const [isUpdatingScope, setIsUpdatingScope] = useState(false); const [selectedTab, setSelectedTab] = useState("info"); - const { agentConfiguration, isAgentConfigurationValidating } = - useAgentConfiguration({ - workspaceId: owner.sId, - agentConfigurationId: assistantId, - }); + const { + agentConfiguration, + isAgentConfigurationValidating, + isAgentConfigurationError, + } = useAgentConfiguration({ + workspaceId: owner.sId, + agentConfigurationId: assistantId, + }); const doUpdateScope = useUpdateAgentScope({ owner, @@ -380,6 +384,18 @@ export function AssistantDetails({ )} )} + {isAgentConfigurationError?.error.type === + "agent_configuration_not_found" && ( + + This is a private assistant that can't be shared with other + workspace members. + + )} diff --git a/front/components/assistant/AssistantsTable.tsx b/front/components/assistant/AssistantsTable.tsx index 8800d3c0cbc5..8410147839d0 100644 --- a/front/components/assistant/AssistantsTable.tsx +++ b/front/components/assistant/AssistantsTable.tsx @@ -26,7 +26,10 @@ import { useRouter } from "next/router"; import { useMemo, useState } from "react"; import { DeleteAssistantDialog } from "@app/components/assistant/DeleteAssistantDialog"; -import { assistantUsageMessage } from "@app/components/assistant/Usage"; +import { + assistantActiveUsersMessage, + assistantUsageMessage, +} from "@app/components/assistant/Usage"; import { SCOPE_INFO } from "@app/components/assistant_builder/Sharing"; import { classNames, formatTimestampToFriendlyDate } from "@app/lib/utils"; @@ -133,6 +136,29 @@ const getTableColumns = () => { width: "6rem", }, }, + { + header: "Active Users", + accessorKey: "usage.userCount", + cell: (info: CellContext) => ( + + + {info.row.original.usage?.userCount ?? 0} + + } + /> + + ), + meta: { + width: "6rem", + }, + }, { header: "Feedbacks", accessorKey: "feedbacks", @@ -236,6 +262,8 @@ export function AssistantsTable({ name: agentConfiguration.name, usage: agentConfiguration.usage ?? { messageCount: 0, + conversationCount: 0, + userCount: 0, timePeriodSec: 30 * 24 * 60 * 60, }, description: agentConfiguration.description, diff --git a/front/components/assistant/Usage.tsx b/front/components/assistant/Usage.tsx index 32938eb779b0..04cd185f3c98 100644 --- a/front/components/assistant/Usage.tsx +++ b/front/components/assistant/Usage.tsx @@ -52,3 +52,32 @@ export function assistantUsageMessage({ ); } } + +export function assistantActiveUsersMessage({ + usage, + isLoading, + isError, +}: { + usage: AgentUsageType | null; + isLoading: boolean; + isError: boolean; +}): ReactNode { + if (isError) { + return "Error loading usage data."; + } + + if (isLoading) { + return "Loading usage data..."; + } + + if (usage) { + const days = usage.timePeriodSec / (60 * 60 * 24); + const nb = usage.userCount || 0; + + return ( + <> + {nb} active user{pluralize(nb)} over the last {days} days + + ); + } +} diff --git a/front/components/assistant/conversation/AgentMessage.tsx b/front/components/assistant/conversation/AgentMessage.tsx index a5c925fcf6fd..2ef07dedcfb5 100644 --- a/front/components/assistant/conversation/AgentMessage.tsx +++ b/front/components/assistant/conversation/AgentMessage.tsx @@ -13,7 +13,6 @@ import { ConversationMessage, DocumentDuplicateIcon, EyeIcon, - FeedbackSelector, Markdown, Page, Popover, @@ -56,6 +55,7 @@ import type { PluggableList } from "react-markdown/lib/react-markdown"; import { makeDocumentCitation } from "@app/components/actions/retrieval/utils"; import { makeWebsearchResultsCitation } from "@app/components/actions/websearch/utils"; import { AgentMessageActions } from "@app/components/assistant/conversation/actions/AgentMessageActions"; +import { FeedbackSelector } from "@app/components/assistant/conversation/FeedbackSelector"; import { GenerationContext } from "@app/components/assistant/conversation/GenerationContextProvider"; import { CitationsContext, diff --git a/front/components/assistant/conversation/ConversationContainer.tsx b/front/components/assistant/conversation/ConversationContainer.tsx index 1b6e932b4075..c23d51325fe5 100644 --- a/front/components/assistant/conversation/ConversationContainer.tsx +++ b/front/components/assistant/conversation/ConversationContainer.tsx @@ -1,15 +1,14 @@ -import { Page } from "@dust-tt/sparkle"; -import { useSendNotification } from "@dust-tt/sparkle"; +import { Page, useSendNotification } from "@dust-tt/sparkle"; import type { AgentMention, LightAgentConfigurationType, MentionType, Result, SubscriptionType, + UploadedContentFragment, UserType, WorkspaceType, } from "@dust-tt/types"; -import type { UploadedContentFragment } from "@dust-tt/types"; import { Err, Ok } from "@dust-tt/types"; import { useRouter } from "next/router"; import { useCallback, useContext, useEffect, useRef, useState } from "react"; @@ -35,26 +34,22 @@ import { } from "@app/lib/swr/conversations"; interface ConversationContainerProps { - conversationId: string | null; owner: WorkspaceType; subscription: SubscriptionType; user: UserType; isBuilder: boolean; agentIdToMention: string | null; - messageRankToScrollTo: number | undefined; } export function ConversationContainer({ - conversationId, owner, subscription, user, isBuilder, agentIdToMention, - messageRankToScrollTo, }: ConversationContainerProps) { - const [activeConversationId, setActiveConversationId] = - useState(conversationId); + const { activeConversationId } = useConversationsNavigation(); + const [planLimitReached, setPlanLimitReached] = useState(false); const [stickyMentions, setStickyMentions] = useState([]); @@ -76,7 +71,6 @@ export function ConversationContainer({ conversationId: activeConversationId, workspaceId: owner.sId, limit: 50, - startAtRank: messageRankToScrollTo, }); const setInputbarMention = useCallback( @@ -242,7 +236,6 @@ export function ConversationContainer({ undefined, { shallow: true } ); - setActiveConversationId(conversationRes.value.sId); await mutateConversations(); await scrollConversationsToTop(); @@ -310,7 +303,6 @@ export function ConversationContainer({ conversationId={activeConversationId} // TODO(2024-06-20 flav): Fix extra-rendering loop with sticky mentions. onStickyMentionsChange={onStickyMentionsChange} - messageRankToScrollTo={messageRankToScrollTo} /> ) : (
diff --git a/front/components/assistant/conversation/ConversationLayout.tsx b/front/components/assistant/conversation/ConversationLayout.tsx index c42908d1c2ee..7b5332fcb43b 100644 --- a/front/components/assistant/conversation/ConversationLayout.tsx +++ b/front/components/assistant/conversation/ConversationLayout.tsx @@ -1,11 +1,14 @@ import type { SubscriptionType, WorkspaceType } from "@dust-tt/types"; import { useRouter } from "next/router"; -import React, { useCallback, useEffect, useState } from "react"; +import React, { useMemo } from "react"; import RootLayout from "@app/components/app/RootLayout"; import { AssistantDetails } from "@app/components/assistant/AssistantDetails"; import { ConversationErrorDisplay } from "@app/components/assistant/conversation/ConversationError"; -import { ConversationsNavigationProvider } from "@app/components/assistant/conversation/ConversationsNavigationProvider"; +import { + ConversationsNavigationProvider, + useConversationsNavigation, +} from "@app/components/assistant/conversation/ConversationsNavigationProvider"; import { ConversationTitle } from "@app/components/assistant/conversation/ConversationTitle"; import { FileDropProvider } from "@app/components/assistant/conversation/FileUploaderContext"; import { GenerationContextProvider } from "@app/components/assistant/conversation/GenerationContextProvider"; @@ -13,10 +16,7 @@ import { InputBarProvider } from "@app/components/assistant/conversation/input_b import { AssistantSidebarMenu } from "@app/components/assistant/conversation/SidebarMenu"; import AppLayout from "@app/components/sparkle/AppLayout"; import { useURLSheet } from "@app/hooks/useURLSheet"; -import { - useConversation, - useDeleteConversation, -} from "@app/lib/swr/conversations"; +import { useConversation } from "@app/lib/swr/conversations"; export interface ConversationLayoutProps { baseUrl: string; @@ -32,116 +32,81 @@ export default function ConversationLayout({ children: React.ReactNode; pageProps: ConversationLayoutProps; }) { - const { baseUrl, conversationId, owner, subscription } = pageProps; - - const router = useRouter(); + const { baseUrl, owner, subscription } = pageProps; - const [detailViewContent, setDetailViewContent] = useState(""); - const [activeConversationId, setActiveConversationId] = useState( - conversationId !== "new" ? conversationId : null + return ( + + + + {children} + + + ); +} +const ConversationLayoutContent = ({ + owner, + subscription, + baseUrl, + children, +}: any) => { + const router = useRouter(); const { onOpenChange: onOpenChangeAssistantModal } = useURLSheet("assistantDetails"); - - useEffect(() => { - const handleRouteChange = () => { - const assistantSId = router.query.assistantDetails ?? []; - // We use shallow browsing when creating a new conversation. - // Monitor router to update conversation info. - const conversationId = router.query.cId ?? ""; - - if (assistantSId && typeof assistantSId === "string") { - setDetailViewContent(assistantSId); - } else { - setDetailViewContent(""); - } - - if ( - conversationId && - typeof conversationId === "string" && - conversationId !== activeConversationId - ) { - setActiveConversationId( - conversationId !== "new" ? conversationId : null - ); - } - }; - - // Initial check in case the component mounts with the query already set. - handleRouteChange(); - - router.events.on("routeChangeComplete", handleRouteChange); - return () => { - router.events.off("routeChangeComplete", handleRouteChange); - }; - }, [ - router.query, - router.events, - setActiveConversationId, - activeConversationId, - ]); - + const { activeConversationId } = useConversationsNavigation(); const { conversation, conversationError } = useConversation({ conversationId: activeConversationId, workspaceId: owner.sId, }); - const doDelete = useDeleteConversation(owner); - - const onDeleteConversation = useCallback(async () => { - const res = await doDelete(conversation); - if (res) { - void router.push(`/w/${owner.sId}/assistant/new`); + const assistantSId = useMemo(() => { + const sid = router.query.assistantDetails ?? []; + if (sid && typeof sid === "string") { + return sid; } - }, [conversation, doDelete, owner.sId, router]); + return null; + }, [router.query.assistantDetails]); return ( - - - - - ) - } - navChildren={} - > - {conversationError ? ( - - ) : ( - <> - onOpenChangeAssistantModal(false)} - /> - - - {children} - - - - )} - - - - + + + ) + } + navChildren={} + > + {conversationError ? ( + + ) : ( + <> + onOpenChangeAssistantModal(false)} + /> + + {children} + + + )} + + ); -} +}; diff --git a/front/components/assistant/conversation/ConversationTitle.tsx b/front/components/assistant/conversation/ConversationTitle.tsx index 4a2e182a92e9..d34da1e9ff3d 100644 --- a/front/components/assistant/conversation/ConversationTitle.tsx +++ b/front/components/assistant/conversation/ConversationTitle.tsx @@ -10,30 +10,47 @@ import { TrashIcon, XMarkIcon, } from "@dust-tt/sparkle"; -import type { ConversationType } from "@dust-tt/types"; import type { WorkspaceType } from "@dust-tt/types"; +import { useRouter } from "next/router"; import type { MouseEvent } from "react"; -import React, { useRef, useState } from "react"; +import React, { useCallback, useRef, useState } from "react"; import { useSWRConfig } from "swr"; import { ConversationParticipants } from "@app/components/assistant/conversation/ConversationParticipants"; +import { useConversationsNavigation } from "@app/components/assistant/conversation/ConversationsNavigationProvider"; import { DeleteConversationsDialog } from "@app/components/assistant/conversation/DeleteConversationsDialog"; +import { + useConversation, + useDeleteConversation, +} from "@app/lib/swr/conversations"; import { classNames } from "@app/lib/utils"; export function ConversationTitle({ owner, - conversationId, - conversation, - shareLink, - onDelete, + baseUrl, }: { owner: WorkspaceType; - conversationId: string; - conversation: ConversationType | null; - shareLink: string; - onDelete?: (conversationId: string) => void; + baseUrl: string; }) { const { mutate } = useSWRConfig(); + const router = useRouter(); + const { activeConversationId } = useConversationsNavigation(); + + const { conversation } = useConversation({ + conversationId: activeConversationId, + workspaceId: owner.sId, + }); + + const shareLink = `${baseUrl}/w/${owner.sId}/assistant/${activeConversationId}`; + + const doDelete = useDeleteConversation(owner); + + const onDelete = useCallback(async () => { + const res = await doDelete(conversation); + if (res) { + void router.push(`/w/${owner.sId}/assistant/new`); + } + }, [conversation, doDelete, owner.sId, router]); const [copyLinkSuccess, setCopyLinkSuccess] = useState(false); const [isEditingTitle, setIsEditingTitle] = useState(false); @@ -43,57 +60,62 @@ export function ConversationTitle({ const titleInputFocused = useRef(false); const saveButtonFocused = useRef(false); - const handleClick = async () => { + const handleClick = useCallback(async () => { await navigator.clipboard.writeText(shareLink || ""); setCopyLinkSuccess(true); setTimeout(() => { setCopyLinkSuccess(false); }, 1000); - }; + }, [shareLink]); - const onTitleChange = async (title: string) => { - try { - const res = await fetch( - `/api/w/${owner.sId}/assistant/conversations/${conversationId}`, - { - method: "PATCH", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - title, - visibility: conversation?.visibility, - }), + const onTitleChange = useCallback( + async (title: string) => { + try { + const res = await fetch( + `/api/w/${owner.sId}/assistant/conversations/${activeConversationId}`, + { + method: "PATCH", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + title, + visibility: conversation?.visibility, + }), + } + ); + await mutate( + `/api/w/${owner.sId}/assistant/conversations/${activeConversationId}` + ); + void mutate(`/api/w/${owner.sId}/assistant/conversations`); + if (!res.ok) { + throw new Error("Failed to update title"); } - ); - await mutate( - `/api/w/${owner.sId}/assistant/conversations/${conversationId}` - ); - void mutate(`/api/w/${owner.sId}/assistant/conversations`); - if (!res.ok) { - throw new Error("Failed to update title"); + setIsEditingTitle(false); + setEditedTitle(""); + } catch (e) { + alert("Failed to update title"); } - setIsEditingTitle(false); - setEditedTitle(""); - } catch (e) { - alert("Failed to update title"); - } - }; + }, + [activeConversationId, conversation?.visibility, mutate, owner.sId] + ); + + if (!activeConversationId) { + return null; + } return ( <> - {onDelete && ( - setShowDeleteDialog(false)} - onDelete={() => { - setShowDeleteDialog(false); - onDelete(conversationId); - }} - /> - )} + setShowDeleteDialog(false)} + onDelete={() => { + setShowDeleteDialog(false); + void onDelete(); + }} + />
{!isEditingTitle ? ( @@ -177,21 +199,19 @@ export function ConversationTitle({
- {onDelete && ( -
void; owner: WorkspaceType; user: UserType; - messageRankToScrollTo?: number | undefined; } /** @@ -63,7 +70,6 @@ const ConversationViewer = React.forwardRef< onStickyMentionsChange, isInModal = false, isFading = false, - messageRankToScrollTo, }, ref ) { @@ -93,10 +99,6 @@ const ConversationViewer = React.forwardRef< conversationId, workspaceId: owner.sId, limit: DEFAULT_PAGE_LIMIT, - // Make sure that the message rank to scroll to is in the middle of the page. - startAtRank: messageRankToScrollTo - ? Math.max(0, messageRankToScrollTo - Math.floor(DEFAULT_PAGE_LIMIT / 2)) - : undefined, }); const { mutateConversationParticipants } = useConversationParticipants({ @@ -320,57 +322,6 @@ const ConversationViewer = React.forwardRef< useLastMessageGroupObserver(typedGroupedMessages); - // Used for auto-scrolling to the message in the anchor. - const [hasScrolledToMessage, setHasScrolledToMessage] = useState(false); - const [messageShaking, setMessageShaking] = useState(false); - const scrollRef = React.useRef(null); - - // Track index of the group with message sId in anchor. - const groupIndexWithMessageIdInAnchor = useMemo(() => { - const messageToScrollTo = messages - .flatMap((messagePage) => messagePage.messages) - .find((message) => message.rank === messageRankToScrollTo); - if (!messageToScrollTo) { - return -1; - } - return typedGroupedMessages.findIndex((group) => { - return group.some((message) => message.sId === messageToScrollTo.sId); - }); - }, [typedGroupedMessages, messageRankToScrollTo, messages]); - - // Effect: scroll to the message and temporarily highlight if it is the anchor's target - useEffect(() => { - if ( - !messageRankToScrollTo || - !groupIndexWithMessageIdInAnchor || - !scrollRef.current || - hasScrolledToMessage - ) { - return; - } - setTimeout(() => { - if (scrollRef.current) { - setHasScrolledToMessage(true); - // Use ref to scroll to the message - scrollRef.current.scrollIntoView({ - behavior: "instant", - block: "center", - }); - setMessageShaking(true); - - // Have the message blink for a short time - setTimeout(() => { - setMessageShaking(false); - }, 1000); - } - }, 100); - }, [ - hasScrolledToMessage, - messageRankToScrollTo, - groupIndexWithMessageIdInAnchor, - scrollRef, - ]); - return (
)} {(isMessagesLoading || prevFirstMessageId) && ( -
+
)} {conversation && typedGroupedMessages.map((typedGroup, index) => { const isLastGroup = index === typedGroupedMessages.length - 1; - const isGroupInAnchor = index === groupIndexWithMessageIdInAnchor; return ( -
- -
+ messages={typedGroup} + isLastMessageGroup={isLastGroup} + conversationId={conversationId} + feedbacks={feedbacks} + isInModal={isInModal} + owner={owner} + prevFirstMessageId={prevFirstMessageId} + prevFirstMessageRef={prevFirstMessageRef} + user={user} + latestPage={latestPage} + /> ); })}
diff --git a/front/components/assistant/conversation/ConversationsNavigationProvider.tsx b/front/components/assistant/conversation/ConversationsNavigationProvider.tsx index 29d51ece88f5..c31bc166519f 100644 --- a/front/components/assistant/conversation/ConversationsNavigationProvider.tsx +++ b/front/components/assistant/conversation/ConversationsNavigationProvider.tsx @@ -1,22 +1,27 @@ +import { useRouter } from "next/router"; import type { RefObject } from "react"; -import { createContext, useContext, useRef } from "react"; +import { createContext, useCallback, useContext, useMemo, useRef } from "react"; interface ConversationsNavigationContextType { conversationsNavigationRef: RefObject; scrollConversationsToTop: () => void; + activeConversationId: string | null; } const ConversationsNavigationContext = createContext(null); export function ConversationsNavigationProvider({ + initialConversationId, children, }: { + initialConversationId?: string | null; children: React.ReactNode; }) { + const router = useRouter(); const conversationsNavigationRef = useRef(null); - const scrollConversationsToTop = () => { + const scrollConversationsToTop = useCallback(() => { if (conversationsNavigationRef.current) { // Find the ScrollArea viewport const viewport = conversationsNavigationRef.current.querySelector( @@ -29,13 +34,24 @@ export function ConversationsNavigationProvider({ }); } } - }; + }, []); + + const activeConversationId = useMemo(() => { + const conversationId = router.query.cId ?? ""; + + if (conversationId && typeof conversationId === "string") { + return conversationId === "new" ? null : conversationId; + } + + return initialConversationId ?? null; + }, [initialConversationId, router.query.cId]); return ( {children} diff --git a/sparkle/src/components/FeedbackSelector.tsx b/front/components/assistant/conversation/FeedbackSelector.tsx similarity index 87% rename from sparkle/src/components/FeedbackSelector.tsx rename to front/components/assistant/conversation/FeedbackSelector.tsx index fdf55b2b19fb..180bb1e137c2 100644 --- a/sparkle/src/components/FeedbackSelector.tsx +++ b/front/components/assistant/conversation/FeedbackSelector.tsx @@ -1,18 +1,13 @@ +import { Button } from "@dust-tt/sparkle"; +import { Checkbox } from "@dust-tt/sparkle"; +import { Page } from "@dust-tt/sparkle"; +import { PopoverContent, PopoverRoot, PopoverTrigger } from "@dust-tt/sparkle"; +import { Spinner } from "@dust-tt/sparkle"; +import { TextArea } from "@dust-tt/sparkle"; +import { Tooltip } from "@dust-tt/sparkle"; +import { HandThumbDownIcon, HandThumbUpIcon } from "@dust-tt/sparkle"; import React, { useCallback, useEffect, useRef } from "react"; -import { Button } from "@sparkle/components/Button"; -import { Checkbox } from "@sparkle/components/Checkbox"; -import { Page } from "@sparkle/components/Page"; -import { - PopoverContent, - PopoverRoot, - PopoverTrigger, -} from "@sparkle/components/Popover"; -import Spinner from "@sparkle/components/Spinner"; -import { TextArea } from "@sparkle/components/TextArea"; -import { Tooltip } from "@sparkle/components/Tooltip"; -import { HandThumbDownIcon, HandThumbUpIcon } from "@sparkle/icons/solid"; - export type ThumbReaction = "up" | "down"; export type FeedbackType = { @@ -134,10 +129,10 @@ export function FeedbackSelector({ ]); return ( -
+
-
+
{isSubmittingThumb ? ( -
+
) : ( -
+
{lastSelectedThumb === "up" ? "🎉 Glad you liked it! Tell us more?" @@ -196,14 +191,14 @@ export function FeedbackSelector({ ? "What did you like?" : "Tell us what went wrong so we can make this assistant better." } - className="s-mb-4 s-mt-4" + className="mb-4 mt-4" rows={3} value={localFeedbackContent ?? ""} onChange={handleTextAreaChange} /> {popOverInfo} -
+
{ @@ -214,7 +209,7 @@ export function FeedbackSelector({ By clicking, you accept to share your full conversation
-
+
) : (
+
diff --git a/front/components/assistant_builder/DataSourceSelectionSection.tsx b/front/components/assistant_builder/DataSourceSelectionSection.tsx index c72fa99659ab..c7f23f7f876c 100644 --- a/front/components/assistant_builder/DataSourceSelectionSection.tsx +++ b/front/components/assistant_builder/DataSourceSelectionSection.tsx @@ -122,7 +122,7 @@ export default function DataSourceSelectionSection({ return ( { - if (node.dustDocumentId) { + if (node.type === "file") { setDataSourceViewToDisplay( dsConfig.dataSourceView ); - setDocumentToDisplay(node.dustDocumentId); + setDocumentToDisplay(node.internalId); } }} className={classNames( - node.dustDocumentId + node.type === "file" ? "" : "pointer-events-none opacity-0" )} - disabled={!node.dustDocumentId} - variant="ghost" + disabled={node.type !== "file"} + variant="outline" />
} diff --git a/front/components/assistant_builder/FeedbacksSection.tsx b/front/components/assistant_builder/FeedbacksSection.tsx index 1d6bfccc01ab..f6e684463822 100644 --- a/front/components/assistant_builder/FeedbacksSection.tsx +++ b/front/components/assistant_builder/FeedbacksSection.tsx @@ -82,7 +82,9 @@ export const FeedbacksSection = ({ !isAgentConfigurationFeedbacksLoading && (!agentConfigurationFeedbacks || agentConfigurationFeedbacks.length === 0) ) { - return
No feedbacks.
; + return ( +
No feedback yet.
+ ); } if (!agentConfigurationHistory) { @@ -164,7 +166,7 @@ function AgentConfigurationVersionHeader({ ); return ( -
+
{agentConfiguration ? getAgentConfigurationVersionString(agentConfiguration) : `v${agentConfigurationVersion}`} @@ -216,9 +218,9 @@ function FeedbackCard({ owner, feedback }: FeedbackCardProps) { {timeSinceFeedback} ago
{feedback.thumbDirection === "up" ? ( - + ) : ( - + )}
@@ -229,12 +231,13 @@ function FeedbackCard({ owner, feedback }: FeedbackCardProps) { href={conversationUrl ?? ""} icon={ExternalLinkIcon} disabled={!conversationUrl} + tooltip="View conversation" target="_blank" />
{feedback.content && ( -
-
+
+
{feedback.content}
diff --git a/front/components/assistant_builder/SlackIntegration.tsx b/front/components/assistant_builder/SlackIntegration.tsx index 9bcfde333339..638809edbc7d 100644 --- a/front/components/assistant_builder/SlackIntegration.tsx +++ b/front/components/assistant_builder/SlackIntegration.tsx @@ -6,7 +6,7 @@ import { SlackLogo, } from "@dust-tt/sparkle"; import type { - BaseContentNode, + ContentNode, DataSourceType, WorkspaceType, } from "@dust-tt/types"; @@ -52,7 +52,7 @@ export function SlackIntegration({ }, [existingSelection, newSelection]); const customIsNodeChecked = useCallback( - (node: BaseContentNode) => { + (node: ContentNode) => { return ( newSelection?.some((c) => c.slackChannelId === node.internalId) || false ); diff --git a/front/components/assistant_builder/spaces/SpaceSelector.tsx b/front/components/assistant_builder/spaces/SpaceSelector.tsx index 51735902e4fe..631cbaae1b8b 100644 --- a/front/components/assistant_builder/spaces/SpaceSelector.tsx +++ b/front/components/assistant_builder/spaces/SpaceSelector.tsx @@ -1,6 +1,11 @@ import { - Dialog, Icon, + NewDialog, + NewDialogContainer, + NewDialogContent, + NewDialogFooter, + NewDialogHeader, + NewDialogTitle, RadioGroup, RadioGroupChoice, Separator, @@ -118,15 +123,31 @@ export function SpaceSelector({ })} - setAlertIsDialogOpen(false)} - title="Changing source selection" + { + if (!open) { + setAlertIsDialogOpen(false); + } + }} > - An assistant can access one source of data only. The other tools are - using a different source. - + + + Changing source selection + + + An assistant can access one source of data only. The other tools are + using a different source. + + setAlertIsDialogOpen(false), + }} + /> + + ); } diff --git a/front/components/data_source/DocumentUploadOrEditModal.tsx b/front/components/data_source/DocumentUploadOrEditModal.tsx index e665a712aa52..d8aa7157386e 100644 --- a/front/components/data_source/DocumentUploadOrEditModal.tsx +++ b/front/components/data_source/DocumentUploadOrEditModal.tsx @@ -147,7 +147,7 @@ export const DocumentUploadOrEditModal = ({ const body = { name: initialId ?? document.name, title: initialId ?? document.name, - mime_type: document.mimeType ?? "application/octet-stream", + mime_type: document.mimeType ?? "text/plain", timestamp: null, parent_id: null, parents: [initialId ?? document.name], diff --git a/front/components/data_source/MultipleDocumentsUpload.tsx b/front/components/data_source/MultipleDocumentsUpload.tsx index b384934df851..f71760fb5109 100644 --- a/front/components/data_source/MultipleDocumentsUpload.tsx +++ b/front/components/data_source/MultipleDocumentsUpload.tsx @@ -1,14 +1,25 @@ -import { Dialog } from "@dust-tt/sparkle"; +import { + NewDialog, + NewDialogContainer, + NewDialogContent, + NewDialogDescription, + NewDialogHeader, + NewDialogTitle, + Spinner, +} from "@dust-tt/sparkle"; import type { DataSourceViewType, LightWorkspaceType, PlanType, } from "@dust-tt/types"; -import { concurrentExecutor } from "@dust-tt/types"; -import { getSupportedNonImageFileExtensions } from "@dust-tt/types"; +import { + concurrentExecutor, + getSupportedNonImageFileExtensions, +} from "@dust-tt/types"; import type { ChangeEvent } from "react"; -import { useCallback, useEffect, useRef, useState } from "react"; +import React, { useCallback, useEffect, useRef, useState } from "react"; +import { useFileDrop } from "@app/components/assistant/conversation/FileUploaderContext"; import { DocumentLimitPopup } from "@app/components/data_source/DocumentLimitPopup"; import type { FileBlob, @@ -64,12 +75,10 @@ export const MultipleDocumentsUpload = ({ completed: number; }>(null); - const handleFileChange = useCallback( - async ( - e: ChangeEvent & { target: { files: File[] } } - ) => { + const uploadFiles = useCallback( + async (files: File[]) => { // Empty file input - if (!e.target.files || e.target.files.length === 0) { + if (files.length === 0) { close(false); return; } @@ -77,21 +86,22 @@ export const MultipleDocumentsUpload = ({ // Open plan popup if limit is reached if ( plan.limits.dataSources.documents.count != -1 && - e.target.files.length + totalNodesCount > - plan.limits.dataSources.documents.count + files.length + totalNodesCount > plan.limits.dataSources.documents.count ) { setIsLimitPopupOpen(true); return; } setIsBulkFilesUploading({ - total: e.target.files.length, + total: files.length, completed: 0, }); // upload Files and get FileBlobs (only keep successful uploads) // Each individual error triggers a notification - const fileBlobs = (await fileUploaderService.handleFileChange(e))?.filter( + const fileBlobs = ( + await fileUploaderService.handleFilesUpload(files) + )?.filter( (fileBlob: FileBlob): fileBlob is FileBlobWithFileId => !!fileBlob.fileId ); @@ -138,6 +148,33 @@ export const MultipleDocumentsUpload = ({ ] ); + // Process dropped files if any. + const { droppedFiles, setDroppedFiles } = useFileDrop(); + useEffect(() => { + const handleDroppedFiles = async () => { + const droppedFilesCopy = [...droppedFiles]; + if (droppedFilesCopy.length > 0) { + // Make sure the files are cleared after processing + setDroppedFiles([]); + await uploadFiles(droppedFilesCopy); + } + }; + void handleDroppedFiles(); + }, [droppedFiles, setDroppedFiles, uploadFiles]); + + // Handle file change from file input. + const handleFileChange = useCallback( + async ( + e: ChangeEvent & { target: { files: File[] } } + ) => { + const selectedFiles = Array.from( + (e?.target as HTMLInputElement).files ?? [] + ); + await uploadFiles(selectedFiles); + }, + [uploadFiles] + ); + const handleFileInputBlur = useCallback(() => { close(false); }, [close]); @@ -168,26 +205,32 @@ export const MultipleDocumentsUpload = ({ onClose={() => setIsLimitPopupOpen(false)} owner={owner} /> - { - //no-op as we can't cancel file upload - }} - onValidate={() => { - //no-op as we can't cancel file upload - }} - // isSaving is always true since we are showing this Dialog while - // uploading files only - isSaving={true} - isOpen={isBulkFilesUploading !== null} - title="Uploading files" - > - {isBulkFilesUploading && ( - <> - Processing files {isBulkFilesUploading.completed} /{" "} - {isBulkFilesUploading.total} - - )} - + + e.preventDefault()} + > + + Uploading files + + {isBulkFilesUploading && ( + <> + Processing files {isBulkFilesUploading.completed} /{" "} + {isBulkFilesUploading.total} + + )} + + + + {isBulkFilesUploading && ( +
+ +
+ )} + + +
-
)}
diff --git a/front/components/home/HubSpotForm.tsx b/front/components/home/HubSpotForm.tsx new file mode 100644 index 000000000000..8c38913ea4c7 --- /dev/null +++ b/front/components/home/HubSpotForm.tsx @@ -0,0 +1,45 @@ +import React, { useEffect } from "react"; + +declare global { + interface Window { + hbspt?: { + forms: { + create: (config: { + region: string; + portalId: string; + formId: string; + target: string; + }) => void; + }; + }; + } +} + +export default function HubSpotForm() { + useEffect(() => { + const existingScript = document.getElementById("hubspot-script"); + const createForm = () => { + if (window.hbspt && window.hbspt.forms && window.hbspt.forms.create) { + window.hbspt.forms.create({ + region: "eu1", + portalId: "144442587", + formId: "31e790e5-f4d5-4c79-acc5-acd770fe8f84", + target: "#hubspotForm", + }); + } + }; + + if (!existingScript) { + const script = document.createElement("script"); + script.id = "hubspot-script"; + script.src = "https://js-eu1.hsforms.net/forms/v2.js"; + script.defer = true; + script.onload = createForm; + document.body.appendChild(script); + } else { + // If the script is already present, just recreate the form + createForm(); + } + }, []); + return
; +} diff --git a/front/components/home/LandingLayout.tsx b/front/components/home/LandingLayout.tsx index 5f0de2fb30eb..62960d38f12f 100644 --- a/front/components/home/LandingLayout.tsx +++ b/front/components/home/LandingLayout.tsx @@ -99,14 +99,15 @@ export default function LandingLayout({
-
-
- {testSuccessful ? ( -
-
- - -
-
- - - ); -} diff --git a/front/components/providers/AzureOpenAISetup.tsx b/front/components/providers/AzureOpenAISetup.tsx deleted file mode 100644 index 34fb9f0a1466..000000000000 --- a/front/components/providers/AzureOpenAISetup.tsx +++ /dev/null @@ -1,220 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function AzureOpenAISetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [endpoint, setEndpoint] = useState(config ? config.endpoint : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - if (config && config.endpoint.length > 0 && endpoint.length == 0) { - setEndpoint(config.endpoint); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "azure_openai", { - api_key: apiKey, - endpoint, - }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/azure_openai`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - endpoint, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/azure_openai`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup Azure OpenAI - -
-

- To use Azure OpenAI models you must provide your API key - and Endpoint. They can be found in the left menu of your - OpenAI Azure Resource portal (menu item `Keys and - Endpoint`). -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setEndpoint(e.target.value); - setTestSuccessful(false); - }} - /> -
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable Azure OpenAI. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/BrowserlessAPISetup.tsx b/front/components/providers/BrowserlessAPISetup.tsx deleted file mode 100644 index ce93839e330e..000000000000 --- a/front/components/providers/BrowserlessAPISetup.tsx +++ /dev/null @@ -1,214 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function BrowserlessAPISetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "browserlessapi", { - api_key: apiKey, - }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/browserlessapi`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/browserlessapi`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup Browserless API - -
-

- Browserless lets you use headless browsers to scrape web - content. To use Browserless, you must provide your API - key. It can be found{" "} - - here - - . -

-

- Note that it generally takes{" "} - 5 mins for the API - key to become active (an email is sent when it's ready). -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable the Browserless API. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/DeepseekSetup.tsx b/front/components/providers/DeepseekSetup.tsx deleted file mode 100644 index aa9098d5bf04..000000000000 --- a/front/components/providers/DeepseekSetup.tsx +++ /dev/null @@ -1,199 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function DeepseekSetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "deepseek", { - api_key: apiKey, - }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/deepseek`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/deepseek`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup Deepseek - -
-

- To use Deepseek models you must provide your API key. -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError?.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable Deepseek. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/GoogleAiStudioSetup.tsx b/front/components/providers/GoogleAiStudioSetup.tsx deleted file mode 100644 index fb4ec115aff5..000000000000 --- a/front/components/providers/GoogleAiStudioSetup.tsx +++ /dev/null @@ -1,208 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function GoogleAiStudioSetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "google_ai_studio", { - api_key: apiKey, - }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/google_ai_studio`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/google_ai_studio`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup Google AI Studio - -
-

- To use Google AI Studio models you must provide your API - key. It can be found{" "} - - here - -  (you can create a new key specifically for Dust). -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable Google AI Studio. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/MistralAISetup.tsx b/front/components/providers/MistralAISetup.tsx deleted file mode 100644 index 8477a384f179..000000000000 --- a/front/components/providers/MistralAISetup.tsx +++ /dev/null @@ -1,206 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function MistralAISetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "mistral", { api_key: apiKey }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/mistral`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/mistral`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup Mistral AI - -
-

- To use Mistral AI models you must provide your API key. - It can be found{" "} - - here - -  (you can create a new key specifically for Dust). -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable Mistral AI. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/OpenAISetup.tsx b/front/components/providers/OpenAISetup.tsx deleted file mode 100644 index 6bc275eb0ada..000000000000 --- a/front/components/providers/OpenAISetup.tsx +++ /dev/null @@ -1,206 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function OpenAISetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "openai", { api_key: apiKey }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/openai`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/openai`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup OpenAI - -
-

- To use OpenAI models you must provide your API key. It - can be found{" "} - - here - -  (you can create a new key specifically for Dust). -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable OpenAI. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/ProviderSetup.tsx b/front/components/providers/ProviderSetup.tsx new file mode 100644 index 000000000000..2b2b6dfdee84 --- /dev/null +++ b/front/components/providers/ProviderSetup.tsx @@ -0,0 +1,428 @@ +import { + NewDialog, + NewDialogContainer, + NewDialogContent, + NewDialogDescription, + NewDialogFooter, + NewDialogHeader, + NewDialogTitle, +} from "@dust-tt/sparkle"; +import type { WorkspaceType } from "@dust-tt/types"; +import type { MouseEvent } from "react"; +import React, { useEffect, useState } from "react"; +import { useSWRConfig } from "swr"; + +import { checkProvider } from "@app/lib/providers"; + +export type ProviderField = { + name: string; + label?: string; + placeholder: string; + type?: string; +}; + +type ProviderConfig = { + title: string; + fields: { + name: string; + placeholder: string; + type?: string; + }[]; + instructions: React.ReactNode; +}; + +export const MODEL_PROVIDER_CONFIGS: Record = { + openai: { + title: "OpenAI", + fields: [{ name: "api_key", placeholder: "OpenAI API Key" }], + instructions: ( + <> +

+ To use OpenAI models you must provide your API key. It can be found{" "} + + here + + . +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + azure_openai: { + title: "Azure OpenAI", + fields: [ + { name: "endpoint", placeholder: "Azure OpenAI Endpoint" }, + { name: "api_key", placeholder: "Azure OpenAI API Key" }, + ], + instructions: ( + <> +

+ To use Azure OpenAI models you must provide your API key and Endpoint. + They can be found in the left menu of your OpenAI Azure Resource + portal (menu item `Keys and Endpoint`). +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + anthropic: { + title: "Anthropic", + fields: [{ name: "api_key", placeholder: "Anthropic API Key" }], + instructions: ( + <> +

+ To use Anthropic models you must provide your API key. It can be found{" "} + + here + +  (you can create a new key specifically for Dust). +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + mistral: { + title: "Mistral AI", + fields: [{ name: "api_key", placeholder: "Mistral AI API Key" }], + instructions: ( + <> +

+ To use Mistral AI models you must provide your API key. It can be + found{" "} + + here + +  (you can create a new key specifically for Dust). +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + google_ai_studio: { + title: "Google AI Studio", + fields: [{ name: "api_key", placeholder: "Google AI Studio API Key" }], + instructions: ( + <> +

+ To use Google AI Studio models you must provide your API key. It can + be found{" "} + + here + +  (you can create a new key specifically for Dust). +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + togetherai: { + title: "TogetherAI", + fields: [{ name: "api_key", placeholder: "TogetherAI API Key" }], + instructions: ( + <> +

To use TogetherAI models you must provide your API key.

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + deepseek: { + title: "Deepseek", + fields: [{ name: "api_key", placeholder: "Deepseek API Key" }], + instructions: ( + <> +

To use Deepseek models you must provide your API key.

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, +}; + +export const SERVICE_PROVIDER_CONFIGS: Record = { + serpapi: { + title: "SerpAPI Search", + fields: [{ name: "api_key", placeholder: "SerpAPI API Key" }], + instructions: ( + <> +

+ SerpAPI lets you search Google (and other search engines). To use + SerpAPI you must provide your API key. It can be found{" "} + + here + +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + serper: { + title: "Serper Search", + fields: [{ name: "api_key", placeholder: "Serper API Key" }], + instructions: ( + <> +

+ Serper lets you search Google (and other search engines). To use + Serper you must provide your API key. It can be found{" "} + + here + +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + browserlessapi: { + title: "Browserless API", + fields: [{ name: "api_key", placeholder: "Browserless API Key" }], + instructions: ( + <> +

+ Browserless lets you use headless browsers to scrape web content. To + use Browserless, you must provide your API key. It can be found{" "} + + here + + . +

+

+ Note that it generally takes 5 mins{" "} + for the API key to become active (an email is sent when it's ready). +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, +}; + +export interface ProviderSetupProps { + owner: WorkspaceType; + providerId: string; + title: string; + instructions?: React.ReactNode; + fields: ProviderField[]; + config: { [key: string]: string }; + enabled: boolean; + testSuccessMessage?: string; + isOpen: boolean; + onClose: () => void; +} + +export function ProviderSetup({ + owner, + providerId, + title, + instructions, + fields, + config, + enabled, + testSuccessMessage, + isOpen, + onClose, +}: ProviderSetupProps) { + const { mutate } = useSWRConfig(); + const [values, setValues] = useState>({}); + const [testError, setTestError] = useState(""); + const [testSuccessful, setTestSuccessful] = useState(false); + const [testRunning, setTestRunning] = useState(false); + const [enableRunning, setEnableRunning] = useState(false); + + useEffect(() => { + const newValues: Record = {}; + for (const field of fields) { + newValues[field.name] = config[field.name] || ""; + } + setValues(newValues); + setTestSuccessful(false); + setTestError(""); + }, [config, fields]); + + const runTest = async () => { + setTestRunning(true); + setTestError(""); + setTestSuccessful(false); + + const partialConfig: Record = {}; + for (const field of fields) { + partialConfig[field.name] = values[field.name]; + } + + const check = await checkProvider(owner, providerId, partialConfig); + if (!check.ok) { + setTestError(check.error || "Unknown error"); + setTestSuccessful(false); + } else { + setTestError(""); + setTestSuccessful(true); + } + setTestRunning(false); + }; + + const handleEnable = async () => { + setEnableRunning(true); + const payload: Record = {}; + for (const field of fields) { + payload[field.name] = values[field.name]; + } + + await fetch(`/api/w/${owner.sId}/providers/${providerId}`, { + headers: { "Content-Type": "application/json" }, + method: "POST", + body: JSON.stringify({ config: JSON.stringify(payload) }), + }); + setEnableRunning(false); + await mutate(`/api/w/${owner.sId}/providers`); + onClose(); + }; + + const handleDisable = async () => { + await fetch(`/api/w/${owner.sId}/providers/${providerId}`, { + method: "DELETE", + }); + await mutate(`/api/w/${owner.sId}/providers`); + onClose(); + }; + + const renderFields = () => + fields.map((field) => ( +
+ {field.label && ( + + )} + { + setTestSuccessful(false); + const val = e.target.value; + setValues((prev) => ({ ...prev, [field.name]: val })); + }} + /> +
+ )); + + const testDisabled = + fields.some((field) => !values[field.name]) || testRunning; + + const rightButtonProps = testSuccessful + ? { + label: enabled + ? enableRunning + ? "Updating..." + : "Update" + : enableRunning + ? "Enabling..." + : "Enable", + variant: "primary" as const, + disabled: enableRunning, + onClick: handleEnable, + } + : { + label: testRunning ? "Testing..." : "Test", + variant: "primary" as const, + disabled: testDisabled, + onClick: async (event: MouseEvent) => { + event.preventDefault(); + await runTest(); + }, + }; + + return ( + !open && onClose()}> + + + {title} + + {instructions || ( +

Provide the necessary configuration for {title}.

+ )} + + + + +
+ {renderFields()} +
+ {testError ? ( + Error: {testError} + ) : testSuccessful ? ( + + {testSuccessMessage || + `Test succeeded! You can now enable ${title}.`} + + ) : ( +   + )} +
+
+ + + + + + ); +} diff --git a/front/components/providers/SerpAPISetup.tsx b/front/components/providers/SerpAPISetup.tsx deleted file mode 100644 index 21b88fb368b3..000000000000 --- a/front/components/providers/SerpAPISetup.tsx +++ /dev/null @@ -1,206 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function SerpAPISetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "serpapi", { api_key: apiKey }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/serpapi`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/serpapi`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup SerpAPI Search - -
-

- SerpAPI lets you search Google (and other search - engines). To use SerpAPI you must provide your API key. - It can be found{" "} - - here - -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable SerpAPI Search. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/SerperSetup.tsx b/front/components/providers/SerperSetup.tsx deleted file mode 100644 index 27ea09b16ab2..000000000000 --- a/front/components/providers/SerperSetup.tsx +++ /dev/null @@ -1,206 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function SerperSetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "serper", { api_key: apiKey }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/serper`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/serper`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup Serper Search - -
-

- Serper lets you search Google (and other search - engines). To use Serper you must provide your API key. - It can be found{" "} - - here - -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable Serper Search. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/TogetherAISetup.tsx b/front/components/providers/TogetherAISetup.tsx deleted file mode 100644 index d25785738c08..000000000000 --- a/front/components/providers/TogetherAISetup.tsx +++ /dev/null @@ -1,199 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function TogetherAISetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "togetherai", { - api_key: apiKey, - }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/togetherai`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/togetherai`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup TogetherAI - -
-

- To use TogetherAI models you must provide your API key. -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError?.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable TogetherAI. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/types.ts b/front/components/providers/types.ts index 7fb5e4bebe57..72611bbfc253 100644 --- a/front/components/providers/types.ts +++ b/front/components/providers/types.ts @@ -10,6 +10,8 @@ import { CLAUDE_3_5_HAIKU_DEFAULT_MODEL_CONFIG, CLAUDE_3_5_SONNET_DEFAULT_MODEL_CONFIG, DEEPSEEK_CHAT_MODEL_CONFIG, + GEMINI_2_FLASH_PREVIEW_MODEL_CONFIG, + GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_CONFIG, GEMINI_FLASH_DEFAULT_MODEL_CONFIG, GEMINI_PRO_DEFAULT_MODEL_CONFIG, GPT_4_TURBO_MODEL_CONFIG, @@ -53,6 +55,8 @@ export const USED_MODEL_CONFIGS: readonly ModelConfig[] = [ MISTRAL_CODESTRAL_MODEL_CONFIG, GEMINI_PRO_DEFAULT_MODEL_CONFIG, GEMINI_FLASH_DEFAULT_MODEL_CONFIG, + GEMINI_2_FLASH_PREVIEW_MODEL_CONFIG, + GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_CONFIG, TOGETHERAI_LLAMA_3_3_70B_INSTRUCT_TURBO_MODEL_CONFIG, TOGETHERAI_QWEN_2_5_CODER_32B_INSTRUCT_MODEL_CONFIG, TOGETHERAI_QWEN_32B_PREVIEW_MODEL_CONFIG, diff --git a/front/components/spaces/AddConnectionMenu.tsx b/front/components/spaces/AddConnectionMenu.tsx index 1e2657f07269..060410daf1d5 100644 --- a/front/components/spaces/AddConnectionMenu.tsx +++ b/front/components/spaces/AddConnectionMenu.tsx @@ -1,12 +1,12 @@ import { Button, CloudArrowLeftRightIcon, - Dialog, DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger, NewDialog, + NewDialogContainer, NewDialogContent, NewDialogDescription, NewDialogFooter, @@ -282,16 +282,36 @@ export const AddConnectionMenu = ({ return ( availableIntegrations.length > 0 && ( <> - setShowUpgradePopup(false)} - title={`${plan.name} plan`} - onValidate={() => { - void router.push(`/w/${owner.sId}/subscription`); + { + if (!open) { + setShowUpgradePopup(false); + } }} > -

Unlock this managed data source by upgrading your plan.

- + + + ${plan.name} plan + + + Unlock this managed data source by upgrading your plan. + + { + void router.push(`/w/${owner.sId}/subscription`); + }, + }} + /> + + {connectorProvider === "snowflake" ? ( { if (currentAction.action === "DocumentViewRawContent") { - setCurrentDocumentId(currentAction.contentNode?.dustDocumentId ?? ""); + setCurrentDocumentId(currentAction.contentNode?.internalId ?? ""); } }, [currentAction, setCurrentDocumentId]); diff --git a/front/components/spaces/SpaceDataSourceViewContentList.tsx b/front/components/spaces/SpaceDataSourceViewContentList.tsx index 11f1a7704aab..1ecd6a758b73 100644 --- a/front/components/spaces/SpaceDataSourceViewContentList.tsx +++ b/front/components/spaces/SpaceDataSourceViewContentList.tsx @@ -31,8 +31,10 @@ import { useRouter } from "next/router"; import * as React from "react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { FileDropProvider } from "@app/components/assistant/conversation/FileUploaderContext"; import { ConnectorPermissionsModal } from "@app/components/ConnectorPermissionsModal"; import { RequestDataSourceModal } from "@app/components/data_source/RequestDataSourceModal"; +import { DropzoneContainer } from "@app/components/misc/DropzoneContainer"; import type { ContentActionKey, ContentActionsRef, @@ -371,143 +373,150 @@ export const SpaceDataSourceViewContentList = ({ const isEmpty = rows.length === 0 && !isNodesLoading; return ( - <> -
+ - {!isEmpty && ( - <> - { - setPagination( - { pageIndex: 0, pageSize: pagination.pageSize }, - "replace" - ); - setDataSourceSearch(s); - }} - /> - - )} - {isEmpty && emptyContent} - {isFolder(dataSourceView.dataSource) && ( - <> - {((viewType === "tables" && hasDocuments) || - (viewType === "documents" && hasTables)) && ( - - -
- {isNodesLoading && ( -
- + )} + {isManaged(dataSourceView.dataSource) && + connector && + !parentId && + space.kind === "system" && ( +
+ {!isNodesLoading && rows.length === 0 && ( +
Connection ready. Select the data to sync.
+ )} + + { + setShowConnectorPermissionsModal(false); + if (save) { + void mutateContentNodes(); + } + }} + readOnly={false} + isAdmin={isAdmin} + onManageButtonClick={() => { + setShowConnectorPermissionsModal(true); + }} + /> +
+ )}
- )} - {rows.length > 0 && ( - + +
+ )} + {rows.length > 0 && ( + + )} + - )} - - + + ); }; diff --git a/front/components/tables/TablePicker.tsx b/front/components/tables/TablePicker.tsx index 261769f6f857..593d335a6866 100644 --- a/front/components/tables/TablePicker.tsx +++ b/front/components/tables/TablePicker.tsx @@ -62,7 +62,7 @@ export default function TablePicker({ }); const currentTable = currentTableId - ? tables.find((t) => t.dustDocumentId === currentTableId) + ? tables.find((t) => t.internalId === currentTableId) : null; const [searchFilter, setSearchFilter] = useState(""); @@ -139,12 +139,12 @@ export default function TablePicker({ !excludeTables?.some( (et) => et.dataSourceId === dataSource.data_source_id && - et.tableId === t.dustDocumentId + et.tableId === t.internalId ) ) .map((t) => (
{ onTableUpdate(t); diff --git a/front/components/trackers/TrackerDataSourceSelectedTree.tsx b/front/components/trackers/TrackerDataSourceSelectedTree.tsx index f4ebb9d3829a..e2f53af18b2e 100644 --- a/front/components/trackers/TrackerDataSourceSelectedTree.tsx +++ b/front/components/trackers/TrackerDataSourceSelectedTree.tsx @@ -83,7 +83,7 @@ export const TrackerDataSourceSelectedTree = ({ return ( { - if (node.dustDocumentId) { + if (node.type === "file") { setDataSourceViewToDisplay( dsConfig.dataSourceView ); - setDocumentToDisplay(node.dustDocumentId); + setDocumentToDisplay(node.internalId); } }} className={classNames( - node.dustDocumentId + node.type === "file" ? "" : "pointer-events-none opacity-0" )} - disabled={!node.dustDocumentId} - variant="ghost" + disabled={node.type !== "file"} + variant="outline" />
} diff --git a/front/components/workspace/ChangeMemberModal.tsx b/front/components/workspace/ChangeMemberModal.tsx index 1db3ac79e2d5..86964c6a61d6 100644 --- a/front/components/workspace/ChangeMemberModal.tsx +++ b/front/components/workspace/ChangeMemberModal.tsx @@ -129,7 +129,7 @@ export function ChangeMemberModal({ rightButtonProps={{ label: "Yes, revoke", variant: "warning", - onClick: () => async () => { + onClick: async () => { await handleMembersRoleChange({ members: [member], role: "none", diff --git a/front/components/workspace/connection.tsx b/front/components/workspace/connection.tsx index f2b4f3190142..f83858cae755 100644 --- a/front/components/workspace/connection.tsx +++ b/front/components/workspace/connection.tsx @@ -8,6 +8,12 @@ import { Label, LockIcon, Modal, + NewDialog, + NewDialogContainer, + NewDialogContent, + NewDialogFooter, + NewDialogHeader, + NewDialogTitle, Page, Popup, RadioGroup, @@ -889,20 +895,38 @@ function DisableEnterpriseConnectionModal({ } return ( - { - await handleUpdateWorkspace(); + { + if (!open) { + onClose(false); + } }} - onCancel={() => onClose(false)} - validateLabel={`Disable ${strategyHumanReadable} Single Sign On`} - validateVariant="warning" > -
- Anyone with an {strategyHumanReadable} account won't be able to access - your Dust workspace anymore. -
- + + + + Disable ${strategyHumanReadable} Single Sign On + + + + Anyone with an {strategyHumanReadable} account won't be able to access + your Dust workspace anymore. + + { + await handleUpdateWorkspace(); + }, + }} + /> + + ); } diff --git a/front/hooks/useAppKeyboardShortcuts.ts b/front/hooks/useAppKeyboardShortcuts.ts index 82dbe737f202..23f65812ada3 100644 --- a/front/hooks/useAppKeyboardShortcuts.ts +++ b/front/hooks/useAppKeyboardShortcuts.ts @@ -20,7 +20,9 @@ export function useAppKeyboardShortcuts(owner: LightWorkspaceType) { case "/": event.preventDefault(); - void router.push(`/w/${owner.sId}/assistant/new`); + void router.push(`/w/${owner.sId}/assistant/new`, undefined, { + shallow: true, + }); break; } } diff --git a/front/lib/api/assistant/actions/constants.ts b/front/lib/api/assistant/actions/constants.ts index f038fb8f2dd9..d74101224942 100644 --- a/front/lib/api/assistant/actions/constants.ts +++ b/front/lib/api/assistant/actions/constants.ts @@ -29,3 +29,6 @@ export const DEFAULT_CONVERSATION_QUERY_TABLES_ACTION_DATA_DESCRIPTION = `The ta export const DEFAULT_CONVERSATION_SEARCH_ACTION_NAME = "search_conversation_files"; export const DEFAULT_CONVERSATION_SEARCH_ACTION_DATA_DESCRIPTION = `Search within the 'searchable' conversation files as returned by \`${DEFAULT_CONVERSATION_LIST_FILES_ACTION_NAME}\``; + +export const DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY = + "__dust_conversation_history"; diff --git a/front/lib/api/assistant/actions/dust_app_run.ts b/front/lib/api/assistant/actions/dust_app_run.ts index b0fbb273fbd5..5b77b0971d16 100644 --- a/front/lib/api/assistant/actions/dust_app_run.ts +++ b/front/lib/api/assistant/actions/dust_app_run.ts @@ -14,11 +14,17 @@ import type { AgentActionSpecification } from "@dust-tt/types"; import type { SpecificationType } from "@dust-tt/types"; import type { DatasetSchema } from "@dust-tt/types"; import type { Result } from "@dust-tt/types"; -import { BaseAction, getHeaderFromGroupIds } from "@dust-tt/types"; +import { + BaseAction, + getHeaderFromGroupIds, + SUPPORTED_MODEL_CONFIGS, +} from "@dust-tt/types"; import { Err, Ok } from "@dust-tt/types"; +import { DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY } from "@app/lib/api/assistant/actions/constants"; import type { BaseActionRunParams } from "@app/lib/api/assistant/actions/types"; import { BaseActionConfigurationServerRunner } from "@app/lib/api/assistant/actions/types"; +import { renderConversationForModel } from "@app/lib/api/assistant/generation"; import config from "@app/lib/api/config"; import { getDatasetSchema } from "@app/lib/api/datasets"; import type { Authenticator } from "@app/lib/auth"; @@ -154,6 +160,13 @@ export class DustAppRunConfigurationServerRunner extends BaseActionConfiguration if (datasetName) { // We have a dataset name we need to find associated schema. schema = await getDatasetSchema(auth, app, datasetName); + // remove from the schema the magic input key + if (schema) { + schema = schema.filter( + (s) => s.key !== DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY + ); + } + if (!schema) { return new Err( new Error( @@ -257,6 +270,25 @@ export class DustAppRunConfigurationServerRunner extends BaseActionConfiguration } } + // Fetch the dataset schema again to check whether the magic input key is present. + const appSpec = JSON.parse( + app.savedSpecification || "[]" + ) as SpecificationType; + const inputSpec = appSpec.find((b) => b.type === "input"); + const inputConfig = inputSpec ? appConfig[inputSpec.name] : null; + const datasetName: string | null = inputConfig ? inputConfig.dataset : null; + + let schema: DatasetSchema | null = null; + if (datasetName) { + schema = await getDatasetSchema(auth, app, datasetName); + } + let shouldIncludeConversationHistory = false; + if ( + schema?.find((s) => s.key === DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY) + ) { + shouldIncludeConversationHistory = true; + } + // Create the AgentDustAppRunAction object in the database and yield an event for the generation // of the params. We store the action here as the params have been generated, if an error occurs // later on, the action won't have an output but the error will be stored on the parent agent @@ -313,6 +345,57 @@ export class DustAppRunConfigurationServerRunner extends BaseActionConfiguration // As we run the app (using a system API key here), we do force using the workspace credentials so // that the app executes in the exact same conditions in which they were developed. + if (shouldIncludeConversationHistory) { + const model = SUPPORTED_MODEL_CONFIGS.find( + (m) => + m.modelId === agentConfiguration.model.modelId && + m.providerId === agentConfiguration.model.providerId + ); + if (!model) { + yield { + type: "dust_app_run_error", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + error: { + code: "dust_app_run_error", + message: `Model not found: ${agentConfiguration.model.modelId}`, + }, + }; + return; + } + const MIN_GENERATION_TOKENS = 2048; + const allowedTokenCount = model.contextSize - MIN_GENERATION_TOKENS; + const prompt = ""; + + const convoRes = await renderConversationForModel(auth, { + conversation, + model, + prompt, + allowedTokenCount, + excludeImages: true, + }); + if (convoRes.isErr()) { + yield { + type: "dust_app_run_error", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + error: { + code: "dust_app_run_error", + message: `Error rendering conversation for model: ${convoRes.error.message}`, + }, + }; + return; + } + + const renderedConvo = convoRes.value; + const messages = renderedConvo.modelConversation.messages; + + params[DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY] = + JSON.stringify(messages); + } + const runRes = await api.runAppStreamed( { workspaceId: actionConfiguration.appWorkspaceId, diff --git a/front/lib/api/assistant/agent_usage.ts b/front/lib/api/assistant/agent_usage.ts index 3a2504b71ec6..b15903801859 100644 --- a/front/lib/api/assistant/agent_usage.ts +++ b/front/lib/api/assistant/agent_usage.ts @@ -34,13 +34,8 @@ const TTL_KEY_NOT_SET = -1; type AgentUsageCount = { agentId: string; messageCount: number; - timePeriodSec: number; -}; - -type MentionCount = { - agentId: string; - count: number; conversationCount: number; + userCount: number; timePeriodSec: number; }; @@ -89,11 +84,16 @@ export async function getAgentsUsage({ // Retrieve and parse agents usage const agentsUsage = await redis.hGetAll(agentMessageCountKey); return Object.entries(agentsUsage) - .map(([agentId, count]) => ({ - agentId, - messageCount: parseInt(count), - timePeriodSec: RANKING_TIMEFRAME_SEC, - })) + .map(([agentId, value]) => { + const parsed = JSON.parse(value); + return { + agentId, + conversationCount: 0, + userCount: 0, + ...(typeof parsed === "object" ? parsed : { messageCount: parsed }), + timePeriodSec: RANKING_TIMEFRAME_SEC, + }; + }) .sort((a, b) => b.messageCount - a.messageCount) .slice(0, limit); } @@ -134,6 +134,8 @@ export async function getAgentUsage( ? { agentId: agentConfiguration.sId, messageCount: agentUsage, + conversationCount: 0, + userCount: 0, timePeriodSec: RANKING_TIMEFRAME_SEC, } : null; @@ -143,7 +145,7 @@ export async function agentMentionsCount( workspaceId: number, agentConfiguration?: LightAgentConfigurationType, rankingUsageDays: number = RANKING_USAGE_DAYS -): Promise { +): Promise { // We retrieve mentions from conversations in order to optimize the query // Since we need to filter out by workspace id, retrieving mentions first // would lead to retrieve every single messages @@ -155,12 +157,19 @@ export async function agentMentionsCount( ], [ Sequelize.fn("COUNT", Sequelize.literal('"messages->mentions"."id"')), - "count", + "messageCount", ], [ Sequelize.fn("COUNT", Sequelize.literal("DISTINCT conversation.id")), "conversationCount", ], + [ + Sequelize.fn( + "COUNT", + Sequelize.literal('DISTINCT "messages->userMessage"."userId"') + ), + "userCount", + ], ], where: { workspaceId, @@ -185,6 +194,12 @@ export async function agentMentionsCount( }, }, }, + { + model: UserMessage, + as: "userMessage", + required: true, + attributes: [], + }, ], }, ], @@ -196,13 +211,15 @@ export async function agentMentionsCount( return mentions.map((mention) => { const castMention = mention as unknown as { agentConfigurationId: string; - count: number; + messageCount: number; conversationCount: number; + userCount: number; }; return { agentId: castMention.agentConfigurationId, - count: castMention.count, + messageCount: castMention.messageCount, conversationCount: castMention.conversationCount, + userCount: castMention.userCount, timePeriodSec: rankingUsageDays * 24 * 60 * 60, }; }); @@ -210,7 +227,7 @@ export async function agentMentionsCount( export async function storeCountsInRedis( workspaceId: string, - agentMessageCounts: MentionCount[], + agentMessageCounts: AgentUsageCount[], redis: RedisClientType ) { const agentMessageCountKey = _getUsageKey(workspaceId); @@ -225,8 +242,9 @@ export async function storeCountsInRedis( if (!amcByAgentId[agentId]) { amcByAgentId[agentId] = { agentId, - count: 0, + messageCount: 0, conversationCount: 0, + userCount: 0, timePeriodSec: RANKING_TIMEFRAME_SEC, }; } @@ -234,9 +252,15 @@ export async function storeCountsInRedis( const transaction = redis.multi(); - Object.values(amcByAgentId).forEach(({ agentId, count }) => { - transaction.hSet(agentMessageCountKey, agentId, count); - }); + Object.values(amcByAgentId).forEach( + ({ agentId, messageCount, conversationCount, userCount }) => { + transaction.hSet( + agentMessageCountKey, + agentId, + JSON.stringify({ messageCount, conversationCount, userCount }) + ); + } + ); transaction.expire(agentMessageCountKey, MENTION_COUNT_TTL); @@ -261,7 +285,24 @@ export async function signalAgentUsage({ if (agentMessageCountTTL !== TTL_KEY_NOT_EXIST) { // We only want to increment if the counts have already been computed - await redis.hIncrBy(agentMessageCountKey, agentConfigurationId, 1); + const usage = await redis.hGet(agentMessageCountKey, agentConfigurationId); + if (usage) { + const value = JSON.parse(usage); + const newValue = + typeof value === "object" + ? { ...value, messageCount: value.messageCount + 1 } + : { + messageCount: value + 1, + conversationCount: 0, + userCount: 0, + }; + + await redis.hSet( + agentMessageCountKey, + agentConfigurationId, + JSON.stringify(newValue) + ); + } } } @@ -273,7 +314,7 @@ type UsersUsageCount = { export async function getAgentUsers( owner: LightWorkspaceType, - agentConfiguration?: LightAgentConfigurationType, + agentConfiguration: LightAgentConfigurationType, rankingUsageDays: number = RANKING_USAGE_DAYS ): Promise { const mentions = await Conversation.findAll({ diff --git a/front/lib/api/assistant/conversation.ts b/front/lib/api/assistant/conversation.ts index 64ac535d6acb..f6bea16d5ad6 100644 --- a/front/lib/api/assistant/conversation.ts +++ b/front/lib/api/assistant/conversation.ts @@ -32,20 +32,18 @@ import type { import { assertNever, ConversationError, - getSmallWhitelistedModel, - isContentFragmentType, - isProviderWhitelisted, - md5, - removeNulls, -} from "@dust-tt/types"; -import { Err, + getSmallWhitelistedModel, getTimeframeSecondsFromLiteral, isAgentMention, isAgentMessageType, + isContentFragmentType, + isProviderWhitelisted, isUserMessageType, + md5, Ok, rateLimiter, + removeNulls, } from "@dust-tt/types"; import { isEqual, sortBy } from "lodash"; import type { Transaction } from "sequelize"; @@ -426,28 +424,6 @@ export async function getConversation( }); } -export async function getMessageRank( - auth: Authenticator, - messageId: string -): Promise { - const owner = auth.workspace(); - if (!owner) { - throw new Error("Unexpected `auth` without `workspace`."); - } - - const message = await Message.findOne({ - where: { - sId: messageId, - }, - }); - - if (!message) { - return null; - } - - return message.rank; -} - export async function getConversationMessageType( auth: Authenticator, conversation: ConversationType | ConversationWithoutContentType, diff --git a/front/lib/api/data_source_view.ts b/front/lib/api/data_source_view.ts index 7632170f5135..68528e0c42a4 100644 --- a/front/lib/api/data_source_view.ts +++ b/front/lib/api/data_source_view.ts @@ -203,7 +203,6 @@ async function getContentNodesForStaticDataSourceView( } } return { - dustDocumentId: doc.document_id, expandable: false, internalId: doc.document_id, lastUpdatedAt: doc.timestamp, @@ -238,7 +237,6 @@ async function getContentNodesForStaticDataSourceView( const tablesAsContentNodes: DataSourceViewContentNode[] = tablesRes.value.tables.map((table) => ({ - dustDocumentId: table.table_id, expandable: false, internalId: getContentNodeInternalIdFromTableId( dataSourceView, diff --git a/front/lib/api/data_sources.ts b/front/lib/api/data_sources.ts index f563970641ac..dba6cd1c18ee 100644 --- a/front/lib/api/data_sources.ts +++ b/front/lib/api/data_sources.ts @@ -45,13 +45,13 @@ import { getMembers } from "@app/lib/api/workspace"; import type { Authenticator } from "@app/lib/auth"; import { getFeatureFlags } from "@app/lib/auth"; import { DustError } from "@app/lib/error"; +import { Lock } from "@app/lib/lock"; import { DataSourceResource } from "@app/lib/resources/data_source_resource"; import { DataSourceViewResource } from "@app/lib/resources/data_source_view_resource"; import { SpaceResource } from "@app/lib/resources/space_resource"; import { generateRandomModelSId } from "@app/lib/resources/string_ids"; import { ServerSideTracking } from "@app/lib/tracking/server"; import { enqueueUpsertTable } from "@app/lib/upsert_queue"; -import { wakeLock, wakeLockIsFree } from "@app/lib/wake_lock"; import logger from "@app/logger/logger"; import { launchScrubDataSourceWorkflow } from "@app/poke/temporal/client"; @@ -936,12 +936,9 @@ async function getOrCreateConversationDataSource( } const lockName = "conversationDataSource" + conversation.id; - // Handle race condition when we try to create a conversation data source - while (!wakeLockIsFree(lockName)) { - await new Promise((resolve) => setTimeout(resolve, 100)); - } - const res = await wakeLock( + const res = await Lock.executeWithLock( + lockName, async (): Promise< Result< DataSourceResource, @@ -984,8 +981,7 @@ async function getOrCreateConversationDataSource( } return new Ok(dataSource); - }, - lockName + } ); return res; diff --git a/front/lib/content_nodes.ts b/front/lib/content_nodes.ts index c33e4e29d1ff..68452c7b1ef1 100644 --- a/front/lib/content_nodes.ts +++ b/front/lib/content_nodes.ts @@ -6,10 +6,10 @@ import { LockIcon, Square3Stack3DIcon, } from "@dust-tt/sparkle"; -import type { BaseContentNode } from "@dust-tt/types"; +import type { ContentNode } from "@dust-tt/types"; import { assertNever } from "@dust-tt/types"; -function getVisualForFileContentNode(node: BaseContentNode & { type: "file" }) { +function getVisualForFileContentNode(node: ContentNode & { type: "file" }) { if (node.expandable) { return DocumentPileIcon; } @@ -17,7 +17,7 @@ function getVisualForFileContentNode(node: BaseContentNode & { type: "file" }) { return DocumentIcon; } -export function getVisualForContentNode(node: BaseContentNode) { +export function getVisualForContentNode(node: ContentNode) { switch (node.type) { case "channel": if (node.providerVisibility === "private") { @@ -30,7 +30,7 @@ export function getVisualForContentNode(node: BaseContentNode) { case "file": return getVisualForFileContentNode( - node as BaseContentNode & { type: "file" } + node as ContentNode & { type: "file" } ); case "folder": diff --git a/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_retrieval.ts b/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_retrieval.ts index 3b25db694ab2..526e43c30c25 100644 --- a/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_retrieval.ts +++ b/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_retrieval.ts @@ -69,6 +69,7 @@ const DocTrackerRetrievalActionValueSchema = t.array( created: t.Integer, document_id: t.string, timestamp: t.Integer, + title: t.union([t.string, t.null]), tags: t.array(t.string), parents: t.array(t.string), source_url: t.union([t.string, t.null]), @@ -77,12 +78,17 @@ const DocTrackerRetrievalActionValueSchema = t.array( text: t.union([t.string, t.null, t.undefined]), chunk_count: t.Integer, chunks: t.array( - t.type({ - text: t.string, - hash: t.string, - offset: t.Integer, - score: t.number, - }) + t.intersection([ + t.type({ + text: t.string, + hash: t.string, + offset: t.Integer, + score: t.number, + }), + t.partial({ + expanded_offsets: t.array(t.Integer), + }), + ]) ), token_count: t.Integer, }) diff --git a/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_score_docs.ts b/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_score_docs.ts new file mode 100644 index 000000000000..fcaf6876133f --- /dev/null +++ b/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_score_docs.ts @@ -0,0 +1,65 @@ +import * as t from "io-ts"; + +import { callAction } from "@app/lib/actions/helpers"; +import type { Authenticator } from "@app/lib/auth"; +import { cloneBaseConfig, DustProdActionRegistry } from "@app/lib/registry"; + +export async function callDocTrackerScoreDocsAction( + auth: Authenticator, + { + watchedDocDiff, + maintainedDocuments, + prompt, + providerId, + modelId, + }: { + watchedDocDiff: string; + maintainedDocuments: Array<{ + content: string; + title: string | null; + sourceUrl: string | null; + dataSourceId: string; + documentId: string; + }>; + prompt: string | null; + providerId: string; + modelId: string; + } +): Promise { + const action = DustProdActionRegistry["doc-tracker-score-docs"]; + + const config = cloneBaseConfig(action.config); + config.MODEL.provider_id = providerId; + config.MODEL.model_id = modelId; + + const res = await callAction(auth, { + action, + config, + input: { + watched_diff: watchedDocDiff, + maintained_documents: maintainedDocuments, + prompt, + }, + responseValueSchema: DocTrackerScoreDocsActionResultSchema, + }); + + if (res.isErr()) { + throw res.error; + } + + return res.value; +} + +const DocTrackerScoreDocsActionResultSchema = t.array( + t.type({ + documentId: t.string, + dataSourceId: t.string, + score: t.number, + title: t.union([t.string, t.null, t.undefined]), + sourceUrl: t.union([t.string, t.null, t.undefined]), + }) +); + +type DocTrackerScoreDocsActionResult = t.TypeOf< + typeof DocTrackerScoreDocsActionResultSchema +>; diff --git a/front/lib/lock.ts b/front/lib/lock.ts new file mode 100644 index 000000000000..d142fcad5ea3 --- /dev/null +++ b/front/lib/lock.ts @@ -0,0 +1,45 @@ +export class Lock { + private static locks = new Map>(); + + static async executeWithLock( + lockName: string, + callback: () => Promise, + timeoutMs: number = 30000 + ): Promise { + const start = Date.now(); + + if (Lock.locks.has(lockName)) { + const currentLock = Lock.locks.get(lockName); + if (currentLock) { + const remainingTime = timeoutMs - (Date.now() - start); + if (remainingTime <= 0) { + throw new Error(`Lock acquisition timed out for ${lockName}`); + } + + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject(new Error(`Lock acquisition timed out for ${lockName}`)); + }, remainingTime); + }); + + await Promise.race([currentLock, timeoutPromise]); + } + } + + // Initialize resolveLock with a no-op function to satisfy TypeScript + let resolveLock = () => {}; + const lockPromise = new Promise((resolve) => { + resolveLock = resolve; + }); + + Lock.locks.set(lockName, lockPromise); + + try { + const result = await callback(); + return result; + } finally { + Lock.locks.delete(lockName); + resolveLock(); + } + } +} diff --git a/front/lib/models/doc_tracker.ts b/front/lib/models/doc_tracker.ts index d42ffedf1745..3b9af1c38b21 100644 --- a/front/lib/models/doc_tracker.ts +++ b/front/lib/models/doc_tracker.ts @@ -237,11 +237,14 @@ export class TrackerGenerationModel extends SoftDeletableModel; declare dataSourceId: ForeignKey; declare documentId: string; + declare maintainedDocumentDataSourceId: ForeignKey; + declare maintainedDocumentId: string; declare consumedAt: Date | null; declare trackerConfiguration: NonAttribute; declare dataSource: NonAttribute; + declare maintainedDocumentDataSource: NonAttribute; } TrackerGenerationModel.init( @@ -275,6 +278,10 @@ TrackerGenerationModel.init( type: DataTypes.DATE, allowNull: true, }, + maintainedDocumentId: { + type: DataTypes.STRING, + allowNull: true, + }, }, { modelName: "tracker_generation", @@ -300,3 +307,12 @@ TrackerGenerationModel.belongsTo(DataSourceModel, { foreignKey: { allowNull: false }, as: "dataSource", }); + +DataSourceModel.hasMany(TrackerGenerationModel, { + foreignKey: { allowNull: true }, + as: "maintainedDocumentDataSource", +}); +TrackerGenerationModel.belongsTo(DataSourceModel, { + foreignKey: { allowNull: false }, + as: "maintainedDocumentDataSource", +}); diff --git a/front/lib/registry.ts b/front/lib/registry.ts index b51ae750152c..32f1558d44dd 100644 --- a/front/lib/registry.ts +++ b/front/lib/registry.ts @@ -136,6 +136,20 @@ export const DustProdActionRegistry = createActionRegistry({ }, }, }, + "doc-tracker-score-docs": { + app: { + workspaceId: PRODUCTION_DUST_APPS_WORKSPACE_ID, + appId: "N0RrhyTXfq", + appHash: + "ba5637f356c55676c7e175719bbd4fa5059c5a99a519ec75aea78b452e2168dc", + appSpaceId: PRODUCTION_DUST_APPS_SPACE_ID, + }, + config: { + MODEL: { + use_cache: true, + }, + }, + }, "doc-tracker-suggest-changes": { app: { workspaceId: PRODUCTION_DUST_APPS_WORKSPACE_ID, diff --git a/front/lib/resources/tracker_resource.ts b/front/lib/resources/tracker_resource.ts index 243077392f27..7910e389db46 100644 --- a/front/lib/resources/tracker_resource.ts +++ b/front/lib/resources/tracker_resource.ts @@ -236,22 +236,36 @@ export class TrackerConfigurationResource extends ResourceWithSpace = - fetcher; + const conversationFetcher: Fetcher = fetcher; const { data, error, mutate } = useSWRWithDefaults( `/api/w/${workspaceId}/assistant/conversations`, @@ -121,7 +122,6 @@ export function useConversationMessages({ conversationId, workspaceId, limit, - startAtRank, }: { conversationId: string | null; workspaceId: string; @@ -147,10 +147,7 @@ export function useConversationMessages({ } if (previousPageData === null) { - const startAtRankParam = startAtRank - ? `&lastValue=${startAtRank}` - : ""; - return `/api/w/${workspaceId}/assistant/conversations/${conversationId}/messages?orderDirection=desc&orderColumn=rank&limit=${limit}${startAtRankParam}`; + return `/api/w/${workspaceId}/assistant/conversations/${conversationId}/messages?orderDirection=desc&orderColumn=rank&limit=${limit}`; } return `/api/w/${workspaceId}/assistant/conversations/${conversationId}/messages?lastValue=${previousPageData.lastValue}&orderDirection=desc&orderColumn=rank&limit=${limit}`; @@ -208,7 +205,9 @@ export const useDeleteConversation = (owner: LightWorkspaceType) => { workspaceId: owner.sId, }); - const doDelete = async (conversation: ConversationType | null) => { + const doDelete = async ( + conversation: ConversationWithoutContentType | null + ) => { if (!conversation) { return false; } diff --git a/front/lib/wake_lock.ts b/front/lib/wake_lock.ts index bc0f8ed93c62..1266492c2885 100644 --- a/front/lib/wake_lock.ts +++ b/front/lib/wake_lock.ts @@ -1,13 +1,10 @@ import { v4 as uuidv4 } from "uuid"; -export async function wakeLock( - autoCallback: () => Promise, - lockName?: string -): Promise { +export async function wakeLock(autoCallback: () => Promise): Promise { if (!global.wakeLocks) { global.wakeLocks = new Set(); } - lockName ??= uuidv4(); + const lockName = uuidv4(); global.wakeLocks.add(lockName); try { const r = await autoCallback(); @@ -17,11 +14,6 @@ export async function wakeLock( } } -// If a lockName is provided, checks if that lock is free, otherwise checks if all locks are free -export function wakeLockIsFree(lockName?: string): boolean { - if (lockName) { - return !global.wakeLocks || !global.wakeLocks.has(lockName); - } - +export function wakeLockIsFree(): boolean { return !global.wakeLocks || global.wakeLocks.size === 0; } diff --git a/front/migrations/20250110_backfill_slack_mime_types.ts b/front/migrations/20250110_backfill_slack_mime_types.ts new file mode 100644 index 000000000000..de94cc0c039e --- /dev/null +++ b/front/migrations/20250110_backfill_slack_mime_types.ts @@ -0,0 +1,117 @@ +import assert from "assert"; +import _ from "lodash"; +import type { Sequelize } from "sequelize"; +import { QueryTypes } from "sequelize"; + +import { getCorePrimaryDbConnection } from "@app/lib/production_checks/utils"; +import { DataSourceModel } from "@app/lib/resources/storage/models/data_source"; +import type Logger from "@app/logger/logger"; +import { makeScript } from "@app/scripts/helpers"; + +const { CORE_DATABASE_URI } = process.env; +const SELECT_BATCH_SIZE = 512; +const UPDATE_BATCH_SIZE = 32; + +async function backfillDataSource( + frontDataSource: DataSourceModel, + coreSequelize: Sequelize, + nodeType: "thread" | "messages", + execute: boolean, + logger: typeof Logger +) { + const pattern = `^slack-[A-Z0-9]+-${nodeType}-[0-9.\\-]+$`; + const mimeType = `application/vnd.dust.slack.${nodeType}`; + + logger.info({ pattern, mimeType }, "Processing data source"); + + let nextId = 0; + let updatedRowsCount; + do { + const rows: { id: number }[] = await coreSequelize.query( + ` + SELECT dsn.id + FROM data_sources_nodes dsn + JOIN data_sources ds ON ds.id = dsn.data_source + WHERE dsn.id > :nextId + AND ds.data_source_id = :dataSourceId + AND ds.project = :projectId + AND dsn.node_id ~ :pattern + ORDER BY dsn.id + LIMIT :batchSize;`, + { + replacements: { + dataSourceId: frontDataSource.dustAPIDataSourceId, + projectId: frontDataSource.dustAPIProjectId, + batchSize: SELECT_BATCH_SIZE, + nextId, + pattern, + }, + type: QueryTypes.SELECT, + } + ); + + if (rows.length == 0) { + logger.info({ nextId }, `Finished processing data source.`); + break; + } + nextId = rows[rows.length - 1].id; + updatedRowsCount = rows.length; + + // doing smaller chunks to avoid long transactions + const chunks = _.chunk(rows, UPDATE_BATCH_SIZE); + + for (const chunk of chunks) { + if (execute) { + await coreSequelize.query( + `UPDATE data_sources_nodes SET mime_type = :mimeType WHERE id IN (:ids)`, + { + replacements: { + mimeType, + ids: chunk.map((row) => row.id), + }, + } + ); + logger.info( + `Updated chunk from ${chunk[0].id} to ${chunk[chunk.length - 1].id}` + ); + } else { + logger.info( + `Would update chunk from ${chunk[0].id} to ${chunk[chunk.length - 1].id}` + ); + } + } + } while (updatedRowsCount === SELECT_BATCH_SIZE); +} + +makeScript( + { + nodeType: { type: "string", choices: ["thread", "messages"] }, + }, + async ({ nodeType, execute }, logger) => { + assert(CORE_DATABASE_URI, "CORE_DATABASE_URI is required"); + + const coreSequelize = getCorePrimaryDbConnection(); + + if (!["thread", "messages"].includes(nodeType)) { + throw new Error(`Unknown node type: ${nodeType}`); + } + + const frontDataSources = await DataSourceModel.findAll({ + where: { connectorProvider: "slack" }, + }); + logger.info(`Found ${frontDataSources.length} Slack data sources`); + + for (const frontDataSource of frontDataSources) { + await backfillDataSource( + frontDataSource, + coreSequelize, + nodeType as "thread" | "messages", + execute, + logger.child({ + dataSourceId: frontDataSource.id, + connectorId: frontDataSource.connectorId, + }) + ); + } + } +); diff --git a/front/migrations/20250113_backfill_github_mime_types.ts b/front/migrations/20250113_backfill_github_mime_types.ts new file mode 100644 index 000000000000..bc5dff4b9e83 --- /dev/null +++ b/front/migrations/20250113_backfill_github_mime_types.ts @@ -0,0 +1,164 @@ +import type { GithubMimeType } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; +import assert from "assert"; +import type { Sequelize } from "sequelize"; +import { QueryTypes } from "sequelize"; + +import { getCorePrimaryDbConnection } from "@app/lib/production_checks/utils"; +import { DataSourceModel } from "@app/lib/resources/storage/models/data_source"; +import type Logger from "@app/logger/logger"; +import { makeScript } from "@app/scripts/helpers"; + +const { CORE_DATABASE_URI } = process.env; +const BATCH_SIZE = 256; + +type GithubContentNodeType = "REPO_CODE" | "REPO_CODE_DIR" | "REPO_CODE_FILE"; + +/** + * Gets the type of the GitHub content node from its internal id. + * Copy-pasted from connectors/src/connectors/github/lib/utils.ts + */ +function matchGithubInternalIdType(internalId: string): { + type: GithubContentNodeType; + repoId: number; +} { + // All code from repo is selected, format = "github-code-12345678" + if (/^github-code-\d+$/.test(internalId)) { + return { + type: "REPO_CODE", + repoId: parseInt(internalId.replace(/^github-code-/, ""), 10), + }; + } + // A code directory is selected, format = "github-code-12345678-dir-s0Up1n0u" + if (/^github-code-\d+-dir-[a-f0-9]+$/.test(internalId)) { + return { + type: "REPO_CODE_DIR", + repoId: parseInt( + internalId.replace(/^github-code-(\d+)-dir-.*/, "$1"), + 10 + ), + }; + } + // A code file is selected, format = "github-code-12345678-file-s0Up1n0u" + if (/^github-code-\d+-file-[a-f0-9]+$/.test(internalId)) { + return { + type: "REPO_CODE_FILE", + repoId: parseInt( + internalId.replace(/^github-code-(\d+)-file-.*/, "$1"), + 10 + ), + }; + } + throw new Error(`Invalid Github internal id (code-only): ${internalId}`); +} + +function getMimeTypeForNodeId(nodeId: string): GithubMimeType { + switch (matchGithubInternalIdType(nodeId).type) { + case "REPO_CODE": + return MIME_TYPES.GITHUB.CODE_ROOT; + case "REPO_CODE_DIR": + return MIME_TYPES.GITHUB.CODE_DIRECTORY; + case "REPO_CODE_FILE": + return MIME_TYPES.GITHUB.CODE_FILE; + default: + throw new Error(`Unreachable: unrecognized node_id: ${nodeId}`); + } +} + +async function backfillDataSource( + frontDataSource: DataSourceModel, + coreSequelize: Sequelize, + execute: boolean, + logger: typeof Logger +) { + logger.info("Processing data source"); + + let nextId = 0; + let updatedRowsCount; + do { + const rows: { id: number; node_id: string; mime_type: string }[] = + await coreSequelize.query( + ` + SELECT dsn.id, dsn.node_id, dsn.mime_type + FROM data_sources_nodes dsn + JOIN data_sources ds ON ds.id = dsn.data_source + WHERE dsn.id > :nextId + AND ds.data_source_id = :dataSourceId + AND ds.project = :projectId + AND dsn.node_id LIKE 'github-code-%' -- leverages the btree + ORDER BY dsn.id + LIMIT :batchSize;`, + { + replacements: { + dataSourceId: frontDataSource.dustAPIDataSourceId, + projectId: frontDataSource.dustAPIProjectId, + batchSize: BATCH_SIZE, + nextId, + }, + type: QueryTypes.SELECT, + } + ); + + if (rows.length == 0) { + logger.info({ nextId }, `Finished processing data source.`); + break; + } + nextId = rows[rows.length - 1].id; + updatedRowsCount = rows.length; + + if (execute) { + await coreSequelize.query( + `WITH pairs AS ( + SELECT UNNEST(ARRAY[:ids]) as id, UNNEST(ARRAY[:mimeTypes]) as mime_type + ) + UPDATE data_sources_nodes dsn + SET mime_type = p.mime_type + FROM pairs p + WHERE dsn.id = p.id;`, + { + replacements: { + mimeTypes: rows.map((row) => getMimeTypeForNodeId(row.node_id)), + ids: rows.map((row) => row.id), + }, + } + ); + logger.info( + `Updated chunk from ${rows[0].id} to ${rows[rows.length - 1].id}` + ); + } else { + logger.info( + { + nodes: rows.map((row) => ({ + nodeId: row.node_id, + fromMimeType: row.mime_type, + toMimeType: getMimeTypeForNodeId(row.node_id), + })), + }, + `Would update chunk from ${rows[0].id} to ${rows[rows.length - 1].id}` + ); + } + } while (updatedRowsCount === BATCH_SIZE); +} + +makeScript({}, async ({ execute }, logger) => { + assert(CORE_DATABASE_URI, "CORE_DATABASE_URI is required"); + + const coreSequelize = getCorePrimaryDbConnection(); + + const frontDataSources = await DataSourceModel.findAll({ + where: { connectorProvider: "github" }, + }); + logger.info(`Found ${frontDataSources.length} GitHub data sources`); + + for (const frontDataSource of frontDataSources) { + await backfillDataSource( + frontDataSource, + coreSequelize, + execute, + logger.child({ + dataSourceId: frontDataSource.id, + connectorId: frontDataSource.connectorId, + }) + ); + } +}); diff --git a/front/migrations/20250113_backfill_zendesk_hc_mime_types.ts b/front/migrations/20250113_backfill_zendesk_hc_mime_types.ts new file mode 100644 index 000000000000..3395b6451ce4 --- /dev/null +++ b/front/migrations/20250113_backfill_zendesk_hc_mime_types.ts @@ -0,0 +1,96 @@ +import { MIME_TYPES } from "@dust-tt/types"; +import assert from "assert"; +import type { Sequelize } from "sequelize"; +import { QueryTypes } from "sequelize"; + +import { getCorePrimaryDbConnection } from "@app/lib/production_checks/utils"; +import { DataSourceModel } from "@app/lib/resources/storage/models/data_source"; +import type Logger from "@app/logger/logger"; +import { makeScript } from "@app/scripts/helpers"; + +const { CORE_DATABASE_URI } = process.env; +const BATCH_SIZE = 16; + +async function backfillDataSource( + frontDataSource: DataSourceModel, + coreSequelize: Sequelize, + execute: boolean, + logger: typeof Logger +) { + logger.info("Processing data source"); + + let nextId = 0; + let updatedRowsCount; + do { + const rows: { id: number }[] = await coreSequelize.query( + ` + SELECT dsn.id + FROM data_sources_nodes dsn + JOIN data_sources ds ON ds.id = dsn.data_source + WHERE dsn.id > :nextId + AND ds.data_source_id = :dataSourceId + AND ds.project = :projectId + AND dsn.node_id LIKE 'zendesk-help-center-%' + ORDER BY dsn.id + LIMIT :batchSize;`, + { + replacements: { + dataSourceId: frontDataSource.dustAPIDataSourceId, + projectId: frontDataSource.dustAPIProjectId, + batchSize: BATCH_SIZE, + nextId, + }, + type: QueryTypes.SELECT, + } + ); + + if (rows.length == 0) { + logger.info({ nextId }, `Finished processing data source.`); + break; + } + nextId = rows[rows.length - 1].id; + updatedRowsCount = rows.length; + + if (execute) { + await coreSequelize.query( + `UPDATE data_sources_nodes SET mime_type = :mimeType WHERE id IN (:ids)`, + { + replacements: { + mimeType: MIME_TYPES.ZENDESK.HELP_CENTER, + ids: rows.map((row) => row.id), + }, + } + ); + logger.info( + `Updated chunk from ${rows[0].id} to ${rows[rows.length - 1].id}` + ); + } else { + logger.info( + `Would update chunk from ${rows[0].id} to ${rows[rows.length - 1].id}` + ); + } + } while (updatedRowsCount === BATCH_SIZE); +} + +makeScript({}, async ({ execute }, logger) => { + assert(CORE_DATABASE_URI, "CORE_DATABASE_URI is required"); + + const coreSequelize = getCorePrimaryDbConnection(); + + const frontDataSources = await DataSourceModel.findAll({ + where: { connectorProvider: "zendesk" }, + }); + logger.info(`Found ${frontDataSources.length} Zendesk data sources`); + + for (const frontDataSource of frontDataSources) { + await backfillDataSource( + frontDataSource, + coreSequelize, + execute, + logger.child({ + dataSourceId: frontDataSource.id, + connectorId: frontDataSource.connectorId, + }) + ); + } +}); diff --git a/front/migrations/db/migration_140.sql b/front/migrations/db/migration_140.sql new file mode 100644 index 000000000000..23e65f3f70aa --- /dev/null +++ b/front/migrations/db/migration_140.sql @@ -0,0 +1,3 @@ +-- Migration created on Jan 10, 2025 +ALTER TABLE "public"."tracker_generations" ADD COLUMN "maintainedDocumentId" VARCHAR(255); +ALTER TABLE "public"."tracker_generations" ADD COLUMN "maintainedDocumentDataSourceId" INTEGER REFERENCES "public"."data_sources" ("id") ON DELETE RESTRICT; diff --git a/front/next.config.js b/front/next.config.js index 3ecf9b8155c4..b8723723b7be 100644 --- a/front/next.config.js +++ b/front/next.config.js @@ -2,11 +2,11 @@ const path = require("path"); const CONTENT_SECURITY_POLICIES = [ "default-src 'none';", - `script-src 'self' 'unsafe-inline' 'unsafe-eval' *.googletagmanager.com *.google-analytics.com;`, + `script-src 'self' 'unsafe-inline' 'unsafe-eval' *.googletagmanager.com *.google-analytics.com *.hsforms.net;`, `style-src 'self' 'unsafe-inline' *.typekit.net;`, `img-src 'self' data: https:;`, - `connect-src 'self' *.google-analytics.com;`, - `frame-src 'self' *.wistia.net viz.dust.tt;`, + `connect-src 'self' *.google-analytics.com cdn.jsdelivr.net *.hsforms.com;`, + `frame-src 'self' *.wistia.net viz.dust.tt *.hsforms.net;`, `font-src 'self' data: *.typekit.net;`, `object-src 'none';`, `form-action 'self';`, diff --git a/front/package-lock.json b/front/package-lock.json index 057403b60645..a594546469d8 100644 --- a/front/package-lock.json +++ b/front/package-lock.json @@ -5,10 +5,9 @@ "packages": { "": { "dependencies": { - "@aaronhayes/react-use-hubspot-form": "^2.1.1", "@auth0/nextjs-auth0": "^3.5.0", "@dust-tt/client": "file:../sdks/js", - "@dust-tt/sparkle": "^0.2.361", + "@dust-tt/sparkle": "^0.2.362", "@dust-tt/types": "file:../types", "@headlessui/react": "^1.7.7", "@heroicons/react": "^2.0.11", @@ -10519,18 +10518,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@aaronhayes/react-use-hubspot-form": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@aaronhayes/react-use-hubspot-form/-/react-use-hubspot-form-2.1.1.tgz", - "integrity": "sha512-K1ybIAa6b49hTnr08OcYZWI9yqpI/bGzjh2Vnk+H3Lt1udpToa0wJnP9ONmfs+wUdefzX2v0BwQPxtZ7yE6E8A==", - "engines": { - "node": ">=10.16" - }, - "peerDependencies": { - "react": ">=16.8.0", - "react-dom": ">=16.8.0" - } - }, "node_modules/@aashutoshrathi/word-wrap": { "version": "1.2.6", "dev": true, @@ -11467,9 +11454,9 @@ "link": true }, "node_modules/@dust-tt/sparkle": { - "version": "0.2.361", - "resolved": "https://registry.npmjs.org/@dust-tt/sparkle/-/sparkle-0.2.361.tgz", - "integrity": "sha512-vMTP+yfZKKPplkLqG5uOdzGl0BOGHWkZeO8d/hdDSTjC3+YPYjM0xHdWcSsnlKcMNSkFlLou5IWAUmac84FVfA==", + "version": "0.2.362", + "resolved": "https://registry.npmjs.org/@dust-tt/sparkle/-/sparkle-0.2.362.tgz", + "integrity": "sha512-ZC1byoxaN16p9Fb+QUcsp7mnvCX/D2DaFRgSrkhx8nr96w08hQiTznWy5D84T0PzslXVgCtXf/KyqprFjSaZEQ==", "dependencies": { "@emoji-mart/data": "^1.1.2", "@emoji-mart/react": "^1.1.1", diff --git a/front/package.json b/front/package.json index 21e47d031339..801abb1eb090 100644 --- a/front/package.json +++ b/front/package.json @@ -18,10 +18,9 @@ "prepare": "cd .. && husky .husky" }, "dependencies": { - "@aaronhayes/react-use-hubspot-form": "^2.1.1", "@auth0/nextjs-auth0": "^3.5.0", "@dust-tt/client": "file:../sdks/js", - "@dust-tt/sparkle": "^0.2.361", + "@dust-tt/sparkle": "^0.2.362", "@dust-tt/types": "file:../types", "@headlessui/react": "^1.7.7", "@heroicons/react": "^2.0.11", diff --git a/front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts b/front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts index 55e685bda05a..bf0fb46a61db 100644 --- a/front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts +++ b/front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts @@ -127,7 +127,7 @@ async function handler( })) .filter((r) => r.user), mentions: { - messageCount: mentionCounts?.count ?? 0, + messageCount: mentionCounts?.messageCount ?? 0, conversationCount: mentionCounts?.conversationCount ?? 0, timePeriodSec: mentionCounts?.timePeriodSec ?? period * 60 * 60 * 24, }, diff --git a/front/pages/api/w/[wId]/assistant/agent_configurations/index.ts b/front/pages/api/w/[wId]/assistant/agent_configurations/index.ts index 3cef3d3761a3..57ca3488cf5d 100644 --- a/front/pages/api/w/[wId]/assistant/agent_configurations/index.ts +++ b/front/pages/api/w/[wId]/assistant/agent_configurations/index.ts @@ -120,10 +120,7 @@ async function handler( usageMap[agentConfiguration.sId] ? { ...agentConfiguration, - usage: { - messageCount: usageMap[agentConfiguration.sId].messageCount, - timePeriodSec: usageMap[agentConfiguration.sId].timePeriodSec, - }, + usage: _.omit(usageMap[agentConfiguration.sId], ["agentId"]), } : agentConfiguration ); diff --git a/front/pages/api/w/[wId]/providers/[pId]/models.ts b/front/pages/api/w/[wId]/providers/[pId]/models.ts index a10f05446bbc..ae20245dcf68 100644 --- a/front/pages/api/w/[wId]/providers/[pId]/models.ts +++ b/front/pages/api/w/[wId]/providers/[pId]/models.ts @@ -1,4 +1,10 @@ import type { WithAPIErrorResponse } from "@dust-tt/types"; +import { + GEMINI_1_5_FLASH_LATEST_MODEL_ID, + GEMINI_1_5_PRO_LATEST_MODEL_ID, + GEMINI_2_FLASH_PREVIEW_MODEL_ID, + GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_ID, +} from "@dust-tt/types"; import type { NextApiRequest, NextApiResponse } from "next"; import { withSessionAuthenticationForWorkspace } from "@app/lib/api/auth_wrappers"; @@ -237,8 +243,10 @@ async function handler( case "google_ai_studio": return res.status(200).json({ models: [ - { id: "gemini-1.5-flash-latest" }, - { id: "gemini-1.5-pro-latest" }, + { id: GEMINI_1_5_FLASH_LATEST_MODEL_ID }, + { id: GEMINI_1_5_PRO_LATEST_MODEL_ID }, + { id: GEMINI_2_FLASH_PREVIEW_MODEL_ID }, + { id: GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_ID }, ], }); diff --git a/front/pages/api/w/[wId]/spaces/[spaceId]/trackers/index.ts b/front/pages/api/w/[wId]/spaces/[spaceId]/trackers/index.ts index cfc315345632..ec4d930e5726 100644 --- a/front/pages/api/w/[wId]/spaces/[spaceId]/trackers/index.ts +++ b/front/pages/api/w/[wId]/spaces/[spaceId]/trackers/index.ts @@ -12,6 +12,7 @@ import { withSessionAuthenticationForWorkspace } from "@app/lib/api/auth_wrapper import { withResourceFetchingFromRoute } from "@app/lib/api/resource_wrappers"; import type { Authenticator } from "@app/lib/auth"; import { getFeatureFlags } from "@app/lib/auth"; +import { PRODUCTION_DUST_WORKSPACE_ID } from "@app/lib/registry"; import type { SpaceResource } from "@app/lib/resources/space_resource"; import { TrackerConfigurationResource } from "@app/lib/resources/tracker_resource"; import { apiError } from "@app/logger/withlogging"; @@ -106,12 +107,15 @@ async function handler( auth, space ); - if (existingTrackers.length >= 3) { + if ( + owner.sId !== PRODUCTION_DUST_WORKSPACE_ID && + existingTrackers.length >= 1 + ) { return apiError(req, res, { status_code: 400, api_error: { type: "invalid_request_error", - message: "You can't have more than 3 trackers in a space.", + message: "You can't have more than 1 tracker in a space.", }, }); } diff --git a/front/pages/home/contact/index.tsx b/front/pages/home/contact.tsx similarity index 77% rename from front/pages/home/contact/index.tsx rename to front/pages/home/contact.tsx index 49db154efd58..e6182d782417 100644 --- a/front/pages/home/contact/index.tsx +++ b/front/pages/home/contact.tsx @@ -1,7 +1,7 @@ -import { HubspotProvider } from "@aaronhayes/react-use-hubspot-form"; import type { ReactElement } from "react"; import { HeaderContentBlock } from "@app/components/home/ContentBlocks"; +import HubSpotForm from "@app/components/home/HubSpotForm"; import type { LandingLayoutProps } from "@app/components/home/LandingLayout"; import LandingLayout from "@app/components/home/LandingLayout"; import { @@ -9,7 +9,6 @@ import { shapeNames, } from "@app/components/home/Particles"; import TrustedBy from "@app/components/home/TrustedBy"; -import { HubSpotForm } from "@app/pages/home/contact/hubspot/HubSpotForm"; export async function getServerSideProps() { return { @@ -18,7 +17,8 @@ export async function getServerSideProps() { }, }; } -function ContactContent() { + +export default function Contact() { return (
-
+
@@ -45,14 +45,6 @@ function ContactContent() { ); } -export default function Index() { - return ( - - - - ); -} - -Index.getLayout = (page: ReactElement, pageProps: LandingLayoutProps) => { +Contact.getLayout = (page: ReactElement, pageProps: LandingLayoutProps) => { return {page}; }; diff --git a/front/pages/home/contact/hubspot/HubSpotForm.tsx b/front/pages/home/contact/hubspot/HubSpotForm.tsx deleted file mode 100644 index 41332d18a8d1..000000000000 --- a/front/pages/home/contact/hubspot/HubSpotForm.tsx +++ /dev/null @@ -1,32 +0,0 @@ -import { useHubspotForm } from "@aaronhayes/react-use-hubspot-form"; -import { classNames } from "@dust-tt/sparkle"; - -export default function HubspotFormComponent() { - const { loaded, error } = useHubspotForm({ - portalId: "144442587", - formId: "31e790e5-f4d5-4c79-acc5-acd770fe8f84", - target: "#hubspotForm", - }); - - if (error) { - return
Failed to load form
; - } - - return ( - <> - {!loaded && ( -
-
-
- )} -
- - ); -} - -export function HubSpotForm() { - return ; -} diff --git a/front/pages/home/solutions/customer-support.tsx b/front/pages/home/solutions/customer-support.tsx index 14951d7ddaa8..9e1d8c0e3460 100644 --- a/front/pages/home/solutions/customer-support.tsx +++ b/front/pages/home/solutions/customer-support.tsx @@ -108,7 +108,7 @@ export default function CustomerSupport() {
{MainVisualImage()}
- {pageSettings.uptitle && ( -

- Dust for {pageSettings.uptitle} -

- )}

{pageSettings.title}

@@ -137,13 +132,14 @@ export default function CustomerSupport() { icon={RocketIcon} /> -
-
+
{MainVisualImage()}
diff --git a/front/pages/w/[wId]/assistant/[cId]/index.tsx b/front/pages/w/[wId]/assistant/[cId]/index.tsx index 5d1b3ac5ad72..f6e8c49fe21a 100644 --- a/front/pages/w/[wId]/assistant/[cId]/index.tsx +++ b/front/pages/w/[wId]/assistant/[cId]/index.tsx @@ -7,8 +7,8 @@ import { useEffect, useState } from "react"; import { ConversationContainer } from "@app/components/assistant/conversation/ConversationContainer"; import type { ConversationLayoutProps } from "@app/components/assistant/conversation/ConversationLayout"; import ConversationLayout from "@app/components/assistant/conversation/ConversationLayout"; +import { useConversationsNavigation } from "@app/components/assistant/conversation/ConversationsNavigationProvider"; import { CONVERSATION_PARENT_SCROLL_DIV_ID } from "@app/components/assistant/conversation/lib"; -import { getMessageRank } from "@app/lib/api/assistant/conversation"; import config from "@app/lib/api/config"; import { withDefaultUserAuthRequirements } from "@app/lib/iam/session"; @@ -18,7 +18,6 @@ export const getServerSideProps = withDefaultUserAuthRequirements< conversationId: string | null; user: UserType; isBuilder: boolean; - messageRankToScrollTo: number | null; } >(async (context, auth) => { const owner = auth.workspace(); @@ -47,13 +46,6 @@ export const getServerSideProps = withDefaultUserAuthRequirements< const { cId } = context.params; - // Extract messageId from query params and fetch its rank - const { messageId } = context.query; - let messageRankToScrollTo: number | null = null; - if (typeof messageId === "string") { - messageRankToScrollTo = (await getMessageRank(auth, messageId)) ?? null; - } - return { props: { user, @@ -62,7 +54,6 @@ export const getServerSideProps = withDefaultUserAuthRequirements< subscription, baseUrl: config.getClientFacingUrl(), conversationId: getValidConversationId(cId), - messageRankToScrollTo: messageRankToScrollTo, }, }; }); @@ -73,23 +64,24 @@ export default function AssistantConversation({ subscription, user, isBuilder, - messageRankToScrollTo, }: InferGetServerSidePropsType) { const [conversationKey, setConversationKey] = useState(null); const [agentIdToMention, setAgentIdToMention] = useState(null); const router = useRouter(); - const { cId, assistant } = router.query; + + const { activeConversationId } = useConversationsNavigation(); + + const { assistant } = router.query; + // This useEffect handles whether to change the key of the ConversationContainer // or not. Altering the key forces a re-render of the component. A random number // is used in the key to maintain the component during the transition from new // to the conversation view. The key is reset when navigating to a new conversation. useEffect(() => { - const conversationId = getValidConversationId(cId); - - if (conversationId && initialConversationId) { + if (activeConversationId) { // Set conversation id as key if it exists. - setConversationKey(conversationId); - } else if (!conversationId && !initialConversationId) { + setConversationKey(activeConversationId); + } else if (!activeConversationId) { // Force re-render by setting a new key with a random number. setConversationKey(`new_${Math.random() * 1000}`); @@ -109,19 +101,22 @@ export default function AssistantConversation({ } else { setAgentIdToMention(null); } - }, [cId, assistant, setConversationKey, initialConversationId]); + }, [ + assistant, + setConversationKey, + initialConversationId, + activeConversationId, + ]); return ( ); } diff --git a/front/pages/w/[wId]/assistant/labs/transcripts/index.tsx b/front/pages/w/[wId]/assistant/labs/transcripts/index.tsx index 814ac801e9f8..dbee904f1482 100644 --- a/front/pages/w/[wId]/assistant/labs/transcripts/index.tsx +++ b/front/pages/w/[wId]/assistant/labs/transcripts/index.tsx @@ -4,14 +4,19 @@ import { ChatBubbleThoughtIcon, CloudArrowLeftRightIcon, ContentMessage, - Dialog, Input, + NewDialog, + NewDialogContainer, + NewDialogContent, + NewDialogFooter, + NewDialogHeader, + NewDialogTitle, Page, SliderToggle, Spinner, + useSendNotification, XMarkIcon, } from "@dust-tt/sparkle"; -import { useSendNotification } from "@dust-tt/sparkle"; import type { DataSourceViewSelectionConfigurations, DataSourceViewType, @@ -632,20 +637,38 @@ export default function LabsTranscriptsIndex({ pageTitle="Dust - Transcripts processing" navChildren={} > - { - await handleDisconnectProvider(); - setIsDeleteProviderDialogOpened(false); + { + if (!open) { + setIsDeleteProviderDialogOpened(false); + } }} - onCancel={() => setIsDeleteProviderDialogOpened(false)} > -
- This will stop the processing of your meeting transcripts and delete - all history. You can reconnect anytime. -
- + + + Disconnect transcripts provider + + + This will stop the processing of your meeting transcripts and + delete all history. You can reconnect anytime. + + { + await handleDisconnectProvider(); + setIsDeleteProviderDialogOpened(false); + }, + }} + /> + + ( + null + ); + const [isModelProvider, setIsModelProvider] = useState(true); const appWhiteListedProviders = owner.whiteListedProviders ? [...owner.whiteListedProviders, "azure_openai"] : APP_MODEL_PROVIDER_IDS; + const filteredProvidersIdSet = new Set( modelProviders - .filter((provider) => { - return ( + .filter( + (provider) => APP_MODEL_PROVIDER_IDS.includes(provider.providerId) && appWhiteListedProviders.includes(provider.providerId) - ); - }) + ) .map((provider) => provider.providerId) ); const configs = {} as any; - if (!isProvidersLoading && !isProvidersError) { - for (let i = 0; i < providers.length; i++) { - // Extract API key and hide it from the config object to be displayed. - // Store the original API key in a separate property for display use. - const { api_key, ...rest } = JSON.parse(providers[i].config); - configs[providers[i].providerId] = { + for (const provider of providers) { + const { api_key, ...rest } = JSON.parse(provider.config); + configs[provider.providerId] = { ...rest, api_key: "", redactedApiKey: api_key, }; - filteredProvidersIdSet.add(providers[i].providerId); + filteredProvidersIdSet.add(provider.providerId); } } - const filteredProviders = modelProviders.filter((provider) => - filteredProvidersIdSet.has(provider.providerId) + + const filteredProviders = modelProviders.filter((p) => + filteredProvidersIdSet.has(p.providerId) ); - return ( - <> - - - - - - - - - - - <> - -
    - {filteredProviders.map((provider) => ( -
  • -
    -
    -
    -

    - {provider.name} -

    -
    -

    - {configs[provider.providerId] ? "enabled" : "disabled"} -

    -
    -
    - {configs[provider.providerId] && ( -

    - API Key:{" "} - 

    {configs[provider.providerId].redactedApiKey}
    -

    - )} -
    -
    -
    -
    -
    • - ))} -
+ const selectedConfig = selectedProviderId + ? isModelProvider + ? MODEL_PROVIDER_CONFIGS[selectedProviderId] + : SERVICE_PROVIDER_CONFIGS[selectedProviderId] + : null; + + const enabled = + selectedProviderId && configs[selectedProviderId] + ? !!configs[selectedProviderId] + : false; - + {selectedProviderId && selectedConfig && ( + setSelectedProviderId(null)} /> + )} -
    - {serviceProviders.map((provider) => ( -
  • -
    + +
      + {filteredProviders.map((provider) => ( +
    • +
      +

      -
      -
      +
      +
      • + ))} +
    + + +
      + {serviceProviders.map((provider) => ( +
    • +
      +
      +

      + {provider.name} +

      +
      +

      + ? "bg-green-100 text-green-800" + : "bg-gray-100 text-gray-800" + )} + > + {configs[provider.providerId] ? "enabled" : "disabled"} +

      -
      • - ))} -
    - +
    +
    • + ))} +
); } diff --git a/front/pages/w/[wId]/members/index.tsx b/front/pages/w/[wId]/members/index.tsx index acd5bdc01fb9..8537bf0742d5 100644 --- a/front/pages/w/[wId]/members/index.tsx +++ b/front/pages/w/[wId]/members/index.tsx @@ -1,12 +1,17 @@ import { Button, - Dialog, + NewDialog, + NewDialogContainer, + NewDialogContent, + NewDialogFooter, + NewDialogHeader, + NewDialogTitle, Page, PlusIcon, Popup, SearchInput, + useSendNotification, } from "@dust-tt/sparkle"; -import { useSendNotification } from "@dust-tt/sparkle"; import type { PlanType, SubscriptionPerSeatPricing, @@ -313,18 +318,34 @@ function DomainAutoJoinModal({ } return ( - { - await handleUpdateWorkspace(); - onClose(); + { + if (!open) { + onClose(); + } }} - onCancel={() => onClose()} - validateLabel={validateLabel} - validateVariant={validateVariant} > -
{description}
- + + + {title} + + {description} + { + await handleUpdateWorkspace(); + onClose(); + }, + }} + /> + + ); } diff --git a/front/pages/w/[wId]/subscription/index.tsx b/front/pages/w/[wId]/subscription/index.tsx index 222702fed2d9..9a4b84ba1b3c 100644 --- a/front/pages/w/[wId]/subscription/index.tsx +++ b/front/pages/w/[wId]/subscription/index.tsx @@ -2,17 +2,23 @@ import { Button, CardIcon, Chip, - Dialog, DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger, MoreIcon, + NewDialog, + NewDialogContainer, + NewDialogContent, + NewDialogDescription, + NewDialogFooter, + NewDialogHeader, + NewDialogTitle, Page, ShapesIcon, Spinner, + useSendNotification, } from "@dust-tt/sparkle"; -import { useSendNotification } from "@dust-tt/sparkle"; import type { SubscriptionPerSeatPricing, SubscriptionType, @@ -457,29 +463,47 @@ function SkipFreeTrialDialog({ plan: SubscriptionType["plan"]; }) { return ( - { - onValidate(); + { + if (!open) { + onClose(); + } }} - isSaving={isSaving} > - - - Ending your trial will allow you to invite more than{" "} - {plan.limits.users.maxUsers} members to your workspace. - - - {(() => { - if (workspaceSeats === 1) { + + + End trial + + Ending your trial will allow you to invite more than{" "} + {plan.limits.users.maxUsers} members to your workspace. + + + + {isSaving ? ( +
+ +
+ ) : ( + (() => { + if (workspaceSeats === 1) { + return ( + <> + Billing will start immediately for your workspace.
+ Currently: {workspaceSeats} member,{" "} + {getPriceAsString({ + currency: perSeatPricing.seatCurrency, + priceInCents: perSeatPricing.seatPrice, + })} + monthly (excluding taxes). + + ); + } return ( <> - Billing will start immediately for your workspace.
- Currently: {workspaceSeats} member,{" "} + Billing will start immediately for your workspace:. +
+ Currently: {workspaceSeats} members,{" "} {getPriceAsString({ currency: perSeatPricing.seatCurrency, priceInCents: perSeatPricing.seatPrice, @@ -487,23 +511,22 @@ function SkipFreeTrialDialog({ monthly (excluding taxes). ); - } - return ( - <> - Billing will start immediately for your workspace:. -
- Currently: {workspaceSeats} members,{" "} - {getPriceAsString({ - currency: perSeatPricing.seatCurrency, - priceInCents: perSeatPricing.seatPrice, - })} - monthly (excluding taxes). - - ); - })()} - - - + })() + )} + + + + ); } @@ -519,25 +542,43 @@ function CancelFreeTrialDialog({ isSaving: boolean; }) { return ( - { + if (!open) { + onClose(); + } + }} > - - - + + + Cancel subscription + All your workspace data will be deleted and you will lose access to your Dust workspace. - - - - Are you sure you want to cancel ? - - + + + + {isSaving ? ( +
+ +
+ ) : ( +
Are you sure you want to proceed?
+ )} + + + + ); } diff --git a/front/tailwind.config.js b/front/tailwind.config.js index bb1cf88e67c4..695072d73175 100644 --- a/front/tailwind.config.js +++ b/front/tailwind.config.js @@ -22,7 +22,6 @@ module.exports = { }, maxWidth: { 48: "12rem", - 150: "600px", }, scale: { 99: ".99", diff --git a/front/temporal/tracker/activities.ts b/front/temporal/tracker/activities.ts index 16850c9683c5..d8e4f5b40982 100644 --- a/front/temporal/tracker/activities.ts +++ b/front/temporal/tracker/activities.ts @@ -5,7 +5,15 @@ import type { Result, TrackerIdWorkspaceId, } from "@dust-tt/types"; -import { ConnectorsAPI, CoreAPI, Err, Ok } from "@dust-tt/types"; +import { + ConnectorsAPI, + CoreAPI, + Err, + GPT_4O_MODEL_CONFIG, + Ok, + removeNulls, +} from "@dust-tt/types"; +import { Context } from "@temporalio/activity"; import _ from "lodash"; import config from "@app/lib/api/config"; @@ -13,6 +21,7 @@ import { processTrackerNotification } from "@app/lib/api/tracker"; import { Authenticator } from "@app/lib/auth"; import { getDocumentDiff } from "@app/lib/document_upsert_hooks/hooks/data_source_helpers"; import { callDocTrackerRetrievalAction } from "@app/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_retrieval"; +import { callDocTrackerScoreDocsAction } from "@app/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_score_docs"; import { callDocTrackerSuggestChangesAction } from "@app/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_suggest_changes"; import { Workspace } from "@app/lib/models/workspace"; import { DataSourceResource } from "@app/lib/resources/data_source_resource"; @@ -28,10 +37,14 @@ const TRACKER_WATCHED_DOCUMENT_MINIMUM_DIFF_LINE_LENGTH = 4; const TRACKER_WATCHED_DOCUMENT_MAX_DIFF_TOKENS = 4096; // The total number of tokens to show to the model (watched doc diff + maintained scope retrieved tokens) const TRACKER_TOTAL_TARGET_TOKENS = 8192; -// The topK used for the semantic search against the maintained scope. -// TODO(DOC_TRACKER): Decide how we handle this. If the top doc has less than $targetDocumentTokens, -// we could include content from the next doc in the maintained scope. -const TRACKER_MAINTAINED_DOCUMENT_TOP_K = 1; +// The maximum number of chunks to retrieve from the maintained scope. +const TRACKER_MAINTAINED_SCOPE_MAX_TOP_K = 8; + +// The size of the chunks in our data sources. +// TODO(@fontanierh): find a way to ensure this remains true. +const CHUNK_SIZE = 512; + +const TRACKER_SCORE_DOCS_MODEL_CONFIG = GPT_4O_MODEL_CONFIG; export async function getDebounceMsActivity( dataSourceConnectorProvider: ConnectorProvider | null @@ -52,6 +65,11 @@ export async function trackersGenerationActivity( documentHash: string, dataSourceConnectorProvider: ConnectorProvider | null ) { + if (Context.current().info.attempt > 1) { + // TODO(DOC_TRACKER): mechanism to retry "manually" + throw new Error("Too many attempts"); + } + const localLogger = logger.child({ workspaceId, dataSourceId, @@ -66,6 +84,8 @@ export async function trackersGenerationActivity( throw new Error(`Could not find data source ${dataSourceId}`); } + // We start by finding all trackers that are watching the modified document. + const trackers = await getTrackersToRun(auth, dataSource, documentId); if (!trackers.length) { @@ -114,6 +134,9 @@ export async function trackersGenerationActivity( return; } + // We compute the diff between the current version of the document and the version right before the edit + // that triggered the tracker. + const documentDiff = await getDocumentDiff({ dataSource, documentId, @@ -185,6 +208,21 @@ export async function trackersGenerationActivity( const targetMaintainedScopeTokens = TRACKER_TOTAL_TARGET_TOKENS - tokensInDiffCount; + // We don't want to retrieve more than targetMaintainedScopeTokens / CHUNK_SIZE chunks, + // in case all retrieved chunks are from the same document (in which case, we'd have + // more than targetMaintainedScopeTokens tokens for that document). + const maintainedScopeTopK = Math.min( + TRACKER_MAINTAINED_SCOPE_MAX_TOP_K, + Math.floor(targetMaintainedScopeTokens / CHUNK_SIZE) + ); + + if (maintainedScopeTopK === 0) { + throw new Error( + "Unreachable: targetMaintainedScopeTokens is less than CHUNK_SIZE." + ); + } + + // We run each tracker. for (const tracker of trackers) { const trackerLogger = localLogger.child({ trackerId: tracker.sId, @@ -213,71 +251,203 @@ export async function trackersGenerationActivity( (x) => x.filter?.parents?.in ?? null ); + // We retrieve content from the maintained scope based on the diff. const maintainedScopeRetrieval = await callDocTrackerRetrievalAction(auth, { inputText: diffString, targetDocumentTokens: targetMaintainedScopeTokens, - topK: TRACKER_MAINTAINED_DOCUMENT_TOP_K, + topK: maintainedScopeTopK, maintainedScope, parentsInMap, }); - // TODO(DOC_TRACKER): Right now we only handle the top match. - // We may want to support topK > 1 and process more than 1 doc if the top doc has less than - // $targetDocumentTokens. if (maintainedScopeRetrieval.length === 0) { trackerLogger.info("No content retrieved from maintained scope."); continue; } - const content = maintainedScopeRetrieval[0].chunks - .map((c) => c.text) - .join("\n"); - if (!content) { - trackerLogger.info("No content retrieved from maintained scope."); - continue; - } - - const suggestChangesResult = await callDocTrackerSuggestChangesAction( - auth, - { - watchedDocDiff: diffString, - maintainedDocContent: content, - prompt: tracker.prompt, - providerId: tracker.providerId, - modelId: tracker.modelId, + const maintainedDocuments: { + content: string; + sourceUrl: string | null; + title: string | null; + dataSourceId: string; + documentId: string; + }[] = []; + + // For each document retrieved from the maintained scope, we build the content of the document. + // We add "[...]" separators when there is a gap in the chunks (so the model understands that parts of the document are missing). + for (const retrievalDoc of maintainedScopeRetrieval) { + let docContent: string = ""; + const sortedChunks = _.sortBy(retrievalDoc.chunks, (c) => c.offset); + + for (const [i, chunk] of sortedChunks.entries()) { + if (i === 0) { + // If we are at index 0 (i.e the first retrieved chunk), we check whether our chunk includes + // the beginning of the document. If it doesn't, we add a "[...]"" separator. + const allOffsetsInChunk = [ + chunk.offset, + ...(chunk.expanded_offsets ?? []), + ]; + const isBeginningOfDocument = allOffsetsInChunk.includes(0); + if (!isBeginningOfDocument) { + docContent += "[...]\n"; + } + } else { + // If we are not at index 0, we check whether the current chunk is a direct continuation of the previous chunk. + // We do this by checking that the first offset of the current chunk is the last offset of the previous chunk + 1. + const previousChunk = sortedChunks[i - 1]; + const allOffsetsInCurrentChunk = [ + chunk.offset, + ...(chunk.expanded_offsets ?? []), + ]; + const firstOffsetInCurrentChunk = _.min(allOffsetsInCurrentChunk)!; + const allOffsetsInPreviousChunk = [ + previousChunk.offset, + ...(previousChunk.expanded_offsets ?? []), + ]; + const lastOffsetInPreviousChunk = _.max(allOffsetsInPreviousChunk)!; + const hasGap = + firstOffsetInCurrentChunk !== lastOffsetInPreviousChunk + 1; + + if (hasGap) { + docContent += "[...]\n"; + } + } + + // Add the chunk text to the document. + docContent += chunk.text + "\n"; + + if (i === sortedChunks.length - 1) { + // If we are at the last chunk, we check if we have the last offset of the doc. + // If not, we add a "[...]" separator. + const lastChunk = sortedChunks[sortedChunks.length - 1]; + if (lastChunk.offset !== retrievalDoc.chunk_count - 1) { + docContent += "[...]\n"; + } + } } - ); - if (!suggestChangesResult.suggestion) { - trackerLogger.info("No changes suggested."); - continue; + maintainedDocuments.push({ + content: docContent, + sourceUrl: retrievalDoc.source_url, + title: retrievalDoc.title, + dataSourceId: retrievalDoc.data_source_id, + documentId: retrievalDoc.document_id, + }); } - const suggestedChanges = suggestChangesResult.suggestion; - const thinking = suggestChangesResult.thinking; - const confidenceScore = suggestChangesResult.confidence_score; - - trackerLogger.info( - { - confidenceScore, - }, - "Changes suggested." + const contentByDocumentIdentifier = _.mapValues( + _.keyBy( + maintainedDocuments, + (doc) => `${doc.dataSourceId}__${doc.documentId}` + ), + (doc) => doc.content ); - await tracker.addGeneration({ - generation: suggestedChanges, - thinking: thinking ?? null, - dataSourceId, - documentId, + // We find documents for which to run the change suggestion. + // We do this by asking which documents are most relevant to the diff and using the + // logprobs as a score. + const scoreDocsResult = await callDocTrackerScoreDocsAction(auth, { + watchedDocDiff: diffString, + maintainedDocuments, + prompt: tracker.prompt, + providerId: TRACKER_SCORE_DOCS_MODEL_CONFIG.providerId, + modelId: TRACKER_SCORE_DOCS_MODEL_CONFIG.modelId, }); + + // The output of the Dust App above is a list of document for which we want to run the change suggestion. + + for (const { + documentId: maintainedDocumentId, + dataSourceId: maintainedDataSourceId, + score, + } of scoreDocsResult) { + logger.info( + { + maintainedDocumentId, + maintainedDataSourceId, + score, + }, + "Running document tracker suggest changes." + ); + + const content = + contentByDocumentIdentifier[ + `${maintainedDataSourceId}__${maintainedDocumentId}` + ]; + if (!content) { + continue; + } + + const suggestChangesResult = await callDocTrackerSuggestChangesAction( + auth, + { + watchedDocDiff: diffString, + maintainedDocContent: content, + prompt: tracker.prompt, + providerId: tracker.providerId, + modelId: tracker.modelId, + } + ); + + if (!suggestChangesResult.suggestion) { + trackerLogger.info("No changes suggested."); + continue; + } + + const maintainedDocumentDataSource = + await DataSourceResource.fetchByDustAPIDataSourceId( + auth, + maintainedDataSourceId + ); + if (!maintainedDocumentDataSource) { + throw new Error( + `Could not find maintained data source ${maintainedDataSourceId}` + ); + } + + const suggestedChanges = suggestChangesResult.suggestion; + const thinking = suggestChangesResult.thinking; + const confidenceScore = suggestChangesResult.confidence_score; + + trackerLogger.info( + { + confidenceScore, + }, + "Changes suggested." + ); + + await tracker.addGeneration({ + generation: suggestedChanges, + thinking: thinking ?? null, + dataSourceId, + documentId, + maintainedDocumentDataSourceId: maintainedDocumentDataSource.sId, + maintainedDocumentId, + }); + } } } -export async function shouldRunTrackersActivity( - workspaceId: string, - dataSourceId: string, - documentId: string -): Promise { +export async function shouldRunTrackersActivity({ + workspaceId, + dataSourceId, + documentId, + dataSourceConnectorProvider, +}: { + workspaceId: string; + dataSourceId: string; + documentId: string; + dataSourceConnectorProvider: ConnectorProvider | null; +}): Promise { + if ( + dataSourceConnectorProvider === "slack" && + !documentId.includes("-thread-") + ) { + // Special case for Slack -- we only run trackers for threads (and not for "non-threaded messages" + // otherwise we end up running the tracker twice a a thread is initially a non-threaded message. + return false; + } + const auth = await Authenticator.internalBuilderForWorkspace(workspaceId); const dataSource = await DataSourceResource.fetchById(auth, dataSourceId); @@ -315,18 +485,19 @@ async function getTrackersToRun( config.getConnectorsAPIConfig(), logger ); - const parentsResult = await connectorsAPI.getContentNodesParents({ + const parentsResult = await connectorsAPI.getContentNodes({ connectorId: dataSource.connectorId, internalIds: [documentId], + includeParents: true, }); if (parentsResult.isErr()) { throw parentsResult.error; } - docParentIds = [ + docParentIds = removeNulls([ documentId, - ...parentsResult.value.nodes.flatMap((node) => node.parents), - ]; + ...parentsResult.value.nodes.flatMap((node) => node.parentInternalIds), + ]); } return TrackerConfigurationResource.fetchAllWatchedForDocument(auth, { diff --git a/front/temporal/tracker/config.ts b/front/temporal/tracker/config.ts index 0aab30c95634..50a9db704e3e 100644 --- a/front/temporal/tracker/config.ts +++ b/front/temporal/tracker/config.ts @@ -1,4 +1,4 @@ -const RUN_QUEUE_VERSION = 2; +const RUN_QUEUE_VERSION = 3; export const RUN_QUEUE_NAME = `document-tracker-queue-v${RUN_QUEUE_VERSION}`; const TRACKER_NOTIFICATION_QUEUE_VERSION = 1; diff --git a/front/temporal/tracker/workflows.ts b/front/temporal/tracker/workflows.ts index 68538eec4733..998c315d29bf 100644 --- a/front/temporal/tracker/workflows.ts +++ b/front/temporal/tracker/workflows.ts @@ -39,17 +39,18 @@ export async function trackersGenerationWorkflow( documentHash: string, dataSourceConnectorProvider: ConnectorProvider | null ) { - let signaled = true; + let lastUpsertAt = Date.now(); setHandler(newUpsertSignal, () => { - signaled = true; + lastUpsertAt = Date.now(); }); - const shouldRun = await shouldRunTrackersActivity( + const shouldRun = await shouldRunTrackersActivity({ workspaceId, dataSourceId, - documentId - ); + documentId, + dataSourceConnectorProvider, + }); if (!shouldRun) { return; @@ -57,22 +58,21 @@ export async function trackersGenerationWorkflow( const debounceMs = await getDebounceMsActivity(dataSourceConnectorProvider); - while (signaled) { - signaled = false; - await sleep(debounceMs); - - if (signaled) { - continue; - } + function getSleepTime() { + return Math.max(0, lastUpsertAt + debounceMs - Date.now()); + } - await trackersGenerationActivity( - workspaceId, - dataSourceId, - documentId, - documentHash, - dataSourceConnectorProvider - ); + while (getSleepTime() > 0) { + await sleep(getSleepTime()); } + + await trackersGenerationActivity( + workspaceId, + dataSourceId, + documentId, + documentHash, + dataSourceConnectorProvider + ); } /** diff --git a/init_dev_container.sh b/init_dev_container.sh index d3a42eabf2db..cb27a25ffc06 100755 --- a/init_dev_container.sh +++ b/init_dev_container.sh @@ -14,5 +14,5 @@ cd - ## Initializing Elasticsearch indices cd core/ -cargo run --bin elasticsearch_create_index -- --index-name data_sources_nodes --index-version 1 --skip-confirmation +cargo run --bin elasticsearch_create_index -- --index-name data_sources_nodes --index-version 2 --skip-confirmation cd - diff --git a/sdks/js/package-lock.json b/sdks/js/package-lock.json index 042f0a7fbb70..f5af04bb203c 100644 --- a/sdks/js/package-lock.json +++ b/sdks/js/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dust-tt/client", - "version": "1.0.22", + "version": "1.0.23", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dust-tt/client", - "version": "1.0.22", + "version": "1.0.23", "license": "ISC", "dependencies": { "axios": "^1.7.9", diff --git a/sdks/js/package.json b/sdks/js/package.json index e7a2061b790f..c74c731e563d 100644 --- a/sdks/js/package.json +++ b/sdks/js/package.json @@ -1,6 +1,6 @@ { "name": "@dust-tt/client", - "version": "1.0.22", + "version": "1.0.23", "description": "Client for Dust API", "repository": { "type": "git", diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index f18a9a84fbc9..c9a9928b6090 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -43,6 +43,8 @@ const ModelLLMIdSchema = FlexibleEnumSchema< | "codestral-latest" | "gemini-1.5-pro-latest" | "gemini-1.5-flash-latest" + | "gemini-2.0-flash-exp" + | "gemini-2.0-flash-thinking-exp-1219" | "meta-llama/Llama-3.3-70B-Instruct-Turbo" | "Qwen/Qwen2.5-Coder-32B-Instruct" | "Qwen/QwQ-32B-Preview" @@ -666,6 +668,7 @@ const WhitelistableFeaturesSchema = FlexibleEnumSchema< | "openai_o1_custom_assistants_feature" | "openai_o1_high_reasoning_custom_assistants_feature" | "deepseek_feature" + | "google_ai_studio_experimental_models_feature" | "snowflake_connector_feature" | "index_private_slack_channel" | "conversations_jit_actions" @@ -781,6 +784,8 @@ export type AgentConfigurationViewType = z.infer< const AgentUsageTypeSchema = z.object({ messageCount: z.number(), + conversationCount: z.number(), + userCount: z.number(), timePeriodSec: z.number(), }); @@ -945,6 +950,7 @@ export type ConversationVisibility = z.infer< const ConversationWithoutContentSchema = z.object({ id: ModelIdSchema, created: z.number(), + updated: z.number().optional(), owner: WorkspaceSchema, sId: z.string(), title: z.string().nullable(), diff --git a/sparkle/package-lock.json b/sparkle/package-lock.json index 3b99af702f20..5ee3879a3bd6 100644 --- a/sparkle/package-lock.json +++ b/sparkle/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dust-tt/sparkle", - "version": "0.2.361", + "version": "0.2.363", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dust-tt/sparkle", - "version": "0.2.361", + "version": "0.2.363", "license": "ISC", "dependencies": { "@emoji-mart/data": "^1.1.2", diff --git a/sparkle/package.json b/sparkle/package.json index 0ced0c514a22..19cb50bd0fe6 100644 --- a/sparkle/package.json +++ b/sparkle/package.json @@ -1,6 +1,6 @@ { "name": "@dust-tt/sparkle", - "version": "0.2.361", + "version": "0.2.363", "scripts": { "build": "rm -rf dist && npm run tailwind && npm run build:esm && npm run build:cjs", "tailwind": "tailwindcss -i ./src/styles/tailwind.css -o dist/sparkle.css", diff --git a/sparkle/src/components/DataTable.tsx b/sparkle/src/components/DataTable.tsx index 97513b86f810..6de17110ab46 100644 --- a/sparkle/src/components/DataTable.tsx +++ b/sparkle/src/components/DataTable.tsx @@ -363,7 +363,14 @@ DataTable.Row = function Row({
{moreMenuItems && moreMenuItems.length > 0 && ( - + { + event.stopPropagation(); + }} + asChild + > {moreMenuItems?.map((item, index) => ( - + { + event.stopPropagation(); + item.onClick?.(event); + }} + /> ))} diff --git a/sparkle/src/components/index.ts b/sparkle/src/components/index.ts index db73d0adc667..ba27ba1a9f07 100644 --- a/sparkle/src/components/index.ts +++ b/sparkle/src/components/index.ts @@ -61,8 +61,6 @@ export { ElementModal } from "./ElementModal"; export type { EmojiMartData } from "./EmojiPicker"; export { DataEmojiMart, EmojiPicker } from "./EmojiPicker"; export { EmptyCTA, EmptyCTAButton } from "./EmptyCTA"; -export type { FeedbackSelectorProps } from "./FeedbackSelector"; -export { FeedbackSelector } from "./FeedbackSelector"; export { FilterChips } from "./FilterChips"; export { Div3D, Hover3D } from "./Hover3D"; export { Hoverable } from "./Hoverable"; diff --git a/sparkle/src/stories/DataTable.stories.tsx b/sparkle/src/stories/DataTable.stories.tsx index ec4c786e6ebe..56c6c9d759ef 100644 --- a/sparkle/src/stories/DataTable.stories.tsx +++ b/sparkle/src/stories/DataTable.stories.tsx @@ -201,6 +201,7 @@ const columns: ColumnDef[] = [ }, ]; +// TODO: Fix 'Edit' changing the order of the rows export const DataTableExample = () => { const [filter, setFilter] = React.useState(""); const [dialogOpen, setDialogOpen] = React.useState(false); diff --git a/sparkle/src/stories/FeedbackSelector.stories.tsx b/sparkle/src/stories/FeedbackSelector.stories.tsx deleted file mode 100644 index f47c75fba94f..000000000000 --- a/sparkle/src/stories/FeedbackSelector.stories.tsx +++ /dev/null @@ -1,72 +0,0 @@ -import type { Meta, StoryObj } from "@storybook/react"; -import React from "react"; - -import { - FeedbackSelector, - FeedbackSelectorProps, -} from "@sparkle/components/FeedbackSelector"; - -const meta = { - title: "Primitives/FeedbackSelector", - component: FeedbackSelector, - argTypes: { - feedback: { - thumb: "up", - feedbackContent: null, - }, - onSubmitThumb: { - description: "The submit function", - control: { - type: "object", - }, - }, - isSubmittingThumb: { - description: "Whether the thumb is submitting", - control: { - type: "boolean", - }, - }, - }, -} satisfies Meta>; - -export default meta; -type Story = StoryObj; - -// Wrap the story in a component that can use hooks -const ExampleFeedbackComponent = () => { - const [messageFeedback, setMessageFeedback] = - React.useState({ - feedback: null, - onSubmitThumb: async (element) => { - setMessageFeedback((prev) => ({ - ...prev, - feedback: element.shouldRemoveExistingFeedback - ? null - : { - thumb: element.thumb, - feedbackContent: element.feedbackContent, - isConversationShared: element.isConversationShared, - }, - })); - }, - isSubmittingThumb: false, - getPopoverInfo: () =>
Some info here, like the last author
, - }); - - return ; -}; - -export const ExamplePicker: Story = { - args: { - feedback: { - thumb: "up", - feedbackContent: null, - isConversationShared: true, - }, - onSubmitThumb: async (element) => { - console.log(element); - }, - isSubmittingThumb: false, - }, - render: () => , -}; diff --git a/types/src/connectors/admin/cli.ts b/types/src/connectors/admin/cli.ts index b394748e37a3..a432100100e2 100644 --- a/types/src/connectors/admin/cli.ts +++ b/types/src/connectors/admin/cli.ts @@ -262,6 +262,7 @@ export const ZendeskCommandSchema = t.type({ t.literal("count-tickets"), t.literal("resync-tickets"), t.literal("fetch-ticket"), + t.literal("fetch-brand"), ]), args: t.type({ connectorId: t.union([t.number, t.undefined]), @@ -303,6 +304,14 @@ export const ZendeskFetchTicketResponseSchema = t.type({ export type ZendeskFetchTicketResponseType = t.TypeOf< typeof ZendeskFetchTicketResponseSchema >; + +export const ZendeskFetchBrandResponseSchema = t.type({ + brand: t.union([t.UnknownRecord, t.null]), // Zendesk type, can't be iots'd, + brandOnDb: t.union([t.UnknownRecord, t.null]), +}); +export type ZendeskFetchBrandResponseType = t.TypeOf< + typeof ZendeskFetchBrandResponseSchema +>; /** * */ @@ -465,6 +474,7 @@ export const AdminResponseSchema = t.union([ ZendeskCountTicketsResponseSchema, ZendeskResyncTicketsResponseSchema, ZendeskFetchTicketResponseSchema, + ZendeskFetchBrandResponseSchema, ]); export type AdminResponseType = t.TypeOf; diff --git a/types/src/front/api_handlers/public/spaces.ts b/types/src/front/api_handlers/public/spaces.ts index c52214c24d30..cd1a50f1b3b0 100644 --- a/types/src/front/api_handlers/public/spaces.ts +++ b/types/src/front/api_handlers/public/spaces.ts @@ -21,7 +21,6 @@ export type PatchDataSourceViewType = t.TypeOf< >; export type LightContentNode = { - dustDocumentId: string | null; expandable: boolean; internalId: string; lastUpdatedAt: number | null; @@ -29,7 +28,6 @@ export type LightContentNode = { preventSelection?: boolean; sourceUrl: string | null; title: string; - titleWithParentsContext?: string; type: ContentNodeType; }; diff --git a/types/src/front/assistant/agent.ts b/types/src/front/assistant/agent.ts index 4fe2a5195b05..420e6627ed90 100644 --- a/types/src/front/assistant/agent.ts +++ b/types/src/front/assistant/agent.ts @@ -173,6 +173,8 @@ export type AgentsGetViewType = export type AgentUsageType = { messageCount: number; + conversationCount: number; + userCount: number; timePeriodSec: number; }; diff --git a/types/src/front/data_source_view.ts b/types/src/front/data_source_view.ts index 392d763b4d28..82fc37c116cf 100644 --- a/types/src/front/data_source_view.ts +++ b/types/src/front/data_source_view.ts @@ -6,7 +6,7 @@ import { DataSourceWithAgentsUsageType, EditedByUser, } from "./data_source"; -import { BaseContentNode } from "./lib/connectors_api"; +import { ContentNode } from "./lib/connectors_api"; export interface DataSourceViewType { category: DataSourceViewCategory; @@ -26,7 +26,7 @@ export type DataSourceViewsWithDetails = DataSourceViewType & { usage: DataSourceWithAgentsUsageType; }; -export type DataSourceViewContentNode = BaseContentNode & { +export type DataSourceViewContentNode = ContentNode & { parentInternalIds: string[] | null; }; diff --git a/types/src/front/lib/assistant.ts b/types/src/front/lib/assistant.ts index 6a5bdd1414c4..5647caf1352e 100644 --- a/types/src/front/lib/assistant.ts +++ b/types/src/front/lib/assistant.ts @@ -114,6 +114,9 @@ export const MISTRAL_CODESTRAL_MODEL_ID = "codestral-latest" as const; export const GEMINI_1_5_PRO_LATEST_MODEL_ID = "gemini-1.5-pro-latest" as const; export const GEMINI_1_5_FLASH_LATEST_MODEL_ID = "gemini-1.5-flash-latest" as const; +export const GEMINI_2_FLASH_PREVIEW_MODEL_ID = "gemini-2.0-flash-exp" as const; +export const GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_ID = + "gemini-2.0-flash-thinking-exp-1219" as const; export const TOGETHERAI_LLAMA_3_3_70B_INSTRUCT_TURBO_MODEL_ID = "meta-llama/Llama-3.3-70B-Instruct-Turbo" as const; export const TOGETHERAI_QWEN_2_5_CODER_32B_INSTRUCT_MODEL_ID = @@ -145,6 +148,8 @@ export const MODEL_IDS = [ MISTRAL_CODESTRAL_MODEL_ID, GEMINI_1_5_PRO_LATEST_MODEL_ID, GEMINI_1_5_FLASH_LATEST_MODEL_ID, + GEMINI_2_FLASH_PREVIEW_MODEL_ID, + GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_ID, TOGETHERAI_LLAMA_3_3_70B_INSTRUCT_TURBO_MODEL_ID, TOGETHERAI_QWEN_2_5_CODER_32B_INSTRUCT_MODEL_ID, TOGETHERAI_QWEN_32B_PREVIEW_MODEL_ID, @@ -570,6 +575,39 @@ export const GEMINI_FLASH_DEFAULT_MODEL_CONFIG: ModelConfigurationType = { supportsVision: false, }; +export const GEMINI_2_FLASH_PREVIEW_MODEL_CONFIG: ModelConfigurationType = { + providerId: "google_ai_studio", + modelId: GEMINI_2_FLASH_PREVIEW_MODEL_ID, + displayName: "Gemini Flash 2.0", + contextSize: 1_000_000, + recommendedTopK: 64, + recommendedExhaustiveTopK: 128, + largeModel: true, + description: + "Google's lightweight, fast and cost-efficient model (1m context).", + shortDescription: "Google's cost-effective model (preview).", + isLegacy: false, + supportsVision: true, + featureFlag: "google_ai_studio_experimental_models_feature", +}; + +export const GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_CONFIG: ModelConfigurationType = + { + providerId: "google_ai_studio", + modelId: GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_ID, + displayName: "Gemini Flash 2.0 Thinking", + contextSize: 32_000, + recommendedTopK: 64, + recommendedExhaustiveTopK: 128, + largeModel: true, + description: + "Google's lightweight model optimized for reasoning (1m context).", + shortDescription: "Google's reasoning-focused model (preview).", + isLegacy: false, + supportsVision: true, + featureFlag: "google_ai_studio_experimental_models_feature", + }; + export const TOGETHERAI_LLAMA_3_3_70B_INSTRUCT_TURBO_MODEL_CONFIG: ModelConfigurationType = { providerId: "togetherai", @@ -667,6 +705,8 @@ export const SUPPORTED_MODEL_CONFIGS: ModelConfigurationType[] = [ MISTRAL_CODESTRAL_MODEL_CONFIG, GEMINI_PRO_DEFAULT_MODEL_CONFIG, GEMINI_FLASH_DEFAULT_MODEL_CONFIG, + GEMINI_2_FLASH_PREVIEW_MODEL_CONFIG, + GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_CONFIG, TOGETHERAI_LLAMA_3_3_70B_INSTRUCT_TURBO_MODEL_CONFIG, TOGETHERAI_QWEN_2_5_CODER_32B_INSTRUCT_MODEL_CONFIG, TOGETHERAI_QWEN_32B_PREVIEW_MODEL_CONFIG, diff --git a/types/src/front/lib/connectors_api.ts b/types/src/front/lib/connectors_api.ts index a6d6ca64ea28..863598f6bbe0 100644 --- a/types/src/front/lib/connectors_api.ts +++ b/types/src/front/lib/connectors_api.ts @@ -96,26 +96,20 @@ export const contentNodeTypeSortOrder: Record = { * information. More details here: * https://www.notion.so/dust-tt/Design-Doc-Microsoft-ids-parents-c27726652aae45abafaac587b971a41d?pvs=4 */ -export interface BaseContentNode { +export interface ContentNode { internalId: string; // The direct parent ID of this content node parentInternalId: string | null; type: ContentNodeType; title: string; - titleWithParentsContext?: string; sourceUrl: string | null; expandable: boolean; preventSelection?: boolean; permission: ConnectorPermission; - dustDocumentId: string | null; lastUpdatedAt: number | null; providerVisibility?: "public" | "private"; } -export type ContentNode = BaseContentNode & { - provider: ConnectorProvider; -}; - export type ContentNodeWithParentIds = ContentNode & { // A list of all parent IDs up to the root node, including the direct parent // Note: When includeParents is true, this list will be populated @@ -503,36 +497,6 @@ export class ConnectorsAPI { return this._resultFromResponse(res); } - async getContentNodesParents({ - connectorId, - internalIds, - }: { - connectorId: string; - internalIds: string[]; - }): Promise< - ConnectorsAPIResponse<{ - nodes: { - internalId: string; - parents: string[]; - }[]; - }> - > { - const res = await this._fetchWithError( - `${this._url}/connectors/${encodeURIComponent( - connectorId - )}/content_nodes/parents`, - { - method: "POST", - headers: this.getDefaultHeaders(), - body: JSON.stringify({ - internalIds, - }), - } - ); - - return this._resultFromResponse(res); - } - async getContentNodes({ connectorId, includeParents, diff --git a/types/src/shared/feature_flags.ts b/types/src/shared/feature_flags.ts index 6c9ffb5d82ec..e9c2d47200cf 100644 --- a/types/src/shared/feature_flags.ts +++ b/types/src/shared/feature_flags.ts @@ -12,6 +12,7 @@ export const WHITELISTABLE_FEATURES = [ "openai_o1_custom_assistants_feature", "openai_o1_high_reasoning_custom_assistants_feature", "deepseek_feature", + "google_ai_studio_experimental_models_feature", "index_private_slack_channel", "conversations_jit_actions", "disable_run_logs", diff --git a/types/src/shared/internal_mime_types.ts b/types/src/shared/internal_mime_types.ts index 959afb2dd03a..f358b741293d 100644 --- a/types/src/shared/internal_mime_types.ts +++ b/types/src/shared/internal_mime_types.ts @@ -1,98 +1,152 @@ -export const CONFLUENCE_MIME_TYPES = { - SPACE: "application/vnd.dust.confluence.space", - PAGE: "application/vnd.dust.confluence.page", +import { ConnectorProvider } from "../front/data_source"; + +/** + * This is a utility type that indicates that we removed all underscores from a string. + * This is used because we don't want underscores in mime types and remove them from connector providers. + */ +type WithoutUnderscores = T extends `${infer A}_${infer B}` + ? WithoutUnderscores<`${A}${B}`> // operates recursively to remove all underscores + : T; + +/** + * This is a utility type that indicates that we replaced all underscores with dashes in a string. + * We don't want underscores in mime types but want to type out the type with one: MIME_TYPE.CAT.SOU_PI_NOU + */ +type UnderscoreToDash = T extends `${infer A}_${infer B}` + ? UnderscoreToDash<`${A}-${B}`> // operates recursively to replace all underscores + : T; + +/** + * This function generates mime types for a given provider and resource types. + * The mime types are in the format `application/vnd.dust.PROVIDER.RESOURCE_TYPE`. + * Notes: + * - The underscores in the provider name are stripped in the generated mime type. + * - The underscores in the resource type are replaced with dashes in the generated mime type. + */ +function getMimeTypes< + P extends ConnectorProvider, + T extends Uppercase[] +>({ + provider, + resourceTypes, +}: { + provider: P; + resourceTypes: T; +}): { + [K in T[number]]: `application/vnd.dust.${WithoutUnderscores

}.${Lowercase< + UnderscoreToDash + >}`; +} { + return resourceTypes.reduce( + (acc, s) => ({ + ...acc, + [s]: `application/vnd.dust.${provider.replace("_", "")}.${s + .replace("_", "-") + .toLowerCase()}`, + }), + {} as { + [K in T[number]]: `application/vnd.dust.${WithoutUnderscores

}.${Lowercase< + UnderscoreToDash + >}`; + } + ); +} + +export const MIME_TYPES = { + CONFLUENCE: getMimeTypes({ + provider: "confluence", + resourceTypes: ["SPACE", "PAGE"], + }), + GITHUB: getMimeTypes({ + provider: "github", + resourceTypes: [ + "REPOSITORY", + "CODE_ROOT", + "CODE_DIRECTORY", + "CODE_FILE", + "ISSUES", + "ISSUE", + "DISCUSSIONS", + "DISCUSSION", + ], + }), + GOOGLE_DRIVE: getMimeTypes({ + provider: "google_drive", + resourceTypes: ["FOLDER"], // for files and spreadsheets, we keep Google's mime types + }), + INTERCOM: getMimeTypes({ + provider: "intercom", + resourceTypes: [ + "COLLECTION", + "TEAMS_FOLDER", + "CONVERSATION", + "TEAM", + "HELP_CENTER", + "ARTICLE", + ], + }), + MICROSOFT: getMimeTypes({ + provider: "microsoft", + resourceTypes: ["FOLDER"], // for files and spreadsheets, we keep Microsoft's mime types + }), + NOTION: getMimeTypes({ + provider: "notion", + resourceTypes: ["UNKNOWN_FOLDER", "DATABASE", "PAGE"], + }), + SLACK: getMimeTypes({ + provider: "slack", + resourceTypes: ["CHANNEL", "THREAD", "MESSAGES"], + }), + SNOWFLAKE: getMimeTypes({ + provider: "snowflake", + resourceTypes: ["DATABASE", "SCHEMA", "TABLE"], + }), + WEBCRAWLER: getMimeTypes({ + provider: "webcrawler", + resourceTypes: ["FOLDER"], // pages are upserted as text/html, not an internal mime type + }), + ZENDESK: getMimeTypes({ + provider: "zendesk", + resourceTypes: [ + "BRAND", + "HELP_CENTER", + "CATEGORY", + "ARTICLE", + "TICKETS", + "TICKET", + ], + }), }; export type ConfluenceMimeType = - (typeof CONFLUENCE_MIME_TYPES)[keyof typeof CONFLUENCE_MIME_TYPES]; - -export const GITHUB_MIME_TYPES = { - REPOSITORY: "application/vnd.dust.github.repository", - CODE_ROOT: "application/vnd.dust.github.code.root", - CODE_DIRECTORY: "application/vnd.dust.github.code.directory", - CODE_FILE: "application/vnd.dust.github.code.file", - ISSUES: "application/vnd.dust.github.issues", - ISSUE: "application/vnd.dust.github.issue", - DISCUSSIONS: "application/vnd.dust.github.discussions", - DISCUSSION: "application/vnd.dust.github.discussion", -}; + (typeof MIME_TYPES.CONFLUENCE)[keyof typeof MIME_TYPES.CONFLUENCE]; export type GithubMimeType = - (typeof GITHUB_MIME_TYPES)[keyof typeof GITHUB_MIME_TYPES]; - -export const GOOGLE_DRIVE_MIME_TYPES = { - FOLDER: "application/vnd.dust.googledrive.folder", - // for files and spreadsheets, we keep Google's mime types -}; + (typeof MIME_TYPES.GITHUB)[keyof typeof MIME_TYPES.GITHUB]; export type GoogleDriveMimeType = - (typeof GOOGLE_DRIVE_MIME_TYPES)[keyof typeof GOOGLE_DRIVE_MIME_TYPES]; - -export const INTERCOM_MIME_TYPES = { - COLLECTION: "application/vnd.dust.intercom.collection", - CONVERSATIONS: "application/vnd.dust.intercom.teams-folder", - CONVERSATION: "application/vnd.dust.intercom.conversation", - TEAM: "application/vnd.dust.intercom.team", - HELP_CENTER: "application/vnd.dust.intercom.help-center", - ARTICLE: "application/vnd.dust.intercom.article", -}; + (typeof MIME_TYPES.GOOGLE_DRIVE)[keyof typeof MIME_TYPES.GOOGLE_DRIVE]; export type IntercomMimeType = - (typeof INTERCOM_MIME_TYPES)[keyof typeof INTERCOM_MIME_TYPES]; - -export const MICROSOFT_MIME_TYPES = { - FOLDER: "application/vnd.dust.microsoft.folder", - // for files and spreadsheets, we keep Microsoft's mime types -}; + (typeof MIME_TYPES.INTERCOM)[keyof typeof MIME_TYPES.INTERCOM]; export type MicrosoftMimeType = - (typeof MICROSOFT_MIME_TYPES)[keyof typeof MICROSOFT_MIME_TYPES]; - -export const NOTION_MIME_TYPES = { - UNKNOWN_FOLDER: "application/vnd.dust.notion.unknown-folder", - DATABASE: "application/vnd.dust.notion.database", - PAGE: "application/vnd.dust.notion.page", -}; + (typeof MIME_TYPES.MICROSOFT)[keyof typeof MIME_TYPES.MICROSOFT]; export type NotionMimeType = - (typeof NOTION_MIME_TYPES)[keyof typeof NOTION_MIME_TYPES]; - -export const SLACK_MIME_TYPES = { - CHANNEL: "application/vnd.dust.slack.channel", - THREAD: "text/vnd.dust.slack.thread", -}; + (typeof MIME_TYPES.NOTION)[keyof typeof MIME_TYPES.NOTION]; export type SlackMimeType = - (typeof SLACK_MIME_TYPES)[keyof typeof SLACK_MIME_TYPES]; - -export const SNOWFLAKE_MIME_TYPES = { - DATABASE: "application/vnd.snowflake.database", - SCHEMA: "application/vnd.snowflake.schema", - TABLE: "application/vnd.snowflake.table", -}; + (typeof MIME_TYPES.SLACK)[keyof typeof MIME_TYPES.SLACK]; export type SnowflakeMimeType = - (typeof SNOWFLAKE_MIME_TYPES)[keyof typeof SNOWFLAKE_MIME_TYPES]; - -export const WEBCRAWLER_MIME_TYPES = { - FOLDER: "application/vnd.dust.webcrawler.folder", - // pages are upserted as text/html, not an internal mime type -}; + (typeof MIME_TYPES.SNOWFLAKE)[keyof typeof MIME_TYPES.SNOWFLAKE]; export type WebcrawlerMimeType = - (typeof WEBCRAWLER_MIME_TYPES)[keyof typeof WEBCRAWLER_MIME_TYPES]; - -export const ZENDESK_MIME_TYPES = { - BRAND: "application/vnd.dust.zendesk.brand", - HELP_CENTER: "application/vnd.dust.zendesk.helpcenter", - CATEGORY: "application/vnd.dust.zendesk.category", - ARTICLE: "application/vnd.dust.zendesk.article", - TICKETS: "application/vnd.dust.zendesk.tickets", - TICKET: "application/vnd.dust.zendesk.ticket", -}; + (typeof MIME_TYPES.WEBCRAWLER)[keyof typeof MIME_TYPES.WEBCRAWLER]; export type ZendeskMimeType = - (typeof ZENDESK_MIME_TYPES)[keyof typeof ZENDESK_MIME_TYPES]; + (typeof MIME_TYPES.ZENDESK)[keyof typeof MIME_TYPES.ZENDESK]; export type DustMimeType = | ConfluenceMimeType