diff --git a/connectors/migrations/20241211_fix_gdrive_parents.ts b/connectors/migrations/20241211_fix_gdrive_parents.ts index ecbe198c59fb..728cbbfac27e 100644 --- a/connectors/migrations/20241211_fix_gdrive_parents.ts +++ b/connectors/migrations/20241211_fix_gdrive_parents.ts @@ -1,4 +1,4 @@ -import { getGoogleSheetTableId } from "@dust-tt/types"; +import { concurrentExecutor, getGoogleSheetTableId } from "@dust-tt/types"; import { makeScript } from "scripts/helpers"; import { Op } from "sequelize"; @@ -19,6 +19,8 @@ import logger from "@connectors/logger/logger"; import { ConnectorModel } from "@connectors/resources/storage/models/connector_model"; const QUERY_BATCH_SIZE = 1024; +const DOCUMENT_CONCURRENCY = 16; +const TABLE_CONCURRENCY = 16; function getParents( fileId: string | null, @@ -88,67 +90,71 @@ async function migrate({ } ); - for (const file of googleDriveFiles) { - const internalId = file.dustFileId; - const driveFileId = file.driveFileId; - const parents = getParents( - file.parentId, - parentsMap, - childLogger.child({ nodeId: driveFileId }) - ); - if (!parents) { - continue; - } - parents.unshift(driveFileId); - - if (file.mimeType === "application/vnd.google-apps.folder") { - const folder = await getFolderNode({ - dataSourceConfig, - folderId: internalId, - }); - const newParents = parents.map((id) => getDocumentId(id)); - if (!folder || folder.parents.join("/") !== newParents.join("/")) { - childLogger.info( - { folderId: file.driveFileId, parents: newParents }, - "Upsert folder" - ); - - if (execute) { - // upsert repository as folder - await upsertFolderNode({ - dataSourceConfig, - folderId: file.dustFileId, - parents: newParents, - parentId: file.parentId ? getDocumentId(file.parentId) : null, - title: file.name, - }); - } + await concurrentExecutor( + googleDriveFiles, + async (file) => { + const internalId = file.dustFileId; + const driveFileId = file.driveFileId; + const parents = getParents( + file.parentId, + parentsMap, + childLogger.child({ nodeId: driveFileId }) + ); + if (!parents) { + return; } - } else if (file.mimeType === "text/csv") { - const tableId = internalId; - parents.unshift(...parents.map((id) => getDocumentId(id))); - const table = await getTable({ dataSourceConfig, tableId }); - if (table) { - if (table.parents.join("/") !== parents.join("/")) { + parents.unshift(driveFileId); + + if (file.mimeType === "application/vnd.google-apps.folder") { + const folder = await getFolderNode({ + dataSourceConfig, + folderId: internalId, + }); + const newParents = parents.map((id) => getDocumentId(id)); + if (!folder || folder.parents.join("/") !== newParents.join("/")) { childLogger.info( - { - tableId, - parents, - previousParents: table.parents, - }, - "Update parents for table" + { folderId: file.driveFileId, parents: newParents }, + "Upsert folder" ); + if (execute) { - await updateTableParentsField({ + // upsert repository as folder + await upsertFolderNode({ dataSourceConfig, - tableId, - parents, + folderId: file.dustFileId, + parents: newParents, + parentId: file.parentId ? getDocumentId(file.parentId) : null, + title: file.name, }); } } + } else if (file.mimeType === "text/csv") { + const tableId = internalId; + parents.unshift(...parents.map((id) => getDocumentId(id))); + const table = await getTable({ dataSourceConfig, tableId }); + if (table) { + if (table.parents.join("/") !== parents.join("/")) { + childLogger.info( + { + tableId, + parents, + previousParents: table.parents, + }, + "Update parents for table" + ); + if (execute) { + await updateTableParentsField({ + dataSourceConfig, + tableId, + parents, + }); + } + } + } } - } - } + }, + { concurrency: DOCUMENT_CONCURRENCY } + ); nextId = googleDriveFiles[googleDriveFiles.length - 1]?.id; } while (nextId); @@ -167,45 +173,49 @@ async function migrate({ limit: QUERY_BATCH_SIZE, }); - for (const sheet of googleDriveSheets) { - const tableId = getGoogleSheetTableId( - sheet.driveFileId, - sheet.driveSheetId - ); - - const parents = getParents( - sheet.driveFileId, - parentsMap, - childLogger.child({ nodeId: tableId }) - ); - if (!parents) { - continue; - } + await concurrentExecutor( + googleDriveSheets, + async (sheet) => { + const tableId = getGoogleSheetTableId( + sheet.driveFileId, + sheet.driveSheetId + ); - parents.unshift(...parents.map((id) => getDocumentId(id))); - parents.unshift(tableId); - - const table = await getTable({ dataSourceConfig, tableId }); - if (table) { - if (table.parents.join("/") !== parents.join("/")) { - childLogger.info( - { - tableId, - parents, - previousParents: table.parents, - }, - "Update parents for table" - ); - if (execute) { - await updateTableParentsField({ - dataSourceConfig, - tableId, - parents, - }); + const parents = getParents( + sheet.driveFileId, + parentsMap, + childLogger.child({ nodeId: tableId }) + ); + if (!parents) { + return; + } + + parents.unshift(...parents.map((id) => getDocumentId(id))); + parents.unshift(tableId); + + const table = await getTable({ dataSourceConfig, tableId }); + if (table) { + if (table.parents.join("/") !== parents.join("/")) { + childLogger.info( + { + tableId, + parents, + previousParents: table.parents, + }, + "Update parents for table" + ); + if (execute) { + await updateTableParentsField({ + dataSourceConfig, + tableId, + parents, + }); + } } } - } - } + }, + { concurrency: TABLE_CONCURRENCY } + ); nextId = googleDriveSheets[googleDriveSheets.length - 1]?.id; } while (nextId); diff --git a/connectors/migrations/20241216_backfill_confluence_folders.ts b/connectors/migrations/20241216_backfill_confluence_folders.ts new file mode 100644 index 000000000000..e2f3888091ab --- /dev/null +++ b/connectors/migrations/20241216_backfill_confluence_folders.ts @@ -0,0 +1,43 @@ +import { makeScript } from "scripts/helpers"; + +import { makeSpaceInternalId } from "@connectors/connectors/confluence/lib/internal_ids"; +import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config"; +import { concurrentExecutor } from "@connectors/lib/async_utils"; +import { upsertFolderNode } from "@connectors/lib/data_sources"; +import { ConfluenceSpace } from "@connectors/lib/models/confluence"; +import { ConnectorResource } from "@connectors/resources/connector_resource"; + +const FOLDER_CONCURRENCY = 10; + +makeScript({}, async ({ execute }, logger) => { + const connectors = await ConnectorResource.listByType("confluence", {}); + + for (const connector of connectors) { + const confluenceSpaces = await ConfluenceSpace.findAll({ + attributes: ["spaceId", "name"], + where: { connectorId: connector.id }, + }); + const dataSourceConfig = dataSourceConfigFromConnector(connector); + if (execute) { + await concurrentExecutor( + confluenceSpaces, + async (space) => { + await upsertFolderNode({ + dataSourceConfig, + folderId: makeSpaceInternalId(space.spaceId), + parents: [makeSpaceInternalId(space.spaceId)], + title: space.name, + }); + }, + { concurrency: FOLDER_CONCURRENCY } + ); + logger.info( + `Upserted ${confluenceSpaces.length} spaces for connector ${connector.id}` + ); + } else { + logger.info( + `Found ${confluenceSpaces.length} spaces for connector ${connector.id}` + ); + } + } +}); diff --git a/connectors/migrations/20241216_backfill_zendesk_folders.ts b/connectors/migrations/20241216_backfill_zendesk_folders.ts new file mode 100644 index 000000000000..f6e9d02318b1 --- /dev/null +++ b/connectors/migrations/20241216_backfill_zendesk_folders.ts @@ -0,0 +1,95 @@ +import { makeScript } from "scripts/helpers"; + +import { getBrandInternalId } from "@connectors/connectors/zendesk/lib/id_conversions"; +import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config"; +import { concurrentExecutor } from "@connectors/lib/async_utils"; +import { upsertFolderNode } from "@connectors/lib/data_sources"; +import { ConnectorResource } from "@connectors/resources/connector_resource"; +import { + ZendeskBrandResource, + ZendeskCategoryResource, +} from "@connectors/resources/zendesk_resources"; + +const FOLDER_CONCURRENCY = 10; + +makeScript({}, async ({ execute }, logger) => { + const connectors = await ConnectorResource.listByType("zendesk", {}); + + for (const connector of connectors) { + const dataSourceConfig = dataSourceConfigFromConnector(connector); + const connectorId = connector.id; + + const brands = await ZendeskBrandResource.fetchByConnector(connector); + if (execute) { + await concurrentExecutor( + brands, + async (brand) => { + /// same code as in the connector + const brandInternalId = getBrandInternalId({ + connectorId, + brandId: brand.brandId, + }); + await upsertFolderNode({ + dataSourceConfig, + folderId: brandInternalId, + parents: [brandInternalId], + title: brand.name, + }); + + const helpCenterNode = brand.getHelpCenterContentNode(connectorId); + await upsertFolderNode({ + dataSourceConfig, + folderId: helpCenterNode.internalId, + parents: [ + helpCenterNode.internalId, + helpCenterNode.parentInternalId, + ], + title: helpCenterNode.title, + }); + + const ticketsNode = brand.getTicketsContentNode(connectorId); + await upsertFolderNode({ + dataSourceConfig, + folderId: ticketsNode.internalId, + parents: [ticketsNode.internalId, ticketsNode.parentInternalId], + title: ticketsNode.title, + }); + }, + { concurrency: FOLDER_CONCURRENCY } + ); + logger.info( + `Upserted ${brands.length} spaces for connector ${connector.id}` + ); + } else { + logger.info( + `Found ${brands.length} spaces for connector ${connector.id}` + ); + } + + const categories = + await ZendeskCategoryResource.fetchByConnector(connector); + if (execute) { + await concurrentExecutor( + categories, + async (category) => { + /// same code as in the connector + const parents = category.getParentInternalIds(connectorId); + await upsertFolderNode({ + dataSourceConfig: dataSourceConfigFromConnector(connector), + folderId: parents[0], + parents, + title: category.name, + }); + }, + { concurrency: FOLDER_CONCURRENCY } + ); + logger.info( + `Upserted ${brands.length} spaces for connector ${connector.id}` + ); + } else { + logger.info( + `Found ${brands.length} spaces for connector ${connector.id}` + ); + } + } +}); diff --git a/connectors/src/connectors/confluence/temporal/activities.ts b/connectors/src/connectors/confluence/temporal/activities.ts index af5104ea2afd..d5aed2ca4d24 100644 --- a/connectors/src/connectors/confluence/temporal/activities.ts +++ b/connectors/src/connectors/confluence/temporal/activities.ts @@ -18,16 +18,21 @@ import { getConfluencePageParentIds, getSpaceHierarchy, } from "@connectors/connectors/confluence/lib/hierarchy"; -import { makePageInternalId } from "@connectors/connectors/confluence/lib/internal_ids"; +import { + makePageInternalId, + makeSpaceInternalId, +} from "@connectors/connectors/confluence/lib/internal_ids"; import { makeConfluenceDocumentUrl } from "@connectors/connectors/confluence/temporal/workflow_ids"; import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config"; import { concurrentExecutor } from "@connectors/lib/async_utils"; import type { UpsertToDataSourceParams } from "@connectors/lib/data_sources"; import { + deleteFolderNode, deleteFromDataSource, renderDocumentTitleAndContent, renderMarkdownSection, updateDocumentParentsField, + upsertFolderNode, upsertToDatasource, } from "@connectors/lib/data_sources"; import { @@ -196,6 +201,28 @@ export async function confluenceGetSpaceNameActivity({ } } +/** + * Upserts the page in data_sources_folders (core). + */ +export async function confluenceUpsertSpaceFolderActivity({ + connectorId, + spaceId, + spaceName, +}: { + connectorId: ModelId; + spaceId: string; + spaceName: string; +}) { + const connector = await fetchConfluenceConnector(connectorId); + + await upsertFolderNode({ + dataSourceConfig: dataSourceConfigFromConnector(connector), + folderId: makeSpaceInternalId(spaceId), + parents: [makeSpaceInternalId(spaceId)], + title: spaceName, + }); +} + export async function markPageHasVisited({ connectorId, pageId, @@ -814,6 +841,12 @@ export async function confluenceRemoveSpaceActivity( for (const page of allPages) { await deletePage(connectorId, page.pageId, dataSourceConfig); } + + // deleting the folder in data_source_folders (core) + await deleteFolderNode({ + dataSourceConfig, + folderId: makeSpaceInternalId(spaceId), + }); } export async function fetchConfluenceSpaceIdsForConnectorActivity({ diff --git a/connectors/src/connectors/confluence/temporal/workflows.ts b/connectors/src/connectors/confluence/temporal/workflows.ts index 47b260815274..1aed30e58646 100644 --- a/connectors/src/connectors/confluence/temporal/workflows.ts +++ b/connectors/src/connectors/confluence/temporal/workflows.ts @@ -37,6 +37,7 @@ const { fetchConfluenceUserAccountAndConnectorIdsActivity, fetchConfluenceConfigurationActivity, + confluenceUpsertSpaceFolderActivity, getSpaceIdsToSyncActivity, } = proxyActivities({ startToCloseTimeout: "30 minutes", @@ -150,6 +151,12 @@ export async function confluenceSpaceSyncWorkflow( return startConfluenceRemoveSpaceWorkflow(wInfo, connectorId, spaceId); } + await confluenceUpsertSpaceFolderActivity({ + connectorId, + spaceId, + spaceName, + }); + // Get the root level pages for the space. const rootPageRefs = await confluenceGetRootPageRefsActivity({ connectorId, diff --git a/connectors/src/connectors/notion/temporal/activities.ts b/connectors/src/connectors/notion/temporal/activities.ts index ddbeca0a270c..1e32cdc5e3af 100644 --- a/connectors/src/connectors/notion/temporal/activities.ts +++ b/connectors/src/connectors/notion/temporal/activities.ts @@ -2047,6 +2047,7 @@ export async function renderAndUpsertPageFromCache({ }), // TODO(kw_search) remove legacy parents: [...parentIds, ...legacyParentIds], + parentId: parentIds.length > 1 ? parentIds[1] : null, loggerArgs, upsertContext: { sync_type: isFullSync ? "batch" : "incremental", @@ -2603,6 +2604,7 @@ export async function upsertDatabaseStructuredDataFromCache({ tags: [`title:${databaseName}`, "is_database:true"], // TODO(kw_search) remove legacy parents: [databaseDocId, ...parentIds, ...legacyParentIds], + parentId: parentIds.length > 1 ? parentIds[1] : null, loggerArgs, upsertContext: { sync_type: "batch", diff --git a/connectors/src/connectors/zendesk/lib/sync_category.ts b/connectors/src/connectors/zendesk/lib/sync_category.ts index a0cf5bd120d3..dc5efe8b68e7 100644 --- a/connectors/src/connectors/zendesk/lib/sync_category.ts +++ b/connectors/src/connectors/zendesk/lib/sync_category.ts @@ -1,9 +1,16 @@ import type { ModelId } from "@dust-tt/types"; import type { ZendeskFetchedCategory } from "@connectors/@types/node-zendesk"; -import { getArticleInternalId } from "@connectors/connectors/zendesk/lib/id_conversions"; +import { + getArticleInternalId, + getCategoryInternalId, +} from "@connectors/connectors/zendesk/lib/id_conversions"; import { concurrentExecutor } from "@connectors/lib/async_utils"; -import { deleteFromDataSource } from "@connectors/lib/data_sources"; +import { + deleteFolderNode, + deleteFromDataSource, + upsertFolderNode, +} from "@connectors/lib/data_sources"; import { ZendeskArticleResource, ZendeskCategoryResource, @@ -16,13 +23,15 @@ import type { DataSourceConfig } from "@connectors/types/data_source_config"; export async function deleteCategory({ connectorId, categoryId, + brandId, dataSourceConfig, }: { connectorId: number; categoryId: number; + brandId: number; dataSourceConfig: DataSourceConfig; }) { - /// deleting the articles in the data source + // deleting the articles in the data source const articles = await ZendeskArticleResource.fetchByCategoryId({ connectorId, categoryId, @@ -36,11 +45,14 @@ export async function deleteCategory({ ), { concurrency: 10 } ); - /// deleting the articles stored in the db + // deleting the articles stored in the db await ZendeskArticleResource.deleteByCategoryId({ connectorId, categoryId, }); + // deleting the folder in data_sources_folders (core) + const folderId = getCategoryInternalId({ connectorId, brandId, categoryId }); + await deleteFolderNode({ dataSourceConfig, folderId }); // deleting the category stored in the db await ZendeskCategoryResource.deleteByCategoryId({ connectorId, categoryId }); } @@ -53,18 +65,20 @@ export async function syncCategory({ brandId, category, currentSyncDateMs, + dataSourceConfig, }: { connectorId: ModelId; brandId: number; category: ZendeskFetchedCategory; currentSyncDateMs: number; + dataSourceConfig: DataSourceConfig; }): Promise { - const categoryInDb = await ZendeskCategoryResource.fetchByCategoryId({ + let categoryInDb = await ZendeskCategoryResource.fetchByCategoryId({ connectorId, categoryId: category.id, }); if (!categoryInDb) { - await ZendeskCategoryResource.makeNew({ + categoryInDb = await ZendeskCategoryResource.makeNew({ blob: { name: category.name || "Category", url: category.html_url, @@ -84,4 +98,12 @@ export async function syncCategory({ lastUpsertedTs: new Date(currentSyncDateMs), }); } + // upserting a folder to data_sources_folders (core) + const parents = categoryInDb.getParentInternalIds(connectorId); + await upsertFolderNode({ + dataSourceConfig, + folderId: parents[0], + parents, + title: categoryInDb.name, + }); } diff --git a/connectors/src/connectors/zendesk/temporal/activities.ts b/connectors/src/connectors/zendesk/temporal/activities.ts index 89570b60b5e5..1dce501e0248 100644 --- a/connectors/src/connectors/zendesk/temporal/activities.ts +++ b/connectors/src/connectors/zendesk/temporal/activities.ts @@ -1,6 +1,7 @@ import type { ModelId } from "@dust-tt/types"; import _ from "lodash"; +import { getBrandInternalId } from "@connectors/connectors/zendesk/lib/id_conversions"; import { syncArticle } from "@connectors/connectors/zendesk/lib/sync_article"; import { syncCategory } from "@connectors/connectors/zendesk/lib/sync_category"; import { syncTicket } from "@connectors/connectors/zendesk/lib/sync_ticket"; @@ -16,6 +17,7 @@ import { import { ZENDESK_BATCH_SIZE } from "@connectors/connectors/zendesk/temporal/config"; import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config"; import { concurrentExecutor } from "@connectors/lib/async_utils"; +import { upsertFolderNode } from "@connectors/lib/data_sources"; import { ZendeskTimestampCursor } from "@connectors/lib/models/zendesk"; import { syncStarted, syncSucceeded } from "@connectors/lib/sync_status"; import { heartbeat } from "@connectors/lib/temporal"; @@ -122,7 +124,36 @@ export async function syncZendeskBrandActivity({ return { helpCenterAllowed: false, ticketsAllowed: false }; } - // otherwise, we update the brand data and lastUpsertedTs + // upserting three folders to data_sources_folders (core): brand, help center, tickets + const dataSourceConfig = dataSourceConfigFromConnector(connector); + + const brandInternalId = getBrandInternalId({ connectorId, brandId }); + await upsertFolderNode({ + dataSourceConfig, + folderId: brandInternalId, + parents: [brandInternalId], + title: brandInDb.name, + }); + + // using the content node to get one source of truth regarding the parent relationship + const helpCenterNode = brandInDb.getHelpCenterContentNode(connectorId); + await upsertFolderNode({ + dataSourceConfig, + folderId: helpCenterNode.internalId, + parents: [helpCenterNode.internalId, helpCenterNode.parentInternalId], + title: helpCenterNode.title, + }); + + // using the content node to get one source of truth regarding the parent relationship + const ticketsNode = brandInDb.getTicketsContentNode(connectorId); + await upsertFolderNode({ + dataSourceConfig, + folderId: ticketsNode.internalId, + parents: [ticketsNode.internalId, ticketsNode.parentInternalId], + title: ticketsNode.title, + }); + + // updating the entry in db await brandInDb.update({ name: fetchedBrand.name || "Brand", url: fetchedBrand?.url || brandInDb.url, @@ -192,6 +223,7 @@ export async function syncZendeskCategoryBatchActivity({ if (!connector) { throw new Error("[Zendesk] Connector not found."); } + const dataSourceConfig = dataSourceConfigFromConnector(connector); const { accessToken, subdomain } = await getZendeskSubdomainAndAccessToken( connector.connectionId @@ -210,8 +242,15 @@ export async function syncZendeskCategoryBatchActivity({ await concurrentExecutor( categories, - async (category) => - syncCategory({ connectorId, brandId, category, currentSyncDateMs }), + async (category) => { + return syncCategory({ + connectorId, + brandId, + category, + currentSyncDateMs, + dataSourceConfig, + }); + }, { concurrency: 10, onBatchComplete: heartbeat, @@ -278,6 +317,15 @@ export async function syncZendeskCategoryActivity({ return { shouldSyncArticles: false }; } + // upserting a folder to data_sources_folders (core) + const parents = categoryInDb.getParentInternalIds(connectorId); + await upsertFolderNode({ + dataSourceConfig: dataSourceConfigFromConnector(connector), + folderId: parents[0], + parents, + title: categoryInDb.name, + }); + // otherwise, we update the category name and lastUpsertedTs await categoryInDb.update({ name: fetchedCategory.name || "Category", diff --git a/connectors/src/connectors/zendesk/temporal/gc_activities.ts b/connectors/src/connectors/zendesk/temporal/gc_activities.ts index b020e58d6ea3..110282609763 100644 --- a/connectors/src/connectors/zendesk/temporal/gc_activities.ts +++ b/connectors/src/connectors/zendesk/temporal/gc_activities.ts @@ -2,7 +2,11 @@ import type { ModelId } from "@dust-tt/types"; import { getArticleInternalId, + getBrandInternalId, + getCategoryInternalId, + getHelpCenterInternalId, getTicketInternalId, + getTicketsInternalId, } from "@connectors/connectors/zendesk/lib/id_conversions"; import { deleteArticle } from "@connectors/connectors/zendesk/lib/sync_article"; import { deleteCategory } from "@connectors/connectors/zendesk/lib/sync_category"; @@ -16,7 +20,10 @@ import { getZendeskGarbageCollectionWorkflowId } from "@connectors/connectors/ze import { ZENDESK_BATCH_SIZE } from "@connectors/connectors/zendesk/temporal/config"; import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config"; import { concurrentExecutor } from "@connectors/lib/async_utils"; -import { deleteFromDataSource } from "@connectors/lib/data_sources"; +import { + deleteFolderNode, + deleteFromDataSource, +} from "@connectors/lib/data_sources"; import logger from "@connectors/logger/logger"; import { ConnectorResource } from "@connectors/resources/connector_resource"; import { @@ -211,20 +218,20 @@ export async function removeForbiddenCategoriesActivity( }; const batchSize = 2; // we process categories 2 by 2 since each of them typically contains ~50 articles - const categoryIds = + const categoryIdsWithBrand = await ZendeskCategoryResource.fetchReadForbiddenCategoryIds({ connectorId, batchSize, }); logger.info( - { ...loggerArgs, categoryCount: categoryIds.length }, + { ...loggerArgs, categoryCount: categoryIdsWithBrand.length }, "[Zendesk] Removing categories with no permission." ); - for (const categoryId of categoryIds) { - await deleteCategory({ connectorId, categoryId, dataSourceConfig }); + for (const ids of categoryIdsWithBrand) { + await deleteCategory({ connectorId, ...ids, dataSourceConfig }); } - return { hasMore: categoryIds.length === batchSize }; + return { hasMore: categoryIdsWithBrand.length === batchSize }; } /** @@ -244,20 +251,19 @@ export async function removeEmptyCategoriesActivity(connectorId: number) { dataSourceId: dataSourceConfig.dataSourceId, }; - const categoryIds = ( - await ZendeskCategoryResource.fetchIdsForConnector(connectorId) - ).map(({ categoryId }) => categoryId); + const categoryIdsWithBrand = + await ZendeskCategoryResource.fetchIdsForConnector(connectorId); - const categoriesToDelete = new Set(); + const categoriesToDelete = new Set<{ categoryId: number; brandId: number }>(); await concurrentExecutor( - categoryIds, - async (categoryId) => { + categoryIdsWithBrand, + async ({ categoryId, brandId }) => { const articles = await ZendeskArticleResource.fetchByCategoryIdReadOnly({ connectorId, categoryId, }); if (articles.length === 0) { - categoriesToDelete.add(categoryId); + categoriesToDelete.add({ categoryId, brandId }); } }, { concurrency: 10 } @@ -267,8 +273,8 @@ export async function removeEmptyCategoriesActivity(connectorId: number) { "[Zendesk] Removing empty categories." ); - for (const categoryId of categoriesToDelete) { - await deleteCategory({ connectorId, categoryId, dataSourceConfig }); + for (const ids of categoriesToDelete) { + await deleteCategory({ connectorId, ...ids, dataSourceConfig }); } } @@ -291,6 +297,30 @@ export async function deleteBrandsWithNoPermissionActivity( dataSourceId: dataSourceConfig.dataSourceId, }; + // deleting from data_sources_folders (core) + const brands = + await ZendeskBrandResource.fetchBrandsWithNoPermission(connectorId); + + await concurrentExecutor( + brands, + async (brandId) => { + await deleteFolderNode({ + dataSourceConfig, + folderId: getBrandInternalId({ connectorId, brandId }), + }); + await deleteFolderNode({ + dataSourceConfig, + folderId: getHelpCenterInternalId({ connectorId, brandId }), + }); + await deleteFolderNode({ + dataSourceConfig, + folderId: getTicketsInternalId({ connectorId, brandId }), + }); + }, + { concurrency: 10 } + ); + + // deleting from zendesk_brands (connectors) const deletedCount = await ZendeskBrandResource.deleteBrandsWithNoPermission(connectorId); @@ -423,11 +453,27 @@ export async function deleteCategoryBatchActivity({ dataSourceId: dataSourceConfig.dataSourceId, }; - const deletedCount = await ZendeskCategoryResource.deleteByBrandId({ + const categories = await ZendeskCategoryResource.fetchByBrandId({ connectorId, brandId, batchSize: ZENDESK_BATCH_SIZE, }); + + await concurrentExecutor( + categories, + async ({ categoryId, brandId }) => { + await deleteFolderNode({ + dataSourceConfig, + folderId: getCategoryInternalId({ connectorId, brandId, categoryId }), + }); + }, + { concurrency: 10 } + ); + + const deletedCount = await ZendeskCategoryResource.deleteByCategoryIds({ + connectorId, + categoryIds: categories.map((category) => category.categoryId), + }); logger.info( { ...loggerArgs, brandId, deletedCount }, "[Zendesk] Deleting a batch of categories." diff --git a/connectors/src/connectors/zendesk/temporal/incremental_activities.ts b/connectors/src/connectors/zendesk/temporal/incremental_activities.ts index 11e66e60c4d7..2303b95afd15 100644 --- a/connectors/src/connectors/zendesk/temporal/incremental_activities.ts +++ b/connectors/src/connectors/zendesk/temporal/incremental_activities.ts @@ -14,6 +14,7 @@ import { } from "@connectors/connectors/zendesk/lib/zendesk_api"; import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config"; import { concurrentExecutor } from "@connectors/lib/async_utils"; +import { upsertFolderNode } from "@connectors/lib/data_sources"; import { ZendeskTimestampCursor } from "@connectors/lib/models/zendesk"; import logger from "@connectors/logger/logger"; import { ConnectorResource } from "@connectors/resources/connector_resource"; @@ -140,6 +141,14 @@ export async function syncZendeskArticleUpdateBatchActivity({ description: fetchedCategory.description, }, }); + // upserting a folder to data_sources_folders (core) + const parents = category.getParentInternalIds(connectorId); + await upsertFolderNode({ + dataSourceConfig, + folderId: parents[0], + parents, + title: category.name, + }); } else { /// ignoring these to proceed with the other articles, but these might have to be checked at some point logger.error( diff --git a/connectors/src/resources/zendesk_resources.ts b/connectors/src/resources/zendesk_resources.ts index 8781b578e8e2..dde2cef94ca9 100644 --- a/connectors/src/resources/zendesk_resources.ts +++ b/connectors/src/resources/zendesk_resources.ts @@ -24,6 +24,7 @@ import { ZendeskTicket, } from "@connectors/lib/models/zendesk"; import { BaseResource } from "@connectors/resources/base_resource"; +import type { ConnectorResource } from "@connectors/resources/connector_resource"; import type { ReadonlyAttributesType } from "@connectors/resources/storage/types"; // Attributes are marked as read-only to reflect the stateless nature of our Resource. @@ -188,6 +189,15 @@ export class ZendeskBrandResource extends BaseResource { } } + static async fetchByConnector( + connector: ConnectorResource + ): Promise { + const brands = await ZendeskBrand.findAll({ + where: { connectorId: connector.id }, + }); + return brands.map((brand) => new this(this.model, brand.get())); + } + static async fetchByBrandId({ connectorId, brandId, @@ -287,6 +297,16 @@ export class ZendeskBrandResource extends BaseResource { return brands.map((brand) => Number(brand.get().brandId)); } + static async fetchBrandsWithNoPermission( + connectorId: number + ): Promise { + const brands = await ZendeskBrand.findAll({ + where: { connectorId, ticketsPermission: "none" }, + attributes: ["brandId"], + }); + return brands.map((brand) => Number(brand.get().brandId)); + } + static async deleteBrandsWithNoPermission( connectorId: number, transaction?: Transaction @@ -331,7 +351,7 @@ export class ZendeskBrandResource extends BaseResource { getHelpCenterContentNode( connectorId: number, { richTitle = false }: { richTitle?: boolean } = {} - ): ContentNode { + ): ContentNode & { parentInternalId: string } { const { brandId } = this; return { provider: "zendesk", @@ -353,7 +373,7 @@ export class ZendeskBrandResource extends BaseResource { expandable = false, richTitle = false, }: { expandable?: boolean; richTitle?: boolean } = {} - ): ContentNode { + ): ContentNode & { parentInternalId: string } { const { brandId } = this; return { provider: "zendesk", @@ -434,13 +454,25 @@ export class ZendeskCategoryResource extends BaseResource { }: { connectorId: number; batchSize: number; - }): Promise { + }): Promise<{ categoryId: number; brandId: number }[]> { const categories = await ZendeskCategory.findAll({ where: { connectorId, permission: "none" }, attributes: ["categoryId"], limit: batchSize, }); - return categories.map((category) => Number(category.get().categoryId)); + return categories.map((category) => { + const { categoryId, brandId } = category.get(); + return { categoryId, brandId }; + }); + } + + static async fetchByConnector( + connector: ConnectorResource + ): Promise { + const categories = await ZendeskCategory.findAll({ + where: { connectorId: connector.id }, + }); + return categories.map((category) => new this(this.model, category.get())); } static async fetchIdsForConnector( @@ -508,6 +540,26 @@ export class ZendeskCategoryResource extends BaseResource { return categories.map((category) => category.get().brandId); } + static async fetchByBrandId({ + connectorId, + brandId, + batchSize = null, + }: { + connectorId: number; + brandId: number; + batchSize?: number | null; + }): Promise<{ categoryId: number; brandId: number }[]> { + const categories = await ZendeskCategory.findAll({ + attributes: ["categoryId", "brandId"], + where: { connectorId, brandId, permission: "read" }, + ...(batchSize && { limit: batchSize }), + }); + return categories.map((category) => { + const { categoryId, brandId } = category.get(); + return { categoryId, brandId }; + }); + } + static async fetchByBrandIdReadOnly({ connectorId, brandId, @@ -533,18 +585,15 @@ export class ZendeskCategoryResource extends BaseResource { await ZendeskCategory.destroy({ where: { connectorId, categoryId } }); } - static async deleteByBrandId({ + static async deleteByCategoryIds({ connectorId, - brandId, - batchSize = null, + categoryIds, }: { connectorId: number; - brandId: number; - batchSize?: number | null; + categoryIds: number[]; }): Promise { return ZendeskCategory.destroy({ - where: { connectorId, brandId }, - ...(batchSize && { limit: batchSize }), + where: { connectorId, categoryId: { [Op.in]: categoryIds } }, }); } @@ -599,7 +648,7 @@ export class ZendeskCategoryResource extends BaseResource { }; } - getParentInternalIds(connectorId: number): string[] { + getParentInternalIds(connectorId: number): [string, string, string] { /// Categories have two parents: the Help Center and the Brand. const { brandId, categoryId } = this; return [ @@ -685,7 +734,7 @@ export class ZendeskTicketResource extends BaseResource { }; } - getParentInternalIds(connectorId: number): string[] { + getParentInternalIds(connectorId: number): [string, string, string] { const { brandId, ticketId } = this; /// Tickets have two parents: the Tickets and the Brand. return [ @@ -890,7 +939,7 @@ export class ZendeskArticleResource extends BaseResource { }; } - getParentInternalIds(connectorId: number): string[] { + getParentInternalIds(connectorId: number): [string, string, string, string] { const { brandId, categoryId, articleId } = this; /// Articles have three parents: the Category, the Help Center and the Brand. return [ diff --git a/front/components/actions/browse/BrowseActionDetails.tsx b/front/components/actions/browse/BrowseActionDetails.tsx index 87701652d845..076ad29b9f70 100644 --- a/front/components/actions/browse/BrowseActionDetails.tsx +++ b/front/components/actions/browse/BrowseActionDetails.tsx @@ -1,8 +1,8 @@ import { - CitationNew, - CitationNewDescription, - CitationNewIcons, - CitationNewTitle, + Citation, + CitationDescription, + CitationIcons, + CitationTitle, GlobeAltIcon, Icon, } from "@dust-tt/sparkle"; @@ -34,15 +34,15 @@ export function BrowseActionDetails({ {r.responseCode === "200" ? ( - - + + - - {r.browsedUrl} - + + {r.browsedUrl} + {r.content.slice(0, 500)} - - + + ) : ( Cannot fetch content, error code : {r.responseCode}. diff --git a/front/components/actions/retrieval/utils.ts b/front/components/actions/retrieval/utils.tsx similarity index 75% rename from front/components/actions/retrieval/utils.ts rename to front/components/actions/retrieval/utils.tsx index 86597f0677d1..c56e03fedc4d 100644 --- a/front/components/actions/retrieval/utils.ts +++ b/front/components/actions/retrieval/utils.tsx @@ -5,14 +5,17 @@ import { } from "@dust-tt/types"; import type { MarkdownCitation } from "@app/components/markdown/MarkdownCitation"; +import { citationIconMap } from "@app/components/markdown/MarkdownCitation"; export function makeDocumentCitation( document: RetrievalDocumentType ): MarkdownCitation { + const IconComponent = + citationIconMap[getProviderFromRetrievedDocument(document)]; return { href: document.sourceUrl ?? undefined, title: getTitleFromRetrievedDocument(document), - type: getProviderFromRetrievedDocument(document), + icon: , }; } diff --git a/front/components/actions/tables_query/TablesQueryActionDetails.tsx b/front/components/actions/tables_query/TablesQueryActionDetails.tsx index baee6630850e..3e793b221f33 100644 --- a/front/components/actions/tables_query/TablesQueryActionDetails.tsx +++ b/front/components/actions/tables_query/TablesQueryActionDetails.tsx @@ -1,6 +1,6 @@ import { - CitationNew, - CitationNewTitle, + Citation, + CitationTitle, Collapsible, ContentMessage, InformationCircleIcon, @@ -144,9 +144,9 @@ function QueryTablesResults({
Results
- - {title} - + + {title} +
diff --git a/front/components/actions/websearch/utils.ts b/front/components/actions/websearch/utils.tsx similarity index 84% rename from front/components/actions/websearch/utils.ts rename to front/components/actions/websearch/utils.tsx index 70a729edd462..7779b075771f 100644 --- a/front/components/actions/websearch/utils.ts +++ b/front/components/actions/websearch/utils.tsx @@ -1,3 +1,4 @@ +import { DocumentTextStrokeIcon } from "@dust-tt/sparkle"; import type { WebsearchActionType, WebsearchResultType } from "@dust-tt/types"; import type { MarkdownCitation } from "@app/components/markdown/MarkdownCitation"; @@ -9,7 +10,7 @@ export function makeWebsearchResultsCitation( description: result.snippet, href: result.link, title: result.title, - type: "document" as const, + icon: , }; } diff --git a/front/components/assistant/conversation/AgentMessage.tsx b/front/components/assistant/conversation/AgentMessage.tsx index 3c969244a3e0..3a7f580e1278 100644 --- a/front/components/assistant/conversation/AgentMessage.tsx +++ b/front/components/assistant/conversation/AgentMessage.tsx @@ -2,24 +2,8 @@ import type { ConversationMessageSizeType, FeedbackSelectorProps, } from "@dust-tt/sparkle"; -import { CitationNewIndex } from "@dust-tt/sparkle"; -import { - CitationNew, - CitationNewIcons, - CitationNewTitle, - ConfluenceLogo, - DocumentTextIcon, - DriveLogo, - GithubLogo, - Icon, - ImageIcon, - IntercomLogo, - MicrosoftLogo, - NotionLogo, - SlackLogo, - SnowflakeLogo, - ZendeskLogo, -} from "@dust-tt/sparkle"; +import { CitationIndex } from "@dust-tt/sparkle"; +import { Citation, CitationIcons, CitationTitle } from "@dust-tt/sparkle"; import { ArrowPathIcon, Button, @@ -93,20 +77,6 @@ import { useEventSource } from "@app/hooks/useEventSource"; import { useSubmitFunction } from "@app/lib/client/utils"; import { useAgentConfigurationLastAuthor } from "@app/lib/swr/assistants"; -const typeIcons = { - confluence: ConfluenceLogo, - document: DocumentTextIcon, - github: GithubLogo, - google_drive: DriveLogo, - intercom: IntercomLogo, - microsoft: MicrosoftLogo, - zendesk: ZendeskLogo, - notion: NotionLogo, - slack: SlackLogo, - image: ImageIcon, - snowflake: SnowflakeLogo, -}; - function cleanUpCitations(message: string): string { const regex = / ?:cite\[[a-zA-Z0-9, ]+\]/g; return message.replace(regex, ""); @@ -725,13 +695,13 @@ function getCitations({ activeReferences.sort((a, b) => a.index - b.index); return activeReferences.map(({ document, index }) => { return ( - - - {index} - - - {document.title} - + + + {index} + {document.icon} + + {document.title} + ); }); } diff --git a/front/components/assistant/conversation/MessageItem.tsx b/front/components/assistant/conversation/MessageItem.tsx index 14172841d871..9939c85372cb 100644 --- a/front/components/assistant/conversation/MessageItem.tsx +++ b/front/components/assistant/conversation/MessageItem.tsx @@ -1,10 +1,10 @@ -import type { CitationType, FeedbackSelectorProps } from "@dust-tt/sparkle"; +import type { FeedbackSelectorProps } from "@dust-tt/sparkle"; import { Avatar, - CitationNew, - CitationNewIcons, - CitationNewImage, - CitationNewTitle, + Citation, + CitationIcons, + CitationImage, + CitationTitle, DocumentTextIcon, Icon, SlackLogo, @@ -109,7 +109,7 @@ const MessageItem = React.forwardRef( case "user_message": const citations = message.contenFragments ? message.contenFragments.map((contentFragment) => { - const citationType: CitationType = [ + const citationType = [ "dust-application/slack", "text/vnd.dust.attachment.slack.thread", ].includes(contentFragment.contentType) @@ -120,34 +120,34 @@ const MessageItem = React.forwardRef( citationType === "slack" ? SlackLogo : DocumentTextIcon; return ( -
{contentFragment.context.profilePictureUrl && ( - + - + )} {contentFragment.sourceUrl ? ( <> - - + + - + ) : ( - + - + )}
- {contentFragment.title} - + {contentFragment.title} + ); }) : undefined; diff --git a/front/components/assistant/conversation/input_bar/InputBarCitations.tsx b/front/components/assistant/conversation/input_bar/InputBarCitations.tsx index 8b68c5c66a69..d96f21cbb688 100644 --- a/front/components/assistant/conversation/input_bar/InputBarCitations.tsx +++ b/front/components/assistant/conversation/input_bar/InputBarCitations.tsx @@ -1,9 +1,9 @@ import { - CitationNew, - CitationNewClose, - CitationNewIcons, - CitationNewImage, - CitationNewTitle, + Citation, + CitationClose, + CitationIcons, + CitationImage, + CitationTitle, DocumentIcon, Icon, ImageIcon, @@ -25,28 +25,28 @@ export function InputBarCitations({ const isImage = Boolean(blob.preview); nodes.push( <> - {isImage ? ( <> - - + + - + ) : ( - + - + )} - {blob.id} - {blob.id} + fileUploaderService.removeFile(blob.id)} /> - + ); } diff --git a/front/components/home/content/Product/BlogSection.tsx b/front/components/home/content/Product/BlogSection.tsx index e3bdeba325ed..e08b06202091 100644 --- a/front/components/home/content/Product/BlogSection.tsx +++ b/front/components/home/content/Product/BlogSection.tsx @@ -47,7 +47,7 @@ export function BlogSection() { href="https://blog.dust.tt/november-five-ai-transformation-dust/" > Blog Image @@ -59,7 +59,7 @@ export function BlogSection() { href="https://blog.dust.tt/pennylane-dust-customer-support-journey/" > Blog Image @@ -71,7 +71,7 @@ export function BlogSection() { href="https://blog.dust.tt/integrating-ai-workflows-alan/" > Blog Image diff --git a/front/components/markdown/MarkdownCitation.tsx b/front/components/markdown/MarkdownCitation.tsx index 7912285dda6a..61bdcfdca421 100644 --- a/front/components/markdown/MarkdownCitation.tsx +++ b/front/components/markdown/MarkdownCitation.tsx @@ -1,8 +1,54 @@ -import type { CitationType } from "@dust-tt/sparkle"; +import { + ConfluenceLogo, + DocumentTextStrokeIcon, + DriveLogo, + GithubLogo, + ImageStrokeIcon, + IntercomLogo, + MicrosoftLogo, + NotionLogo, + SlackLogo, + SnowflakeLogo, + ZendeskLogo, +} from "@dust-tt/sparkle"; +import type { SVGProps } from "react"; + +const CITATION_ICONS = [ + "confluence", + "document", + "github", + "google_drive", + "intercom", + "microsoft", + "zendesk", + "notion", + "slack", + "image", + "snowflake", +] as const; + +export type CitationIconType = (typeof CITATION_ICONS)[number]; + +export const citationIconMap: Record< + CitationIconType, + (props: SVGProps) => React.JSX.Element +> = { + confluence: ConfluenceLogo, + document: DocumentTextStrokeIcon, + github: GithubLogo, + google_drive: DriveLogo, + intercom: IntercomLogo, + microsoft: MicrosoftLogo, + zendesk: ZendeskLogo, + notion: NotionLogo, + slack: SlackLogo, + image: ImageStrokeIcon, + snowflake: SnowflakeLogo, +}; export interface MarkdownCitation { description?: string; href?: string; title: string; - type: CitationType; + icon: React.JSX.Element; } diff --git a/front/lib/api/assistant/configuration.ts b/front/lib/api/assistant/configuration.ts index 4b18efa80d34..fa2f2c389a75 100644 --- a/front/lib/api/assistant/configuration.ts +++ b/front/lib/api/assistant/configuration.ts @@ -347,6 +347,7 @@ async function fetchWorkspaceAgentConfigurationsWithoutActions( } const relations = await AgentUserRelation.findAll({ where: { + workspaceId: owner.id, userId, favorite: true, }, @@ -614,8 +615,8 @@ export async function getAgentConfigurations({ }), ]); - // Filter out agents that the user does not have access to - // user should be in all groups that are in the agent's groupIds + // Filter out agents that the user does not have access to user should be in all groups that are + // in the agent's groupIds const allowedAgentConfigurations = dangerouslySkipPermissionFiltering ? allAgentConfigurations : allAgentConfigurations diff --git a/front/lib/api/assistant/get_favorite_states.ts b/front/lib/api/assistant/get_favorite_states.ts index 48aab8c4a5a8..c2927479f8c0 100644 --- a/front/lib/api/assistant/get_favorite_states.ts +++ b/front/lib/api/assistant/get_favorite_states.ts @@ -19,6 +19,7 @@ export async function getFavoriteStates( const relations = await AgentUserRelation.findAll({ where: { + workspaceId: auth.getNonNullableWorkspace().id, agentConfiguration: { [Op.in]: configurationIds }, userId: user.id, }, diff --git a/front/lib/connector_providers.ts b/front/lib/connector_providers.ts index 85e1533a0779..35aaccc48509 100644 --- a/front/lib/connector_providers.ts +++ b/front/lib/connector_providers.ts @@ -187,7 +187,7 @@ export const CONNECTOR_CONFIGURATIONS: Record< zendesk: { name: "Zendesk", connectorProvider: "zendesk", - status: "rolling_out", + status: "built", hide: false, description: "Authorize access to Zendesk for indexing tickets from your support center and articles from your help center.", @@ -197,7 +197,6 @@ export const CONNECTOR_CONFIGURATIONS: Record< logoComponent: ZendeskLogo, isNested: true, isSearchEnabled: false, - rollingOutFlag: "zendesk_connector_feature", }, }; diff --git a/front/migrations/20241211_parents_migrator.ts b/front/migrations/20241211_parents_migrator.ts index 5f185bd4701e..09d876459558 100644 --- a/front/migrations/20241211_parents_migrator.ts +++ b/front/migrations/20241211_parents_migrator.ts @@ -34,7 +34,7 @@ type ProviderMigrator = { ) => { parents: string[]; parentId: string | null }; }; -const QUERY_BATCH_SIZE = 256; +const QUERY_BATCH_SIZE = 10_000; const DOCUMENT_CONCURRENCY = 16; const TABLE_CONCURRENCY = 16; @@ -264,6 +264,7 @@ async function migrateDocument({ dataSource, coreDocument, execute, + skipIfParentsAreAlreadyCorrect, }: { action: MigratorAction; migrator: ProviderMigrator; @@ -274,6 +275,7 @@ async function migrateDocument({ document_id: string; }; execute: boolean; + skipIfParentsAreAlreadyCorrect: boolean; }) { let newParents = coreDocument.parents; let newParentId: string | null = null; @@ -295,6 +297,21 @@ async function migrateDocument({ throw e; } + if ( + skipIfParentsAreAlreadyCorrect && + newParents.every((x, i) => x === coreDocument.parents[i]) + ) { + logger.info( + { + documentId: coreDocument.document_id, + fromParents: coreDocument.parents, + toParents: newParents, + }, + `SKIP document (parents are already correct)` + ); + return new Ok(undefined); + } + if (execute) { const updateRes = await withRetries( async () => { @@ -417,11 +434,13 @@ async function migrateDataSource({ migrator, dataSource, execute, + skipIfParentsAreAlreadyCorrect, }: { action: MigratorAction; migrator: ProviderMigrator; dataSource: DataSourceModel; execute: boolean; + skipIfParentsAreAlreadyCorrect: boolean; }) { const corePrimary = getCorePrimaryDbConnection(); @@ -449,7 +468,7 @@ async function migrateDataSource({ for (;;) { const [coreDocumentRows] = (await corePrimary.query( "SELECT id, parents, document_id, timestamp FROM data_sources_documents " + - "WHERE data_source = ? AND STATUS = ? AND timestamp > ? " + + "WHERE data_source = ? AND STATUS = ? AND timestamp >= ? " + "ORDER BY timestamp ASC LIMIT ?", { replacements: [ @@ -480,6 +499,7 @@ async function migrateDataSource({ migrator, dataSource, coreDocument, + skipIfParentsAreAlreadyCorrect, execute, }), { concurrency: DOCUMENT_CONCURRENCY } @@ -555,12 +575,14 @@ async function migrateAll({ migrator, nextDataSourceId, execute, + skipIfParentsAreAlreadyCorrect, }: { provider: ConnectorProvider; action: MigratorAction; migrator: ProviderMigrator; nextDataSourceId: number; execute: boolean; + skipIfParentsAreAlreadyCorrect: boolean; }) { // retrieve all data sources for the provider const dataSources = await DataSourceModel.findAll({ @@ -584,6 +606,7 @@ async function migrateAll({ action, dataSource, execute, + skipIfParentsAreAlreadyCorrect, }); } else { logger.info({ dataSourceId: dataSource.id }, "SKIP"); @@ -601,13 +624,24 @@ makeScript( type: "string", required: true, }, + skipIfParentsAreAlreadyCorrect: { + type: "boolean", + required: false, + default: false, + }, nextDataSourceId: { type: "number", required: false, default: 0, }, }, - async ({ provider, action, nextDataSourceId, execute }) => { + async ({ + provider, + action, + nextDataSourceId, + execute, + skipIfParentsAreAlreadyCorrect, + }) => { if (!isMigratorAction(action)) { console.error( `Invalid action ${action}, supported actions are "transform" and "clean"` @@ -630,6 +664,7 @@ makeScript( migrator, nextDataSourceId, execute, + skipIfParentsAreAlreadyCorrect, }); } ); diff --git a/front/package-lock.json b/front/package-lock.json index 2849e84dad9c..d5b3734ef06a 100644 --- a/front/package-lock.json +++ b/front/package-lock.json @@ -7,7 +7,7 @@ "dependencies": { "@auth0/nextjs-auth0": "^3.5.0", "@dust-tt/client": "file:../sdks/js", - "@dust-tt/sparkle": "^0.2.339", + "@dust-tt/sparkle": "^0.2.341", "@dust-tt/types": "file:../types", "@headlessui/react": "^1.7.7", "@heroicons/react": "^2.0.11", @@ -11452,9 +11452,9 @@ "link": true }, "node_modules/@dust-tt/sparkle": { - "version": "0.2.339", - "resolved": "https://registry.npmjs.org/@dust-tt/sparkle/-/sparkle-0.2.339.tgz", - "integrity": "sha512-5SaV+/T077ioLs5y53/o6c9xRsZuubPEcnDpdIG1u00Da12zi8zUnYepplnkCOLKaSje+1KQK1Iw5XH6mkVTEQ==", + "version": "0.2.341", + "resolved": "https://registry.npmjs.org/@dust-tt/sparkle/-/sparkle-0.2.341.tgz", + "integrity": "sha512-0jecvnBu4Pp6hHyWKXYv0dIwToIiUAJlkitd6LZQAEU8O5cJIM1vYnym3gk70roVl5zyv2eMapObYDkhL61kvw==", "dependencies": { "@emoji-mart/data": "^1.1.2", "@emoji-mart/react": "^1.1.1", diff --git a/front/package.json b/front/package.json index 382662e59b6f..7f6c98ecf34b 100644 --- a/front/package.json +++ b/front/package.json @@ -20,7 +20,7 @@ "dependencies": { "@auth0/nextjs-auth0": "^3.5.0", "@dust-tt/client": "file:../sdks/js", - "@dust-tt/sparkle": "^0.2.339", + "@dust-tt/sparkle": "^0.2.341", "@dust-tt/types": "file:../types", "@headlessui/react": "^1.7.7", "@heroicons/react": "^2.0.11", diff --git a/front/pages/api/v1/w/[wId]/assistant/agent_configurations.ts b/front/pages/api/v1/w/[wId]/assistant/agent_configurations.ts index 9791edf436c1..46bf643127c5 100644 --- a/front/pages/api/v1/w/[wId]/assistant/agent_configurations.ts +++ b/front/pages/api/v1/w/[wId]/assistant/agent_configurations.ts @@ -1,8 +1,5 @@ import type { GetAgentConfigurationsResponseType } from "@dust-tt/client"; -import type { - LightAgentConfigurationType, - WithAPIErrorResponse, -} from "@dust-tt/types"; +import type { WithAPIErrorResponse } from "@dust-tt/types"; import { isLeft } from "fp-ts/lib/Either"; import * as t from "io-ts"; import * as reporter from "io-ts-reporters"; @@ -140,18 +137,13 @@ async function handler( auth, agents: agentConfigurations, }); - agentConfigurations = await Promise.all( - agentConfigurations.map( - async ( - agentConfiguration, - index - ): Promise => { - return { - ...agentConfiguration, - lastAuthors: recentAuthors[index], - }; - } - ) + agentConfigurations = agentConfigurations.map( + (agentConfiguration, index) => { + return { + ...agentConfiguration, + lastAuthors: recentAuthors[index], + }; + } ); } diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index 8cc61a80cc97..95822e1c329f 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -655,7 +655,6 @@ const WhitelistableFeaturesSchema = FlexibleEnumSchema< | "openai_o1_feature" | "openai_o1_mini_feature" | "snowflake_connector_feature" - | "zendesk_connector_feature" | "index_private_slack_channel" | "conversations_jit_actions" >(); diff --git a/sparkle/package-lock.json b/sparkle/package-lock.json index 99ada6ec1066..00f6d6379b9f 100644 --- a/sparkle/package-lock.json +++ b/sparkle/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dust-tt/sparkle", - "version": "0.2.340", + "version": "0.2.341", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dust-tt/sparkle", - "version": "0.2.340", + "version": "0.2.341", "license": "ISC", "dependencies": { "@emoji-mart/data": "^1.1.2", diff --git a/sparkle/package.json b/sparkle/package.json index 4e3d743bdf8f..6ae0c9cc44b3 100644 --- a/sparkle/package.json +++ b/sparkle/package.json @@ -1,6 +1,6 @@ { "name": "@dust-tt/sparkle", - "version": "0.2.340", + "version": "0.2.341", "scripts": { "build": "rm -rf dist && npm run tailwind && npm run build:esm && npm run build:cjs", "tailwind": "tailwindcss -i ./src/styles/tailwind.css -o dist/sparkle.css", diff --git a/sparkle/src/components/Citation.tsx b/sparkle/src/components/Citation.tsx index f73afb52196b..1694c352bb0f 100644 --- a/sparkle/src/components/Citation.tsx +++ b/sparkle/src/components/Citation.tsx @@ -1,182 +1,238 @@ import React, { ReactNode } from "react"; import { - Avatar, - Card, - Icon, - IconButton, + Button, Card, CardProps, Spinner, Tooltip, -} from "@sparkle/components"; -import { XCircleIcon } from "@sparkle/icons"; -import { DocumentTextStrokeIcon, ImageStrokeIcon } from "@sparkle/icons/stroke"; -import { classNames } from "@sparkle/lib/utils"; -import { - ConfluenceLogo, - DriveLogo, - GithubLogo, - IntercomLogo, - MicrosoftLogo, - NotionLogo, - SlackLogo, - SnowflakeLogo, - ZendeskLogo, -} from "@sparkle/logo/platforms"; - -export type CitationType = - | "confluence" - | "document" - | "github" - | "google_drive" - | "image" - | "intercom" - | "microsoft" - | "zendesk" - | "notion" - | "slack" - | "snowflake"; - -const typeIcons = { - confluence: ConfluenceLogo, - document: DocumentTextStrokeIcon, - github: GithubLogo, - google_drive: DriveLogo, - intercom: IntercomLogo, - microsoft: MicrosoftLogo, - zendesk: ZendeskLogo, - notion: NotionLogo, - slack: SlackLogo, - image: ImageStrokeIcon, - snowflake: SnowflakeLogo, -}; +} from "@sparkle/components/"; +import { XMarkIcon } from "@sparkle/icons"; +import { cn } from "@sparkle/lib/utils"; -const typeSizing = { - fixed: { xs: "s-w-48", sm: "s-w-64" }, - fluid: "s-w-full", +type CitationProps = CardProps & { + children: React.ReactNode; + isLoading?: boolean; + tooltip?: string; }; -interface CitationProps { - avatarSrc?: string; - description?: string; - href?: string; - imgSrc?: string; - index?: ReactNode; - isBlinking?: boolean; - isLoading?: boolean; - onClose?: () => void; - size?: "xs" | "sm"; - sizing?: "fixed" | "fluid"; - title: string; - type?: CitationType; -} +const Citation = React.forwardRef( + ( + { children, variant = "primary", isLoading, className, tooltip, ...props }, + ref + ) => { + const cardButton = ( + + {children} + {isLoading && } + + ); + + if (tooltip) { + return ; + } + + return cardButton; + } +); -export function Citation({ - avatarSrc, - description, - href, - imgSrc, - index, - isBlinking = false, - isLoading, - onClose, - size = "sm", - sizing = "fixed", - title, - type = "document", -}: CitationProps) { - const cardContent = ( - <> - {type === "image" && imgSrc && ( -
+Citation.displayName = "Citation"; + +const CitationIndex = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ children, className, ...props }, ref) => { + return ( +
- {avatarSrc && } - {index && ( -
- {index} -
- )} + {...props} + > + {children} +
+ ); +}); +CitationIndex.displayName = "CitationIndex"; - {!isLoading && ( - - )} -
- {onClose && ( -
- -
- )} +const CitationGrid = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes +>(({ children, className, ...props }, ref) => { + return ( +
+
+ {children}
+
+ ); +}); +CitationGrid.displayName = "CitationGrid"; + +interface CitationCloseProps + extends React.ButtonHTMLAttributes { + className?: string; + onClick?: (e: React.MouseEvent) => void; +} + +const CitationClose = React.forwardRef( + ({ className, onClick, ...props }, ref) => { + return ( +
+ Example of dynamic grid + + alert("Card clicked")}> + + 1 + + + Hello + + alert("Close action clicked")}> + + 2 + + + Hello + + alert("Close action clicked")}> + + 3 + + + Hello + + alert("Close action clicked")}> + + 4 + + + Hello + +
); diff --git a/sparkle/src/stories/CitationNew.stories.tsx b/sparkle/src/stories/CitationNew.stories.tsx deleted file mode 100644 index bff55916f5b1..000000000000 --- a/sparkle/src/stories/CitationNew.stories.tsx +++ /dev/null @@ -1,162 +0,0 @@ -import type { Meta } from "@storybook/react"; -import React from "react"; - -import { - Button, - CitationNew, - CitationNewClose, - CitationNewDescription, - CitationNewGrid, - CitationNewIcons, - CitationNewImage, - CitationNewIndex, - CitationNewTitle, - Icon, - Popover, -} from "@sparkle/components"; -import { - DocumentIcon, - ExternalLinkIcon, - GlobeAltIcon, - ImageIcon, - TableIcon, -} from "@sparkle/icons"; -import { NotionLogo, SlackLogo } from "@sparkle/logo"; - -const meta = { - title: "Components/CitationNew", - component: CitationNew, -} satisfies Meta; - -export default meta; - -export const CitationsExample = () => ( -
- Example of attachement -
- alert("Card clicked")} className="s-w-48"> - - - - Slack thread - - @ed at 16:32 This is the latest ve - - - alert("Card clicked")} className="s-w-48"> - - - - extract_financa.csv - - alert("Card clicked")} className="s-w-48"> - - - - Linkedin, Edouard Wautier - - - alert("Card clicked")} className="s-w-48"> - - - - - screenshot.png - - - - - - - screenshot.png - -
- Example of dissmissable attachements -
- alert("Card clicked")} className="s-w-48"> - - - - Slack thread - alert("Close clicked")} /> - - @ed at 16:32 This is the latest ve - - - alert("Card clicked")} className="s-w-48"> - - - - alert("Close clicked")} /> - extract_financa.csv - - alert("Card clicked")} className="s-w-48"> - - - - alert("Close clicked")} /> - Linkedin, Edouard Wautier - - - alert("Card clicked")} className="s-w-48"> - - - - - alert("Close clicked")} /> - screenshot.png - -
- Example of citations in markdown -
- 1} - content={ - <> - - 1 - - - Hello -
- Example of dynamic grid - - alert("Card clicked")}> - - 1 - - - Hello - - alert("Close action clicked")}> - - 2 - - - Hello - - alert("Close action clicked")}> - - 3 - - - Hello - - alert("Close action clicked")}> - - 4 - - - Hello - - -
-); diff --git a/sparkle/src/stories/ConversationMessage.stories.tsx b/sparkle/src/stories/ConversationMessage.stories.tsx index 90daa93c7560..6d21beaec621 100644 --- a/sparkle/src/stories/ConversationMessage.stories.tsx +++ b/sparkle/src/stories/ConversationMessage.stories.tsx @@ -4,9 +4,13 @@ import React from "react"; import { Button, Citation, + CitationIcons, + CitationTitle, ConversationMessage, + GithubIcon, + Icon, MagnifyingGlassIcon, - ZoomableImageCitationWrapper, + SlackLogo, } from "../index_with_tw_base"; const meta = { @@ -43,47 +47,20 @@ export const ConversationExample = () => { />, ]} citations={[ - , - , - - , - , - , - , + + + + + + Source: Thread on #general message from @ed + + , + + + + + Title + , ]} > To conditionally render the citations only if a citations React node @@ -120,6 +97,7 @@ export const ConversationExample = () => { { />, ]} citations={[ - , - , - - , + + + + + + Source: Thread on #general message from @ed + + , + + + + + Title + , ]} > To conditionally render the citations only if a citations React node diff --git a/sparkle/src/stories/PaginatedCitationsGrid.stories.tsx b/sparkle/src/stories/PaginatedCitationsGrid.stories.tsx index 66e05f912daa..f06bd0e4c8c2 100644 --- a/sparkle/src/stories/PaginatedCitationsGrid.stories.tsx +++ b/sparkle/src/stories/PaginatedCitationsGrid.stories.tsx @@ -1,9 +1,7 @@ import type { Meta } from "@storybook/react"; import React from "react"; -import { CitationType } from "@sparkle/components/Citation"; - -import { PaginatedCitationsGrid } from "../index_with_tw_base"; +import { DocumentIcon, PaginatedCitationsGrid } from "../index_with_tw_base"; const meta = { title: "Modules/PaginatedCitationsGrid", @@ -16,7 +14,7 @@ function makeCitationItems(items: number) { return Array.from({ length: items }, (_, idx) => ({ title: `test ${idx + 1}`, href: "empty", - type: "document" as CitationType, + icon: , })); } diff --git a/sparkle/src/stories/Tree.stories.tsx b/sparkle/src/stories/Tree.stories.tsx index eaf7e3d1edf8..f753d09e0527 100644 --- a/sparkle/src/stories/Tree.stories.tsx +++ b/sparkle/src/stories/Tree.stories.tsx @@ -469,6 +469,16 @@ export const TreeExample = () => { }, }} /> + { + return; + }, + }} + /> diff --git a/types/src/shared/feature_flags.ts b/types/src/shared/feature_flags.ts index e22dabe70543..003c07bdac3e 100644 --- a/types/src/shared/feature_flags.ts +++ b/types/src/shared/feature_flags.ts @@ -7,7 +7,6 @@ export const WHITELISTABLE_FEATURES = [ "use_app_for_header_detection", "openai_o1_feature", "openai_o1_mini_feature", - "zendesk_connector_feature", "index_private_slack_channel", "conversations_jit_actions", "labs_trackers",