diff --git a/front/lib/api/vaults.ts b/front/lib/api/vaults.ts index 64f97651c99e..0369153a823a 100644 --- a/front/lib/api/vaults.ts +++ b/front/lib/api/vaults.ts @@ -1,16 +1,111 @@ +import type { + ConnectorsAPIError, + ContentNodesViewType, + ContentNodeType, + CoreAPIError, + ResourceCategory, + Result, + WithAPIErrorResponse, +} from "@dust-tt/types"; import { ConnectorsAPI, CoreAPI, Ok } from "@dust-tt/types"; +import type { NextApiRequest, NextApiResponse } from "next"; import config from "@app/lib/api/config"; import type { DataSourceResource } from "@app/lib/resources/data_source_resource"; import logger from "@app/logger/logger"; +import { apiError } from "@app/logger/withlogging"; + +export type LightContentNode = { + internalId: string; + parentInternalId: string | null; + type: ContentNodeType; + title: string; + expandable: boolean; + preventSelection?: boolean; + dustDocumentId: string | null; + lastUpdatedAt: number | null; +}; + +export type GetDataSourceContentResponseBody = { + nodes: LightContentNode[]; +}; + +export const getDataSourceCategory = ( + dataSource: DataSourceResource +): ResourceCategory => { + if (!dataSource.connectorProvider) { + return "files"; + } + + if (dataSource.connectorProvider === "webcrawler") { + return "webfolder"; + } + + return "managed"; +}; + +export const getContentHandler = async ( + req: NextApiRequest, + res: NextApiResponse>, + dataSource: DataSourceResource, + rootIds: string[] | null +): Promise => { + const viewType = req.query.viewType; + if ( + !viewType || + typeof viewType !== "string" || + (viewType !== "tables" && viewType !== "documents") + ) { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: "Invalid viewType. Required: tables | documents", + }, + }); + } + + let parentId: string | null = null; + if (req.query.parentId && typeof req.query.parentId === "string") { + parentId = req.query.parentId; + } + + const limit = req.query.limit ? parseInt(req.query.limit as string) : 10; + const offset = req.query.offset ? parseInt(req.query.offset as string) : 0; + + const content = dataSource.connectorId + ? await getManagedDataSourceContent( + dataSource.connectorId, + "read", + rootIds, + parentId, + viewType + ) + : await getUnmanagedDataSourceContent(dataSource, viewType, limit, offset); + + if (content.isErr()) { + return apiError(req, res, { + status_code: 500, + api_error: { + type: "internal_server_error", + message: `An error occurred while retrieving the data source permissions.`, + }, + }); + } + + res.status(200).json({ + nodes: content.value, + }); + return; +}; export const getManagedDataSourceContent = async ( connectorId: string, permission: "read" | "write" | "read_write" | "none", rootIds: string[] | null, parentId: string | null, - viewType: "tables" | "documents" -) => { + viewType: ContentNodesViewType +): Promise> => { const connectorsAPI = new ConnectorsAPI( config.getConnectorsAPIConfig(), logger @@ -58,10 +153,10 @@ export const getManagedDataSourceContent = async ( export const getUnmanagedDataSourceContent = async ( dataSource: DataSourceResource, - viewType: "tables" | "documents", + viewType: ContentNodesViewType, limit: number, offset: number -) => { +): Promise> => { const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger); if (viewType === "documents") { diff --git a/front/lib/resources/vault_resource.ts b/front/lib/resources/vault_resource.ts index add7313d6442..6d96fe418ae2 100644 --- a/front/lib/resources/vault_resource.ts +++ b/front/lib/resources/vault_resource.ts @@ -173,10 +173,10 @@ export class VaultResource extends BaseResource { return new this(VaultModel, vault.get()); } - static async fetchByName( + static async isNameAvailable( auth: Authenticator, name: string - ): Promise { + ): Promise { const owner = auth.getNonNullableWorkspace(); const vault = await this.model.findOne({ @@ -186,11 +186,7 @@ export class VaultResource extends BaseResource { }, }); - if (!vault) { - return null; - } - - return new this(VaultModel, vault.get()); + return !vault; } async delete( diff --git a/front/lib/swr.ts b/front/lib/swr.ts index df086d998bbb..3e4c8afed057 100644 --- a/front/lib/swr.ts +++ b/front/lib/swr.ts @@ -51,6 +51,10 @@ import type { GetLabsTranscriptsConfigurationResponseBody } from "@app/pages/api import type { GetMembersResponseBody } from "@app/pages/api/w/[wId]/members"; import type { GetProvidersResponseBody } from "@app/pages/api/w/[wId]/providers"; import type { GetSubscriptionsResponseBody } from "@app/pages/api/w/[wId]/subscriptions"; +import type { GetVaultsResponseBody } from "@app/pages/api/w/[wId]/vaults"; +import type { GetVaultResponseBody } from "@app/pages/api/w/[wId]/vaults/[vId]"; +import type { GetVaultDataSourceViewsResponseBody } from "@app/pages/api/w/[wId]/vaults/[vId]/data_source_views"; +import type { GetVaultDataSourcesResponseBody } from "@app/pages/api/w/[wId]/vaults/[vId]/data_sources"; import type { GetWorkspaceAnalyticsResponse } from "@app/pages/api/w/[wId]/workspace-analytics"; const DEFAULT_SWR_CONFIG: SWRConfiguration = { @@ -1261,3 +1265,81 @@ export function useLabsTranscriptsConfiguration({ mutateTranscriptsConfiguration: mutate, }; } + +export function useVaults({ workspaceId }: { workspaceId: string }) { + const vaultsFetcher: Fetcher = fetcher; + + const { data, error } = useSWRWithDefaults( + `/api/w/${workspaceId}/vaults`, + vaultsFetcher + ); + + return { + vaults: data ? data.vaults : null, + isVaultsLoading: !error && !data, + isVaultsError: error, + }; +} + +export function useVaultInfo({ + workspaceId, + vaultId, + disabled, +}: { + workspaceId: string; + vaultId: string; + disabled?: boolean; +}) { + const vaultsCategoriesFetcher: Fetcher = fetcher; + + const { data, error } = useSWRWithDefaults( + disabled ? null : `/api/w/${workspaceId}/vaults/${vaultId}`, + vaultsCategoriesFetcher + ); + + return { + vaultInfo: data ? data.vault : null, + isVaultInfoLoading: !error && !data, + isVaultInfoError: error, + }; +} + +export function useVaultDataSourceOrViews({ + workspaceId, + vaultId, + category, + type, + disabled, +}: { + workspaceId: string; + vaultId: string; + category: string; + type: "data_sources" | "data_source_views"; + disabled?: boolean; +}) { + const vaultsDataSourcesFetcher: Fetcher< + GetVaultDataSourcesResponseBody | GetVaultDataSourceViewsResponseBody + > = fetcher; + + const { data, error } = useSWRWithDefaults( + disabled + ? null + : `/api/w/${workspaceId}/vaults/${vaultId}/${type}?category=${category}`, + vaultsDataSourcesFetcher + ); + + console.log(data, error); + if (data && data.dataSources) { + return { + vaultDataSources: data ? data.dataSources : null, + isVaultDataSourcesLoading: !error && !data, + isVaultDataSourcesError: error, + }; + } else { + return { + vaultDataSources: data ? data.dataSourceViews : null, + isVaultDataSourcesLoading: !error && !data, + isVaultDataSourcesError: error, + }; + } +} diff --git a/front/pages/api/w/[wId]/vaults/[vId]/categories.ts b/front/pages/api/w/[wId]/vaults/[vId]/categories.ts deleted file mode 100644 index 71cf2aed33aa..000000000000 --- a/front/pages/api/w/[wId]/vaults/[vId]/categories.ts +++ /dev/null @@ -1,91 +0,0 @@ -import type { ResourceCategory, WithAPIErrorResponse } from "@dust-tt/types"; -import type { NextApiRequest, NextApiResponse } from "next"; - -import { withSessionAuthenticationForWorkspace } from "@app/lib/api/wrappers"; -import type { Authenticator } from "@app/lib/auth"; -import { VaultResource } from "@app/lib/resources/vault_resource"; -import { apiError } from "@app/logger/withlogging"; -import { getDataSourceViewsInfo } from "@app/pages/api/w/[wId]/vaults/[vId]/data_source_views"; -import { getDataSourceInfos } from "@app/pages/api/w/[wId]/vaults/[vId]/data_sources"; - -export type VaultCategoryInfo = { - category: ResourceCategory; - usage: number; - count: number; -}; - -export type GetVaultDataSourceCategoriesResponseBody = { - categories: VaultCategoryInfo[]; -}; - -async function handler( - req: NextApiRequest, - res: NextApiResponse< - WithAPIErrorResponse - >, - auth: Authenticator -): Promise { - const owner = auth.workspace(); - if (!owner) { - return apiError(req, res, { - status_code: 404, - api_error: { - type: "workspace_not_found", - message: "The workspace you requested was not found.", - }, - }); - } - - const vault = await VaultResource.fetchById(auth, req.query.vId as string); - - if ( - !vault || - (!auth.isAdmin() && !auth.hasPermission([vault.acl()], "read")) - ) { - return apiError(req, res, { - status_code: 404, - api_error: { - type: "vault_not_found", - message: "The vault you requested was not found.", - }, - }); - } - - switch (req.method) { - case "GET": - const all = [ - ...(await getDataSourceInfos(auth, vault)), - ...(await getDataSourceViewsInfo(auth, vault)), - ]; - - const categories = all.reduce((acc, dataSource) => { - const value = acc.find((i) => i.category === dataSource.category); - if (value) { - value.count += 1; - value.usage += dataSource.usage; - } else { - acc.push({ - category: dataSource.category, - count: 1, - usage: dataSource.usage, - }); - } - return acc; - }, [] as VaultCategoryInfo[]); - - res.status(200).json({ - categories, - }); - return; - default: - return apiError(req, res, { - status_code: 405, - api_error: { - type: "method_not_supported_error", - message: "The method passed is not supported, GET is expected.", - }, - }); - } -} - -export default withSessionAuthenticationForWorkspace(handler); diff --git a/front/pages/api/w/[wId]/vaults/[vId]/contentHandler.ts b/front/pages/api/w/[wId]/vaults/[vId]/contentHandler.ts deleted file mode 100644 index 3c8955955c36..000000000000 --- a/front/pages/api/w/[wId]/vaults/[vId]/contentHandler.ts +++ /dev/null @@ -1,79 +0,0 @@ -import type { ContentNodeType, WithAPIErrorResponse } from "@dust-tt/types"; -import type { NextApiRequest, NextApiResponse } from "next"; - -import { - getManagedDataSourceContent, - getUnmanagedDataSourceContent, -} from "@app/lib/api/vaults"; -import type { DataSourceResource } from "@app/lib/resources/data_source_resource"; -import { apiError } from "@app/logger/withlogging"; - -export type LightContentNode = { - internalId: string; - parentInternalId: string | null; - type: ContentNodeType; - title: string; - expandable: boolean; - preventSelection?: boolean; - dustDocumentId: string | null; - lastUpdatedAt: number | null; -}; - -export type GetDataSourceContentResponseBody = { - nodes: LightContentNode[]; -}; - -export const getContentHandler = async ( - req: NextApiRequest, - res: NextApiResponse>, - dataSource: DataSourceResource, - rootNodes: string[] | null -): Promise => { - const viewType = req.query.viewType; - if ( - !viewType || - typeof viewType !== "string" || - (viewType !== "tables" && viewType !== "documents") - ) { - return apiError(req, res, { - status_code: 400, - api_error: { - type: "invalid_request_error", - message: "Invalid viewType. Required: tables | documents", - }, - }); - } - - let parentId: string | null = null; - if (req.query.parentId && typeof req.query.parentId === "string") { - parentId = req.query.parentId; - } - - const limit = req.query.limit ? parseInt(req.query.limit as string) : 10; - const offset = req.query.offset ? parseInt(req.query.offset as string) : 0; - - const content = dataSource.connectorId - ? await getManagedDataSourceContent( - dataSource.connectorId, - "read", - rootNodes, - parentId, - viewType - ) - : await getUnmanagedDataSourceContent(dataSource, viewType, limit, offset); - - if (content.isErr()) { - return apiError(req, res, { - status_code: 500, - api_error: { - type: "internal_server_error", - message: `An error occurred while retrieving the data source permissions.`, - }, - }); - } - - res.status(200).json({ - nodes: content.value, - }); - return; -}; diff --git a/front/pages/api/w/[wId]/vaults/[vId]/data_source_views/[dsvId]/content.ts b/front/pages/api/w/[wId]/vaults/[vId]/data_source_views/[dsvId]/content.ts index 2df084b3f074..f2d4d1049ac9 100644 --- a/front/pages/api/w/[wId]/vaults/[vId]/data_source_views/[dsvId]/content.ts +++ b/front/pages/api/w/[wId]/vaults/[vId]/data_source_views/[dsvId]/content.ts @@ -1,13 +1,13 @@ import type { WithAPIErrorResponse } from "@dust-tt/types"; import type { NextApiRequest, NextApiResponse } from "next"; +import type { GetDataSourceContentResponseBody } from "@app/lib/api/vaults"; +import { getContentHandler } from "@app/lib/api/vaults"; import { withSessionAuthenticationForWorkspace } from "@app/lib/api/wrappers"; import type { Authenticator } from "@app/lib/auth"; import { DataSourceViewResource } from "@app/lib/resources/data_source_view_resource"; import { VaultResource } from "@app/lib/resources/vault_resource"; import { apiError } from "@app/logger/withlogging"; -import type { GetDataSourceContentResponseBody } from "@app/pages/api/w/[wId]/vaults/[vId]/contentHandler"; -import { getContentHandler } from "@app/pages/api/w/[wId]/vaults/[vId]/contentHandler"; async function handler( req: NextApiRequest, @@ -25,28 +25,21 @@ async function handler( }); } - const vault = await VaultResource.fetchById(auth, req.query.vId as string); - - if ( - !vault || - (!auth.isAdmin() && !auth.hasPermission([vault.acl()], "read")) - ) { - return apiError(req, res, { - status_code: 404, - api_error: { - type: "vault_not_found", - message: "The vault you requested was not found.", - }, - }); - } - const dataSourceView = await DataSourceViewResource.fetchById( auth, req.query.dsvId as string ); + const dataSource = dataSourceView?.dataSource; + const vault = dataSource?.vault; - if (!dataSourceView || !dataSource || dataSourceView.vaultId !== vault.id) { + if ( + !dataSourceView || + !dataSource || + !vault || + req.query.vId !== vault.sId || + (!auth.isAdmin() && !auth.hasPermission([vault.acl()], "read")) + ) { return apiError(req, res, { status_code: 404, api_error: { diff --git a/front/pages/api/w/[wId]/vaults/[vId]/data_source_views/index.ts b/front/pages/api/w/[wId]/vaults/[vId]/data_source_views/index.ts index 5a3bf853fd5e..ba9e65e76e5e 100644 --- a/front/pages/api/w/[wId]/vaults/[vId]/data_source_views/index.ts +++ b/front/pages/api/w/[wId]/vaults/[vId]/data_source_views/index.ts @@ -1,10 +1,7 @@ -import type { - ResourceCategory, - ResourceInfo, - WithAPIErrorResponse, -} from "@dust-tt/types"; +import type { ResourceInfo, WithAPIErrorResponse } from "@dust-tt/types"; import type { NextApiRequest, NextApiResponse } from "next"; +import { getDataSourceCategory } from "@app/lib/api/vaults"; import { withSessionAuthenticationForWorkspace } from "@app/lib/api/wrappers"; import type { Authenticator } from "@app/lib/auth"; import type { DataSourceResource } from "@app/lib/resources/data_source_resource"; @@ -16,20 +13,6 @@ export type GetVaultDataSourceViewsResponseBody = { dataSourceViews: ResourceInfo[]; }; -export const getDataSourceCategory = ( - dataSource: DataSourceResource -): ResourceCategory => { - if (!dataSource.connectorProvider) { - return "files"; - } - - if (dataSource.connectorProvider === "webcrawler") { - return "webfolder"; - } - - return "managed"; -}; - export const getDataSourceViewsInfo = async ( auth: Authenticator, vault: VaultResource diff --git a/front/pages/api/w/[wId]/vaults/[vId]/data_sources/[dsId]/content.ts b/front/pages/api/w/[wId]/vaults/[vId]/data_sources/[dsId]/content.ts index e6127e09ef8b..48f69f349c04 100644 --- a/front/pages/api/w/[wId]/vaults/[vId]/data_sources/[dsId]/content.ts +++ b/front/pages/api/w/[wId]/vaults/[vId]/data_sources/[dsId]/content.ts @@ -1,13 +1,13 @@ import type { WithAPIErrorResponse } from "@dust-tt/types"; import type { NextApiRequest, NextApiResponse } from "next"; +import type { GetDataSourceContentResponseBody } from "@app/lib/api/vaults"; +import { getContentHandler } from "@app/lib/api/vaults"; import { withSessionAuthenticationForWorkspace } from "@app/lib/api/wrappers"; import type { Authenticator } from "@app/lib/auth"; import { DataSourceResource } from "@app/lib/resources/data_source_resource"; import { VaultResource } from "@app/lib/resources/vault_resource"; import { apiError } from "@app/logger/withlogging"; -import type { GetDataSourceContentResponseBody } from "@app/pages/api/w/[wId]/vaults/[vId]/contentHandler"; -import { getContentHandler } from "@app/pages/api/w/[wId]/vaults/[vId]/contentHandler"; async function handler( req: NextApiRequest, @@ -25,27 +25,19 @@ async function handler( }); } - const vault = await VaultResource.fetchById(auth, req.query.vId as string); - - if ( - !vault || - (!auth.isAdmin() && !auth.hasPermission([vault.acl()], "read")) - ) { - return apiError(req, res, { - status_code: 404, - api_error: { - type: "vault_not_found", - message: "The vault you requested was not found.", - }, - }); - } - const dataSource = await DataSourceResource.fetchByName( auth, req.query.dsId as string ); - if (!dataSource || dataSource.vaultId !== vault.id) { + const vault = dataSource?.vault; + + if ( + !dataSource || + !vault || + req.query.vId !== vault.sId || + (!auth.isAdmin() && !auth.hasPermission([vault.acl()], "read")) + ) { return apiError(req, res, { status_code: 404, api_error: { diff --git a/front/pages/api/w/[wId]/vaults/[vId]/data_sources/index.ts b/front/pages/api/w/[wId]/vaults/[vId]/data_sources/index.ts index 4b0ac7b642ba..fa8e6ed45d84 100644 --- a/front/pages/api/w/[wId]/vaults/[vId]/data_sources/index.ts +++ b/front/pages/api/w/[wId]/vaults/[vId]/data_sources/index.ts @@ -1,10 +1,7 @@ -import type { - ResourceCategory, - ResourceInfo, - WithAPIErrorResponse, -} from "@dust-tt/types"; +import type { ResourceInfo, WithAPIErrorResponse } from "@dust-tt/types"; import type { NextApiRequest, NextApiResponse } from "next"; +import { getDataSourceCategory } from "@app/lib/api/vaults"; import { withSessionAuthenticationForWorkspace } from "@app/lib/api/wrappers"; import type { Authenticator } from "@app/lib/auth"; import { DataSourceResource } from "@app/lib/resources/data_source_resource"; @@ -15,20 +12,6 @@ export type GetVaultDataSourcesResponseBody = { dataSources: ResourceInfo[]; }; -export const getDataSourceCategory = ( - dataSource: DataSourceResource -): ResourceCategory => { - if (!dataSource.connectorProvider) { - return "files"; - } - - if (dataSource.connectorProvider === "webcrawler") { - return "webfolder"; - } - - return "managed"; -}; - export const getDataSourceInfos = async ( auth: Authenticator, vault: VaultResource diff --git a/front/pages/api/w/[wId]/vaults/[vId]/index.ts b/front/pages/api/w/[wId]/vaults/[vId]/index.ts index 88029ced8aeb..a480d2382fdf 100644 --- a/front/pages/api/w/[wId]/vaults/[vId]/index.ts +++ b/front/pages/api/w/[wId]/vaults/[vId]/index.ts @@ -12,9 +12,16 @@ import { GroupResource } from "@app/lib/resources/group_resource"; import { UserResource } from "@app/lib/resources/user_resource"; import { VaultResource } from "@app/lib/resources/vault_resource"; import { apiError } from "@app/logger/withlogging"; +import { getDataSourceViewsInfo } from "@app/pages/api/w/[wId]/vaults/[vId]/data_source_views"; +import { getDataSourceInfos } from "@app/pages/api/w/[wId]/vaults/[vId]/data_sources"; + +export type VaultCategoryInfo = { + usage: number; + count: number; +}; export type GetVaultResponseBody = { - vault: VaultType; + vault: VaultType & { categories: { [key: string]: VaultCategoryInfo } }; }; async function handler( @@ -61,7 +68,34 @@ async function handler( switch (req.method) { case "GET": - res.status(200).json({ vault: vault.toJSON() }); + const all = [ + ...(await getDataSourceInfos(auth, vault)), + ...(await getDataSourceViewsInfo(auth, vault)), + ]; + + const categories = all.reduce( + (acc, dataSource) => { + const value = acc[dataSource.category]; + if (value) { + value.count += 1; + value.usage += dataSource.usage; + } else { + acc[dataSource.category] = { + count: 1, + usage: dataSource.usage, + }; + } + return acc; + }, + {} as { [key: string]: VaultCategoryInfo } + ); + + res.status(200).json({ + vault: { + ...vault.toJSON(), + categories, + }, + }); return; case "PATCH": if (!auth.isAdmin() || !auth.isBuilder()) { diff --git a/front/pages/api/w/[wId]/vaults/index.ts b/front/pages/api/w/[wId]/vaults/index.ts index b4c6156aa96c..a41ec744a289 100644 --- a/front/pages/api/w/[wId]/vaults/index.ts +++ b/front/pages/api/w/[wId]/vaults/index.ts @@ -94,6 +94,13 @@ async function handler( type: "regular", }); + const vault = await VaultResource.makeNew({ + name, + kind: "regular", + workspaceId: owner.id, + groupId: group.id, + }); + if (members) { await Promise.all( members?.map(async (member) => { @@ -105,14 +112,7 @@ async function handler( ); } - const vault = await VaultResource.makeNew({ - name, - kind: "regular", - workspaceId: owner.id, - groupId: group.id, - }); return res.status(200).json({ vault: vault.toJSON() }); - default: return apiError(req, res, { status_code: 405,