From efd186748ddf2e473e18cb62ee2bb77b9601b777 Mon Sep 17 00:00:00 2001 From: Aric Lasry Date: Fri, 22 Dec 2023 16:20:34 +0100 Subject: [PATCH] Agent leaderboard --- front/lib/api/assistant/agent_usage.ts | 6 +- front/lib/redis.ts | 11 ++ front/lib/swr.ts | 27 +++++ .../agent_configurations/leaderboard.ts | 114 ++++++++++++++++++ front/pages/w/[wId]/assistant/gallery.tsx | 99 ++++++++++++--- .../internal/agent_configuration.ts | 11 ++ types/src/front/assistant/agent.ts | 4 + 7 files changed, 253 insertions(+), 19 deletions(-) create mode 100644 front/pages/api/w/[wId]/assistant/agent_configurations/leaderboard.ts diff --git a/front/lib/api/assistant/agent_usage.ts b/front/lib/api/assistant/agent_usage.ts index 46de05bb571c7..116a05399b952 100644 --- a/front/lib/api/assistant/agent_usage.ts +++ b/front/lib/api/assistant/agent_usage.ts @@ -172,9 +172,11 @@ async function populateUsageIfNeeded({ export async function getAgentUsage({ workspaceId, agentConfigurationId, + providedRedis, }: { workspaceId: string; agentConfigurationId: string; + providedRedis?: Awaited>; }): Promise { let redis: Awaited> | null = null; @@ -184,7 +186,7 @@ export async function getAgentUsage({ }); try { - redis = await redisClient(); + redis = providedRedis ?? (await redisClient()); await populateUsageIfNeeded({ agentConfigurationId, workspaceId, @@ -210,7 +212,7 @@ export async function getAgentUsage({ timePeriodSec: rankingTimeframeSec, }; } finally { - if (redis) { + if (redis && !providedRedis) { await redis.quit(); } } diff --git a/front/lib/redis.ts b/front/lib/redis.ts index 7e2966714378c..590f4e98960f7 100644 --- a/front/lib/redis.ts +++ b/front/lib/redis.ts @@ -14,3 +14,14 @@ export async function redisClient() { return client; } + +export async function safeRedisClient( + fn: (client: Awaited>) => PromiseLike +): Promise { + const client = await redisClient(); + try { + return await fn(client); + } finally { + await client.quit(); + } +} diff --git a/front/lib/swr.ts b/front/lib/swr.ts index 1d671146bfb0f..133f01cc10cec 100644 --- a/front/lib/swr.ts +++ b/front/lib/swr.ts @@ -21,6 +21,7 @@ import { GetRunBlockResponseBody } from "@app/pages/api/w/[wId]/apps/[aId]/runs/ import { GetRunStatusResponseBody } from "@app/pages/api/w/[wId]/apps/[aId]/runs/[runId]/status"; import { GetAgentConfigurationsResponseBody } from "@app/pages/api/w/[wId]/assistant/agent_configurations"; import { GetAgentUsageResponseBody } from "@app/pages/api/w/[wId]/assistant/agent_configurations/[aId]/usage"; +import { GetAgentConfigurationsLeaderboardResponseBody } from "@app/pages/api/w/[wId]/assistant/agent_configurations/leaderboard"; import { GetAgentNamesResponseBody } from "@app/pages/api/w/[wId]/assistant/agent_configurations/names"; import { GetDataSourcesResponseBody } from "@app/pages/api/w/[wId]/data_sources"; import { GetDocumentsResponseBody } from "@app/pages/api/w/[wId]/data_sources/[name]/documents"; @@ -468,6 +469,32 @@ export function useAgentConfigurations({ }; } +export function useAgentsLeaderboard({ + workspaceId, + agentsGetView, +}: { + workspaceId: string; + agentsGetView: AgentsGetViewType; +}) { + const agentsLeaderboardFetcher: Fetcher = + fetcher; + const viewQueryString = + typeof agentsGetView === "string" + ? `view=${agentsGetView}` + : `conversationId=${agentsGetView.conversationId}`; + const { data, error, mutate } = useSWR( + `/api/w/${workspaceId}/assistant/agent_configurations/leaderboard?${viewQueryString}`, + agentsLeaderboardFetcher + ); + + return { + agentsLeaderboard: data ? data.agentConfigurationsWithUsage : [], + isAgentsLeaderboardLoading: !error && !data, + isAgentsLeaderboardError: error, + mutateAgentsLeaderboard: mutate, + }; +} + export function useAgentUsage({ workspaceId, agentConfigurationId, diff --git a/front/pages/api/w/[wId]/assistant/agent_configurations/leaderboard.ts b/front/pages/api/w/[wId]/assistant/agent_configurations/leaderboard.ts new file mode 100644 index 0000000000000..7896f459dccb9 --- /dev/null +++ b/front/pages/api/w/[wId]/assistant/agent_configurations/leaderboard.ts @@ -0,0 +1,114 @@ +import { + AgentConfigurationTypeWithUsage, + GetAgentConfigurationsLeaderboardQuerySchema, +} from "@dust-tt/types"; +import { ReturnedAPIErrorType } from "@dust-tt/types"; +import { isLeft } from "fp-ts/lib/Either"; +import * as reporter from "io-ts-reporters"; +import { NextApiRequest, NextApiResponse } from "next"; + +import { getAgentUsage } from "@app/lib/api/assistant/agent_usage"; +import { getAgentConfigurations } from "@app/lib/api/assistant/configuration"; +import { Authenticator, getSession } from "@app/lib/auth"; +import { safeRedisClient } from "@app/lib/redis"; +import { apiError, withLogging } from "@app/logger/withlogging"; + +export type GetAgentConfigurationsLeaderboardResponseBody = { + agentConfigurationsWithUsage: AgentConfigurationTypeWithUsage[]; +}; + +async function handler( + req: NextApiRequest, + res: NextApiResponse< + GetAgentConfigurationsLeaderboardResponseBody | ReturnedAPIErrorType + > +): Promise { + const session = await getSession(req, res); + const auth = await Authenticator.fromSession( + session, + req.query.wId as string + ); + const owner = auth.workspace(); + if (!owner) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "workspace_not_found", + message: "The workspace you're trying to modify was not found.", + }, + }); + } + + switch (req.method) { + case "GET": { + if (!auth.isUser()) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "app_auth_error", + message: "Only the workspace users can see Assistants.", + }, + }); + } + const queryValidation = + GetAgentConfigurationsLeaderboardQuerySchema.decode(req.query); + if (isLeft(queryValidation)) { + const pathError = reporter.formatValidationErrors(queryValidation.left); + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: `Invalid query parameters: ${pathError}`, + }, + }); + } + const { view } = queryValidation.right; + + if (view === "admin_internal" && !auth.isDustSuperUser()) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "app_auth_error", + message: "Only Dust Super Users can see admin_internal agents.", + }, + }); + } + const agentConfigurations = await getAgentConfigurations(auth, view); + const agentConfigurationsWithUsage = await safeRedisClient( + async (client) => { + return await Promise.all( + agentConfigurations.map( + async ( + agentConfiguration + ): Promise => { + return { + ...agentConfiguration, + usage: await getAgentUsage({ + providedRedis: client, + workspaceId: owner.sId, + agentConfigurationId: agentConfiguration.sId, + }), + }; + } + ) + ); + } + ); + + return res + .status(200) + .json({ agentConfigurationsWithUsage: agentConfigurationsWithUsage }); + } + + default: + return apiError(req, res, { + status_code: 405, + api_error: { + type: "method_not_supported_error", + message: "The method passed is not supported, GET is expected.", + }, + }); + } +} + +export default withLogging(handler); diff --git a/front/pages/w/[wId]/assistant/gallery.tsx b/front/pages/w/[wId]/assistant/gallery.tsx index 1c12dcca6fcfd..449d26d7cb35c 100644 --- a/front/pages/w/[wId]/assistant/gallery.tsx +++ b/front/pages/w/[wId]/assistant/gallery.tsx @@ -1,4 +1,4 @@ -import { Page, Searchbar, Tab } from "@dust-tt/sparkle"; +import { Button, DropdownMenu, Page, Searchbar, Tab } from "@dust-tt/sparkle"; import { AgentConfigurationType, AgentsGetViewType, @@ -17,7 +17,7 @@ import AppLayout from "@app/components/sparkle/AppLayout"; import { AppLayoutSimpleCloseTitle } from "@app/components/sparkle/AppLayoutTitle"; import { subNavigationConversations } from "@app/components/sparkle/navigation"; import { Authenticator, getSession, getUserFromSession } from "@app/lib/auth"; -import { useAgentConfigurations } from "@app/lib/swr"; +import { useAgentConfigurations, useAgentsLeaderboard } from "@app/lib/swr"; import { subFilter } from "@app/lib/utils"; const { GA_TRACKING_ID = "" } = process.env; @@ -91,15 +91,48 @@ export default function AssistantsGallery({ workspaceId: owner.sId, agentsGetView, }); - const [assistantSearch, setAssistantSearch] = useState(""); - const filtered = agentConfigurations.filter((a) => { - return ( - subFilter(assistantSearch.toLowerCase(), a.name.toLowerCase()) && - a.status === "active" - ); + const { agentsLeaderboard } = useAgentsLeaderboard({ + workspaceId: owner.sId, + agentsGetView, }); + const [orderBy, setOrderBy] = useState<"name" | "usage">("name"); + const [assistantSearch, setAssistantSearch] = useState(""); + + let agentsToDisplay: AgentConfigurationType[] = []; + + switch (orderBy) { + case "name": { + agentsToDisplay = agentConfigurations + .filter((a) => { + return ( + subFilter(assistantSearch.toLowerCase(), a.name.toLowerCase()) && + a.status === "active" + ); + }) + .sort((a, b) => { + return a.name.localeCompare(b.name); + }); + break; + } + case "usage": { + agentsToDisplay = agentsLeaderboard + .filter((a) => { + return ( + subFilter(assistantSearch.toLowerCase(), a.name.toLowerCase()) && + a.status === "active" + ); + }) + .sort((a, b) => { + return b.usage.messageCount - a.usage.messageCount; + }); + break; + } + default: + assertNever(orderBy); + } + const [showDetails, setShowDetails] = useState( null ); @@ -179,17 +212,49 @@ export default function AssistantsGallery({
- { - setAssistantSearch(s); - }} - /> +
+
+ { + setAssistantSearch(s); + }} + /> +
+
+ + +
+
- {filtered.map((a) => ( + {agentsToDisplay.map((a) => (