From f1bfcec130731ea960afe2ad32e808439af5192c Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Fri, 22 Nov 2024 15:52:58 +0800 Subject: [PATCH 001/114] readme: add ossinsight widget (#396) --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 99b7633bb..28d75f2e0 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,14 @@ You can reach out to us on [@TiDB_Developer](https://twitter.com/TiDB_Developer) We welcome contributions from the community. If you are interested in contributing to the project, please read the [Contributing Guidelines](/CONTRIBUTING.md). + + + + Performance Stats of pingcap/autoflow - Last 28 days + + + + ## License TiDB.AI is open-source under the Apache License, Version 2.0. You can [find it here](/LICENSE.txt). From c2aad9ea9fe26a2d9b932c4bbaac8e1bf893aeb1 Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Fri, 22 Nov 2024 16:26:58 +0800 Subject: [PATCH 002/114] fix(frontend): update trace url --- .../app/src/components/chat/debug-info.tsx | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/frontend/app/src/components/chat/debug-info.tsx b/frontend/app/src/components/chat/debug-info.tsx index cd74369f5..5921836f7 100644 --- a/frontend/app/src/components/chat/debug-info.tsx +++ b/frontend/app/src/components/chat/debug-info.tsx @@ -8,6 +8,7 @@ import { Separator } from '@/components/ui/separator'; import { differenceInSeconds } from 'date-fns'; import { WorkflowIcon } from 'lucide-react'; import 'react-json-view-lite/dist/index.css'; +import { useMemo } from 'react'; export interface DebugInfoProps { group: ChatMessageGroup; @@ -19,12 +20,26 @@ export function DebugInfo ({ group }: DebugInfoProps) { const createdAt = useChatMessageField(group.assistant, 'created_at'); const finishedAt = useChatMessageField(group.assistant, 'finished_at'); + const stackVMUrl = useMemo(() => { + if (traceURL) { + try { + const url = new URL(traceURL); + if (url.host === 'stackvm.tidb.ai') { + const id = url.searchParams.get('task_id'); + return `https://stackvm-ui.vercel.app/tasks/${id}`; + } + } catch { + return undefined; + } + } + }, [traceURL]); + return (
{traceURL &&
- + - Langfuse Tracing + Tracing URL
} {/**/} From 6d49fca91e68965f8d3d088bd2204fced7e0dfd2 Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Fri, 22 Nov 2024 16:50:53 +0800 Subject: [PATCH 003/114] refactor(frontend): support create chat engine (#393) --- frontend/app/src/api/chat-engines.ts | 100 +++-- .../(main)/(admin)/chat-engines/[id]/page.tsx | 25 +- .../(main)/(admin)/chat-engines/new/page.tsx | 18 + .../app/src/app/(main)/(admin)/layout.tsx | 13 +- .../api/[[...fallback_placeholder]]/route.ts | 1 + .../chat-engine/chat-engine-details.tsx | 32 -- .../chat-engine-options-details.tsx | 48 --- .../chat-engine/chat-engines-table.tsx | 65 +-- .../chat-engine/create-chat-engine-form.tsx | 223 ++++++++++ .../chat-engine/edit-boolean-form.tsx | 40 -- .../chat-engine/edit-kg-boolean-form.tsx | 49 --- .../chat-engine/edit-kg-integer-form.tsx | 49 --- .../components/chat-engine/edit-llm-form.tsx | 38 -- .../components/chat-engine/edit-name-form.tsx | 35 -- .../chat-engine/edit-option-boolean-form.tsx | 39 -- .../edit-options-llm-prompt-form.tsx | 49 --- .../chat-engine/edit-property-form.tsx | 60 --- .../chat-engine/edit-reranker-form.tsx | 37 -- .../chat-engine/edit-token-form.tsx | 39 -- .../components/chat-engine/edit-url-form.tsx | 39 -- .../chat-engine/knowledge-graph-details.tsx | 20 - .../components/chat-engine/llm-details.tsx | 41 -- .../components/chat-engine/prompt-viewer.tsx | 27 -- .../chat-engine/update-chat-engine-form.tsx | 398 ++++++++++++++++++ .../app/src/components/chat/debug-info.tsx | 9 +- .../app/src/components/chat/message-input.tsx | 2 +- .../components/chat/use-message-feedback.ts | 15 +- frontend/app/src/components/date-format.tsx | 2 +- .../app/src/components/document-viewer.tsx | 5 +- frontend/app/src/components/form/biz.tsx | 25 ++ .../src/components/form/control-widget.tsx | 2 +- .../app/src/components/form/field-layout.tsx | 67 ++- .../components/form/widgets/PromptInput.tsx | 29 ++ frontend/app/src/components/grid/Grid.tsx | 18 + .../src/components/knowledge-base/KBInfo.tsx | 11 + .../src/components/knowledge-base/hooks.ts | 7 +- .../settings-form/GeneralSettingsField.tsx | 14 +- 37 files changed, 883 insertions(+), 808 deletions(-) create mode 100644 frontend/app/src/app/(main)/(admin)/chat-engines/new/page.tsx delete mode 100644 frontend/app/src/components/chat-engine/chat-engine-details.tsx delete mode 100644 frontend/app/src/components/chat-engine/chat-engine-options-details.tsx create mode 100644 frontend/app/src/components/chat-engine/create-chat-engine-form.tsx delete mode 100644 frontend/app/src/components/chat-engine/edit-boolean-form.tsx delete mode 100644 frontend/app/src/components/chat-engine/edit-kg-boolean-form.tsx delete mode 100644 frontend/app/src/components/chat-engine/edit-kg-integer-form.tsx delete mode 100644 frontend/app/src/components/chat-engine/edit-llm-form.tsx delete mode 100644 frontend/app/src/components/chat-engine/edit-name-form.tsx delete mode 100644 frontend/app/src/components/chat-engine/edit-option-boolean-form.tsx delete mode 100644 frontend/app/src/components/chat-engine/edit-options-llm-prompt-form.tsx delete mode 100644 frontend/app/src/components/chat-engine/edit-property-form.tsx delete mode 100644 frontend/app/src/components/chat-engine/edit-reranker-form.tsx delete mode 100644 frontend/app/src/components/chat-engine/edit-token-form.tsx delete mode 100644 frontend/app/src/components/chat-engine/edit-url-form.tsx delete mode 100644 frontend/app/src/components/chat-engine/knowledge-graph-details.tsx delete mode 100644 frontend/app/src/components/chat-engine/llm-details.tsx delete mode 100644 frontend/app/src/components/chat-engine/prompt-viewer.tsx create mode 100644 frontend/app/src/components/chat-engine/update-chat-engine-form.tsx create mode 100644 frontend/app/src/components/form/widgets/PromptInput.tsx create mode 100644 frontend/app/src/components/grid/Grid.tsx create mode 100644 frontend/app/src/components/knowledge-base/KBInfo.tsx diff --git a/frontend/app/src/api/chat-engines.ts b/frontend/app/src/api/chat-engines.ts index 821b5ad4e..70b03fe85 100644 --- a/frontend/app/src/api/chat-engines.ts +++ b/frontend/app/src/api/chat-engines.ts @@ -15,11 +15,22 @@ export interface ChatEngine { is_default: boolean; } +export interface CreateChatEngineParams { + name: string; + engine_options: ChatEngineOptions; + llm_id?: number | null; + fast_llm_id?: number | null; + reranker_id?: number | null; +} + export interface ChatEngineOptions { - external_engine_config?: object | null; + external_engine_config?: { + stream_chat_api_url?: string | null + } | null; + clarify_question?: boolean | null; knowledge_base?: ChatEngineKnowledgeBaseOptions | null; - knowledge_graph: ChatEngineKnowledgeGraphOptions; - llm: ChatEngineLLMOptions; + knowledge_graph?: ChatEngineKnowledgeGraphOptions | null; + llm?: ChatEngineLLMOptions | null; post_verification_url?: string | null; post_verification_token?: string | null; hide_sources?: boolean | null; @@ -30,63 +41,69 @@ export interface ChatEngineKnowledgeBaseOptions { } export interface ChatEngineKnowledgeGraphOptions { - depth: number; - enabled: boolean; - include_meta: boolean; - with_degree: boolean; - using_intent_search: boolean; + depth?: number | null; + enabled?: boolean | null; + include_meta?: boolean | null; + with_degree?: boolean | null; + using_intent_search?: boolean | null; } export type ChatEngineLLMOptions = { - condense_question_prompt: string; - text_qa_prompt: string; - refine_prompt: string; - - intent_graph_knowledge: string - normal_graph_knowledge: string - - // provider: string; - // reranker_provider: string; - // reranker_top_k: number; + condense_question_prompt?: string | null + condense_answer_prompt?: string | null + text_qa_prompt?: string | null + refine_prompt?: string | null + intent_graph_knowledge?: string | null + normal_graph_knowledge?: string | null + clarifying_question_prompt?: string | null + generate_goal_prompt?: string | null + further_questions_prompt?: string | null } export interface LinkedKnowledgeBaseOptions { - id: number; + id?: number | null; } const kbOptionsSchema = z.object({ - linked_knowledge_base: z.object({ id: number() }).nullable().optional(), -}); + linked_knowledge_base: z.object({ id: number().nullable().optional() }).nullable().optional(), +}).passthrough(); const kgOptionsSchema = z.object({ - depth: z.number(), - enabled: z.boolean(), - include_meta: z.boolean(), - with_degree: z.boolean(), - using_intent_search: z.boolean(), -}) satisfies ZodType; + depth: z.number().nullable().optional(), + enabled: z.boolean().nullable().optional(), + include_meta: z.boolean().nullable().optional(), + with_degree: z.boolean().nullable().optional(), + using_intent_search: z.boolean().nullable().optional(), +}).passthrough() satisfies ZodType; const llmOptionsSchema = z.object({ - condense_question_prompt: z.string(), - text_qa_prompt: z.string(), - refine_prompt: z.string(), - intent_graph_knowledge: z.string(), - normal_graph_knowledge: z.string(), + condense_question_prompt: z.string().nullable().optional(), + condense_answer_prompt: z.string().nullable().optional(), + text_qa_prompt: z.string().nullable().optional(), + refine_prompt: z.string().nullable().optional(), + intent_graph_knowledge: z.string().nullable().optional(), + normal_graph_knowledge: z.string().nullable().optional(), + clarifying_question_prompt: z.string().nullable().optional(), + generate_goal_prompt: z.string().nullable().optional(), + further_questions_prompt: z.string().nullable().optional(), // provider: z.string(), // reranker_provider: z.string(), // reranker_top_k: z.number(), }).passthrough() as ZodType; const chatEngineOptionsSchema = z.object({ - external_engine_config: z.object({}).passthrough().nullable().optional(), - kbOptionsSchema: kbOptionsSchema.nullable().optional(), - knowledge_graph: kgOptionsSchema, - llm: llmOptionsSchema, + external_engine_config: z.object({ + stream_chat_api_url: z.string().optional().nullable(), + }).nullable().optional(), + clarify_question: z.boolean().nullable().optional(), + knowledge_base: kbOptionsSchema.nullable().optional(), + knowledge_graph: kgOptionsSchema.nullable().optional(), + llm: llmOptionsSchema.nullable().optional(), post_verification_url: z.string().nullable().optional(), post_verification_token: z.string().nullable().optional(), hide_sources: z.boolean().nullable().optional(), -}) satisfies ZodType; +}).passthrough() satisfies ZodType; const chatEngineSchema = z.object({ id: z.number(), @@ -101,6 +118,13 @@ const chatEngineSchema = z.object({ is_default: z.boolean(), }) satisfies ZodType; +export async function getDefaultChatEngineOptions (): Promise { + return await fetch(requestUrl('/api/v1/admin/chat-engines-default-config'), { + headers: await authenticationHeaders(), + }) + .then(handleResponse(chatEngineOptionsSchema)); +} + export async function listChatEngines ({ page = 1, size = 10 }: PageParams = {}): Promise> { return await fetch(requestUrl('/api/v1/admin/chat-engines', { page, size }), { headers: await authenticationHeaders(), @@ -127,7 +151,7 @@ export async function updateChatEngine (id: number, partial: Partial) { +export async function createChatEngine (create: CreateChatEngineParams) { return await fetch(requestUrl(`/api/v1/admin/chat-engines`), { method: 'POST', headers: { diff --git a/frontend/app/src/app/(main)/(admin)/chat-engines/[id]/page.tsx b/frontend/app/src/app/(main)/(admin)/chat-engines/[id]/page.tsx index b3a92f02d..9df187362 100644 --- a/frontend/app/src/app/(main)/(admin)/chat-engines/[id]/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/chat-engines/[id]/page.tsx @@ -1,21 +1,22 @@ -import { getChatEngine } from '@/api/chat-engines'; +import { getChatEngine, getDefaultChatEngineOptions } from '@/api/chat-engines'; import { AdminPageHeading } from '@/components/admin-page-heading'; -import { ChatEngineDetails } from '@/components/chat-engine/chat-engine-details'; -import { Card, CardContent } from '@/components/ui/card'; +import { UpdateChatEngineForm } from '@/components/chat-engine/update-chat-engine-form'; export default async function ChatEnginePage ({ params }: { params: { id: string } }) { - const chatEngine = await getChatEngine(parseInt(params.id)); + const [chatEngine, defaultChatEngineOptions] = await Promise.all([ + getChatEngine(parseInt(params.id)), + getDefaultChatEngineOptions(), + ]); return ( <> - -
- - - - - -
+ + ); } diff --git a/frontend/app/src/app/(main)/(admin)/chat-engines/new/page.tsx b/frontend/app/src/app/(main)/(admin)/chat-engines/new/page.tsx new file mode 100644 index 000000000..afa3b6fb4 --- /dev/null +++ b/frontend/app/src/app/(main)/(admin)/chat-engines/new/page.tsx @@ -0,0 +1,18 @@ +import { getDefaultChatEngineOptions } from '@/api/chat-engines'; +import { AdminPageHeading } from '@/components/admin-page-heading'; +import { CreateChatEngineForm } from '@/components/chat-engine/create-chat-engine-form'; + +export default async function NewChatEnginePage () { + const defaultOptions = await getDefaultChatEngineOptions(); + return ( + <> + + + + ); +} diff --git a/frontend/app/src/app/(main)/(admin)/layout.tsx b/frontend/app/src/app/(main)/(admin)/layout.tsx index d7fdbbfb6..02cf7d3a5 100644 --- a/frontend/app/src/app/(main)/(admin)/layout.tsx +++ b/frontend/app/src/app/(main)/(admin)/layout.tsx @@ -1,23 +1,12 @@ import { AdminPageLayout } from '@/components/admin-page-layout'; -import { VersionStatus } from '@/components/VersionStatus'; import { requireAuth } from '@/lib/auth'; -import { type ReactNode, Suspense } from 'react'; +import { type ReactNode } from 'react'; export default async function Layout ({ children }: { children: ReactNode }) { await requireAuth(); return ( {children} -
- {'Frontend build '} - {process.env.GIT_BRANCH} - {' / '} - {process.env.GIT_VERSION} - {': '} - Loading version status...}> - - -
); } \ No newline at end of file diff --git a/frontend/app/src/app/api/[[...fallback_placeholder]]/route.ts b/frontend/app/src/app/api/[[...fallback_placeholder]]/route.ts index de5ca82a3..decaecd77 100644 --- a/frontend/app/src/app/api/[[...fallback_placeholder]]/route.ts +++ b/frontend/app/src/app/api/[[...fallback_placeholder]]/route.ts @@ -35,6 +35,7 @@ function handler (request: NextRequest) { return response; }, error => { console.error('[proxy]', request.method, newUrl.toString(), error); + return Promise.reject(error); }); } diff --git a/frontend/app/src/components/chat-engine/chat-engine-details.tsx b/frontend/app/src/components/chat-engine/chat-engine-details.tsx deleted file mode 100644 index 75026cdb7..000000000 --- a/frontend/app/src/components/chat-engine/chat-engine-details.tsx +++ /dev/null @@ -1,32 +0,0 @@ -import type { ChatEngine } from '@/api/chat-engines'; -import { ChatEngineOptionsDetails } from '@/components/chat-engine/chat-engine-options-details'; -import { EditBooleanForm } from '@/components/chat-engine/edit-boolean-form'; -import { EditLlmForm } from '@/components/chat-engine/edit-llm-form'; -import { EditNameForm } from '@/components/chat-engine/edit-name-form'; -import { EditRerankerForm } from '@/components/chat-engine/edit-reranker-form'; -import { EditTokenForm } from '@/components/chat-engine/edit-token-form'; -import { EditUrlForm } from '@/components/chat-engine/edit-url-form'; -import { LlmInfo } from '@/components/llm/LlmInfo'; -import { OptionDetail } from '@/components/option-detail'; -import { RerankerInfo } from '@/components/reranker/RerankerInfo'; -import { Separator } from '@/components/ui/separator'; -import { format } from 'date-fns'; - -export function ChatEngineDetails ({ chatEngine }: { chatEngine: ChatEngine }) { - return ( -
-
- - } /> - } editPanel={} /> - } editPanel={} /> - } editPanel={} /> - - - } /> -
- - -
- ); -} diff --git a/frontend/app/src/components/chat-engine/chat-engine-options-details.tsx b/frontend/app/src/components/chat-engine/chat-engine-options-details.tsx deleted file mode 100644 index 66304d097..000000000 --- a/frontend/app/src/components/chat-engine/chat-engine-options-details.tsx +++ /dev/null @@ -1,48 +0,0 @@ -import type { ChatEngine, ChatEngineOptions } from '@/api/chat-engines'; -import { EditBooleanForm } from '@/components/chat-engine/edit-boolean-form'; -import { EditOptionBooleanForm } from '@/components/chat-engine/edit-option-boolean-form'; -import { EditTokenForm } from '@/components/chat-engine/edit-token-form'; -import { EditUrlForm } from '@/components/chat-engine/edit-url-form'; -import { ChatEngineKnowledgeGraphDetails } from '@/components/chat-engine/knowledge-graph-details'; -import { ChatEngineLLMDetails } from '@/components/chat-engine/llm-details'; -import { OptionDetail } from '@/components/option-detail'; -import { Separator } from '@/components/ui/separator'; - -export function ChatEngineOptionsDetails ({ - detailed = true, - editable, - options, -}: { - detailed?: boolean - editable?: ChatEngine - options: ChatEngineOptions -}) { - return ( - <> -
-
Knowledge Graph
- -
- -
-
LLM
- -
- -
-
UI
-
- } /> -
-
- -
-
Post Verification
-
- } /> - {editable && } />} -
-
- - ); -} \ No newline at end of file diff --git a/frontend/app/src/components/chat-engine/chat-engines-table.tsx b/frontend/app/src/components/chat-engine/chat-engines-table.tsx index 327ff8487..110219ba8 100644 --- a/frontend/app/src/components/chat-engine/chat-engines-table.tsx +++ b/frontend/app/src/components/chat-engine/chat-engines-table.tsx @@ -6,15 +6,11 @@ import { boolean } from '@/components/cells/boolean'; import { datetime } from '@/components/cells/datetime'; import { link } from '@/components/cells/link'; import { mono } from '@/components/cells/mono'; -import { DangerousActionButton } from '@/components/dangerous-action-button'; import { DataTableRemote } from '@/components/data-table-remote'; -import { usePush } from '@/components/nextjs/app-router-hooks'; -import { Button } from '@/components/ui/button'; -import { useDataTable } from '@/components/use-data-table'; +import { NextLink } from '@/components/nextjs/NextLink'; import type { ColumnDef } from '@tanstack/react-table'; import { createColumnHelper } from '@tanstack/table-core'; import { CopyIcon, TrashIcon } from 'lucide-react'; -import { useState } from 'react'; import { toast } from 'sonner'; const helper = createColumnHelper(); @@ -63,6 +59,7 @@ const columns = [ export function ChatEnginesTable () { return ( New Chat Engine} columns={columns} apiKey="api.chat-engines.list" api={listChatEngines} @@ -70,61 +67,3 @@ export function ChatEnginesTable () { /> ); } - -function ChatEngineActions ({ chatEngine }: { chatEngine: ChatEngine }) { - return ( - - - - - ); -} - -function CloneButton ({ chatEngine }: { chatEngine: ChatEngine }) { - const [cloning, setCloning] = useState(false); - const [navigating, push] = usePush(); - - return ( - - ); -} - -function DeleteButton ({ chatEngine }: { chatEngine: ChatEngine }) { - const { reload } = useDataTable(); - - return ( - { - await deleteChatEngine(chatEngine.id); - reload?.(); - }} - variant="ghost" - className="text-xs text-destructive hover:text-destructive hover:bg-destructive/20" - > - - Delete - - ); -} diff --git a/frontend/app/src/components/chat-engine/create-chat-engine-form.tsx b/frontend/app/src/components/chat-engine/create-chat-engine-form.tsx new file mode 100644 index 000000000..24b476cea --- /dev/null +++ b/frontend/app/src/components/chat-engine/create-chat-engine-form.tsx @@ -0,0 +1,223 @@ +'use client'; + +import { type ChatEngineOptions, createChatEngine } from '@/api/chat-engines'; +import { KBSelect, LLMSelect, RerankerSelect } from '@/components/form/biz'; +import { FormInput, FormSwitch } from '@/components/form/control-widget'; +import { FormFieldBasicLayout, FormFieldContainedLayout } from '@/components/form/field-layout'; +import { FormRootError } from '@/components/form/root-error'; +import { handleSubmitHelper } from '@/components/form/utils'; +import { PromptInput } from '@/components/form/widgets/PromptInput'; +import { Grid2, Grid3 } from '@/components/grid/Grid'; +import { Button } from '@/components/ui/button'; +import { Form } from '@/components/ui/form'; +import { Separator } from '@/components/ui/separator'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { capitalCase } from 'change-case-all'; +import { useRouter } from 'next/navigation'; +import { type ReactNode, useTransition } from 'react'; +import { useForm } from 'react-hook-form'; +import { toast } from 'sonner'; +import { z } from 'zod'; + +const schema = z.object({ + name: z.string().min(1), + llm_id: z.number().optional(), + fast_llm_id: z.number().optional(), + reranker_id: z.number().optional(), + engine_options: z.object({ + knowledge_base: z.object({ + linked_knowledge_base: z.object({ + id: z.number(), + }), + }), + knowledge_graph: z.object({ + depth: z.string().pipe(z.coerce.number().min(1)).optional(), + }).passthrough(), + llm: z.object({}).passthrough(), + }).passthrough(), +}); + +export function CreateChatEngineForm ({ defaultChatEngineOptions }: { defaultChatEngineOptions: ChatEngineOptions }) { + const [transitioning, startTransition] = useTransition(); + const router = useRouter(); + + const form = useForm>({ + resolver: zodResolver(schema), + disabled: transitioning, + defaultValues: { + name: '', + engine_options: {}, + }, + }); + + const handleSubmit = handleSubmitHelper(form, async data => { + // TODO: refactor types + const ce = await createChatEngine(data as never); + startTransition(() => { + router.push(`/chat-engines/${ce.id}`); + }); + }, errors => { + toast.error('Validation failed', { + description: 'Please check your chat engine configurations.', + classNames: { + toast: 'group-[.toaster]:bg-destructive group-[.toaster]:text-destructive-foreground', + description: 'group-[.toast]:text-destructive-foreground/70', + }, + }); + }); + + return ( +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + {(['intent_graph_knowledge', 'normal_graph_knowledge'] as const).map(field => ( + + + + ))} + + +
+ +
+ + {(['condense_question_prompt', 'condense_answer_prompt', 'text_qa_prompt', 'refine_prompt'] as const).map(field => ( + + + + ))} + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + ); +} + +function Section ({ noSeparator, title, children }: { noSeparator?: boolean, title: ReactNode, children: ReactNode }) { + return ( + <> + {!noSeparator && } +
+

{title}

+ {children} +
+ + ); +} + +function SubSection ({ title, children }: { title: ReactNode, children: ReactNode }) { + return ( +
+

{title}

+ {children} +
+ ); +} + +const llmPromptFields = [ + 'condense_question_prompt', + 'condense_answer_prompt', + 'text_qa_prompt', + 'refine_prompt', + 'intent_graph_knowledge', + 'normal_graph_knowledge', + 'clarifying_question_prompt', + 'generate_goal_prompt', + 'further_questions_prompt', +] as const; + +const llmPromptDescriptions: { [P in typeof llmPromptFields[number]]: string } = { + 'condense_question_prompt': '/// Description TBD', + 'condense_answer_prompt': '/// Description TBD', + 'text_qa_prompt': '/// Description TBD', + 'refine_prompt': '/// Description TBD', + 'intent_graph_knowledge': '/// Description TBD', + 'normal_graph_knowledge': '/// Description TBD', + 'clarifying_question_prompt': '/// Description TBD', + 'generate_goal_prompt': '/// Description TBD', + 'further_questions_prompt': '/// Description TBD', +}; diff --git a/frontend/app/src/components/chat-engine/edit-boolean-form.tsx b/frontend/app/src/components/chat-engine/edit-boolean-form.tsx deleted file mode 100644 index 108dafe63..000000000 --- a/frontend/app/src/components/chat-engine/edit-boolean-form.tsx +++ /dev/null @@ -1,40 +0,0 @@ -'use client'; - -import { type ChatEngine, updateChatEngine } from '@/api/chat-engines'; -import { EditPropertyForm } from '@/components/chat-engine/edit-property-form'; -import { FormSwitch } from '@/components/form/control-widget'; -import { useRefresh } from '@/components/nextjs/app-router-hooks'; -import { toast } from 'sonner'; -import { z } from 'zod'; - -const booleanSchema = z.boolean(); - -type KeyOfType = keyof { [P in keyof T as T[P] extends Value ? P : never]: any } - -export interface EditBooleanFormProps { - chatEngine: ChatEngine; - type: KeyOfType; -} - -export function EditBooleanForm ({ type, chatEngine }: EditBooleanFormProps) { - const [refreshing, refresh] = useRefresh(); - - return ( - <> - { - await updateChatEngine(chatEngine.id, data); - refresh(); - toast('ChatEngine successfully updated.'); - }} - disabled={refreshing} - > - - - - ); -} diff --git a/frontend/app/src/components/chat-engine/edit-kg-boolean-form.tsx b/frontend/app/src/components/chat-engine/edit-kg-boolean-form.tsx deleted file mode 100644 index 7f992d9fb..000000000 --- a/frontend/app/src/components/chat-engine/edit-kg-boolean-form.tsx +++ /dev/null @@ -1,49 +0,0 @@ -'use client'; - -import { type ChatEngine, type ChatEngineOptions, updateChatEngine } from '@/api/chat-engines'; -import { EditPropertyForm } from '@/components/chat-engine/edit-property-form'; -import { FormSwitch } from '@/components/form/control-widget'; -import { useRefresh } from '@/components/nextjs/app-router-hooks'; -import { toast } from 'sonner'; -import { z } from 'zod'; - -const booleanSchema = z.boolean(); - -type KeyOfType = keyof { [P in keyof T as T[P] extends Value ? P : never]: any } - -export interface EditBooleanFormProps { - chatEngine: ChatEngine; - type: KeyOfType; -} - -export function EditKgBooleanForm ({ type, chatEngine }: EditBooleanFormProps) { - const [refreshing, refresh] = useRefresh(); - - return ( - <> - { - const options: ChatEngineOptions = { - knowledge_graph: { - ...chatEngine.engine_options.knowledge_graph, - ...data, - }, - llm: { ...chatEngine.engine_options.llm }, - }; - await updateChatEngine(chatEngine.id, { - engine_options: options, - }); - refresh(); - toast('ChatEngine successfully updated.'); - }} - disabled={refreshing} - > - - - - ); -} diff --git a/frontend/app/src/components/chat-engine/edit-kg-integer-form.tsx b/frontend/app/src/components/chat-engine/edit-kg-integer-form.tsx deleted file mode 100644 index 4ad4c4f4a..000000000 --- a/frontend/app/src/components/chat-engine/edit-kg-integer-form.tsx +++ /dev/null @@ -1,49 +0,0 @@ -'use client'; - -import { type ChatEngine, type ChatEngineOptions, updateChatEngine } from '@/api/chat-engines'; -import { EditPropertyForm } from '@/components/chat-engine/edit-property-form'; -import { FormInput } from '@/components/form/control-widget'; -import { useRefresh } from '@/components/nextjs/app-router-hooks'; -import { toast } from 'sonner'; -import { z } from 'zod'; - -const booleanSchema = z.coerce.number().int(); - -type KeyOfType = keyof { [P in keyof T as T[P] extends Value ? P : never]: any } - -export interface EditIntegerFormProps { - chatEngine: ChatEngine; - type: KeyOfType; -} - -export function EditKgIntegerForm ({ type, chatEngine }: EditIntegerFormProps) { - const [refreshing, refresh] = useRefresh(); - - return ( - <> - { - const options: ChatEngineOptions = { - knowledge_graph: { - ...chatEngine.engine_options.knowledge_graph, - ...data, - }, - llm: { ...chatEngine.engine_options.llm }, - }; - await updateChatEngine(chatEngine.id, { - engine_options: options, - }); - refresh(); - toast('ChatEngine successfully updated.'); - }} - disabled={refreshing} - > - - - - ); -} diff --git a/frontend/app/src/components/chat-engine/edit-llm-form.tsx b/frontend/app/src/components/chat-engine/edit-llm-form.tsx deleted file mode 100644 index fe094dd4b..000000000 --- a/frontend/app/src/components/chat-engine/edit-llm-form.tsx +++ /dev/null @@ -1,38 +0,0 @@ -'use client'; - -import { type ChatEngine, updateChatEngine } from '@/api/chat-engines'; -import { EditPropertyForm } from '@/components/chat-engine/edit-property-form'; -import { LLMSelect } from '@/components/form/biz'; -import { useRefresh } from '@/components/nextjs/app-router-hooks'; -import { toast } from 'sonner'; -import { z } from 'zod'; - -const schema = z.number().nullable(); - -export interface EditLlmFormProps { - chatEngine: ChatEngine; - type: 'llm_id' | 'fast_llm_id'; -} - -export function EditLlmForm ({ type, chatEngine }: EditLlmFormProps) { - const [refreshing, refresh] = useRefresh(); - - return ( - <> - { - await updateChatEngine(chatEngine.id, data); - refresh(); - toast(`ChatEngine successfully updated.`); - }} - inline - disabled={refreshing} - > - - - - ); -} \ No newline at end of file diff --git a/frontend/app/src/components/chat-engine/edit-name-form.tsx b/frontend/app/src/components/chat-engine/edit-name-form.tsx deleted file mode 100644 index 266f6957b..000000000 --- a/frontend/app/src/components/chat-engine/edit-name-form.tsx +++ /dev/null @@ -1,35 +0,0 @@ -'use client'; - -import { type ChatEngine, updateChatEngine } from '@/api/chat-engines'; -import { EditPropertyForm } from '@/components/chat-engine/edit-property-form'; -import { FormInput } from '@/components/form/control-widget'; -import { useRefresh } from '@/components/nextjs/app-router-hooks'; -import { toast } from 'sonner'; -import { z } from 'zod'; - -export interface EditNameFormProps { - chatEngine: ChatEngine; -} - -const stringSchema = z.string(); - -export function EditNameForm ({ chatEngine }: EditNameFormProps) { - const [refreshing, refresh] = useRefresh(); - - return ( - { - await updateChatEngine(chatEngine.id, data); - refresh(); - toast('ChatEngine\'s name successfully updated.'); - }} - disabled={refreshing} - > - - - ); -} diff --git a/frontend/app/src/components/chat-engine/edit-option-boolean-form.tsx b/frontend/app/src/components/chat-engine/edit-option-boolean-form.tsx deleted file mode 100644 index 0af4839c0..000000000 --- a/frontend/app/src/components/chat-engine/edit-option-boolean-form.tsx +++ /dev/null @@ -1,39 +0,0 @@ -'use client'; - -import { type ChatEngine, type ChatEngineOptions, updateChatEngine } from '@/api/chat-engines'; -import { EditPropertyForm } from '@/components/chat-engine/edit-property-form'; -import { FormSwitch } from '@/components/form/control-widget'; -import { useRefresh } from '@/components/nextjs/app-router-hooks'; -import { toast } from 'sonner'; -import { z } from 'zod'; - -type KeyOfType = keyof { [P in keyof T as T[P] extends Value ? P : never]: any } - -export interface EditOptionBooleanFormProps

> { - property: P; - chatEngine: ChatEngine; -} - -const booleanSchema = z.boolean(); - -export function EditOptionBooleanForm

> ({ property, chatEngine }: EditOptionBooleanFormProps

) { - const [refreshing, refresh] = useRefresh(); - - return ( - { - const options = { ...chatEngine.engine_options, ...data }; - await updateChatEngine(chatEngine.id, { engine_options: options }); - refresh(); - toast(`ChatEngine's option ${property} successfully updated.`); - }} - disabled={refreshing} - > - - - ); -} diff --git a/frontend/app/src/components/chat-engine/edit-options-llm-prompt-form.tsx b/frontend/app/src/components/chat-engine/edit-options-llm-prompt-form.tsx deleted file mode 100644 index a96635c76..000000000 --- a/frontend/app/src/components/chat-engine/edit-options-llm-prompt-form.tsx +++ /dev/null @@ -1,49 +0,0 @@ -'use client'; - -import { type ChatEngine, updateChatEngine } from '@/api/chat-engines'; -import { EditPropertyForm } from '@/components/chat-engine/edit-property-form'; -import { FormTextarea } from '@/components/form/control-widget'; -import { useRefresh } from '@/components/nextjs/app-router-hooks'; -import { toast } from 'sonner'; -import { z } from 'zod'; - -export interface EditOptionsLlmPromptFormProps { - chatEngine: ChatEngine; - type: 'intent_graph_knowledge' - | 'normal_graph_knowledge' - | 'condense_question_prompt' - | 'refine_prompt' - | 'text_qa_prompt'; -} - -const schema = z.string().min(1); - -export function EditOptionsLlmPromptForm ({ chatEngine, type }: EditOptionsLlmPromptFormProps) { - const [refreshing, refresh] = useRefresh(); - - return ( - <> - { - const chatEngineOptions = { - ...chatEngine.engine_options, - llm: { - ...chatEngine.engine_options.llm, - ...data, - }, - }; - await updateChatEngine(chatEngine.id, { engine_options: chatEngineOptions }); - refresh(); - toast(`ChatEngine successfully updated.`); - }} - disabled={refreshing} - > - - - - ); -} \ No newline at end of file diff --git a/frontend/app/src/components/chat-engine/edit-property-form.tsx b/frontend/app/src/components/chat-engine/edit-property-form.tsx deleted file mode 100644 index 1a6df74c0..000000000 --- a/frontend/app/src/components/chat-engine/edit-property-form.tsx +++ /dev/null @@ -1,60 +0,0 @@ -import { FormFieldInlineLayout } from '@/components/form/field-layout'; -import { useManagedDialog } from '@/components/managed-dialog'; -import { Button } from '@/components/ui/button'; -import { Form } from '@/components/ui/form'; -import { cn } from '@/lib/utils'; -import { zodResolver } from '@hookform/resolvers/zod'; -import { type ReactElement, useMemo } from 'react'; -import { type DefaultValues, useForm } from 'react-hook-form'; -import { z, type ZodType } from 'zod'; - -export interface EditPropertyFormProps { - className?: string; - object: T; - property: P; - schema: ZodType; - description?: string; - disabled?: boolean; - children: ReactElement; - inline?: boolean; - - onSubmit (value: Record): void; -} - -export function EditPropertyForm ({ className, object, property, schema, onSubmit, inline, disabled, description, children }: EditPropertyFormProps) { - const { setOpen } = useManagedDialog(); - - const resolver = useMemo(() => { - return zodResolver(z.object({ - [property]: schema, - })); - }, [schema, property]); - - const form = useForm>({ - resolver, - disabled, - defaultValues: { - [property]: object[property] ?? '', - } as DefaultValues>, - }); - - const handleSubmit = form.handleSubmit(async (data) => { - onSubmit(data); - setOpen(false); - }); - - return ( -

- - - {children} - -
- -
-
- - ); -} \ No newline at end of file diff --git a/frontend/app/src/components/chat-engine/edit-reranker-form.tsx b/frontend/app/src/components/chat-engine/edit-reranker-form.tsx deleted file mode 100644 index c15654d37..000000000 --- a/frontend/app/src/components/chat-engine/edit-reranker-form.tsx +++ /dev/null @@ -1,37 +0,0 @@ -'use client'; - -import { type ChatEngine, updateChatEngine } from '@/api/chat-engines'; -import { EditPropertyForm } from '@/components/chat-engine/edit-property-form'; -import { RerankerSelect } from '@/components/form/biz'; -import { useRefresh } from '@/components/nextjs/app-router-hooks'; -import { toast } from 'sonner'; -import { z } from 'zod'; - -const schema = z.number().nullable(); - -export interface EditRerankerFormProps { - chatEngine: ChatEngine; -} - -export function EditRerankerForm ({ chatEngine }: EditRerankerFormProps) { - const [refreshing, refresh] = useRefresh(); - - return ( - <> - { - await updateChatEngine(chatEngine.id, data); - refresh(); - toast(`ChatEngine successfully updated.`); - }} - inline - disabled={refreshing} - > - - - - ); -} \ No newline at end of file diff --git a/frontend/app/src/components/chat-engine/edit-token-form.tsx b/frontend/app/src/components/chat-engine/edit-token-form.tsx deleted file mode 100644 index b040901bf..000000000 --- a/frontend/app/src/components/chat-engine/edit-token-form.tsx +++ /dev/null @@ -1,39 +0,0 @@ -'use client'; - -import { type ChatEngine, type ChatEngineOptions, updateChatEngine } from '@/api/chat-engines'; -import { EditPropertyForm } from '@/components/chat-engine/edit-property-form'; -import { FormInput } from '@/components/form/control-widget'; -import { useRefresh } from '@/components/nextjs/app-router-hooks'; -import { toast } from 'sonner'; -import { z } from 'zod'; - -type KeyOfType = keyof { [P in keyof T as T[P] extends Value ? P : never]: any } - -export interface EditNameFormProps

> { - property: P; - chatEngine: ChatEngine; -} - -const stringSchema = z.string(); - -export function EditTokenForm

> ({ property, chatEngine }: EditNameFormProps

) { - const [refreshing, refresh] = useRefresh(); - - return ( - { - const options = { ...chatEngine.engine_options, ...data }; - await updateChatEngine(chatEngine.id, { engine_options: options }); - refresh(); - toast(`ChatEngine's ${property} successfully updated.`); - }} - disabled={refreshing} - > - - - ); -} diff --git a/frontend/app/src/components/chat-engine/edit-url-form.tsx b/frontend/app/src/components/chat-engine/edit-url-form.tsx deleted file mode 100644 index c183695c1..000000000 --- a/frontend/app/src/components/chat-engine/edit-url-form.tsx +++ /dev/null @@ -1,39 +0,0 @@ -'use client'; - -import { type ChatEngine, type ChatEngineOptions, updateChatEngine } from '@/api/chat-engines'; -import { EditPropertyForm } from '@/components/chat-engine/edit-property-form'; -import { FormInput } from '@/components/form/control-widget'; -import { useRefresh } from '@/components/nextjs/app-router-hooks'; -import { toast } from 'sonner'; -import { z } from 'zod'; - -type KeyOfType = keyof { [P in keyof T as T[P] extends Value ? P : never]: any } - -export interface EditNameFormProps

> { - property: P; - chatEngine: ChatEngine; -} - -const stringSchema = z.string().url(); - -export function EditUrlForm

> ({ property, chatEngine }: EditNameFormProps

) { - const [refreshing, refresh] = useRefresh(); - - return ( - { - const options = { ...chatEngine.engine_options, ...data }; - await updateChatEngine(chatEngine.id, { engine_options: options }); - refresh(); - toast(`ChatEngine's option ${property} successfully updated.`); - }} - disabled={refreshing} - > - - - ); -} diff --git a/frontend/app/src/components/chat-engine/knowledge-graph-details.tsx b/frontend/app/src/components/chat-engine/knowledge-graph-details.tsx deleted file mode 100644 index 2107e3768..000000000 --- a/frontend/app/src/components/chat-engine/knowledge-graph-details.tsx +++ /dev/null @@ -1,20 +0,0 @@ -import type { ChatEngine, ChatEngineKnowledgeGraphOptions } from '@/api/chat-engines'; -import { EditKgBooleanForm } from '@/components/chat-engine/edit-kg-boolean-form'; -import { EditKgIntegerForm } from '@/components/chat-engine/edit-kg-integer-form'; -import { OptionDetail } from '@/components/option-detail'; - -export function ChatEngineKnowledgeGraphDetails ({ detailed, editable, options }: { detailed: boolean, editable?: ChatEngine, options: ChatEngineKnowledgeGraphOptions }) { - return ( -

- } /> - {(detailed || options.enabled) && ( - <> - } /> - } /> - } /> - } /> - - )} -
- ); -} diff --git a/frontend/app/src/components/chat-engine/llm-details.tsx b/frontend/app/src/components/chat-engine/llm-details.tsx deleted file mode 100644 index aa28c71ad..000000000 --- a/frontend/app/src/components/chat-engine/llm-details.tsx +++ /dev/null @@ -1,41 +0,0 @@ -import type { ChatEngine, ChatEngineLLMOptions } from '@/api/chat-engines'; -import { EditOptionsLlmPromptForm } from '@/components/chat-engine/edit-options-llm-prompt-form'; -import { PromptViewer } from '@/components/chat-engine/prompt-viewer'; -import { OptionDetail } from '@/components/option-detail'; -import { Button } from '@/components/ui/button'; -import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogTrigger } from '@/components/ui/dialog'; -import { ScrollArea } from '@/components/ui/scroll-area'; - -export function ChatEngineLLMDetails ({ editable, options }: { editable?: ChatEngine, options: ChatEngineLLMOptions }) { - return ( -
- {options.condense_question_prompt && } editPanel={editable && } />} - {options.refine_prompt && } editPanel={editable && } />} - {options.text_qa_prompt && } editPanel={editable && } />} - {options.intent_graph_knowledge && } editPanel={editable && } />} - {options.normal_graph_knowledge && } editPanel={editable && } />} -
- ); -} - -function PromptPreviewDialog ({ title, value }: { title: string, value: string }) { - return ( - - - - - - - - {title} - - - - - - - - ); -} diff --git a/frontend/app/src/components/chat-engine/prompt-viewer.tsx b/frontend/app/src/components/chat-engine/prompt-viewer.tsx deleted file mode 100644 index 8577cb128..000000000 --- a/frontend/app/src/components/chat-engine/prompt-viewer.tsx +++ /dev/null @@ -1,27 +0,0 @@ -'use client'; - -import Highlight from 'highlight.js/lib/core'; -import django from 'highlight.js/lib/languages/django'; -import { useEffect, useState } from 'react'; -import '../code-theme.scss'; - -Highlight.registerLanguage('django', django); - -export function PromptViewer ({ value: propValue }: { value: string }) { - const [value, setValue] = useState(propValue); - - useEffect(() => { - setValue(propValue); - try { - const { value: result } = Highlight.highlight('django', propValue); - setValue(result); - } catch { - } - }, [propValue]); - - return ( - -
-    
-  );
-}
\ No newline at end of file
diff --git a/frontend/app/src/components/chat-engine/update-chat-engine-form.tsx b/frontend/app/src/components/chat-engine/update-chat-engine-form.tsx
new file mode 100644
index 000000000..89512d11c
--- /dev/null
+++ b/frontend/app/src/components/chat-engine/update-chat-engine-form.tsx
@@ -0,0 +1,398 @@
+'use client';
+
+import { type ChatEngine, type ChatEngineKnowledgeGraphOptions, type ChatEngineLLMOptions, type ChatEngineOptions, updateChatEngine } from '@/api/chat-engines';
+import { KBSelect, LLMSelect, RerankerSelect } from '@/components/form/biz';
+import { FormInput, FormSwitch } from '@/components/form/control-widget';
+import { FormFieldBasicLayout, FormFieldContainedLayout } from '@/components/form/field-layout';
+import { PromptInput } from '@/components/form/widgets/PromptInput';
+import { Grid2, Grid3 } from '@/components/grid/Grid';
+import { fieldAccessor, GeneralSettingsField, type GeneralSettingsFieldAccessor, GeneralSettingsForm, shallowPick } from '@/components/settings-form';
+import { Separator } from '@/components/ui/separator';
+import type { KeyOfType } from '@/lib/typing-utils';
+import { capitalCase } from 'change-case-all';
+import { format } from 'date-fns';
+import { useRouter } from 'next/navigation';
+import { type ReactNode, useTransition } from 'react';
+import { z } from 'zod';
+
+export function UpdateChatEngineForm ({ chatEngine, defaultChatEngineOptions }: { chatEngine: ChatEngine, defaultChatEngineOptions: ChatEngineOptions }) {
+  const [transitioning, startTransition] = useTransition();
+  const router = useRouter();
+
+  return (
+     {
+        if (updatableFields.includes(path[0] as any)) {
+          const partial = shallowPick(data, path as [(typeof updatableFields)[number], ...any[]]);
+          await updateChatEngine(chatEngine.id, partial);
+          startTransition(() => {
+            router.refresh();
+          });
+        } else {
+          throw new Error(`${path.map(p => String(p)).join('.')} is not updatable currently.`);
+        }
+      }}
+    >
+      
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {(['intent_graph_knowledge', 'normal_graph_knowledge'] as const).map(type => ( + + + + + + ))} + + +
+ +
+ + {(['condense_question_prompt', 'condense_answer_prompt', 'text_qa_prompt', 'refine_prompt'] as const).map(type => ( + + + + + + ))} + +
+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+ ); +} + +const updatableFields = ['name', 'llm_id', 'fast_llm_id', 'reranker_id', 'engine_options', 'is_default'] as const; + +function optionAccessor (key: K): GeneralSettingsFieldAccessor { + return { + path: ['engine_options', key], + get (engine) { + return engine.engine_options[key]; + }, + set (engine, value) { + return { + ...engine, + engine_options: { + ...engine.engine_options, + [key]: value, + }, + }; + }, + }; +} + +function kgOptionAccessor (key: K): GeneralSettingsFieldAccessor { + return { + path: ['engine_options', 'knowledge_graph', key], + get (engine) { + return engine.engine_options.knowledge_graph?.[key]; + }, + set (engine, value) { + return { + ...engine, + engine_options: { + ...engine.engine_options, + knowledge_graph: { + ...engine.engine_options.knowledge_graph, + [key]: value, + }, + }, + }; + }, + }; +} + +function llmOptionAccessor (key: K): GeneralSettingsFieldAccessor { + return { + path: ['engine_options', 'llm', key], + get (engine) { + return engine.engine_options.llm?.[key]; + }, + set (engine, value) { + return { + ...engine, + engine_options: { + ...engine.engine_options, + llm: { + ...engine.engine_options.llm, + [key]: value, + }, + }, + }; + }, + }; +} + +const getDatetimeAccessor = (key: KeyOfType): GeneralSettingsFieldAccessor => { + return { + path: [key], + get (data) { + return format(data[key], 'yyyy-MM-dd HH:mm:ss'); + }, + set () { + throw new Error(`update ${key} is not supported`); + }, + }; +}; + +const idAccessor = fieldAccessor('id'); + +const createdAccessor = getDatetimeAccessor('created_at'); +const updatedAccessor = getDatetimeAccessor('updated_at'); +const neverSchema = z.never(); + +const nameAccessor = fieldAccessor('name'); +const nameSchema = z.string().min(1); + +const clarifyAccessor = optionAccessor('clarify_question'); +const clarifyAccessorSchema = z.boolean().nullable().optional(); + +const isDefaultAccessor = fieldAccessor('is_default'); +const isDefaultSchema = z.boolean(); + +const getIdAccessor = (id: KeyOfType) => fieldAccessor>(id); +const idSchema = z.number().nullable(); +const llmIdAccessor = getIdAccessor('llm_id'); +const fastLlmIdAccessor = getIdAccessor('fast_llm_id'); +const rerankerIdAccessor = getIdAccessor('reranker_id'); + +const kbAccessor: GeneralSettingsFieldAccessor = { + path: ['engine_options'], + get (data) { + return data.engine_options.knowledge_base?.linked_knowledge_base?.id ?? null; + }, + set (data, id) { + return { + ...data, + engine_options: { + ...data.engine_options, + knowledge_base: { + linked_knowledge_base: { id }, + }, + }, + }; + }, +}; +const kbSchema = z.number(); + +const kgEnabledAccessor = kgOptionAccessor('enabled'); +const kgEnabledSchema = z.boolean().nullable(); + +const kgWithDegreeAccessor = kgOptionAccessor('with_degree'); +const kgWithDegreeSchema = z.boolean().nullable(); + +const kgIncludeMetaAccessor = kgOptionAccessor('include_meta'); +const kgIncludeMetaSchema = z.boolean().nullable(); + +const kgUsingIntentSearchAccessor = kgOptionAccessor('using_intent_search'); +const kgUsingIntentSearchSchema = z.boolean().nullable(); + +const kgDepthAccessor = kgOptionAccessor('depth'); +const kgDepthSchema = z.string().pipe(z.coerce.number().int().min(1)).nullable(); + +const hideSourcesAccessor = optionAccessor('hide_sources'); +const hideSourcesSchema = z.boolean().nullable(); + +const llmPromptFields = [ + 'condense_question_prompt', + 'condense_answer_prompt', + 'text_qa_prompt', + 'refine_prompt', + 'intent_graph_knowledge', + 'normal_graph_knowledge', + 'clarifying_question_prompt', + 'generate_goal_prompt', + 'further_questions_prompt', +] as const; + +const llmAccessor: { [P in (typeof llmPromptFields[number])]: GeneralSettingsFieldAccessor } = Object.fromEntries(llmPromptFields.map(name => [name, llmOptionAccessor(name)])) as never; +const llmSchema = z.string().nullable(); + +const postVerificationUrlAccessor = optionAccessor('post_verification_url'); +const postVerificationUrlSchema = z.string().nullable(); + +const postVerificationTokenAccessor = optionAccessor('post_verification_token'); +const postVerificationTokenSchema = z.string().nullable(); + +const externalEngineAccessor: GeneralSettingsFieldAccessor = { + path: ['engine_options'], + get (engine) { + return engine.engine_options.external_engine_config?.stream_chat_api_url ?? null; + }, + set (engine, value) { + return { + ...engine, + engine_options: { + ...engine.engine_options, + external_engine_config: { + stream_chat_api_url: value, + }, + }, + }; + }, +}; +const externalEngineSchema = z.string().nullable(); + +function Section ({ noSeparator, title, children }: { noSeparator?: boolean, title: ReactNode, children: ReactNode }) { + return ( + <> + {!noSeparator && } +
+

{title}

+ {children} +
+ + ); +} + +function SubSection ({ title, children }: { title: ReactNode, children: ReactNode }) { + return ( +
+

{title}

+ {children} +
+ ); +} diff --git a/frontend/app/src/components/chat/debug-info.tsx b/frontend/app/src/components/chat/debug-info.tsx index 5921836f7..d5fc61555 100644 --- a/frontend/app/src/components/chat/debug-info.tsx +++ b/frontend/app/src/components/chat/debug-info.tsx @@ -1,10 +1,8 @@ -import { ChatEngineOptionsDetails } from '@/components/chat-engine/chat-engine-options-details'; import { type ChatMessageGroup, useChatInfo, useChatMessageField, useCurrentChatController } from '@/components/chat/chat-hooks'; import { KnowledgeGraphDebugInfo } from '@/components/chat/knowledge-graph-debug-info'; import { DateFormat } from '@/components/date-format'; import { OptionDetail } from '@/components/option-detail'; // import { MessageLangfuse } from '@/components/chat/message-langfuse'; -import { Separator } from '@/components/ui/separator'; import { differenceInSeconds } from 'date-fns'; import { WorkflowIcon } from 'lucide-react'; import 'react-json-view-lite/dist/index.css'; @@ -52,16 +50,11 @@ export function DebugInfo ({ group }: DebugInfoProps) { } /> } /> +
)} - - {chat?.engine_options && ( -
- -
- )} ); } diff --git a/frontend/app/src/components/chat/message-input.tsx b/frontend/app/src/components/chat/message-input.tsx index 3bb15f288..2e526a0b3 100644 --- a/frontend/app/src/components/chat/message-input.tsx +++ b/frontend/app/src/components/chat/message-input.tsx @@ -72,7 +72,7 @@ export function MessageInput ({ {item.is_default ? default : item.name} {item.engine_options.external_engine_config ? External Engine (StackVM) - : item.engine_options.knowledge_graph.enabled + : item.engine_options.knowledge_graph?.enabled !== false /* TODO: require default config */ ? Knowledge graph enabled : undefined} diff --git a/frontend/app/src/components/chat/use-message-feedback.ts b/frontend/app/src/components/chat/use-message-feedback.ts index 9f45655a9..8fc67cd11 100644 --- a/frontend/app/src/components/chat/use-message-feedback.ts +++ b/frontend/app/src/components/chat/use-message-feedback.ts @@ -7,12 +7,7 @@ export interface UseMessageFeedbackReturns { feedbackData: FeedbackParams | undefined; disabled: boolean; - // source?: ContentSource; - // sourceLoading: boolean; - - feedback (action: 'like' | 'dislike', /*details: Record, */comment: string): Promise; - - // deleteFeedback (): Promise; + feedback (action: 'like' | 'dislike', comment: string): Promise; } export function useMessageFeedback (messageId: number | undefined, enabled: boolean = true): UseMessageFeedbackReturns { @@ -22,8 +17,6 @@ export function useMessageFeedback (messageId: number | undefined, enabled: bool const [acting, setActing] = useState(false); const disabled = messageId == null && isValidating || isLoading || acting || !enabled; - // const contentData = useSWR((enabled && !disabled) ? ['get', `/api/v1/chats/${chatId}/messages/${messageId}/content-sources`] : undefined, fetcher, { keepPreviousData: true, revalidateIfStale: false, revalidateOnReconnect: false }); - return { feedbackData: feedback, disabled, @@ -35,11 +28,5 @@ export function useMessageFeedback (messageId: number | undefined, enabled: bool await postFeedback(messageId, { feedback_type: action, comment }).finally(() => setActing(false)); setFeedback({ feedback_type: action, comment }); }, - // deleteFeedback: () => { - // setActing(true); - // return deleteFeedback(chatId, messageId).finally(() => setActing(false)); - // }, - // source: contentData.data, - // sourceLoading: contentData.isLoading || contentData.isValidating, }; } diff --git a/frontend/app/src/components/date-format.tsx b/frontend/app/src/components/date-format.tsx index 6a38253d1..ccc190328 100644 --- a/frontend/app/src/components/date-format.tsx +++ b/frontend/app/src/components/date-format.tsx @@ -8,4 +8,4 @@ export function DateFormat ({ className, date, format: formatStr = 'yyyy-MM-dd H {date ? isNaN(date.getTime()) ? 'Invalid Date' : format(date, formatStr) : '-'} ); -} \ No newline at end of file +} diff --git a/frontend/app/src/components/document-viewer.tsx b/frontend/app/src/components/document-viewer.tsx index c7e73e707..8210957ec 100644 --- a/frontend/app/src/components/document-viewer.tsx +++ b/frontend/app/src/components/document-viewer.tsx @@ -1,11 +1,10 @@ -import { PromptViewer } from '@/components/chat-engine/prompt-viewer'; import { Button } from '@/components/ui/button'; import { Dialog, DialogContent, DialogHeader, DialogTitle, DialogTrigger } from '@/components/ui/dialog'; import { ScrollArea } from '@/components/ui/scroll-area'; -import { useEffect, useState } from 'react'; import Highlight from 'highlight.js/lib/core'; import markdown from 'highlight.js/lib/languages/markdown'; +import { useEffect, useState } from 'react'; import './code-theme.scss'; export interface DocumentPreviewProps { @@ -33,7 +32,7 @@ export function DocumentPreviewDialog ({ title, name, mime, content }: { title: return ( - diff --git a/frontend/app/src/components/form/biz.tsx b/frontend/app/src/components/form/biz.tsx index 967c550e0..c64566656 100644 --- a/frontend/app/src/components/form/biz.tsx +++ b/frontend/app/src/components/form/biz.tsx @@ -1,9 +1,11 @@ import { type EmbeddingModel, listEmbeddingModels } from '@/api/embedding-models'; +import { type KnowledgeBase, type KnowledgeBaseSummary, listKnowledgeBases } from '@/api/knowledge-base'; import { listLlms, type LLM } from '@/api/llms'; import type { ProviderOption } from '@/api/providers'; import { listRerankers, type Reranker } from '@/api/rerankers'; import { EmbeddingModelInfo } from '@/components/embedding-models/EmbeddingModelInfo'; import { FormSelect, type FormSelectConfig, type FormSelectProps } from '@/components/form/control-widget'; +import { KBInfo } from '@/components/knowledge-base/KBInfo'; import { LlmInfo } from '@/components/llm/LlmInfo'; import { RerankerInfo } from '@/components/reranker/RerankerInfo'; import { forwardRef } from 'react'; @@ -105,3 +107,26 @@ export const ProviderSelect = forwardRef(({ }); ProviderSelect.displayName = 'ProviderSelect'; + + +export const KBSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { + const { data: kbs, isLoading, error } = useSWR('api.knowledge-bases.list-all', () => listKnowledgeBases({ size: 100 })); + + return ( + (), + renderOption: option => (), + key: 'id', + } satisfies FormSelectConfig} + /> + ); +}); + +KBSelect.displayName = 'KBSelect'; diff --git a/frontend/app/src/components/form/control-widget.tsx b/frontend/app/src/components/form/control-widget.tsx index 818c6d031..824847234 100644 --- a/frontend/app/src/components/form/control-widget.tsx +++ b/frontend/app/src/components/form/control-widget.tsx @@ -115,7 +115,7 @@ export const FormSelect = forwardRef(({ config, placeholde - + {config.options.map(option => ( {config.renderOption(option)} diff --git a/frontend/app/src/components/form/field-layout.tsx b/frontend/app/src/components/form/field-layout.tsx index 8cbbdcce4..6581f9b84 100644 --- a/frontend/app/src/components/form/field-layout.tsx +++ b/frontend/app/src/components/form/field-layout.tsx @@ -1,6 +1,8 @@ import type { FormControlWidgetProps } from '@/components/form/control-widget'; import { Button } from '@/components/ui/button'; +import { Collapsible, CollapsibleContent, CollapsibleTrigger } from '@/components/ui/collapsible'; import { FormControl, FormDescription, FormField, FormItem, FormLabel, FormMessage } from '@/components/ui/form'; +import { cn } from '@/lib/utils'; import { MinusIcon, PlusIcon } from 'lucide-react'; import { cloneElement, type ReactElement, type ReactNode } from 'react'; import { ControllerRenderProps, FieldPath, type FieldPathByValue, type FieldPathValue, FieldValues } from 'react-hook-form'; @@ -11,18 +13,21 @@ export interface FormFieldLayoutProps< > { name: TName; label: ReactNode; + required?: boolean; description?: ReactNode; + // value = props.value ?? fallbackValue + fallbackValue?: FieldPathValue; children: ((props: ControllerRenderProps) => ReactNode) | ReactElement>; } function renderWidget< TFieldValues extends FieldValues = FieldValues, TName extends FieldPath = FieldPath -> (children: FormFieldLayoutProps['children'], props: ControllerRenderProps) { +> (children: FormFieldLayoutProps['children'], { value, ...props }: ControllerRenderProps, fallbackValue?: FieldPathValue) { if (typeof children === 'function') { - return children(props); + return children({ value: value ?? fallbackValue as never, ...props }); } else { - return cloneElement(children, props); + return cloneElement(children, { value: value ?? fallbackValue as never, ...props }); } } @@ -33,6 +38,8 @@ export function FormFieldBasicLayout< name, label, description, + required, + fallbackValue, children, }: FormFieldLayoutProps) { return ( @@ -40,9 +47,12 @@ export function FormFieldBasicLayout< name={name} render={({ field }) => ( - {label} + + {label} + {required && *} + - {renderWidget(children, field)} + {renderWidget(children, field, fallbackValue)} {description && {description}} @@ -83,21 +93,27 @@ export function FormFieldContainedLayout< name, label, description, + required, + fallbackValue, children, -}: FormFieldLayoutProps) { + unimportant = false, +}: FormFieldLayoutProps & { unimportant?: boolean }) { return ( name={name} render={({ field }) => (
- {label} + + {label} + {required && *} + {description && {description} }
- {renderWidget(children, field)} + {renderWidget(children, field, fallbackValue)}
)} @@ -185,3 +201,38 @@ export function FormPrimitiveArrayFieldBasicLayout< /> ); } + +export function FormCollapsedBasicLayout< + TFieldValues extends FieldValues = FieldValues, + TName extends FieldPath = FieldPath +> ({ + name, + label, + description, + children, + fallbackValue, +}: FormFieldLayoutProps) { + return ( + + name={name} + render={({ field }) => ( + + + + + + {label} + + + + {renderWidget(children, field, fallbackValue)} + + + {description && {description}} + + + + )} + /> + ); +} diff --git a/frontend/app/src/components/form/widgets/PromptInput.tsx b/frontend/app/src/components/form/widgets/PromptInput.tsx new file mode 100644 index 000000000..23bbbd324 --- /dev/null +++ b/frontend/app/src/components/form/widgets/PromptInput.tsx @@ -0,0 +1,29 @@ +import { type FormControlWidgetProps, FormTextarea } from '@/components/form/control-widget'; +import { buttonVariants } from '@/components/ui/button'; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle, DialogTrigger } from '@/components/ui/dialog'; +import { cn } from '@/lib/utils'; +import { forwardRef } from 'react'; + +export interface PromptInputProps extends FormControlWidgetProps { + className?: string; +} + +export const PromptInput = forwardRef(({ className, ...props }: PromptInputProps, ref) => { + return ( + + + {'Edit prompt'} + ({props.value.length} characters) + + + + Update Prompt + + + + + + ); +}); + +PromptInput.displayName = 'PromptInput'; diff --git a/frontend/app/src/components/grid/Grid.tsx b/frontend/app/src/components/grid/Grid.tsx new file mode 100644 index 000000000..d6be9470f --- /dev/null +++ b/frontend/app/src/components/grid/Grid.tsx @@ -0,0 +1,18 @@ +import { cn } from '@/lib/utils'; +import type { ReactNode } from 'react'; + +export function Grid2 ({ children, className }: { className?: string, children: ReactNode }) { + return ( +
+ {children} +
+ ); +} + +export function Grid3 ({ children, className }: { className?: string, children: ReactNode }) { + return ( +
+ {children} +
+ ); +} diff --git a/frontend/app/src/components/knowledge-base/KBInfo.tsx b/frontend/app/src/components/knowledge-base/KBInfo.tsx new file mode 100644 index 000000000..e9bb2cf62 --- /dev/null +++ b/frontend/app/src/components/knowledge-base/KBInfo.tsx @@ -0,0 +1,11 @@ +'use client'; + +import { useKnowledgeBase } from '@/components/knowledge-base/hooks'; + +export function KBInfo ({ className, detailed = false, id }: { className?: string, detailed?: boolean, id: number | undefined | null }) { + const { knowledgeBase, isLoading } = useKnowledgeBase(id); + + return ( + {knowledgeBase?.name} + ); +} diff --git a/frontend/app/src/components/knowledge-base/hooks.ts b/frontend/app/src/components/knowledge-base/hooks.ts index 0512adee0..5c1d3597f 100644 --- a/frontend/app/src/components/knowledge-base/hooks.ts +++ b/frontend/app/src/components/knowledge-base/hooks.ts @@ -1,7 +1,12 @@ -import { getKnowledgeGraphIndexProgress, listKnowledgeBases } from '@/api/knowledge-base'; +import { getKnowledgeBaseById, getKnowledgeGraphIndexProgress, listKnowledgeBases } from '@/api/knowledge-base'; import { useKB } from '@/app/(main)/(admin)/knowledge-bases/[id]/context'; import useSWR from 'swr'; +export function useKnowledgeBase (id: number | null | undefined) { + const { data: knowledgeBase, ...rest } = useSWR(id != null && `api.knowledge-bases.get?id=${id}`, () => getKnowledgeBaseById(id!)) + return { knowledgeBase, ...rest } +} + export function useKnowledgeBaseDatasource (id: number) { const { data_sources } = useKB(); diff --git a/frontend/app/src/components/settings-form/GeneralSettingsField.tsx b/frontend/app/src/components/settings-form/GeneralSettingsField.tsx index 1da2d8e85..2a3814f6f 100644 --- a/frontend/app/src/components/settings-form/GeneralSettingsField.tsx +++ b/frontend/app/src/components/settings-form/GeneralSettingsField.tsx @@ -53,11 +53,17 @@ export function GeneralSettingsField ({ return (
- + { + event.preventDefault(); + form.reset({ + value: accessor.get(data), + }); + }}> {children} - {!readonly && ( -
- {form.formState.dirtyFields.value && } + {!readonly && form.formState.dirtyFields.value && ( +
+ +
)} From bf0fbd327fb41a77cd3c9cf15b5dbd32b28542e9 Mon Sep 17 00:00:00 2001 From: Mini256 Date: Mon, 25 Nov 2024 11:55:08 +0800 Subject: [PATCH 004/114] fix: remove useless debug log --- backend/app/rag/knowledge_graph/graph_store/tidb_graph_store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/app/rag/knowledge_graph/graph_store/tidb_graph_store.py b/backend/app/rag/knowledge_graph/graph_store/tidb_graph_store.py index 050a9f0f5..24ce51c33 100644 --- a/backend/app/rag/knowledge_graph/graph_store/tidb_graph_store.py +++ b/backend/app/rag/knowledge_graph/graph_store/tidb_graph_store.py @@ -536,7 +536,6 @@ def search_relationships_weight( relationship_meta_filters: Dict = {}, session: Optional[Session] = None, ) -> List[SQLModel]: - logger.info("debug") # select the relationships to rank subquery = ( select( From db009bc86ae19e6eec2e71fac973a7ad28570779 Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Mon, 25 Nov 2024 12:50:57 +0800 Subject: [PATCH 005/114] refine(frontend): support create models while selecting (#401) --- .../chat-engine/create-chat-engine-form.tsx | 9 +- .../CreateEmbeddingModelForm.tsx | 25 ++- frontend/app/src/components/form/biz.tsx | 167 +++++++++++++++--- .../src/components/form/control-widget.tsx | 121 ++++++++++++- frontend/app/src/components/form/utils.ts | 9 +- .../create-knowledge-base-form.tsx | 7 +- .../app/src/components/llm/CreateLLMForm.tsx | 24 ++- .../reranker/CreateRerankerForm.tsx | 24 ++- 8 files changed, 304 insertions(+), 82 deletions(-) diff --git a/frontend/app/src/components/chat-engine/create-chat-engine-form.tsx b/frontend/app/src/components/chat-engine/create-chat-engine-form.tsx index 24b476cea..12634c7a5 100644 --- a/frontend/app/src/components/chat-engine/create-chat-engine-form.tsx +++ b/frontend/app/src/components/chat-engine/create-chat-engine-form.tsx @@ -14,7 +14,7 @@ import { Separator } from '@/components/ui/separator'; import { zodResolver } from '@hookform/resolvers/zod'; import { capitalCase } from 'change-case-all'; import { useRouter } from 'next/navigation'; -import { type ReactNode, useTransition } from 'react'; +import { type ReactNode, useId, useTransition } from 'react'; import { useForm } from 'react-hook-form'; import { toast } from 'sonner'; import { z } from 'zod'; @@ -40,6 +40,7 @@ const schema = z.object({ export function CreateChatEngineForm ({ defaultChatEngineOptions }: { defaultChatEngineOptions: ChatEngineOptions }) { const [transitioning, startTransition] = useTransition(); const router = useRouter(); + const id = useId(); const form = useForm>({ resolver: zodResolver(schema), @@ -56,7 +57,7 @@ export function CreateChatEngineForm ({ defaultChatEngineOptions }: { defaultCha startTransition(() => { router.push(`/chat-engines/${ce.id}`); }); - }, errors => { + }, () => { toast.error('Validation failed', { description: 'Please check your chat engine configurations.', classNames: { @@ -68,7 +69,7 @@ export function CreateChatEngineForm ({ defaultChatEngineOptions }: { defaultCha return (
- +
@@ -171,7 +172,7 @@ export function CreateChatEngineForm ({ defaultChatEngineOptions }: { defaultCha - + ); diff --git a/frontend/app/src/components/embedding-models/CreateEmbeddingModelForm.tsx b/frontend/app/src/components/embedding-models/CreateEmbeddingModelForm.tsx index 683895122..034074484 100644 --- a/frontend/app/src/components/embedding-models/CreateEmbeddingModelForm.tsx +++ b/frontend/app/src/components/embedding-models/CreateEmbeddingModelForm.tsx @@ -5,16 +5,16 @@ import { ProviderSelect } from '@/components/form/biz'; import { FormInput, FormSwitch } from '@/components/form/control-widget'; import { FormFieldBasicLayout, FormFieldContainedLayout } from '@/components/form/field-layout'; import { FormRootError } from '@/components/form/root-error'; +import { handleSubmitHelper } from '@/components/form/utils'; import { CodeInput } from '@/components/form/widgets/CodeInput'; import { ProviderDescription } from '@/components/provider-description'; import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@/components/ui/accordion'; import { Button } from '@/components/ui/button'; import { Form } from '@/components/ui/form'; -import { getErrorMessage } from '@/lib/errors'; import { zodJsonText } from '@/lib/zod'; import { zodResolver } from '@hookform/resolvers/zod'; import { Loader2Icon } from 'lucide-react'; -import { useEffect } from 'react'; +import { useEffect, useId } from 'react'; import { useForm } from 'react-hook-form'; import { toast } from 'sonner'; import useSWR from 'swr'; @@ -38,6 +38,7 @@ const dictCredentialForm = unsetForm.extend({ }); export function CreateEmbeddingModelForm ({ transitioning, onCreated }: { transitioning?: boolean, onCreated?: (embeddingModel: EmbeddingModel) => void }) { + const id = useId(); const { data: options, isLoading, error } = useSWR('api.embedding-models.list-options', listEmbeddingModelOptions); const form = useForm({ @@ -83,25 +84,20 @@ export function CreateEmbeddingModelForm ({ transitioning, onCreated }: { transi } }, [provider]); - const handleSubmit = form.handleSubmit(async (values) => { + const handleSubmit = handleSubmitHelper(form, async (values) => { const { error, success } = await testEmbeddingModel(values); if (!success) { - form.setError('root', { message: error || 'Unknown error' }); - return; - } - try { - const embeddingModel = await createEmbeddingModel(values); - toast('Embedding Model successfully created.'); - onCreated?.(embeddingModel); - } catch (error) { - form.setError('root', { message: getErrorMessage(error) }); + throw new Error(error || 'Test Embedding Model failed.'); } + const embeddingModel = await createEmbeddingModel(values); + toast('Embedding Model successfully created.'); + onCreated?.(embeddingModel); }); return ( <>
- + @@ -139,9 +135,8 @@ export function CreateEmbeddingModelForm ({ transitioning, onCreated }: { transi - - diff --git a/frontend/app/src/components/form/biz.tsx b/frontend/app/src/components/form/biz.tsx index c64566656..90e30eea6 100644 --- a/frontend/app/src/components/form/biz.tsx +++ b/frontend/app/src/components/form/biz.tsx @@ -1,74 +1,196 @@ import { type EmbeddingModel, listEmbeddingModels } from '@/api/embedding-models'; -import { type KnowledgeBase, type KnowledgeBaseSummary, listKnowledgeBases } from '@/api/knowledge-base'; +import { type KnowledgeBaseSummary, listKnowledgeBases } from '@/api/knowledge-base'; import { listLlms, type LLM } from '@/api/llms'; import type { ProviderOption } from '@/api/providers'; import { listRerankers, type Reranker } from '@/api/rerankers'; -import { EmbeddingModelInfo } from '@/components/embedding-models/EmbeddingModelInfo'; -import { FormSelect, type FormSelectConfig, type FormSelectProps } from '@/components/form/control-widget'; +import { CreateEmbeddingModelForm } from '@/components/embedding-models/CreateEmbeddingModelForm'; +import { FormCombobox, type FormComboboxConfig, type FormComboboxProps, FormSelect, type FormSelectConfig, type FormSelectProps } from '@/components/form/control-widget'; import { KBInfo } from '@/components/knowledge-base/KBInfo'; -import { LlmInfo } from '@/components/llm/LlmInfo'; +import { CreateLLMForm } from '@/components/llm/CreateLLMForm'; +import { ManagedDialog } from '@/components/managed-dialog'; +import { ManagedPanelContext } from '@/components/managed-panel'; +import { CreateRerankerForm } from '@/components/reranker/CreateRerankerForm'; import { RerankerInfo } from '@/components/reranker/RerankerInfo'; +import { DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog'; +import { PlusIcon } from 'lucide-react'; import { forwardRef } from 'react'; import useSWR from 'swr'; -export const EmbeddingModelSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { +export const EmbeddingModelSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { // TODO - const { data: embeddingModels, isLoading, error } = useSWR('api.embedding-models.list-all', () => listEmbeddingModels({ size: 100 })); + const { data: embeddingModels, isLoading, mutate, error } = useSWR('api.embedding-models.list-all', () => listEmbeddingModels({ size: 100 })); return ( - [option.name, option.provider, option.model], loading: isLoading, error, - renderValue: option => (), - renderOption: option => (), + renderValue: option => ({option.name} [{option.vector_dimension}]), + renderOption: option => ( +
+
{option.name}
+
+ {option.provider}:{option.model} [{option.vector_dimension}] +
+
+ ), + renderCreateOption: (Wrapper, onCreated) => ( + + + {({ setOpen }) => ( + <> + setOpen(true)}> + + + Create New Embedding Model + + + + + + Create New Embedding Model + + + + { + mutate(); + onCreated(embeddingModel); + setOpen(false); + }} + /> + + + )} + + + ), key: 'id', - } satisfies FormSelectConfig} + } satisfies FormComboboxConfig} /> ); }); EmbeddingModelSelect.displayName = 'EmbeddingModelSelect'; -export const LLMSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { - const { data: llms, isLoading, error } = useSWR('api.llms.list-all', () => listLlms({ size: 100 })); +export const LLMSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { + const { data: llms, isLoading, mutate, error } = useSWR('api.llms.list-all', () => listLlms({ size: 100 })); return ( - (), - renderOption: option => (), + renderValue: option => ({option.name}), + renderOption: option => ( +
+
{option.name}
+
+ {option.provider}:{option.model} +
+
+ ), + renderCreateOption: (Wrapper, onCreated) => ( + + + {({ setOpen }) => ( + <> + setOpen(true)}> + + + Create New LLM + + + + + + Create New LLM + + + + { + mutate(); + onCreated(llm); + setOpen(false); + }} + /> + + + )} + + + ), + optionKeywords: option => [option.name, option.provider, option.model], key: 'id', - } satisfies FormSelectConfig} + } satisfies FormComboboxConfig} /> ); }); LLMSelect.displayName = 'LLMSelect'; -export const RerankerSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { - const { data: rerankers, isLoading, error } = useSWR('api.rerankers.list-all', () => listRerankers({ size: 100 })); +export const RerankerSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { + const { data: rerankers, mutate, isLoading, error } = useSWR('api.rerankers.list-all', () => listRerankers({ size: 100 })); return ( - [option.name, option.provider, option.model], loading: isLoading, error, - renderValue: option => (), - renderOption: option => (), + renderValue: option => ({option.name}), + renderOption: option => ( +
+
{option.name}
+
+ {option.provider}:{option.model} +
+
+ ), + renderCreateOption: (Wrapper, onCreated) => ( + + + {({ setOpen }) => ( + <> + setOpen(true)}> + + + Create New Reranker + + + + + + Create New Reranker + + + + { + mutate(); + onCreated(reranker); + setOpen(false); + }} + /> + + + )} + + + ), key: 'id', - } satisfies FormSelectConfig} + } satisfies FormComboboxConfig} /> ); }); @@ -108,7 +230,6 @@ export const ProviderSelect = forwardRef(({ ProviderSelect.displayName = 'ProviderSelect'; - export const KBSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { const { data: kbs, isLoading, error } = useSWR('api.knowledge-bases.list-all', () => listKnowledgeBases({ size: 100 })); diff --git a/frontend/app/src/components/form/control-widget.tsx b/frontend/app/src/components/form/control-widget.tsx index 824847234..a8e62bf55 100644 --- a/frontend/app/src/components/form/control-widget.tsx +++ b/frontend/app/src/components/form/control-widget.tsx @@ -1,13 +1,16 @@ +import { Command, CommandEmpty, CommandGroup, CommandInput, CommandItem, CommandList } from '@/components/ui/command'; +import { Popover, PopoverContent } from '@/components/ui/popover'; import { Select, SelectContent, SelectItem } from '@/components/ui/select'; import { Switch } from '@/components/ui/switch'; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip'; import { getErrorMessage } from '@/lib/errors'; import { cn } from '@/lib/utils'; +import * as PopoverPrimitive from '@radix-ui/react-popover'; import * as SelectPrimitive from '@radix-ui/react-select'; import type { SwitchProps } from '@radix-ui/react-switch'; -import { ChevronDown, Loader2Icon, TriangleAlertIcon, XCircleIcon } from 'lucide-react'; +import { CheckIcon, ChevronDown, Loader2Icon, TriangleAlertIcon, XCircleIcon } from 'lucide-react'; import * as React from 'react'; -import { forwardRef, type ReactElement, type ReactNode } from 'react'; +import { type FC, forwardRef, type ReactElement, type ReactNode } from 'react'; import { ControllerRenderProps, FieldPath, FieldValues } from 'react-hook-form'; export interface FormControlWidgetProps< @@ -39,12 +42,12 @@ export const FormSwitch = forwardRef(({ value, onChange, . FormSwitch.displayName = 'FormSwitch'; -export type FormSelectConfig = { - loading?: boolean - error?: unknown - options: T[] - key: keyof T - clearable?: boolean +export interface FormSelectConfig { + loading?: boolean; + error?: unknown; + options: T[]; + key: keyof T; + clearable?: boolean; itemClassName?: string; renderOption: (option: T) => ReactNode; renderValue?: (option: T) => ReactNode; @@ -127,3 +130,105 @@ export const FormSelect = forwardRef(({ config, placeholde }); FormSelect.displayName = 'FormSelect'; + +export interface FormComboboxConfig extends FormSelectConfig { + optionKeywords: (option: T) => string[]; + renderCreateOption?: (wrapper: FC<{ onSelect: () => void, children: ReactNode }>, onCreated: (item: T) => void) => ReactNode; +} + +export interface FormComboboxProps extends FormControlWidgetProps { + children?: ReactElement; + placeholder?: string; + config: FormComboboxConfig; +} + +export const FormCombobox = forwardRef(({ config, placeholder, value, onChange, name, disabled, children, ...props }, ref) => { + const isConfigReady = !config.loading && !config.error; + const current = config.options.find(option => option[config.key] === value); + + return ( + +
+ span]:line-clamp-1', + )} + asChild={!!children} + > + {config.loading + ? + : !!config.error + ? {getErrorMessage(config.error)} + : (children ? children : current ? (config.renderValue ?? config.renderOption)(current) : {placeholder}) + } + + {config.loading + ? + : config.error + ? + : } + + + + + {(config.clearable !== false && current != null && !disabled) && } + + + Clear select + + + +
+ + + + + + {config.renderCreateOption && config.renderCreateOption( + ({ onSelect, children }) => ( + + {children} + + ), + (item) => { + onChange?.(item[config.key]); + })} + {config.options.map(option => ( + { + const item = config.options.find(option => String(option[config.key]) === value); + if (item) { + onChange?.(item[config.key]); + } + }} + > + {config.renderOption(option)} + + + ))} + + + Empty List + + + + +
+ ); +}); + +FormCombobox.displayName = 'FormCombobox'; diff --git a/frontend/app/src/components/form/utils.ts b/frontend/app/src/components/form/utils.ts index 7339617d4..04f65b235 100644 --- a/frontend/app/src/components/form/utils.ts +++ b/frontend/app/src/components/form/utils.ts @@ -1,4 +1,5 @@ import { getErrorMessage } from '@/lib/errors'; +import { FormEvent } from 'react'; import type { UseFormReturn } from 'react-hook-form'; export function handleSubmitHelper> ( @@ -6,7 +7,7 @@ export function handleSubmitHelper> ( onValid: Parameters[0], onInvalid?: Parameters[1], ) { - return form.handleSubmit( + const handleSubmit = form.handleSubmit( async (data) => { try { await onValid(data); @@ -16,4 +17,10 @@ export function handleSubmitHelper> ( }, onInvalid, ); + + return (event: FormEvent) => { + // Prevent submit event being propagated to parent react components. + event.stopPropagation(); + void handleSubmit(event); + }; } diff --git a/frontend/app/src/components/knowledge-base/create-knowledge-base-form.tsx b/frontend/app/src/components/knowledge-base/create-knowledge-base-form.tsx index 793ebe2b2..4a4455df9 100644 --- a/frontend/app/src/components/knowledge-base/create-knowledge-base-form.tsx +++ b/frontend/app/src/components/knowledge-base/create-knowledge-base-form.tsx @@ -10,13 +10,14 @@ import { Button } from '@/components/ui/button'; import { Form } from '@/components/ui/form'; import { zodResolver } from '@hookform/resolvers/zod'; import { useRouter } from 'next/navigation'; -import { useTransition } from 'react'; +import { useId, useTransition } from 'react'; import { useForm } from 'react-hook-form'; import { z } from 'zod'; export function CreateKnowledgeBaseForm ({}: {}) { const [transitioning, startTransition] = useTransition(); const router = useRouter(); + const id = useId(); const form = useForm>({ resolver: zodResolver(createKnowledgeBaseParamsSchema), @@ -76,7 +77,7 @@ export function CreateKnowledgeBaseForm ({}: {}) { return ( - + @@ -93,7 +94,7 @@ export function CreateKnowledgeBaseForm ({}: {}) { - + ); diff --git a/frontend/app/src/components/llm/CreateLLMForm.tsx b/frontend/app/src/components/llm/CreateLLMForm.tsx index 60a4a2f3b..6c54b060b 100644 --- a/frontend/app/src/components/llm/CreateLLMForm.tsx +++ b/frontend/app/src/components/llm/CreateLLMForm.tsx @@ -5,16 +5,16 @@ import { ProviderSelect } from '@/components/form/biz'; import { FormInput, FormSwitch } from '@/components/form/control-widget'; import { FormFieldBasicLayout, FormFieldContainedLayout } from '@/components/form/field-layout'; import { FormRootError } from '@/components/form/root-error'; +import { handleSubmitHelper } from '@/components/form/utils'; import { CodeInput } from '@/components/form/widgets/CodeInput'; import { ProviderDescription } from '@/components/provider-description'; import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@/components/ui/accordion'; import { Button } from '@/components/ui/button'; import { Form } from '@/components/ui/form'; -import { getErrorMessage } from '@/lib/errors'; import { zodJsonText } from '@/lib/zod'; import { zodResolver } from '@hookform/resolvers/zod'; import { Loader2Icon } from 'lucide-react'; -import { useEffect } from 'react'; +import { useEffect, useId } from 'react'; import { useForm } from 'react-hook-form'; import { toast } from 'sonner'; import useSWR from 'swr'; @@ -38,6 +38,7 @@ const dictCredentialForm = unsetForm.extend({ }); export function CreateLLMForm ({ transitioning, onCreated }: { transitioning?: boolean, onCreated?: (llm: LLM) => void }) { + const id = useId(); const { data: options, isLoading, error } = useSWR('api.llms.list-options', listLlmOptions); const form = useForm({ @@ -87,25 +88,20 @@ export function CreateLLMForm ({ transitioning, onCreated }: { transitioning?: b } }, [provider]); - const handleSubmit = form.handleSubmit(async (values) => { + const handleSubmit = handleSubmitHelper(form, async (values) => { const { error, success } = await testLlm(values); if (!success) { - form.setError('root', { message: error || 'Unknown error' }); - return; - } - try { - const llm = await createLlm(values); - toast('LLM successfully created.'); - onCreated?.(llm); - } catch (error) { - form.setError('root', { message: getErrorMessage(error) }); + throw new Error(error || 'Test LLM failed'); } + const llm = await createLlm(values); + toast(`LLM ${llm.name} successfully created.`); + onCreated?.(llm); }); return ( <>
- + @@ -141,7 +137,7 @@ export function CreateLLMForm ({ transitioning, onCreated }: { transitioning?: b - diff --git a/frontend/app/src/components/reranker/CreateRerankerForm.tsx b/frontend/app/src/components/reranker/CreateRerankerForm.tsx index 52b11b490..fc92b5e28 100644 --- a/frontend/app/src/components/reranker/CreateRerankerForm.tsx +++ b/frontend/app/src/components/reranker/CreateRerankerForm.tsx @@ -5,16 +5,16 @@ import { ProviderSelect } from '@/components/form/biz'; import { FormInput, FormSwitch } from '@/components/form/control-widget'; import { FormFieldBasicLayout, FormFieldContainedLayout } from '@/components/form/field-layout'; import { FormRootError } from '@/components/form/root-error'; +import { handleSubmitHelper } from '@/components/form/utils'; import { CodeInput } from '@/components/form/widgets/CodeInput'; import { ProviderDescription } from '@/components/provider-description'; import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@/components/ui/accordion'; import { Button } from '@/components/ui/button'; import { Form } from '@/components/ui/form'; -import { getErrorMessage } from '@/lib/errors'; import { zodJsonText } from '@/lib/zod'; import { zodResolver } from '@hookform/resolvers/zod'; import { Loader2Icon } from 'lucide-react'; -import { useEffect } from 'react'; +import { useEffect, useId } from 'react'; import { useForm } from 'react-hook-form'; import { toast } from 'sonner'; import useSWR from 'swr'; @@ -39,6 +39,7 @@ const dictCredentialForm = unsetForm.extend({ }); export function CreateRerankerForm ({ transitioning, onCreated }: { transitioning?: boolean, onCreated?: (reranker: Reranker) => void }) { + const id = useId(); const { data: options, isLoading, error } = useSWR('api.rerankers.list-options', listRerankerOptions); const form = useForm({ @@ -91,25 +92,20 @@ export function CreateRerankerForm ({ transitioning, onCreated }: { transitionin } }, [provider]); - const handleSubmit = form.handleSubmit(async (values) => { + const handleSubmit = handleSubmitHelper(form, async (values) => { const { error, success } = await testReranker(values); if (!success) { - form.setError('root', { message: error || 'Unknown error' }); - return; - } - try { - const reranker = await createReranker(values); - toast('Reranker successfully created.'); - onCreated?.(reranker); - } catch (error) { - form.setError('root', { message: getErrorMessage(error) }); + throw new Error(error || 'Test Reranker failed'); } + const reranker = await createReranker(values); + toast('Reranker successfully created.'); + onCreated?.(reranker); }); return ( <> - + @@ -148,7 +144,7 @@ export function CreateRerankerForm ({ transitioning, onCreated }: { transitionin - From 495f236941e82197873a869a5839b21a927dd59f Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Mon, 25 Nov 2024 14:56:00 +0800 Subject: [PATCH 006/114] fix(frontend): close popover after select item --- frontend/app/src/components/form/control-widget.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frontend/app/src/components/form/control-widget.tsx b/frontend/app/src/components/form/control-widget.tsx index a8e62bf55..232a6847f 100644 --- a/frontend/app/src/components/form/control-widget.tsx +++ b/frontend/app/src/components/form/control-widget.tsx @@ -10,7 +10,7 @@ import * as SelectPrimitive from '@radix-ui/react-select'; import type { SwitchProps } from '@radix-ui/react-switch'; import { CheckIcon, ChevronDown, Loader2Icon, TriangleAlertIcon, XCircleIcon } from 'lucide-react'; import * as React from 'react'; -import { type FC, forwardRef, type ReactElement, type ReactNode } from 'react'; +import { type FC, forwardRef, type ReactElement, type ReactNode, useState } from 'react'; import { ControllerRenderProps, FieldPath, FieldValues } from 'react-hook-form'; export interface FormControlWidgetProps< @@ -143,11 +143,12 @@ export interface FormComboboxProps extends FormControlWidgetProps { } export const FormCombobox = forwardRef(({ config, placeholder, value, onChange, name, disabled, children, ...props }, ref) => { + const [open, setOpen] = useState(false); const isConfigReady = !config.loading && !config.error; const current = config.options.find(option => option[config.key] === value); return ( - +
(({ config, placeh ), (item) => { onChange?.(item[config.key]); + setOpen(false); })} {config.options.map(option => ( (({ config, placeh const item = config.options.find(option => String(option[config.key]) === value); if (item) { onChange?.(item[config.key]); + setOpen(false); } }} > From d974e25024f86be58079f4a74de2e60dda6ee750 Mon Sep 17 00:00:00 2001 From: Cheese Date: Tue, 26 Nov 2024 08:43:32 +0800 Subject: [PATCH 007/114] feat: added badge of the trending (#407) --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 28d75f2e0..9442797b8 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ ## Introduction +pingcap%2Fautoflow | Trendshift + An open source GraphRAG (Knowledge Graph) built on top of [TiDB Vector](https://www.pingcap.com/ai?utm_source=tidb.ai&utm_medium=community) and [LlamaIndex](https://github.com/run-llama/llama_index) and [DSPy](https://github.com/stanfordnlp/dspy). - **Live Demo**: [TiDB.AI](https://tidb.ai) From ef1c7ec439e90a9b1b13006e73ef6ae5e6e17150 Mon Sep 17 00:00:00 2001 From: Mini256 Date: Tue, 26 Nov 2024 11:48:26 +0800 Subject: [PATCH 008/114] feat: add need_migration field for bootstrap status (#403) --- backend/app/api/routes/index.py | 42 ++++++------------------- backend/app/api/routes/models.py | 24 ++++++++++++++ backend/app/rag/chat.py | 54 ++++++++++++++++++++++++++------ 3 files changed, 77 insertions(+), 43 deletions(-) create mode 100644 backend/app/api/routes/models.py diff --git a/backend/app/api/routes/index.py b/backend/app/api/routes/index.py index fa26de557..7fd7cb253 100644 --- a/backend/app/api/routes/index.py +++ b/backend/app/api/routes/index.py @@ -1,10 +1,10 @@ from fastapi import APIRouter -from pydantic import BaseModel from sqlmodel import text from app.api.deps import SessionDep +from app.api.routes.models import SystemConfigStatusResponse from app.site_settings import SiteSetting -from app.rag.chat import check_rag_required_config, check_rag_optional_config +from app.rag.chat import check_rag_required_config, check_rag_optional_config, check_rag_config_need_migration router = APIRouter() @@ -20,38 +20,14 @@ def site_config() -> dict: return SiteSetting.get_client_settings() -class RequiredConfigStatus(BaseModel): - default_llm: bool - default_embedding_model: bool - datasource: bool - knowledge_base: bool - - -class OptionalConfigStatus(BaseModel): - langfuse: bool - default_reranker: bool - - -class SystemConfigStatusResponse(BaseModel): - required: RequiredConfigStatus - optional: OptionalConfigStatus - - @router.get("/system/bootstrap-status") def system_bootstrap_status(session: SessionDep) -> SystemConfigStatusResponse: - has_default_llm, has_default_embedding_model, has_datasource, has_knowledge_base = ( - check_rag_required_config(session) - ) - langfuse, default_reranker = check_rag_optional_config(session) + required_config_check_status = check_rag_required_config(session) + optional_config_check_status = check_rag_optional_config(session) + need_migration_status = check_rag_config_need_migration(session) + return SystemConfigStatusResponse( - required=RequiredConfigStatus( - default_llm=has_default_llm, - default_embedding_model=has_default_embedding_model, - datasource=has_datasource, - knowledge_base=has_knowledge_base - ), - optional=OptionalConfigStatus( - langfuse=langfuse, - default_reranker=default_reranker, - ), + required=required_config_check_status, + optional=optional_config_check_status, + need_migration=need_migration_status ) diff --git a/backend/app/api/routes/models.py b/backend/app/api/routes/models.py new file mode 100644 index 000000000..d1a0e6bb8 --- /dev/null +++ b/backend/app/api/routes/models.py @@ -0,0 +1,24 @@ +from pydantic import BaseModel + + +class RequiredConfigStatus(BaseModel): + default_llm: bool + default_embedding_model: bool + default_chat_engine: bool + datasource: bool + knowledge_base: bool + + +class OptionalConfigStatus(BaseModel): + langfuse: bool + default_reranker: bool + + +class NeedMigrationStatus(BaseModel): + chat_engines_without_kb_configured: list[int] + + +class SystemConfigStatusResponse(BaseModel): + required: RequiredConfigStatus + optional: OptionalConfigStatus + need_migration: NeedMigrationStatus diff --git a/backend/app/rag/chat.py b/backend/app/rag/chat.py index 50b90f494..d23305d97 100644 --- a/backend/app/rag/chat.py +++ b/backend/app/rag/chat.py @@ -14,6 +14,7 @@ from llama_index.core.base.embeddings.base import BaseEmbedding from llama_index.core.llms import LLM from pydantic import BaseModel +from sqlalchemy import text from sqlmodel import Session, select, func, SQLModel from llama_index.core import VectorStoreIndex from llama_index.core.base.llms.base import ChatMessage @@ -25,6 +26,7 @@ from langfuse import Langfuse from langfuse.llama_index import LlamaIndexCallbackHandler +from app.api.routes.models import RequiredConfigStatus, OptionalConfigStatus, NeedMigrationStatus from app.models import ( User, Document as DBDocument, @@ -38,7 +40,7 @@ RerankerModel as DBRerankerModel, Chunk as DBChunk, Entity as DBEntity, - Relationship as DBRelationship, + Relationship as DBRelationship, ChatEngine, ) from app.core.config import settings from app.models.chunk import get_kb_chunk_model @@ -1102,11 +1104,18 @@ def get_chat_message_subgraph( return entities, relations -def check_rag_required_config(session: Session) -> tuple[bool, bool, bool, bool]: - # Check if llm, embedding model, and datasource are configured - # If any of them is missing, the rag can not work +def check_rag_required_config(session: Session) -> RequiredConfigStatus: + """ + Check if the required configuration items have been configured, it any of them is + missing, the RAG application can not complete its work. + """ has_default_llm = session.scalar(select(func.count(DBLLM.id))) > 0 has_default_embedding_model = (session.scalar(select(func.count(DBEmbeddingModel.id))) > 0) + has_default_chat_engine = ( + session.scalar(select(func.count(ChatEngine.id)).where(ChatEngine.is_default == True)) > 0 + ) + + # TODO: Remove it after the multiple KB feature is stable. has_datasource = session.scalar(select(func.count(DBDataSource.id))) > 0 try: @@ -1115,18 +1124,43 @@ def check_rag_required_config(session: Session) -> tuple[bool, bool, bool, bool] has_knowledge_base = False logger.exception(e) - return has_default_llm, has_default_embedding_model, has_datasource, has_knowledge_base + return RequiredConfigStatus( + default_llm=has_default_llm, + default_embedding_model=has_default_embedding_model, + default_chat_engine=has_default_chat_engine, + datasource=has_datasource, + knowledge_base=has_knowledge_base + ) -def check_rag_optional_config(session: Session) -> tuple[bool, bool]: +def check_rag_optional_config(session: Session) -> OptionalConfigStatus: langfuse = bool( SiteSetting.langfuse_host and SiteSetting.langfuse_secret_key and SiteSetting.langfuse_public_key ) default_reranker = session.scalar(select(func.count(DBRerankerModel.id))) > 0 - return langfuse, default_reranker + return OptionalConfigStatus( + langfuse=langfuse, + default_reranker=default_reranker, + ) + +def check_rag_config_need_migration(session: Session) -> NeedMigrationStatus: + """ + Check if any configuration needs to be migrated. + """ + chat_engines_without_kb_configured = ( + session.exec( + select(ChatEngine.id) + .where(ChatEngine.deleted_at == None) + .where(text("NOT JSON_CONTAINS_PATH(engine_options, 'one', '$.knowledge_base')")) + ) + ) + + return NeedMigrationStatus( + chat_engines_without_kb_configured=chat_engines_without_kb_configured, + ) class LLMRecommendQuestions(BaseModel): """recommend questions respond model""" @@ -1134,9 +1168,9 @@ class LLMRecommendQuestions(BaseModel): def get_chat_message_recommend_questions( - db_session: Session, - chat_message: DBChatMessage, - engine_name: str = "default", + db_session: Session, + chat_message: DBChatMessage, + engine_name: str = "default", ) -> List[str]: chat_engine_config = ChatEngineConfig.load_from_db(db_session, engine_name) _fast_llm = chat_engine_config.get_fast_llama_llm(db_session) From 6fe61f15969177d22cfb4aa8504989dea18617ee Mon Sep 17 00:00:00 2001 From: wd0517 Date: Tue, 26 Nov 2024 08:06:37 +0000 Subject: [PATCH 009/114] deploy: release local-embedding-reranker v4 --- docker-compose-cn.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-cn.yml b/docker-compose-cn.yml index ca130f2df..15252c007 100644 --- a/docker-compose-cn.yml +++ b/docker-compose-cn.yml @@ -59,7 +59,7 @@ services: max-file: "6" local-embedding-reranker: - image: registry.cn-beijing.aliyuncs.com/pingcap-ee/tidb.ai-local-embedding-reranker:v3-with-cache + image: registry.cn-beijing.aliyuncs.com/pingcap-ee/tidb.ai-local-embedding-reranker:v4-with-cache ports: - 5001:5001 environment: diff --git a/docker-compose.yml b/docker-compose.yml index b5d89cbc5..23fea8205 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,7 +59,7 @@ services: max-file: "6" local-embedding-reranker: - image: tidbai/local-embedding-reranker:v3-with-cache + image: tidbai/local-embedding-reranker:v4-with-cache ports: - 5001:5001 environment: From 9a86bd95204a437bef83b06030e623a3d6ee65a4 Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Tue, 26 Nov 2024 20:31:31 +0800 Subject: [PATCH 010/114] ui(frontend): use new shadcn's sidebar (#409) close #400 --- frontend/app/src/app/(main)/layout.tsx | 51 +- frontend/app/src/app/(main)/nav.tsx | 102 ++- frontend/app/src/app/globals.css | 20 +- .../app/src/components/admin-page-heading.tsx | 4 +- frontend/app/src/components/branding.tsx | 9 +- .../app/src/components/chat/chats-history.tsx | 20 +- .../src/components/site-header-actions.tsx | 48 ++ frontend/app/src/components/site-header.tsx | 30 + .../app/src/components/site-nav-footer.tsx | 110 --- frontend/app/src/components/site-nav.tsx | 149 ++-- frontend/app/src/components/ui/button.tsx | 2 +- frontend/app/src/components/ui/input.tsx | 6 +- frontend/app/src/components/ui/sidebar.tsx | 763 ++++++++++++++++++ frontend/app/src/components/use-size.ts | 6 + frontend/app/src/hooks/use-mobile.tsx | 19 + frontend/app/tailwind.config.ts | 57 +- .../packages/widget-react/tailwind.config.ts | 6 +- frontend/pnpm-lock.yaml | 2 +- 18 files changed, 1097 insertions(+), 307 deletions(-) create mode 100644 frontend/app/src/components/site-header-actions.tsx create mode 100644 frontend/app/src/components/site-header.tsx delete mode 100644 frontend/app/src/components/site-nav-footer.tsx create mode 100644 frontend/app/src/components/ui/sidebar.tsx create mode 100644 frontend/app/src/hooks/use-mobile.tsx diff --git a/frontend/app/src/app/(main)/layout.tsx b/frontend/app/src/app/(main)/layout.tsx index 82979ac6e..da229e875 100644 --- a/frontend/app/src/app/(main)/layout.tsx +++ b/frontend/app/src/app/(main)/layout.tsx @@ -1,54 +1,31 @@ 'use client'; -import { Nav, NavDrawer } from '@/app/(main)/nav'; -import { Branding } from '@/components/branding'; -import { SiteNavActionBar, SiteNavFooter, type SiteSocialsType } from '@/components/site-nav-footer'; -import { ScrollArea } from '@/components/ui/scroll-area'; +import { SiteSidebar } from '@/app/(main)/nav'; +import { SiteHeaderLargeScreen, SiteHeaderSmallScreen } from '@/components/site-header'; +import { SidebarProvider, SidebarTrigger } from '@/components/ui/sidebar'; import { useSettingContext } from '@/components/website-setting-provider'; -import Link from 'next/link'; -import { ReactNode, useMemo } from 'react'; +import { cn } from '@/lib/utils'; +import { ReactNode, useState } from 'react'; export default function Layout ({ children }: { children: ReactNode }) { + const [sidebarOpen, setSidebarOpen] = useState(true); const setting = useSettingContext(); - const socialMemo: SiteSocialsType = useMemo( - () => ({ - discord: setting.social_discord, - twitter: setting.social_twitter, - github: setting.social_github, - }), - [setting], - ); - return ( <> -
-
- - -
-
-
- -
+ + +
+
- -
+ + +
{children}
-
+ ); } diff --git a/frontend/app/src/app/(main)/nav.tsx b/frontend/app/src/app/(main)/nav.tsx index f802c268c..f98e44e85 100644 --- a/frontend/app/src/app/(main)/nav.tsx +++ b/frontend/app/src/app/(main)/nav.tsx @@ -1,26 +1,46 @@ 'use client'; +import { logout } from '@/api/auth'; import { listChatEngines } from '@/api/chat-engines'; -import { listLlms } from '@/api/llms'; +import type { PublicWebsiteSettings } from '@/api/site-settings'; import { useAuth } from '@/components/auth/AuthProvider'; +import { Branding } from '@/components/branding'; import { ChatNewDialog } from '@/components/chat/chat-new-dialog'; import { ChatsHistory } from '@/components/chat/chats-history'; import { useKnowledgeBases } from '@/components/knowledge-base/hooks'; import { type NavGroup, SiteNav } from '@/components/site-nav'; -import { SiteNavFooter } from '@/components/site-nav-footer'; import { useBootstrapStatus } from '@/components/system/BootstrapStatusProvider'; +import { Avatar, AvatarFallback } from '@/components/ui/avatar'; import { Button } from '@/components/ui/button'; -import { Drawer, DrawerContent, DrawerTrigger } from '@/components/ui/drawer'; -import { ScrollArea } from '@/components/ui/scroll-area'; +import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from '@/components/ui/dropdown-menu'; +import { Sidebar, SidebarContent, SidebarFooter, SidebarHeader } from '@/components/ui/sidebar'; import { Skeleton } from '@/components/ui/skeleton'; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip'; import { useHref } from '@/components/use-href'; -import { ActivitySquareIcon, AlertTriangleIcon, BinaryIcon, BotMessageSquareIcon, BrainCircuitIcon, CogIcon, ComponentIcon, FilesIcon, HomeIcon, KeyRoundIcon, LibraryBigIcon, LibraryIcon, MenuIcon, MessageCircleQuestionIcon, MessagesSquareIcon, ShuffleIcon } from 'lucide-react'; +import { ActivitySquareIcon, AlertTriangleIcon, BinaryIcon, BotMessageSquareIcon, BrainCircuitIcon, CogIcon, ComponentIcon, FilesIcon, HomeIcon, KeyRoundIcon, LibraryBigIcon, LibraryIcon, LogInIcon, MessageCircleQuestionIcon, MessagesSquareIcon, ShuffleIcon } from 'lucide-react'; +import NextLink from 'next/link'; import Link from 'next/link'; +import { useRouter } from 'next/navigation'; import type { ReactNode } from 'react'; import useSWR from 'swr'; -export function Nav () { +export function SiteSidebar ({ setting }: { setting: PublicWebsiteSettings }) { + return ( + + + + + + + + + + + + ); +} + +function NavContent () { const { required } = useBootstrapStatus(); const href = useHref(); const auth = useAuth(); @@ -54,7 +74,7 @@ export function Nav () { icon: ComponentIcon, details: (!required.default_llm || !required.default_embedding_model) && , children: [ - { href: '/llms', title: 'LLMs', icon: BrainCircuitIcon, details: !required.default_llm ? You need to configure at least one Default LLM. : }, + { href: '/llms', title: 'LLMs', icon: BrainCircuitIcon, details: !required.default_llm ? You need to configure at least one Default LLM. : undefined }, { href: '/embedding-models', title: 'Embedding Models', icon: BinaryIcon, details: !required.default_embedding_model && You need to configure at least one Default Embedding Model. }, { href: '/reranker-models', title: 'Reranker Models', icon: ShuffleIcon }, ], @@ -85,27 +105,51 @@ export function Nav () { } return ( - <> - - + ); } -export function NavDrawer () { +function NavFooter () { + const href = useHref(); + const user = useAuth().me; + const router = useRouter(); + + if (!user) { + return ( + + ); + } return ( - - - - - - -
+
+ + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + ); +} -
- - {(['condense_question_prompt', 'condense_answer_prompt', 'text_qa_prompt', 'refine_prompt'] as const).map(field => ( - - - - ))} - -
+function SectionTabTrigger ({ value, required }: { value: string, required?: boolean }) { + const fields = useFormSectionFields(value); + const form = useFormContext(); -
- - - - - - - - - - - - - - - - - - - - - - - - - - -
+ const validated = useMemo(() => { + let validated = true; - + for (let field of Array.from(fields)) { + const error = get(form.formState.errors, field); + if (error) { + validated = false; + break; + } + } + return validated; + }, [form.formState.errors, fields]); - - - + return ( + + + {value} + + {required && *} + ); } -function Section ({ noSeparator, title, children }: { noSeparator?: boolean, title: ReactNode, children: ReactNode }) { +function Section ({ title, children }: { title: string, children: ReactNode }) { return ( - <> - {!noSeparator && } -
-

{title}

+ + {children} -
- + + ); } diff --git a/frontend/app/src/components/chat-engine/update-chat-engine-form.tsx b/frontend/app/src/components/chat-engine/update-chat-engine-form.tsx index 89512d11c..08da1c222 100644 --- a/frontend/app/src/components/chat-engine/update-chat-engine-form.tsx +++ b/frontend/app/src/components/chat-engine/update-chat-engine-form.tsx @@ -5,9 +5,8 @@ import { KBSelect, LLMSelect, RerankerSelect } from '@/components/form/biz'; import { FormInput, FormSwitch } from '@/components/form/control-widget'; import { FormFieldBasicLayout, FormFieldContainedLayout } from '@/components/form/field-layout'; import { PromptInput } from '@/components/form/widgets/PromptInput'; -import { Grid2, Grid3 } from '@/components/grid/Grid'; +import { SecondaryNavigatorItem, SecondaryNavigatorLayout, SecondaryNavigatorList, SecondaryNavigatorMain } from '@/components/secondary-navigator-list'; import { fieldAccessor, GeneralSettingsField, type GeneralSettingsFieldAccessor, GeneralSettingsForm, shallowPick } from '@/components/settings-form'; -import { Separator } from '@/components/ui/separator'; import type { KeyOfType } from '@/lib/typing-utils'; import { capitalCase } from 'change-case-all'; import { format } from 'date-fns'; @@ -36,8 +35,22 @@ export function UpdateChatEngineForm ({ chatEngine, defaultChatEngineOptions }: } }} > -
-
+ + + + Info + + + Retrieval + + + Generation + + + Features + + +
@@ -53,49 +66,43 @@ export function UpdateChatEngineForm ({ chatEngine, defaultChatEngineOptions }: - - - - + + + + + + + + + + + + + + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + +
-
+
@@ -115,48 +122,42 @@ export function UpdateChatEngineForm ({ chatEngine, defaultChatEngineOptions }: - + - - - - - - - - - - - - - - - - - - - {(['intent_graph_knowledge', 'normal_graph_knowledge'] as const).map(type => ( - - - - - - ))} - - -
- -
- - {(['condense_question_prompt', 'condense_answer_prompt', 'text_qa_prompt', 'refine_prompt'] as const).map(type => ( + + + + + + + + + + + + + + + + {(['intent_graph_knowledge', 'normal_graph_knowledge'] as const).map(type => ( ))} - + +
+ +
+ {(['condense_question_prompt', 'condense_answer_prompt', 'text_qa_prompt', 'refine_prompt'] as const).map(type => ( + + + + + + ))}
@@ -178,18 +179,16 @@ export function UpdateChatEngineForm ({ chatEngine, defaultChatEngineOptions }: - - - - - - - - - - - - + + + + + + + + + + @@ -199,7 +198,8 @@ export function UpdateChatEngineForm ({ chatEngine, defaultChatEngineOptions }:
-
+ + ); } @@ -376,14 +376,12 @@ const externalEngineAccessor: GeneralSettingsFieldAccessor - {!noSeparator && } -
-

{title}

+ {children} -
+ ); } diff --git a/frontend/app/src/components/form-sections.tsx b/frontend/app/src/components/form-sections.tsx new file mode 100644 index 000000000..8a0ac5140 --- /dev/null +++ b/frontend/app/src/components/form-sections.tsx @@ -0,0 +1,65 @@ +import { createContext, type Dispatch, type ReactNode, type SetStateAction, useContext, useEffect, useRef, useState } from 'react'; +import { FormProvider, useFormContext } from 'react-hook-form'; + +type FieldsMap = Map>; +type FormSectionsContextValues = readonly [FieldsMap, Dispatch>]; +const FormSectionsContext = createContext(undefined); + +const EMPTY_SET = new Set(); + +export function FormSectionsProvider ({ children }: { children?: ReactNode }) { + const context = useState>>(() => new Map()); + return ( + + {children} + + ); +} + +export function useFormSectionFields (section: string): ReadonlySet { + const [map] = useContext(FormSectionsContext) ?? []; + return map?.get(section) ?? EMPTY_SET; +} + +export function FormSection ({ value, children }: { value: string, children?: ReactNode }) { + const { ...form } = useFormContext(); + const [_, setMap] = useContext(FormSectionsContext) ?? []; + const deferred = useRef<(() => void)[]>([]); + + useEffect(() => { + if (deferred.current.length > 0) { + const current = [...deferred.current]; + deferred.current.splice(0, deferred.current.length); + + for (let fn of current) { + fn(); + } + } + }); + + return ( + { + deferred.current.push(() => { + setMap?.(map => { + if (map?.get(value)?.has(name)) { + return map; + } + if (!map) { + return new Map().set(value, new Set(name)); + } else { + return new Map(map.set(value, new Set(map.get(value)).add(name))); + } + }); + }); + return form.control.register(name, options); + }, + }} + > + {children} + + ); +} \ No newline at end of file diff --git a/frontend/app/src/components/nextjs/NextLink.tsx b/frontend/app/src/components/nextjs/NextLink.tsx index ba105efe3..e9f9846d9 100644 --- a/frontend/app/src/components/nextjs/NextLink.tsx +++ b/frontend/app/src/components/nextjs/NextLink.tsx @@ -23,7 +23,7 @@ export const NextLink = forwardRef(({ classNam return; } if (event.ctrlKey || event.shiftKey || event.metaKey || event.altKey) { - event.persist() + event.persist(); return; } onClick?.(event); @@ -45,7 +45,7 @@ export const NextLink = forwardRef(({ classNam & Pick>(({ className, style, children, defaultValue, ...props }, ref) => { + return ( + +
+ {children} +
+
+ ); +}); +SecondaryNavigatorLayout.displayName = 'SecondaryNavigatorLayout'; + +export const SecondaryNavigatorList = forwardRef>(({ className, children, ...props }, ref) => { + return ( + +
+ {children} +
+
+ ); +}); +SecondaryNavigatorList.displayName = 'SecondaryNavigatorList'; + +export function SecondaryNavigatorLink ({ pathname, children }: { pathname: string, children: ReactNode }) { + const current = usePathname(); + const active = current === pathname; + + return ( + + + {children} + + + ); +} + +export const SecondaryNavigatorItem = forwardRef(({ value, className, children, ...props }, ref) => { + return ( + + + + ); +}); + +SecondaryNavigatorItem.displayName = 'SecondaryNavigatorTabsTrigger'; + +export const SecondaryNavigatorMain = forwardRef & { value?: string, strategy?: 'forceMount' | 'hidden' | 'mount' }>(({ value, strategy = 'mount', className, ...props }, ref) => { + const classNames = cn('flex-1 overflow-x-hidden', className); + if (value == null) { + return
; + } else { + return ( + + ); + } +}); + +SecondaryNavigatorMain.displayName = 'SecondaryNavigatorContent'; From 8d1087ec6724135d1dfad43de118832531371e59 Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Wed, 27 Nov 2024 12:05:52 +0800 Subject: [PATCH 017/114] ui(frontend): add docs url for admin pages (#411) --- .../(main)/(admin)/chat-engines/[id]/page.tsx | 2 +- .../(main)/(admin)/chat-engines/new/page.tsx | 2 +- .../app/(main)/(admin)/chat-engines/page.tsx | 6 +++++- .../(admin)/embedding-models/create/page.tsx | 2 +- .../(main)/(admin)/embedding-models/page.tsx | 10 ++++++---- .../src/app/(main)/(admin)/feedbacks/page.tsx | 6 +++++- .../(main)/(admin)/knowledge-bases/page.tsx | 6 +++++- .../src/app/(main)/(admin)/llms/[id]/page.tsx | 2 +- .../app/(main)/(admin)/llms/create/page.tsx | 2 +- .../app/src/app/(main)/(admin)/llms/page.tsx | 7 ++++++- .../(admin)/reranker-models/[id]/page.tsx | 2 +- .../(admin)/reranker-models/create/page.tsx | 2 +- .../(main)/(admin)/reranker-models/page.tsx | 7 ++++++- .../(main)/(admin)/site-settings/layout.tsx | 6 +++++- .../src/app/(main)/(user)/api-keys/page.tsx | 6 +++++- frontend/app/src/app/(main)/(user)/c/page.tsx | 6 +++++- .../app/src/components/admin-page-heading.tsx | 18 +++++++++--------- 17 files changed, 64 insertions(+), 28 deletions(-) diff --git a/frontend/app/src/app/(main)/(admin)/chat-engines/[id]/page.tsx b/frontend/app/src/app/(main)/(admin)/chat-engines/[id]/page.tsx index 9df187362..4b99adf07 100644 --- a/frontend/app/src/app/(main)/(admin)/chat-engines/[id]/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/chat-engines/[id]/page.tsx @@ -12,7 +12,7 @@ export default async function ChatEnginePage ({ params }: { params: { id: string <> diff --git a/frontend/app/src/app/(main)/(admin)/chat-engines/new/page.tsx b/frontend/app/src/app/(main)/(admin)/chat-engines/new/page.tsx index afa3b6fb4..26cc6f7c9 100644 --- a/frontend/app/src/app/(main)/(admin)/chat-engines/new/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/chat-engines/new/page.tsx @@ -8,7 +8,7 @@ export default async function NewChatEnginePage () { <> diff --git a/frontend/app/src/app/(main)/(admin)/chat-engines/page.tsx b/frontend/app/src/app/(main)/(admin)/chat-engines/page.tsx index 777ec7f74..5f84e4fdd 100644 --- a/frontend/app/src/app/(main)/(admin)/chat-engines/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/chat-engines/page.tsx @@ -4,7 +4,11 @@ import { ChatEnginesTable } from '@/components/chat-engine/chat-engines-table'; export default function ChatEnginesPage () { return ( <> - + ); diff --git a/frontend/app/src/app/(main)/(admin)/embedding-models/create/page.tsx b/frontend/app/src/app/(main)/(admin)/embedding-models/create/page.tsx index caedf3f6b..0a02ef15f 100644 --- a/frontend/app/src/app/(main)/(admin)/embedding-models/create/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/embedding-models/create/page.tsx @@ -14,7 +14,7 @@ export default function Page () { diff --git a/frontend/app/src/app/(main)/(admin)/embedding-models/page.tsx b/frontend/app/src/app/(main)/(admin)/embedding-models/page.tsx index c9558fddd..6792ee2e9 100644 --- a/frontend/app/src/app/(main)/(admin)/embedding-models/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/embedding-models/page.tsx @@ -1,16 +1,18 @@ 'use client'; import { AdminPageHeading } from '@/components/admin-page-heading'; -import { ConfigViewer } from '@/components/config-viewer'; -import { DateFormat } from '@/components/date-format'; import { EmbeddingModelsTable } from '@/components/embedding-models/EmbeddingModelsTable'; -import { OptionDetail } from '@/components/option-detail'; export default function EmbeddingModelPage () { return ( <> - + ); diff --git a/frontend/app/src/app/(main)/(admin)/feedbacks/page.tsx b/frontend/app/src/app/(main)/(admin)/feedbacks/page.tsx index af00a05cf..ce6fa976f 100644 --- a/frontend/app/src/app/(main)/(admin)/feedbacks/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/feedbacks/page.tsx @@ -4,7 +4,11 @@ import { FeedbacksTable } from '@/components/feedbacks/feedbacks-table'; export default function ChatEnginesPage () { return ( <> - + ); diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/page.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/page.tsx index 2be458b07..47476ca11 100644 --- a/frontend/app/src/app/(main)/(admin)/knowledge-bases/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/knowledge-bases/page.tsx @@ -14,7 +14,11 @@ export default function KnowledgeBasesPage () { return ( <> - + New Knowledge Base diff --git a/frontend/app/src/app/(main)/(admin)/llms/[id]/page.tsx b/frontend/app/src/app/(main)/(admin)/llms/[id]/page.tsx index b3ee8e7d4..6c628174a 100644 --- a/frontend/app/src/app/(main)/(admin)/llms/[id]/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/llms/[id]/page.tsx @@ -21,7 +21,7 @@ export default function Page ({ params }: { params: { id: string } }) { }, ]} /> diff --git a/frontend/app/src/app/(main)/(admin)/llms/create/page.tsx b/frontend/app/src/app/(main)/(admin)/llms/create/page.tsx index ad9e94273..bcc015d52 100644 --- a/frontend/app/src/app/(main)/(admin)/llms/create/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/llms/create/page.tsx @@ -14,7 +14,7 @@ export default function Page () { diff --git a/frontend/app/src/app/(main)/(admin)/llms/page.tsx b/frontend/app/src/app/(main)/(admin)/llms/page.tsx index 297225476..dc69e7aa8 100644 --- a/frontend/app/src/app/(main)/(admin)/llms/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/llms/page.tsx @@ -4,7 +4,12 @@ import { LLMsTable } from '@/components/llm/LLMsTable'; export default function Page () { return ( <> - + ); diff --git a/frontend/app/src/app/(main)/(admin)/reranker-models/[id]/page.tsx b/frontend/app/src/app/(main)/(admin)/reranker-models/[id]/page.tsx index a4ccb637e..8f49be176 100644 --- a/frontend/app/src/app/(main)/(admin)/reranker-models/[id]/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/reranker-models/[id]/page.tsx @@ -21,7 +21,7 @@ export default function Page ({ params }: { params: { id: string } }) { }, ]} /> diff --git a/frontend/app/src/app/(main)/(admin)/reranker-models/create/page.tsx b/frontend/app/src/app/(main)/(admin)/reranker-models/create/page.tsx index ea5da3d0c..ed553abcd 100644 --- a/frontend/app/src/app/(main)/(admin)/reranker-models/create/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/reranker-models/create/page.tsx @@ -12,7 +12,7 @@ export default function Page () { diff --git a/frontend/app/src/app/(main)/(admin)/reranker-models/page.tsx b/frontend/app/src/app/(main)/(admin)/reranker-models/page.tsx index 17e16a1ca..478d7d9df 100644 --- a/frontend/app/src/app/(main)/(admin)/reranker-models/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/reranker-models/page.tsx @@ -4,7 +4,12 @@ import RerankerModelsTable from '@/components/reranker/RerankerModelsTable'; export default function Page () { return ( <> - + ); diff --git a/frontend/app/src/app/(main)/(admin)/site-settings/layout.tsx b/frontend/app/src/app/(main)/(admin)/site-settings/layout.tsx index 870cf1ad6..7876fa0f6 100644 --- a/frontend/app/src/app/(main)/(admin)/site-settings/layout.tsx +++ b/frontend/app/src/app/(main)/(admin)/site-settings/layout.tsx @@ -7,7 +7,11 @@ import { type ReactNode } from 'react'; export default function SiteSettingsLayout ({ children }: { children: ReactNode }) { return (
- + diff --git a/frontend/app/src/app/(main)/(user)/api-keys/page.tsx b/frontend/app/src/app/(main)/(user)/api-keys/page.tsx index 6da7bd5ae..42bd713f0 100644 --- a/frontend/app/src/app/(main)/(user)/api-keys/page.tsx +++ b/frontend/app/src/app/(main)/(user)/api-keys/page.tsx @@ -41,7 +41,11 @@ export default function ChatEnginesPage () { const [recentlyCreated, setRecentlyCreated] = useState(); return ( <> - + {recentlyCreated && ( diff --git a/frontend/app/src/app/(main)/(user)/c/page.tsx b/frontend/app/src/app/(main)/(user)/c/page.tsx index eaca783b9..f18a8cdd2 100644 --- a/frontend/app/src/app/(main)/(user)/c/page.tsx +++ b/frontend/app/src/app/(main)/(user)/c/page.tsx @@ -7,7 +7,11 @@ export default async function ConversationsListPage () { return ( <> - + ); diff --git a/frontend/app/src/components/admin-page-heading.tsx b/frontend/app/src/components/admin-page-heading.tsx index 07f1d6914..3247cc85d 100644 --- a/frontend/app/src/components/admin-page-heading.tsx +++ b/frontend/app/src/components/admin-page-heading.tsx @@ -1,20 +1,23 @@ import { Breadcrumb, BreadcrumbItem, BreadcrumbLink, BreadcrumbList, BreadcrumbPage, BreadcrumbSeparator } from '@/components/ui/breadcrumb'; +import { HelpCircleIcon } from 'lucide-react'; import Link from 'next/link'; import { Fragment, type ReactNode } from 'react'; export interface BreadcrumbItem { title: ReactNode; url?: string; + docsUrl?: string; } export interface TableHeadingProps { + /** + * @deprecated + */ title?: ReactNode; - description?: ReactNode; - actions?: ReactNode; breadcrumbs?: BreadcrumbItem[]; } -export function AdminPageHeading ({ title, description, actions, breadcrumbs }: TableHeadingProps) { +export function AdminPageHeading ({ title, breadcrumbs }: TableHeadingProps) { return (
{breadcrumbs && ( @@ -29,6 +32,9 @@ export function AdminPageHeading ({ title, description, actions, breadcrumbs }: : index === breadcrumbs.length - 1 ? {item.title} : {item.title}} + {item.docsUrl + ? + : undefined} ))} @@ -37,12 +43,6 @@ export function AdminPageHeading ({ title, description, actions, breadcrumbs }: )}
{title &&

{title}

} -

- {description} -

-
-
- {actions}
) From a9ea734c619fd61462a48ac66ffa2f770ea87411 Mon Sep 17 00:00:00 2001 From: Mini256 Date: Wed, 27 Nov 2024 13:29:44 +0800 Subject: [PATCH 018/114] feat: add knowledge base datasource manage API (#413) Add KB datasource management API: - `GET: /admin/knowledge_bases/{kb_id}/datasources` - `POST: /admin/knowledge_bases/{kb_id}/datasources/{datasource_id}` - `PUT: /admin/knowledge_bases/{kb_id}/datasources/{datasource_id}` - `DELETE: /admin/knowledge_bases/{kb_id}/datasources/{datasource_id}` The data sources will be the exclusive resources at the knowledge base level. **The old datasource management API will be deprecated.** --- backend/Makefile | 2 +- backend/app/api/admin_routes/data_source.py | 19 ++- .../admin_routes/embedding_model/models.py | 10 +- .../admin_routes/embedding_model/routes.py | 18 +-- .../knowledge_base/data_sources/__init__.py | 0 .../knowledge_base/data_sources/models.py | 32 ++++ .../knowledge_base/data_sources/routes.py | 135 ++++++++++++++++ .../api/admin_routes/knowledge_base/models.py | 41 +---- .../api/admin_routes/knowledge_base/routes.py | 46 +++--- backend/app/api/main.py | 7 +- backend/app/api/routes/chat.py | 15 +- backend/app/exceptions.py | 24 ++- backend/app/models/knowledge_base.py | 10 ++ backend/app/rag/build_index.py | 2 +- backend/app/rag/chat.py | 147 +++++++----------- backend/app/rag/chat_config.py | 14 +- backend/app/rag/knowledge_base/index_store.py | 8 +- .../graph_store/tidb_graph_editor.py | 43 +++-- backend/app/rag/retrieve.py | 15 +- backend/app/repositories/embedding_model.py | 18 ++- backend/app/repositories/knowledge_base.py | 72 ++++++++- backend/app/repositories/llm.py | 34 ++-- backend/app/tasks/__init__.py | 4 +- backend/app/tasks/knowledge_base.py | 146 +++++++++++++---- 24 files changed, 587 insertions(+), 275 deletions(-) create mode 100644 backend/app/api/admin_routes/knowledge_base/data_sources/__init__.py create mode 100644 backend/app/api/admin_routes/knowledge_base/data_sources/models.py create mode 100644 backend/app/api/admin_routes/knowledge_base/data_sources/routes.py diff --git a/backend/Makefile b/backend/Makefile index 4c8273839..550d103e0 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -18,4 +18,4 @@ run_dev_server: run_dev_celery_worker: @echo "Running celery..." - @rye run celery -A app.celery worker --pool solo --concurrency=1 --loglevel=DEBUG + @rye run celery -A app.celery worker diff --git a/backend/app/api/admin_routes/data_source.py b/backend/app/api/admin_routes/data_source.py index ec723f2e0..1da656825 100644 --- a/backend/app/api/admin_routes/data_source.py +++ b/backend/app/api/admin_routes/data_source.py @@ -34,7 +34,7 @@ def name_must_not_be_blank(cls, v: str) -> str: return v -@router.post("/admin/datasources") +@router.post("/admin/datasources", deprecated=True) def create_datasource( session: SessionDep, user: CurrentSuperuserDep, request: DataSourceCreate ) -> DataSource: @@ -52,7 +52,7 @@ def create_datasource( return data_source -@router.get("/admin/datasources") +@router.get("/admin/datasources", deprecated=True) def list_datasources( session: SessionDep, user: CurrentSuperuserDep, @@ -61,11 +61,11 @@ def list_datasources( return data_source_repo.paginate(session, params) -@router.get("/admin/datasources/{data_source_id}") +@router.get("/admin/datasources/{data_source_id}", deprecated=True) def get_datasource( - session: SessionDep, - user: CurrentSuperuserDep, - data_source_id: int, + session: SessionDep, + user: CurrentSuperuserDep, + data_source_id: int, ) -> DataSource: data_source = data_source_repo.get(session, data_source_id) if data_source is None: @@ -75,8 +75,7 @@ def get_datasource( ) return data_source - -@router.delete("/admin/datasources/{data_source_id}") +@router.delete("/admin/datasources/{data_source_id}", deprecated=True) def delete_datasource( session: SessionDep, user: CurrentSuperuserDep, @@ -142,10 +141,10 @@ def retry_failed_tasks( data_source = data_source_repo.get(session, data_source_id) if data_source is None: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND) - for docuemnt_id in data_source_repo.set_failed_vector_index_tasks_to_pending( + for document_id in data_source_repo.set_failed_vector_index_tasks_to_pending( session, data_source ): - build_vector_index_from_document.delay(data_source_id, docuemnt_id) + build_vector_index_from_document.delay(data_source_id, document_id) for chunk_id, document_id in data_source_repo.set_failed_kg_index_tasks_to_pending( session, data_source ): diff --git a/backend/app/api/admin_routes/embedding_model/models.py b/backend/app/api/admin_routes/embedding_model/models.py index 08fc4a2f9..ef1ab3e4b 100644 --- a/backend/app/api/admin_routes/embedding_model/models.py +++ b/backend/app/api/admin_routes/embedding_model/models.py @@ -1,6 +1,6 @@ from typing import Any -from pydantic import BaseModel +from pydantic import BaseModel, field_validator from typing_extensions import Optional from app.models.embed_model import DEFAULT_VECTOR_DIMENSION @@ -11,11 +11,17 @@ class EmbeddingModelCreate(BaseModel): name: str provider: EmbeddingProvider model: str - vector_dimension: int = DEFAULT_VECTOR_DIMENSION + vector_dimension: int config: dict | list | None credentials: Any is_default: Optional[bool] = False + @field_validator("vector_dimension") + def name_must_not_be_blank(cls, v: int) -> int: + if v <= 0: + raise ValueError("The vector dimension of the Embedding model should be at least greater than 1.") + return v + class EmbeddingModelUpdate(BaseModel): name: Optional[str] = None diff --git a/backend/app/api/admin_routes/embedding_model/routes.py b/backend/app/api/admin_routes/embedding_model/routes.py index d4706a1f1..dd1df07f1 100644 --- a/backend/app/api/admin_routes/embedding_model/routes.py +++ b/backend/app/api/admin_routes/embedding_model/routes.py @@ -14,7 +14,7 @@ from app.exceptions import EmbeddingModelNotFoundError, InternalServerError from app.rag.chat_config import get_embedding_model from app.rag.embed_model_option import EmbeddingModelOption, admin_embed_model_options -from app.repositories.embedding_model import embedding_model_repo +from app.repositories.embedding_model import embed_model_repo router = APIRouter() logger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ def create_embedding_model( create: EmbeddingModelCreate, ) -> EmbeddingModelDetail: try: - return embedding_model_repo.create(session, create) + return embed_model_repo.create(session, create) except Exception as e: logger.exception(e) raise InternalServerError() @@ -56,7 +56,7 @@ def test_embedding_model( expected_length = create.vector_dimension if len(embedding) != expected_length: raise ValueError( - f"Currently we only support {expected_length} dims embedding, got {len(embedding)} dims." + f"Embedding model is configured with {expected_length} dimensions, but got vector embedding with {len(embedding)} dimensions." ) success = True error = "" @@ -72,7 +72,7 @@ def list_embedding_models( user: CurrentSuperuserDep, params: Params = Depends() ) -> Page[EmbeddingModelItem]: - return embedding_model_repo.paginate(session, params) + return embed_model_repo.paginate(session, params) @router.get("/admin/embedding-models/{model_id}") @@ -82,7 +82,7 @@ def get_embedding_model_detail( model_id: int ) -> EmbeddingModelDetail: try: - return embedding_model_repo.must_get(session, model_id) + return embed_model_repo.must_get(session, model_id) except EmbeddingModelNotFoundError as e: raise e except Exception as e: @@ -98,8 +98,8 @@ def update_embedding_model( update: EmbeddingModelUpdate, ) -> EmbeddingModelDetail: try: - embed_model = embedding_model_repo.must_get(session, model_id) - embedding_model_repo.update(session, embed_model, update) + embed_model = embed_model_repo.must_get(session, model_id) + embed_model_repo.update(session, embed_model, update) return embed_model except EmbeddingModelNotFoundError as e: raise e @@ -115,8 +115,8 @@ def set_default_embedding_model( model_id: int ) -> EmbeddingModelDetail: try: - embed_model = embedding_model_repo.must_get(session, model_id) - embedding_model_repo.set_default_model(session, model_id) + embed_model = embed_model_repo.must_get(session, model_id) + embed_model_repo.set_default_model(session, model_id) session.refresh(embed_model) return embed_model except EmbeddingModelNotFoundError as e: diff --git a/backend/app/api/admin_routes/knowledge_base/data_sources/__init__.py b/backend/app/api/admin_routes/knowledge_base/data_sources/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/app/api/admin_routes/knowledge_base/data_sources/models.py b/backend/app/api/admin_routes/knowledge_base/data_sources/models.py new file mode 100644 index 000000000..cd6ee0ec5 --- /dev/null +++ b/backend/app/api/admin_routes/knowledge_base/data_sources/models.py @@ -0,0 +1,32 @@ +from pydantic import BaseModel, field_validator + +from app.models import DataSourceType + + +class KBDataSource(BaseModel): + """ + Represents a linked data source for a knowledge base. + """ + id: int + name: str + data_source_type: DataSourceType + config: dict | list + + +class KBDataSourceMutable(BaseModel): + name: str + + @field_validator("name") + def name_must_not_be_blank(cls, v: str) -> str: + if not v.strip(): + raise ValueError("Please provide a name for the data source") + return v + + +class KBDataSourceCreate(KBDataSourceMutable): + data_source_type: DataSourceType + config: dict | list + + +class KBDataSourceUpdate(KBDataSourceMutable): + pass \ No newline at end of file diff --git a/backend/app/api/admin_routes/knowledge_base/data_sources/routes.py b/backend/app/api/admin_routes/knowledge_base/data_sources/routes.py new file mode 100644 index 000000000..4a93edca5 --- /dev/null +++ b/backend/app/api/admin_routes/knowledge_base/data_sources/routes.py @@ -0,0 +1,135 @@ +import logging + +from fastapi import APIRouter, Depends +from fastapi_pagination import Params, Page + +from app.api.admin_routes.knowledge_base.data_sources.models import KBDataSourceUpdate, KBDataSource +from app.api.admin_routes.knowledge_base.models import KBDataSourceCreate +from app.api.deps import SessionDep, CurrentSuperuserDep +from app.exceptions import ( + InternalServerError, + KBDataSourceNotFoundError, + KnowledgeBaseNotFoundError +) +from app.models import DataSource +from app.repositories import knowledge_base_repo +from app.tasks.knowledge_base import ( + import_documents_from_kb_datasource, + purge_kb_datasource_related_resources +) + + +router = APIRouter() +logger = logging.getLogger(__name__) + + +@router.post("/admin/knowledge_bases/{kb_id}/datasources") +def create_kb_datasource( + session: SessionDep, + user: CurrentSuperuserDep, + kb_id: int, + create: KBDataSourceCreate +) -> KBDataSource: + try: + kb = knowledge_base_repo.must_get(session, kb_id) + new_data_source = DataSource( + name=create.name, + description="", + data_source_type=create.data_source_type, + config=create.config, + ) + new_data_source = knowledge_base_repo.add_kb_datasource(session, kb, new_data_source) + + import_documents_from_kb_datasource.delay(kb_id, new_data_source.id) + + return new_data_source + except KnowledgeBaseNotFoundError as e: + raise e + except Exception as e: + logger.error(f"Failed to create data source for knowledge base #{kb_id}: {e}", exc_info=e) + raise InternalServerError() + + +@router.put("/admin/knowledge_bases/{kb_id}/datasources/{data_source_id}") +def update_kb_datasource( + session: SessionDep, + user: CurrentSuperuserDep, + kb_id: int, + data_source_id: int, + update: KBDataSourceUpdate, +) -> KBDataSource: + try: + kb = knowledge_base_repo.must_get(session, kb_id) + + data_source = kb.must_get_data_source_by_id(data_source_id) + data_source.name = update.name + + session.add(data_source) + session.commit() + session.refresh(data_source) + + return data_source + except KnowledgeBaseNotFoundError as e: + raise e + except KBDataSourceNotFoundError as e: + raise e + except Exception as e: + logger.error(f"Failed to update data source #{data_source_id}: {e}", exc_info=e) + raise InternalServerError() + + +@router.get("/admin/knowledge_bases/{kb_id}/datasources/{data_source_id}") +def get_kb_datasource( + session: SessionDep, + user: CurrentSuperuserDep, + kb_id: int, + data_source_id: int, +) -> KBDataSource: + try: + kb = knowledge_base_repo.must_get(session, kb_id) + return kb.must_get_data_source_by_id(data_source_id) + except KnowledgeBaseNotFoundError as e: + raise e + except KBDataSourceNotFoundError as e: + raise e + except Exception as e: + logger.error(f"Failed to get data source #{data_source_id}: {e}", exc_info=e) + raise InternalServerError() + + +@router.get("/admin/knowledge_bases/{kb_id}/datasources") +def list_kb_datasources( + session: SessionDep, + user: CurrentSuperuserDep, + kb_id: int, + params: Params = Depends(), +) -> Page[KBDataSource]: + return knowledge_base_repo.list_kb_datasources(session, kb_id, params) + + +@router.delete("/admin/knowledge_bases/{kb_id}/datasources/{data_source_id}") +def remove_kb_datasource( + session: SessionDep, + user: CurrentSuperuserDep, + kb_id: int, + data_source_id: int, +): + try: + kb = knowledge_base_repo.must_get(session, kb_id) + data_source = kb.must_get_data_source_by_id(data_source_id) + + # Flag the data source to be deleted, it will be deleted completely by the background job. + knowledge_base_repo.remove_kb_datasource(session, kb, data_source) + + purge_kb_datasource_related_resources.delay(kb_id, data_source_id) + + return { + "detail": "success" + } + except KnowledgeBaseNotFoundError as e: + raise e + except KBDataSourceNotFoundError as e: + raise e + except Exception as e: + logger.error(f"Failed to remove data source #{data_source_id} from knowledge base #{kb_id}: {e}", exc_info=e) + raise InternalServerError() diff --git a/backend/app/api/admin_routes/knowledge_base/models.py b/backend/app/api/admin_routes/knowledge_base/models.py index a5a3dfd54..9ccb0c441 100644 --- a/backend/app/api/admin_routes/knowledge_base/models.py +++ b/backend/app/api/admin_routes/knowledge_base/models.py @@ -1,38 +1,21 @@ from datetime import datetime from typing import Optional from uuid import UUID -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, field_validator, Field +from app.api.admin_routes.knowledge_base.data_sources.models import KBDataSource, KBDataSourceCreate from app.api.admin_routes.models import EmbeddingModelDescriptor, LLMDescriptor, UserDescriptor from app.exceptions import KBNoVectorIndexConfiguredError from app.models import KgIndexStatus -from app.models.data_source import DataSourceType from app.models.knowledge_base import IndexMethod -from app.models.patch.sql_model import SQLModel - -class KBDataSourceCreate(BaseModel): - id: Optional[int] = None +class KnowledgeBaseCreate(BaseModel): name: str - data_source_type: DataSourceType - config: dict | list - - @field_validator("name") - def name_must_not_be_blank(cls, v: str) -> str: - if not v.strip(): - raise ValueError("Please provide a name for the data source") - return v - - # TODO: verify DataSource config. - - -class KnowledgeBaseCreate(SQLModel): - name: str - description: str - index_methods: list[IndexMethod] + description: Optional[str] = None + index_methods: list[IndexMethod] = Field(default_factory=lambda: [IndexMethod.VECTOR]) llm_id: Optional[int] = None embedding_model_id: Optional[int] = None - data_sources: list[KBDataSourceCreate] + data_sources: Optional[list[KBDataSourceCreate]] = Field(default_factory=list) @field_validator("name") def name_must_not_be_blank(cls, v: str) -> str: @@ -54,16 +37,6 @@ class KnowledgeBaseUpdate(BaseModel): description: Optional[str] = None -class KBDataSource(BaseModel): - """ - Represents a linked data source for a knowledge base. - """ - id: int - name: str - data_source_type: DataSourceType - config: dict | list - - class KnowledgeBaseDetail(BaseModel): """ Represents a detailed view of a knowledge base. @@ -71,6 +44,8 @@ class KnowledgeBaseDetail(BaseModel): id: int name: str description: str + documents_total: int + data_sources_total: int # Notice: By default, SQLModel will not serialize list type relationships. # https://github.com/fastapi/sqlmodel/issues/37#issuecomment-2093607242 data_sources: list[KBDataSource] diff --git a/backend/app/api/admin_routes/knowledge_base/routes.py b/backend/app/api/admin_routes/knowledge_base/routes.py index 0a1212bef..b1588fe02 100644 --- a/backend/app/api/admin_routes/knowledge_base/routes.py +++ b/backend/app/api/admin_routes/knowledge_base/routes.py @@ -1,15 +1,14 @@ import logging from typing import Annotated -from fastapi import APIRouter, Depends, logger, Query +from fastapi import APIRouter, Depends, Query from fastapi_pagination import Params, Page from app.models.chunk import get_kb_chunk_model from app.rag.knowledge_base.index_store import init_kb_tidb_vector_store, init_kb_tidb_graph_store from app.repositories.chunk import ChunkRepo -from app.repositories.embedding_model import embedding_model_repo -from app.repositories.llm import get_default_db_llm - +from app.repositories.embedding_model import embed_model_repo +from app.repositories.llm import llm_repo from .models import ( KnowledgeBaseDetail, @@ -20,9 +19,7 @@ from app.exceptions import ( InternalServerError, KnowledgeBaseNotFoundError, - KBNoVectorIndexConfiguredError, - KBNoLLMConfiguredError, - KBNoEmbedModelConfiguredError + KBNoVectorIndexConfiguredError ) from app.models import ( KnowledgeBase, @@ -33,8 +30,11 @@ build_index_for_document, ) from app.repositories import knowledge_base_repo, data_source_repo, document_repo -from app.tasks.knowledge_base import import_documents_for_knowledge_base, purge_knowledge_base_related_resources, \ - stats_for_knowledge_base +from app.tasks.knowledge_base import ( + import_documents_for_knowledge_base, + stats_for_knowledge_base, + purge_knowledge_base_related_resources +) from ..document.models import DocumentItem, DocumentFilters router = APIRouter() @@ -52,34 +52,24 @@ def create_knowledge_base( data_source_repo.create(session, DataSource( name=data_source.name, description='', + user_id=user.id, data_source_type=data_source.data_source_type, config=data_source.config, )) for data_source in create.data_sources ] - db_llm_id = create.llm_id - if not db_llm_id: - default_llm = get_default_db_llm(session) - if default_llm: - db_llm_id = default_llm.id - else: - raise KBNoLLMConfiguredError() - - db_embed_model_id = create.embedding_model_id - if not db_embed_model_id: - default_embed_model = embedding_model_repo.get_default_model(session) - if default_embed_model: - db_embed_model_id = default_embed_model.id - else: - raise KBNoEmbedModelConfiguredError() + if not create.llm_id: + create.llm_id = llm_repo.must_get_default_llm(session).id + if not create.embedding_model_id: + create.embedding_model_id = embed_model_repo.must_get_default_model(session).id knowledge_base = KnowledgeBase( name=create.name, description=create.description, index_methods=create.index_methods, - llm_id=db_llm_id, - embedding_model_id=db_embed_model_id, + llm_id=create.llm_id, + embedding_model_id=create.embedding_model_id, data_sources=data_sources, created_by=user.id, updated_by=user.id, @@ -97,7 +87,7 @@ def create_knowledge_base( except KBNoVectorIndexConfiguredError as e: raise e except Exception as e: - logging.exception(e) + logger.exception(e) raise InternalServerError() @@ -141,7 +131,7 @@ def update_knowledge_base_setting( except KBNoVectorIndexConfiguredError as e: raise e except Exception as e: - logging.exception(e) + logger.exception(e) raise InternalServerError() diff --git a/backend/app/api/main.py b/backend/app/api/main.py index bbf1f5810..80df60625 100644 --- a/backend/app/api/main.py +++ b/backend/app/api/main.py @@ -9,7 +9,8 @@ feedback, ) from app.api.admin_routes.knowledge_base.routes import router as admin_knowledge_base_router -from app.api.admin_routes.knowledge_base.graph.routes import router as admin_knowledge_base_graph_router +from app.api.admin_routes.knowledge_base.graph.routes import router as admin_kb_graph_router +from app.api.admin_routes.knowledge_base.data_sources.routes import router as admin_kb_data_source_router from app.api.admin_routes.document.routes import router as admin_document_router from app.api.admin_routes.embedding_model.routes import router as admin_embedding_model_router from app.api.admin_routes import ( @@ -46,9 +47,9 @@ api_router.include_router(admin_upload.router, tags=["admin/upload"]) api_router.include_router(admin_data_source.router, tags=["admin/data_source"]) api_router.include_router(admin_knowledge_base_router, tags=["admin/knowledge_base"]) -api_router.include_router(admin_knowledge_base_graph_router, tags=["admin/knowledge_base/graph_editor"]) +api_router.include_router(admin_kb_graph_router, tags=["admin/knowledge_base/graph_editor"]) +api_router.include_router(admin_kb_data_source_router, tags=["admin/knowledge_base/data_source"]) api_router.include_router(admin_llm.router, tags=["admin/llm"]) - api_router.include_router(admin_embedding_model_router, tags=["admin/embedding_model"]) api_router.include_router(admin_retrieve.router, tags=["admin/retrieve"]) api_router.include_router(admin_stats.router, tags=["admin/stats"]) diff --git a/backend/app/api/routes/chat.py b/backend/app/api/routes/chat.py index fea536864..dd1680e17 100644 --- a/backend/app/api/routes/chat.py +++ b/backend/app/api/routes/chat.py @@ -12,7 +12,9 @@ from fastapi_pagination import Params, Page from app.api.deps import SessionDep, OptionalUserDep, CurrentUserDep -from app.repositories import chat_repo +from app.rag.chat_config import get_default_embedding_model, ChatEngineConfig +from app.rag.knowledge_base.config import get_kb_embed_model +from app.repositories import chat_repo, knowledge_base_repo from app.models import Chat, ChatUpdate from app.rag.chat import ( ChatService, @@ -193,7 +195,16 @@ def get_chat_subgraph(session: SessionDep, user: OptionalUserDep, chat_message_i if not user_can_view_chat(chat_message.chat, user): raise HTTPException(status_code=HTTPStatus.FORBIDDEN, detail="Access denied") - entities, relations = get_chat_message_subgraph(session, chat_message) + engine_options = chat_message.chat.engine_options + chat_engine_config = ChatEngineConfig.validate(engine_options) + + if chat_engine_config.knowledge_base: + kb = knowledge_base_repo.must_get(session, chat_engine_config.knowledge_base.linked_knowledge_base.id) + embed_model = get_kb_embed_model(session, kb) + else: + embed_model = get_default_embedding_model(session) + + entities, relations = get_chat_message_subgraph(session, chat_message, embed_model) return SubgraphResponse(entities=entities, relationships=relations) diff --git a/backend/app/exceptions.py b/backend/app/exceptions.py index 143ce0092..6c6aed70b 100644 --- a/backend/app/exceptions.py +++ b/backend/app/exceptions.py @@ -18,8 +18,22 @@ class ChatNotFound(ChatException): class DBLLMNotFoundError(HTTPException): status_code = 404 - def __init__(self, knowledge_base_id: int): - self.detail = f"llm #{knowledge_base_id} is not found" + def __init__(self, llm_id: int): + self.detail = f"llm #{llm_id} is not found" + + +class DefaultLLMNotFoundError(HTTPException): + status_code = 404 + + def __init__(self): + self.detail = f"default llm is not found" + + +class DefaultEmbeddingModelNotFoundError(HTTPException): + status_code = 404 + + def __init__(self): + self.detail = f"default embedding model is not found" class KnowledgeBaseNotFoundError(HTTPException): @@ -28,6 +42,12 @@ class KnowledgeBaseNotFoundError(HTTPException): def __init__(self, knowledge_base_id: int): self.detail = f"knowledge base #{knowledge_base_id} is not found" +class KBDataSourceNotFoundError(HTTPException): + status_code = 404 + + def __init__(self, kb_id: int, data_source_id: int): + self.detail = f"data source #{data_source_id} is not found in knowledge base #{kb_id}" + class KBNoLLMConfiguredError(HTTPException): status_code = 500 diff --git a/backend/app/models/knowledge_base.py b/backend/app/models/knowledge_base.py index 1afb27bf6..38be60603 100644 --- a/backend/app/models/knowledge_base.py +++ b/backend/app/models/knowledge_base.py @@ -13,6 +13,7 @@ SQLModel, ) +from app.exceptions import KBDataSourceNotFoundError from app.models.auth import User from app.models.data_source import DataSource from app.models.embed_model import EmbeddingModel @@ -94,3 +95,12 @@ class KnowledgeBase(SQLModel, table=True): def __hash__(self): return hash(self.id) + + def get_data_source_by_id(self, data_source_id: int) -> Optional[DataSource]: + return next((ds for ds in self.data_sources if ds.id == data_source_id and not ds.deleted_at), None) + + def must_get_data_source_by_id(self, data_source_id: int) -> DataSource: + data_source = self.get_data_source_by_id(data_source_id) + if data_source is None: + raise KBDataSourceNotFoundError(self.id, data_source_id) + return data_source diff --git a/backend/app/rag/build_index.py b/backend/app/rag/build_index.py index 901c02ef4..d8fb8796b 100644 --- a/backend/app/rag/build_index.py +++ b/backend/app/rag/build_index.py @@ -87,7 +87,7 @@ def build_kg_index_for_chunk(self, session: Session, db_chunk: Type[DBChunk]): graph_store = get_kb_tidb_graph_store(session, self._knowledge_base) graph_index: KnowledgeGraphIndex = KnowledgeGraphIndex.from_existing( - dspy_lm=self._dspy_lm, kg_store=graph_store + dspy_lm=self._dspy_lm, kg_store=graph_store, ) node = db_chunk.to_llama_text_node() diff --git a/backend/app/rag/chat.py b/backend/app/rag/chat.py index d23305d97..fd8502596 100644 --- a/backend/app/rag/chat.py +++ b/backend/app/rag/chat.py @@ -52,6 +52,7 @@ ChatEvent, ) from app.models.relationship import get_kb_relationship_model +from app.rag.knowledge_base.config import get_kb_embed_model from app.rag.knowledge_graph.graph_store import TiDBGraphStore from app.rag.vector_store.tidb_vector_store import TiDBVectorStore from app.rag.knowledge_graph.graph_store.tidb_graph_editor import legacy_tidb_graph_editor @@ -77,15 +78,15 @@ class ChatService: _relationship_db_model: Type[SQLModel] = DBRelationship def __init__( - self, - *, - db_session: Session, - user: User, - browser_id: str, - origin: str, - chat_messages: List[ChatMessage], - engine_name: str = "default", - chat_id: Optional[UUID] = None, + self, + *, + db_session: Session, + user: User, + browser_id: str, + origin: str, + chat_messages: List[ChatMessage], + engine_name: str = "default", + chat_id: Optional[UUID] = None, ) -> None: self.db_session = db_session self.user = user @@ -157,6 +158,7 @@ def __init__( self._node_postprocessors = [self._metadata_filter] self._similarity_top_k = 10 + # Langfuse self.langfuse_host = SiteSetting.langfuse_host self.langfuse_secret_key = SiteSetting.langfuse_secret_key self.langfuse_public_key = SiteSetting.langfuse_public_key @@ -164,13 +166,16 @@ def __init__( self.langfuse_host and self.langfuse_secret_key and self.langfuse_public_key ) + # TODO: Support multiple knowledge base retrieve. if self.chat_engine_config.knowledge_base: - # TODO: Support multiple knowledge base retrieve. linked_knowledge_base = self.chat_engine_config.knowledge_base.linked_knowledge_base kb = knowledge_base_repo.must_get(db_session, linked_knowledge_base.id) + self._embed_model = get_kb_embed_model(db_session, kb) self._chunk_db_model = get_kb_chunk_model(kb) self._entity_db_model = get_kb_entity_model(kb) self._relationship_db_model = get_kb_relationship_model(kb) + else: + self._embed_model = get_default_embedding_model(db_session) def chat(self) -> Generator[ChatEvent | str, None, None]: try: @@ -188,13 +193,13 @@ def chat(self) -> Generator[ChatEvent | str, None, None]: ) def _search_kg( - self, - kg_config: KnowledgeGraphOption, - fast_dspy_lm: dspy.LM, - embed_model: BaseEmbedding, - get_llamaindex_callback_manager: Callable[[], Optional[CallbackManager]], - trace_url: str, - annotation_silent: bool = False, + self, + kg_config: KnowledgeGraphOption, + fast_dspy_lm: dspy.LM, + embed_model: BaseEmbedding, + get_llamaindex_callback_manager: Callable[[], Optional[CallbackManager]], + trace_url: str, + annotation_silent: bool = False, ) -> Generator[ChatEvent | str, None, Tuple[List[dict], List[dict], List[dict], dict, str]]: """ Search the knowledge graph for relevant entities, relationships, and chunks. @@ -308,12 +313,12 @@ def _search_kg( return entities, relations, chunks, graph_data_source_ids, graph_knowledges_context def _get_llamaindex_callback_manager( - self, - langfuse: Optional[Langfuse] = None, - trace_id: Optional[str] = None, - llm: Optional[LLM] = None, - fast_llm: Optional[LLM] = None, - embed_model: Optional[BaseEmbedding] = None, + self, + langfuse: Optional[Langfuse] = None, + trace_id: Optional[str] = None, + llm: Optional[LLM] = None, + fast_llm: Optional[LLM] = None, + embed_model: Optional[BaseEmbedding] = None, ) -> CallbackManager: # Why we don't use high-level decorator `observe()` as \ # `https://langfuse.com/docs/integrations/llama-index/get-started` suggested? @@ -373,12 +378,12 @@ class ClarityResult(BaseModel): clarifying_question: str def _refine_or_early_stop( - self, - get_llamaindex_callback_manager: Callable[[], Optional[CallbackManager]], - fast_llm: LLM, - graph_knowledges_context: str, - refined_question_prompt: Optional[str] = None, - annotation_silent: bool = False, + self, + get_llamaindex_callback_manager: Callable[[], Optional[CallbackManager]], + fast_llm: LLM, + graph_knowledges_context: str, + refined_question_prompt: Optional[str] = None, + annotation_silent: bool = False, ) -> Generator[ChatEvent | str, None, Tuple[bool, str, str]]: """ Determine whether to refine the user question or early stop the conversation with a clarifying question. @@ -473,13 +478,13 @@ def _refine_or_early_stop( return False, "", refined_question def _gen_answer_via_llama_index( - self, - get_llamaindex_callback_manager: Callable[[], Optional[CallbackManager]], - refined_question: str, - graph_knowledges_context: str, - llm: LLM, - embed_model: BaseEmbedding, - annotation_silent: bool = False, + self, + get_llamaindex_callback_manager: Callable[[], Optional[CallbackManager]], + refined_question: str, + graph_knowledges_context: str, + llm: LLM, + embed_model: BaseEmbedding, + annotation_silent: bool = False, ) -> Generator[ChatEvent | str, None, Tuple[StreamingResponse, List[dict]]]: if not annotation_silent: yield ChatEvent( @@ -547,13 +552,13 @@ def _gen_answer_via_llama_index( return response, source_documents def _chat_finish( - self, - db_assistant_message: ChatMessage, - db_user_message: ChatMessage, - response_text: str, - source_documents: List[dict], - graph_data_source_ids: dict, - annotation_silent: bool = False, + self, + db_assistant_message: ChatMessage, + db_user_message: ChatMessage, + response_text: str, + source_documents: List[dict], + graph_data_source_ids: dict, + annotation_silent: bool = False, ): if not annotation_silent: yield ChatEvent( @@ -614,7 +619,7 @@ def _chat(self) -> Generator[ChatEvent | str, None, None]: ), ) - _embed_model = get_default_embedding_model(self.db_session) + # TODO: remove to __init__? _llm = self.chat_engine_config.get_llama_llm(self.db_session) _fast_llm = self.chat_engine_config.get_fast_llama_llm(self.db_session) _fast_dspy_lm = self.chat_engine_config.get_fast_dspy_lm(self.db_session) @@ -634,7 +639,7 @@ def _get_llamaindex_callback_manager_in_chat() -> CallbackManager: trace_id=trace_id, llm=_llm, fast_llm=_fast_llm, - embed_model=_embed_model, + embed_model=self._embed_model, ) # 1. Retrieve entities, relations, and chunks from the knowledge graph @@ -642,7 +647,7 @@ def _get_llamaindex_callback_manager_in_chat() -> CallbackManager: entities, relations, chunks, graph_data_source_ids, graph_knowledges_context = yield from self._search_kg( kg_config=kg_config, fast_dspy_lm=_fast_dspy_lm, - embed_model=_embed_model, + embed_model=self._embed_model, trace_url=trace_url, get_llamaindex_callback_manager=_get_llamaindex_callback_manager_in_chat, ) @@ -667,7 +672,7 @@ def _get_llamaindex_callback_manager_in_chat() -> CallbackManager: refined_question=refined_question, graph_knowledges_context=graph_knowledges_context, llm=_llm, - embed_model=_embed_model, + embed_model=self._embed_model, ) response_text = "" @@ -719,27 +724,13 @@ def _external_chat(self) -> Generator[ChatEvent | str, None, None]: ), ) - _embed_model = get_default_embedding_model(self.db_session) + # TODO: remove to __init__? _fast_dspy_lm = self.chat_engine_config.get_fast_dspy_lm(self.db_session) _fast_llm = self.chat_engine_config.get_fast_llama_llm(self.db_session) # retrieve entities, relations, and chunks from the knowledge graph # this retrieve progress is only for the clarifying question checking try: - """ - kg_config = self.chat_engine_config.knowledge_graph - _, _, _, graph_data_source_ids, graph_knowledges_context = yield from self._search_kg( - kg_config=kg_config, - fast_dspy_lm=_fast_dspy_lm, - embed_model=_embed_model, - trace_url="", - get_llamaindex_callback_manager=lambda: self._get_llamaindex_callback_manager( - fast_llm=_fast_llm, - embed_model=_embed_model, - ), - annotation_silent=True, - ) - """ graph_data_source_ids = [] graph_knowledges_context = "" @@ -747,7 +738,7 @@ def _external_chat(self) -> Generator[ChatEvent | str, None, None]: early_stop, clarifying_question, goal = yield from self._refine_or_early_stop( get_llamaindex_callback_manager=lambda: self._get_llamaindex_callback_manager( fast_llm=_fast_llm, - embed_model=_embed_model, + embed_model=self._embed_model, ), fast_llm=_fast_llm, graph_knowledges_context=graph_knowledges_context, @@ -810,31 +801,6 @@ def _external_chat(self) -> Generator[ChatEvent | str, None, None]: except Exception as e: logger.error(f"Failed to get task_id from chunk: {e}") - """ - try: - response_text = "" - final_answer_gen = _fast_llm.stream( - get_prompt_by_jinja2_template( - self.chat_engine_config.llm.condense_answer_prompt, - chat_history=self.chat_history, - question=self.user_question, - agent_answer=stackvm_response_text, - ) - ) - for word in final_answer_gen: - response_text += word - yield ChatEvent( - event_type=ChatEventType.TEXT_PART, - payload=word, - ) - except Exception as e: - for word in stackvm_response_text: - yield ChatEvent( - event_type=ChatEventType.TEXT_PART, - payload=word, - ) - logger.error(f"Failed to refine question: {e}") - """ response_text = stackvm_response_text base_url = stream_chat_api_url.replace('/api/stream_execute_vm', '') db_assistant_message.content = response_text @@ -1031,8 +997,9 @@ def get_graph_data_from_langfuse(trace_url: str): def get_chat_message_subgraph( session: Session, chat_message: DBChatMessage, + embed_model: BaseEmbedding, entity_db_model: SQLModel = DBEntity, - relationship_db_model: SQLModel = DBRelationship + relationship_db_model: SQLModel = DBRelationship, ) -> Tuple[List, List]: if chat_message.role != MessageRole.USER: return [], [] @@ -1089,7 +1056,7 @@ def get_chat_message_subgraph( graph_store = TiDBGraphStore( dspy_lm=chat_engine_config.get_fast_dspy_lm(session), session=session, - embed_model=get_default_embedding_model(session), + embed_model=embed_model, entity_db_model=entity_db_model, relationship_db_model=relationship_db_model, ) diff --git a/backend/app/rag/chat_config.py b/backend/app/rag/chat_config.py index 5ae344478..a2ac0b8c6 100644 --- a/backend/app/rag/chat_config.py +++ b/backend/app/rag/chat_config.py @@ -30,9 +30,9 @@ from app.rag.node_postprocessor.baisheng_reranker import BaishengRerank from app.rag.node_postprocessor.local_reranker import LocalRerank from app.rag.embeddings.local_embedding import LocalEmbedding -from app.repositories import chat_engine_repo -from app.repositories.embedding_model import embedding_model_repo -from app.repositories.llm import get_default_db_llm +from app.repositories import chat_engine_repo, knowledge_base_repo +from app.repositories.embedding_model import embed_model_repo +from app.repositories.llm import llm_repo from app.types import LLMProvider, EmbeddingProvider, RerankerProvider from app.rag.default_prompt import ( DEFAULT_INTENT_GRAPH_KNOWLEDGE, @@ -256,9 +256,7 @@ def get_llm( def get_default_llm(session: Session) -> LLM: - db_llm = get_default_db_llm(session) - if not db_llm: - raise ValueError("No default LLM found in DB") + db_llm = llm_repo.must_get_default_llm(session) return get_llm( db_llm.provider, db_llm.model, @@ -316,9 +314,7 @@ def get_embedding_model( def get_default_embedding_model(session: Session) -> BaseEmbedding: - db_embed_model = embedding_model_repo.get_default_model(session) - if not db_embed_model: - raise ValueError("No default embedding model found in DB") + db_embed_model = embed_model_repo.must_get_default_model(session) return get_embedding_model( db_embed_model.provider, db_embed_model.model, diff --git a/backend/app/rag/knowledge_base/index_store.py b/backend/app/rag/knowledge_base/index_store.py index 6ff83b386..f64f53235 100644 --- a/backend/app/rag/knowledge_base/index_store.py +++ b/backend/app/rag/knowledge_base/index_store.py @@ -50,7 +50,7 @@ def init_kb_tidb_graph_store(session: Session, kb: KnowledgeBase) -> TiDBGraphSt def get_kb_tidb_graph_editor(session: Session, kb: KnowledgeBase) -> TiDBGraphEditor: - entity_model = get_kb_entity_model(kb) - relationship_model = get_kb_relationship_model(kb) - init_kb_tidb_vector_store(session, kb) - return TiDBGraphEditor(entity_model, relationship_model) + entity_db_model = get_kb_entity_model(kb) + relationship_db_model = get_kb_relationship_model(kb) + embed_model = get_kb_embed_model(session, kb) + return TiDBGraphEditor(entity_db_model, relationship_db_model, embed_model) diff --git a/backend/app/rag/knowledge_graph/graph_store/tidb_graph_editor.py b/backend/app/rag/knowledge_graph/graph_store/tidb_graph_editor.py index bf5a31106..846aa18e1 100644 --- a/backend/app/rag/knowledge_graph/graph_store/tidb_graph_editor.py +++ b/backend/app/rag/knowledge_graph/graph_store/tidb_graph_editor.py @@ -1,5 +1,8 @@ from typing import Optional, Tuple, List, Type +from llama_index.core.embeddings import resolve_embed_model +from llama_index.core.embeddings.utils import EmbedType +from llama_index.embeddings.openai import OpenAIEmbedding, OpenAIEmbeddingModelType from sqlmodel import Session, select, SQLModel from sqlalchemy.orm import joinedload from sqlalchemy.orm.attributes import flag_modified @@ -14,7 +17,6 @@ from app.rag.knowledge_graph.graph_store.tidb_graph_store import TiDBGraphStore from app.rag.knowledge_graph.schema import Relationship as RelationshipAIModel -from app.rag.chat_config import get_default_embedding_model from app.staff_action import create_staff_action_log @@ -22,10 +24,21 @@ class TiDBGraphEditor: _entity_db_model: Type[SQLModel] _relationship_db_model: Type[SQLModel] - - def __init__(self, entity_model: Type[SQLModel], relationship_model: Type[SQLModel]): - self._entity_db_model = entity_model - self._relationship_db_model = relationship_model + def __init__( + self, + entity_db_model: Type[SQLModel], + relationship_db_model: Type[SQLModel], + embed_model: Optional[EmbedType] = None, + ): + self._entity_db_model = entity_db_model + self._relationship_db_model = relationship_db_model + + if embed_model: + self._embed_model = resolve_embed_model(embed_model) + else: + self._embed_model = OpenAIEmbedding( + model=OpenAIEmbeddingModelType.TEXT_EMBED_3_SMALL + ) def get_entity(self, session: Session, entity_id: int) -> Optional[SQLModel]: @@ -38,11 +51,10 @@ def update_entity(self, session: Session, entity: SQLModel, new_entity: dict) -> if value is not None: setattr(entity, key, value) flag_modified(entity, key) - embed_model = get_default_embedding_model(session) entity.description_vec = get_entity_description_embedding( - entity.name, entity.description, embed_model + entity.name, entity.description, self._embed_model ) - entity.meta_vec = get_entity_metadata_embedding(entity.meta, embed_model) + entity.meta_vec = get_entity_metadata_embedding(entity.meta, self._embed_model) for relationship in session.exec( select(self._relationship_db_model) .options( @@ -60,7 +72,7 @@ def update_entity(self, session: Session, entity: SQLModel, new_entity: dict) -> relationship.target_entity.name, relationship.target_entity.description, relationship.description, - embed_model, + self._embed_model, ) session.add(relationship) session.commit() @@ -137,14 +149,13 @@ def update_relationship( if value is not None: setattr(relationship, key, value) flag_modified(relationship, key) - embed_model = get_default_embedding_model(session) relationship.description_vec = get_relationship_description_embedding( relationship.source_entity.name, relationship.source_entity.description, relationship.target_entity.name, relationship.target_entity.description, relationship.description, - embed_model, + self._embed_model, ) session.commit() session.refresh(relationship) @@ -162,8 +173,7 @@ def update_relationship( def search_similar_entities(self, session: Session, query: str, top_k: int = 10) -> list: - embed_model = get_default_embedding_model(session) - embedding = get_query_embedding(query, embed_model) + embedding = get_query_embedding(query, self._embed_model) return session.exec( select(self._entity_db_model) .where(self._entity_db_model.entity_type == EntityType.original) @@ -181,16 +191,15 @@ def create_synopsis_entity( meta: dict, related_entities_ids: List[int], ) -> SQLModel: - embed_model = get_default_embedding_model(session) # with session.begin(): synopsis_entity = self._entity_db_model( name=name, description=description, description_vec=get_entity_description_embedding( - name, description, embed_model + name, description, self._embed_model ), meta=meta, - meta_vec=get_entity_metadata_embedding(meta, embed_model), + meta_vec=get_entity_metadata_embedding(meta, self._embed_model), entity_type=EntityType.synopsis, synopsis_info={ "entities": related_entities_ids, @@ -201,7 +210,7 @@ def create_synopsis_entity( graph_store = TiDBGraphStore( dspy_lm=None, session=session, - embed_model=embed_model, + embed_model=self._embed_model, entity_db_model=self._entity_db_model, relationship_db_model=self._relationship_db_model ) diff --git a/backend/app/rag/retrieve.py b/backend/app/rag/retrieve.py index d149e32aa..0ecd269d3 100644 --- a/backend/app/rag/retrieve.py +++ b/backend/app/rag/retrieve.py @@ -17,6 +17,7 @@ from app.rag.chat_config import ChatEngineConfig, get_default_embedding_model from app.models.relationship import get_kb_relationship_model from app.models.patch.sql_model import SQLModel +from app.rag.knowledge_base.config import get_kb_embed_model from app.rag.knowledge_graph import KnowledgeGraphIndex from app.rag.knowledge_graph.graph_store import TiDBGraphStore from app.rag.vector_store.tidb_vector_store import TiDBVectorStore @@ -50,6 +51,10 @@ def __init__( self._chunk_model = get_kb_chunk_model(kb) self._entity_model = get_kb_entity_model(kb) self._relationship_model = get_kb_relationship_model(kb) + self._embed_model = get_kb_embed_model(self.db_session, kb) + else: + self._embed_model = get_default_embedding_model(self.db_session) + def retrieve(self, question: str, top_k: int = 10) -> List[DBDocument]: """ @@ -68,7 +73,7 @@ def retrieve(self, question: str, top_k: int = 10) -> List[DBDocument]: logger.exception(e) def _retrieve(self, question: str, top_k: int) -> List[DBDocument]: - _embed_model = get_default_embedding_model(self.db_session) + # TODO: move to __init__? _llm = self.chat_engine_config.get_llama_llm(self.db_session) _fast_llm = self.chat_engine_config.get_fast_llama_llm(self.db_session) _fast_dspy_lm = self.chat_engine_config.get_fast_dspy_lm(self.db_session) @@ -79,7 +84,7 @@ def _retrieve(self, question: str, top_k: int) -> List[DBDocument]: graph_store = TiDBGraphStore( dspy_lm=_fast_dspy_lm, session=self.db_session, - embed_model=_embed_model, + embed_model=self._embed_model, entity_db_model=self._entity_model, relationship_db_model=self._relationship_model, ) @@ -141,7 +146,7 @@ def _retrieve(self, question: str, top_k: int) -> List[DBDocument]: vector_store = TiDBVectorStore(session=self.db_session, chunk_db_model=self._chunk_model) vector_index = VectorStoreIndex.from_vector_store( vector_store, - embed_model=_embed_model, + embed_model=self._embed_model, ) retrieve_engine = vector_index.as_retriever( @@ -158,12 +163,10 @@ def _retrieve(self, question: str, top_k: int) -> List[DBDocument]: return source_documents def _embedding_retrieve(self, question: str, top_k: int) -> List[NodeWithScore]: - _embed_model = get_default_embedding_model(self.db_session) - vector_store = TiDBVectorStore(session=self.db_session, chunk_db_model=self._chunk_model) vector_index = VectorStoreIndex.from_vector_store( vector_store, - embed_model=_embed_model + embed_model=self._embed_model ) retrieve_engine = vector_index.as_retriever( diff --git a/backend/app/repositories/embedding_model.py b/backend/app/repositories/embedding_model.py index e9db6996f..f74f49796 100644 --- a/backend/app/repositories/embedding_model.py +++ b/backend/app/repositories/embedding_model.py @@ -3,10 +3,10 @@ from fastapi_pagination import Params, Page from fastapi_pagination.ext.sqlmodel import paginate from sqlalchemy.orm.attributes import flag_modified -from sqlmodel import Session, select, update, SQLModel +from sqlmodel import Session, select, update from app.api.admin_routes.embedding_model.models import EmbeddingModelUpdate, EmbeddingModelCreate -from app.exceptions import EmbeddingModelNotFoundError +from app.exceptions import DefaultEmbeddingModelNotFoundError, EmbeddingModelNotFoundError from app.models import EmbeddingModel from app.repositories.base_repo import BaseRepo @@ -17,11 +17,11 @@ class EmbeddingModelRepo(BaseRepo): def create(self, session: Session, create: EmbeddingModelCreate): # If there is currently no model, the first model is # automatically set as the default model. - if not embedding_model_repo.exists_any_model(session): + if not embed_model_repo.exists_any_model(session): create.is_default = True if create.is_default: - embedding_model_repo.unset_default_model(session) + embed_model_repo.unset_default_model(session) embed_model = EmbeddingModel( name=create.name, @@ -37,6 +37,7 @@ def create(self, session: Session, create: EmbeddingModelCreate): return embed_model + def exists_any_model(self, session: Session) -> bool: stmt = select(EmbeddingModel).with_for_update().limit(1) return session.exec(stmt).one_or_none() is not None @@ -65,6 +66,13 @@ def get_default_model(self, session: Session) -> Type[EmbeddingModel]: return session.exec(stmt).first() + def must_get_default_model(self, session: Session) -> Type[EmbeddingModel]: + embed_model = self.get_default_model(session) + if embed_model is None: + raise DefaultEmbeddingModelNotFoundError() + return embed_model + + def unset_default_model(self, session: Session): session.exec( update(EmbeddingModel) @@ -101,4 +109,4 @@ def update( return embed_model -embedding_model_repo = EmbeddingModelRepo() +embed_model_repo = EmbeddingModelRepo() diff --git a/backend/app/repositories/knowledge_base.py b/backend/app/repositories/knowledge_base.py index 491723d54..eae0c6222 100644 --- a/backend/app/repositories/knowledge_base.py +++ b/backend/app/repositories/knowledge_base.py @@ -1,13 +1,14 @@ from typing import Optional, Type from datetime import datetime, UTC +from sqlalchemy import delete from sqlalchemy.orm.attributes import flag_modified from sqlmodel import select, Session, func, update from fastapi_pagination import Params, Page from fastapi_pagination.ext.sqlmodel import paginate from app.api.admin_routes.knowledge_base.models import VectorIndexError, KGIndexError, KnowledgeBaseUpdate -from app.exceptions import KnowledgeBaseNotFoundError +from app.exceptions import KBDataSourceNotFoundError, KnowledgeBaseNotFoundError from app.models import ( KnowledgeBase, Document, @@ -15,6 +16,7 @@ KgIndexStatus, Chunk, KnowledgeBaseDataSource, ) from app.models.chunk import get_kb_chunk_model +from app.models.data_source import DataSource from app.models.entity import get_kb_entity_model from app.models.knowledge_base import IndexMethod from app.models.relationship import get_kb_relationship_model @@ -35,16 +37,16 @@ def paginate(self, session: Session, params: Params | None = Params()) -> Page[K ) return paginate(session, query, params) - def get(self, session: Session, knowledge_base_id: int, include_soft_deleted: bool = True) -> Optional[KnowledgeBase]: + def get(self, session: Session, knowledge_base_id: int, show_soft_deleted: bool = True) -> Optional[KnowledgeBase]: stmt = select(KnowledgeBase).where(KnowledgeBase.id == knowledge_base_id) - if not include_soft_deleted: + if not show_soft_deleted: stmt = stmt.where(KnowledgeBase.deleted_at == None) return session.exec(stmt).first() - def must_get(self, session: Session, knowledge_base_id: int) -> Optional[KnowledgeBase]: - kb = self.get(session, knowledge_base_id) + def must_get(self, session: Session, knowledge_base_id: int, show_soft_deleted: bool = True) -> Optional[KnowledgeBase]: + kb = self.get(session, knowledge_base_id, show_soft_deleted) if kb is None: raise KnowledgeBaseNotFoundError(knowledge_base_id) return kb @@ -195,7 +197,6 @@ def set_failed_chunks_status_to_pending(self, session: Session, kb: KnowledgeBas return chunk_ids - def list_vector_index_built_errors( self, session: Session, @@ -227,7 +228,6 @@ def list_vector_index_built_errors( ] ) - def list_kg_index_built_errors( self, session: Session, @@ -263,6 +263,64 @@ def list_kg_index_built_errors( for row in rows ]) + def get_kb_datasource( + self, + session: Session, + kb: KnowledgeBase, + datasource_id: int, + show_soft_deleted: bool = False + ) -> DataSource: + stmt = select(DataSource).where(DataSource.id == datasource_id) + if not show_soft_deleted: + stmt = stmt.where(DataSource.deleted_at == None) + return session.exec(stmt).first() + + def must_get_kb_datasource( + self, + session: Session, + kb: KnowledgeBase, + datasource_id: int, + show_soft_deleted: bool = False + ) -> DataSource: + data_source = self.get_kb_datasource(session, kb, datasource_id, show_soft_deleted) + if data_source is None: + raise KBDataSourceNotFoundError(kb.id, datasource_id) + return data_source + + def add_kb_datasource(self, session: Session, kb: KnowledgeBase, data_source: DataSource) -> DataSource: + session.add(data_source) + kb.data_sources.append(data_source) + + session.add(kb) + session.commit() + session.refresh(data_source) + + return data_source + + def list_kb_datasources(self, session: Session, kb_id: int, params: Params | None = Params()) -> Page[DataSource]: + query = ( + select(DataSource) + .join(KnowledgeBaseDataSource) + .where( + DataSource.deleted_at == None, + KnowledgeBaseDataSource.knowledge_base_id == kb_id + ) + .order_by(DataSource.created_at.desc()) + ) + return paginate(session, query, params) + + def remove_kb_datasource(self, session: Session, kb: KnowledgeBase, data_source: DataSource) -> None: + # Flag the data source to be deleted. + data_source.deleted_at = datetime.now(UTC) + session.add(data_source) + + # Remove the data source from the knowledge base. + stmt = delete(KnowledgeBaseDataSource).where( + KnowledgeBaseDataSource.knowledge_base_id == kb.id, + KnowledgeBaseDataSource.data_source_id == data_source.id + ) + session.exec(stmt) + session.commit() knowledge_base_repo = KnowledgeBaseRepo() diff --git a/backend/app/repositories/llm.py b/backend/app/repositories/llm.py index 189d34725..4b4d0133a 100644 --- a/backend/app/repositories/llm.py +++ b/backend/app/repositories/llm.py @@ -2,23 +2,35 @@ from sqlmodel import select, Session -from app.exceptions import DBLLMNotFoundError +from app.exceptions import DefaultLLMNotFoundError from app.models import ( LLM as DBLLM ) -def get_db_llm(session: Session, llm_id: int) -> Type[DBLLM] | None: - return session.get(DBLLM, llm_id) +class LLMRepo: + model_cls: DBLLM + def get_db_llm(self, session: Session, llm_id: int) -> Type[DBLLM] | None: + return session.get(DBLLM, llm_id) -def must_get_llm(session: Session, llm_id: int) -> Type[DBLLM]: - db_llm = get_db_llm(session, llm_id) - if db_llm is None: - raise DBLLMNotFoundError(llm_id) - return db_llm + def must_get_llm(self, session: Session, llm_id: int) -> Type[DBLLM]: + db_llm = self.get_db_llm(session, llm_id) + if db_llm is None: + raise DefaultLLMNotFoundError(llm_id) + return db_llm -def get_default_db_llm(session: Session) -> Type[DBLLM] | None: - stmt = select(DBLLM).where(DBLLM.is_default == True).order_by(DBLLM.updated_at.desc()).limit(1) - return session.exec(stmt).first() + + def get_default_llm(self, session: Session) -> Type[DBLLM] | None: + stmt = select(DBLLM).where(DBLLM.is_default == True).order_by(DBLLM.updated_at.desc()).limit(1) + return session.exec(stmt).first() + + + def must_get_default_llm(self, session: Session) -> Type[DBLLM]: + db_llm = self.get_default_llm(session) + if db_llm is None: + raise DefaultLLMNotFoundError() + return db_llm + +llm_repo = LLMRepo() \ No newline at end of file diff --git a/backend/app/tasks/__init__.py b/backend/app/tasks/__init__.py index 64676aa1f..bc5b84323 100644 --- a/backend/app/tasks/__init__.py +++ b/backend/app/tasks/__init__.py @@ -8,7 +8,7 @@ ) from .knowledge_base import ( import_documents_for_knowledge_base, - purge_knowledge_base_related_resources, + purge_kb_datasource_related_resources, ) from .build_index import ( build_index_for_document, @@ -22,7 +22,7 @@ "build_vector_index_from_document", "build_kg_index_from_chunk", "import_documents_for_knowledge_base", - "purge_knowledge_base_related_resources", + "purge_kb_datasource_related_resources", "import_documents_from_datasource", "purge_datasource_related_resources", ] diff --git a/backend/app/tasks/knowledge_base.py b/backend/app/tasks/knowledge_base.py index ecd8dbfc6..461dbc006 100644 --- a/backend/app/tasks/knowledge_base.py +++ b/backend/app/tasks/knowledge_base.py @@ -1,6 +1,6 @@ from celery.utils.log import get_task_logger from sqlalchemy import delete -from sqlmodel import Session +from sqlmodel import Session, select from app.celery import app as celery_app from app.core.db import engine @@ -10,37 +10,61 @@ ) from app.models.knowledge_base import KnowledgeBase from app.rag.datasource import get_data_source_loader -from app.repositories import knowledge_base_repo +from app.repositories import knowledge_base_repo, data_source_repo from .build_index import build_index_for_document +from ..models.chunk import get_kb_chunk_model +from ..models.entity import get_kb_entity_model +from ..models.relationship import get_kb_relationship_model from ..rag.knowledge_base.index_store import get_kb_tidb_vector_store, get_kb_tidb_graph_store logger = get_task_logger(__name__) @celery_app.task -def import_documents_for_knowledge_base(knowledge_base_id: int): +def import_documents_for_knowledge_base(kb_id: int): try: with Session(engine) as session: - kb = knowledge_base_repo.must_get(session, knowledge_base_id) + kb = knowledge_base_repo.must_get(session, kb_id) data_sources = kb.data_sources for data_source in data_sources: - loader = get_data_source_loader( - session, - knowledge_base_id, - data_source.data_source_type, - data_source.id, - data_source.user_id, - data_source.config, - ) - for document in loader.load_documents(): - session.add(document) - session.commit() + import_documents_from_kb_datasource(kb.id, data_source.id) - build_index_for_document.delay(kb.id, document.id) - logger.info(f"Successfully imported documents for knowledge base #{knowledge_base_id}") + logger.info(f"Successfully imported documents for knowledge base #{kb_id}") except KnowledgeBaseNotFoundError: - logger.error(f"Knowledge base #{knowledge_base_id} is not found") + logger.error(f"Knowledge base #{kb_id} is not found") + except Exception as e: + logger.exception(f"Failed to import documents for knowledge base #{kb_id}", exc_info=e) + + +@celery_app.task +def import_documents_from_kb_datasource(kb_id: int, data_source_id: int): + try: + with Session(engine) as session: + kb = knowledge_base_repo.must_get(session, kb_id) + data_source = knowledge_base_repo.must_get_kb_datasource(session, kb, data_source_id) + + logger.info(f"Loading documents from data source #{data_source_id} for knowledge base #{kb_id}") + loader = get_data_source_loader( + session, + kb_id, + data_source.data_source_type, + data_source.id, + data_source.user_id, + data_source.config, + ) + + for document in loader.load_documents(): + session.add(document) + session.commit() + + build_index_for_document.delay(kb_id, document.id) + + stats_for_knowledge_base.delay(kb_id) + logger.info(f"Successfully imported documents for from datasource #{data_source_id}") except Exception as e: - logger.exception(f"Failed to import documents for knowledge base #{knowledge_base_id}", exc_info=e) + logger.exception( + f"Failed to import documents from data source #{data_source_id} of knowledge base #{kb_id}", + exc_info=e + ) @celery_app.task @@ -67,7 +91,8 @@ def stats_for_knowledge_base(knowledge_base_id: int): @celery_app.task def purge_knowledge_base_related_resources(knowledge_base_id: int): - """Purge all resources related to a knowledge base. + """ + Purge all resources related to a knowledge base. Related resources: - documents @@ -79,11 +104,7 @@ def purge_knowledge_base_related_resources(knowledge_base_id: int): """ with Session(engine) as session: - knowledge_base = knowledge_base_repo.get(session, knowledge_base_id, include_soft_deleted=True) - if knowledge_base is None: - logger.error(f"Knowledge base with id {knowledge_base_id} not found") - return - + knowledge_base = knowledge_base_repo.must_get(session, knowledge_base_id, show_soft_deleted=True) assert knowledge_base.deleted_at is not None data_source_ids = [datasource.id for datasource in knowledge_base.data_sources] @@ -104,15 +125,15 @@ def purge_knowledge_base_related_resources(knowledge_base_id: int): session.exec(stmt) logger.info(f"Deleted documents of knowledge base #{knowledge_base_id} successfully.") - # Delete data source links. - stmt = delete(KnowledgeBaseDataSource).where(KnowledgeBase.id == knowledge_base_id) - session.exec(stmt) - logger.info(f"Deleted linked data sources of knowledge base #{knowledge_base_id} successfully.") + # Delete data sources and links. + if len(data_source_ids) > 0: + stmt = delete(KnowledgeBaseDataSource).where(KnowledgeBase.id == knowledge_base_id) + session.exec(stmt) + logger.info(f"Deleted linked data sources of knowledge base #{knowledge_base_id} successfully.") - # Delete data sources. - stmt = delete(DataSource).where(DataSource.id.in_(data_source_ids)) - session.exec(stmt) - logger.info(f"Deleted data sources {', '.join([f'#{did}' for did in data_source_ids])} successfully.") + stmt = delete(DataSource).where(DataSource.id.in_(data_source_ids)) + session.exec(stmt) + logger.info(f"Deleted data sources {', '.join([f'#{did}' for did in data_source_ids])} successfully.") # Delete knowledge base. stmt = delete(KnowledgeBase).where(KnowledgeBase.id == knowledge_base_id) @@ -120,3 +141,62 @@ def purge_knowledge_base_related_resources(knowledge_base_id: int): logger.info(f"Deleted knowledge base #{knowledge_base_id} successfully.") session.commit() + + +@celery_app.task +def purge_kb_datasource_related_resources(kb_id: int, datasource_id: int): + """ + Purge all resources related to the deleted datasource in the knowledge base. + """ + + with Session(engine) as session: + kb = knowledge_base_repo.must_get(session, kb_id, show_soft_deleted=True) + datasource = knowledge_base_repo.must_get_kb_datasource(session, kb, datasource_id, show_soft_deleted=True) + assert datasource.deleted_at is not None + + chunk_model = get_kb_chunk_model(kb) + entity_model = get_kb_entity_model(kb) + relationship_model = get_kb_relationship_model(kb) + + doc_ids_subquery = select(Document.id).where(Document.data_source_id == datasource_id) + chunk_ids_subquery = select(chunk_model.id).where(chunk_model.document_id.in_(doc_ids_subquery)) + + # Delete relationships generated by the chunks of the deleted data source. + stmt = delete(relationship_model).where(relationship_model.chunk_id.in_(chunk_ids_subquery)) + session.exec(stmt) + logger.info(f"Deleted relationships generated by chunks from data source #{datasource_id} successfully.") + + # Delete orphaned entities that are not referenced by any relationships. + orphaned_entity_ids = ( + select(entity_model.id) + .outerjoin( + relationship_model, + (relationship_model.target_entity_id == entity_model.id) | + (relationship_model.source_entity_id == entity_model.id) + ) + .where( + relationship_model.id.is_(None) + ) + .scalar_subquery() + ) + stmt = delete(entity_model).where(entity_model.id.in_(orphaned_entity_ids)) + session.exec(stmt) + logger.info(f"Deleted orphaned entities successfully.") + + # Delete chunks from deleted data source. + stmt = delete(chunk_model).where(chunk_model.document_id.in_(doc_ids_subquery)) + session.exec(stmt) + logger.info(f"Deleted chunks from data source #{datasource_id} successfully.") + + # Delete documents. + stmt = delete(Document).where(Document.data_source_id == datasource_id) + session.exec(stmt) + logger.info(f"Deleted documents from data source #{datasource_id} successfully.") + + # Delete data sources. + session.delete(datasource) + logger.info(f"Deleted data source #{datasource_id} successfully.") + + session.commit() + + stats_for_knowledge_base.delay(kb_id) \ No newline at end of file From 463ada38cf6c55cd0be5e2859f1f8919dac97af5 Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Wed, 27 Nov 2024 13:33:09 +0800 Subject: [PATCH 019/114] feat(frontend): knowledge base datasource management pages (#408) --- frontend/app/src/api/datasources.ts | 73 +++---- .../datasources/[id]/documents/page.tsx | 38 ---- .../(main)/(admin)/datasources/[id]/page.tsx | 21 -- .../app/(main)/(admin)/datasources/page.tsx | 13 -- .../src/app/(main)/(admin)/documents/page.tsx | 14 -- .../[id]/(special)/data-sources/new/page.tsx | 33 ++++ .../[id]/(tabs)/data-sources/page.tsx | 57 ++++++ .../[id]/(tabs)/index-progress/page.tsx | 12 ++ .../create-synopsis-entity/page.tsx | 2 +- .../(tabs)/knowledge-graph-explorer/page.tsx | 11 ++ .../knowledge-bases/[id]/(tabs)/layout.tsx | 59 ++++++ .../[id]/{ => (tabs)}/page.tsx | 0 .../[id]/{ => (tabs)}/settings/page.tsx | 0 .../knowledge-bases/[id]/(tabs)/tabs.tsx | 43 ++++ .../(admin)/knowledge-bases/[id]/context.tsx | 4 - .../[id]/data-sources/page.tsx | 28 --- .../[id]/index-progress/page.tsx | 17 -- .../[id]/knowledge-graph-explorer/page.tsx | 16 -- .../(admin)/knowledge-bases/[id]/layout.tsx | 22 --- .../(admin)/knowledge-bases/[id]/tabs.tsx | 74 ------- .../(main)/(admin)/knowledge-bases/page.tsx | 11 +- .../(main)/(admin)/site-settings/layout.tsx | 2 +- frontend/app/src/app/(main)/nav.tsx | 24 +-- .../app/src/components/cells/reference.tsx | 2 +- .../app/src/components/chat-engine/hooks.ts | 7 + .../src/components/chat/chat-controller.ts | 2 +- .../app/src/components/chat/message-input.tsx | 7 +- .../datasource/DatasourceDeprecationAlert.tsx | 16 -- .../datasource/DatasourceDetails.tsx | 57 ------ .../components/datasource/DatasourceName.tsx | 15 -- .../components/datasource/DatasourceTable.tsx | 46 ----- .../datasource/create-datasource-form.tsx | 187 ++++++++++++++++++ .../components/datasource/datasource-card.tsx | 108 ++++++++++ .../datasource/datasource-create-option.tsx | 36 ++++ .../app/src/components/datasource/hooks.ts | 8 - .../datasource/no-datasource-placeholder.tsx | 9 + .../src/components/datasource/schema.test.ts | 24 --- .../app/src/components/datasource/schema.ts | 9 - .../app/src/components/datasource/types.ts | 6 - .../datasource/update-datasource-form.tsx | 44 +++++ .../documents/DocumentDeprecationAlert.tsx | 16 -- .../documents/documents-table-filters.tsx | 46 ++--- .../components/documents/documents-table.tsx | 44 +++-- .../app/src/components/documents/hooks.ts | 0 .../embedding-models/EmbeddingModelInfo.tsx | 4 +- .../embedding-models/EmbeddingModelsTable.tsx | 7 +- .../src/components/embedding-models/hooks.tsx | 15 +- frontend/app/src/components/form/biz.tsx | 96 ++++++--- .../src/components/form/control-widget.tsx | 7 +- .../src/components/knowledge-base/KBInfo.tsx | 11 -- .../create-knowledge-base-form.tsx | 48 +---- .../knowledge-base/datasource-details.tsx | 54 ----- .../form-create-data-sources.tsx | 139 ------------- .../src/components/knowledge-base/hooks.ts | 46 ++--- .../knowledge-base/knowledge-base-card.tsx | 9 +- .../knowledge-base/knowledge-base-index.tsx | 2 - .../knowledge-base-settings-form.tsx | 2 + frontend/app/src/components/llm/LLMsTable.tsx | 7 +- frontend/app/src/components/llm/LlmInfo.tsx | 4 +- frontend/app/src/components/llm/hooks.ts | 15 +- .../src/components/model-component-info.tsx | 16 +- .../src/components/reranker/RerankerInfo.tsx | 4 +- .../reranker/RerankerModelsTable.tsx | 7 +- frontend/app/src/components/reranker/hooks.ts | 15 +- frontend/app/src/lib/request/index.ts | 1 + .../app/src/lib/request/list-all-helper.ts | 30 +++ 66 files changed, 893 insertions(+), 909 deletions(-) delete mode 100644 frontend/app/src/app/(main)/(admin)/datasources/[id]/documents/page.tsx delete mode 100644 frontend/app/src/app/(main)/(admin)/datasources/[id]/page.tsx delete mode 100644 frontend/app/src/app/(main)/(admin)/datasources/page.tsx delete mode 100644 frontend/app/src/app/(main)/(admin)/documents/page.tsx create mode 100644 frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(special)/data-sources/new/page.tsx create mode 100644 frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/data-sources/page.tsx create mode 100644 frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/index-progress/page.tsx rename frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/{ => (tabs)}/knowledge-graph-explorer/create-synopsis-entity/page.tsx (96%) create mode 100644 frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/knowledge-graph-explorer/page.tsx create mode 100644 frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/layout.tsx rename frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/{ => (tabs)}/page.tsx (100%) rename frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/{ => (tabs)}/settings/page.tsx (100%) create mode 100644 frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/tabs.tsx delete mode 100644 frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/data-sources/page.tsx delete mode 100644 frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/index-progress/page.tsx delete mode 100644 frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/knowledge-graph-explorer/page.tsx delete mode 100644 frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/layout.tsx delete mode 100644 frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/tabs.tsx create mode 100644 frontend/app/src/components/chat-engine/hooks.ts delete mode 100644 frontend/app/src/components/datasource/DatasourceDeprecationAlert.tsx delete mode 100644 frontend/app/src/components/datasource/DatasourceDetails.tsx delete mode 100644 frontend/app/src/components/datasource/DatasourceName.tsx delete mode 100644 frontend/app/src/components/datasource/DatasourceTable.tsx create mode 100644 frontend/app/src/components/datasource/create-datasource-form.tsx create mode 100644 frontend/app/src/components/datasource/datasource-card.tsx create mode 100644 frontend/app/src/components/datasource/datasource-create-option.tsx delete mode 100644 frontend/app/src/components/datasource/hooks.ts create mode 100644 frontend/app/src/components/datasource/no-datasource-placeholder.tsx delete mode 100644 frontend/app/src/components/datasource/schema.test.ts delete mode 100644 frontend/app/src/components/datasource/schema.ts delete mode 100644 frontend/app/src/components/datasource/types.ts create mode 100644 frontend/app/src/components/datasource/update-datasource-form.tsx delete mode 100644 frontend/app/src/components/documents/DocumentDeprecationAlert.tsx delete mode 100644 frontend/app/src/components/documents/hooks.ts delete mode 100644 frontend/app/src/components/knowledge-base/KBInfo.tsx delete mode 100644 frontend/app/src/components/knowledge-base/datasource-details.tsx delete mode 100644 frontend/app/src/components/knowledge-base/form-create-data-sources.tsx create mode 100644 frontend/app/src/lib/request/list-all-helper.ts diff --git a/frontend/app/src/api/datasources.ts b/frontend/app/src/api/datasources.ts index b039d6631..d2ac5750b 100644 --- a/frontend/app/src/api/datasources.ts +++ b/frontend/app/src/api/datasources.ts @@ -8,16 +8,6 @@ export interface DatasourceBase { name: string; } -interface DeprecatedDatasourceBase extends DatasourceBase { - created_at: Date; - updated_at: Date; - user_id: string | null; - build_kg_index: boolean; - llm_id: number | null; -} - -export type DeprecatedDatasource = DeprecatedDatasourceBase & DatasourceSpec - type DatasourceSpec = ({ data_source_type: 'file' config: { file_id: number, file_name: string }[] @@ -43,18 +33,6 @@ export interface BaseCreateDatasourceParams { name: string; } -export interface DeprecatedBaseCreateDatasourceParams extends BaseCreateDatasourceParams { - description: string; - /** - * @deprecated - */ - build_kg_index: boolean; - /** - * @deprecated - */ - llm_id: number | null; -} - export type CreateDatasourceSpecParams = ({ data_source_type: 'file' config: { file_id: number, file_name: string }[] @@ -66,7 +44,7 @@ export type CreateDatasourceSpecParams = ({ config: { url: string } }); -export type CreateDatasourceParams = DeprecatedBaseCreateDatasourceParams & CreateDatasourceSpecParams; +export type CreateDatasourceParams = BaseCreateDatasourceParams & CreateDatasourceSpecParams; export interface Upload { created_at?: Date; @@ -92,16 +70,6 @@ export type DatasourceKgIndexError = { error: string | null } -const deprecatedBaseDatasourceSchema = z.object({ - id: z.number(), - name: z.string(), - created_at: zodJsonDate(), - updated_at: zodJsonDate(), - user_id: z.string().nullable(), - build_kg_index: z.boolean(), - llm_id: z.number().nullable(), -}); - const datasourceSpecSchema = z.discriminatedUnion('data_source_type', [ z.object({ data_source_type: z.literal('file'), @@ -123,9 +91,6 @@ const datasourceSpecSchema = z.discriminatedUnion('data_source_type', [ })], ) satisfies ZodType; -export const deprecatedDatasourceSchema = deprecatedBaseDatasourceSchema - .and(datasourceSpecSchema) satisfies ZodType; - export const datasourceSchema = z.object({ id: z.number(), name: z.string(), @@ -141,8 +106,7 @@ const uploadSchema = z.object({ created_at: zodJsonDate().optional(), updated_at: zodJsonDate().optional(), }) satisfies ZodType; - -const datasourceOverviewSchema = z.object({ +z.object({ vector_index: indexSchema, documents: totalSchema, chunks: totalSchema, @@ -150,34 +114,45 @@ const datasourceOverviewSchema = z.object({ relationships: totalSchema.optional(), }) satisfies ZodType; -export async function listDataSources ({ page = 1, size = 10 }: PageParams = {}): Promise> { - return fetch(requestUrl('/api/v1/admin/datasources', { page, size }), { +export async function listDataSources (kbId: number, { page = 1, size = 10 }: PageParams = {}): Promise> { + return fetch(requestUrl(`/api/v1/admin/knowledge_bases/${kbId}/datasources`, { page, size }), { headers: await authenticationHeaders(), - }).then(handleResponse(zodPage(deprecatedDatasourceSchema))); + }).then(handleResponse(zodPage(datasourceSchema))); } -export async function getDatasource (id: number): Promise { - return fetch(requestUrl(`/api/v1/admin/datasources/${id}`), { +export async function getDatasource (kbId: number, id: number): Promise { + return fetch(requestUrl(`/api/v1/admin/knowledge_bases/${kbId}/datasources/${id}`), { headers: await authenticationHeaders(), - }).then(handleResponse(deprecatedDatasourceSchema)); + }).then(handleResponse(datasourceSchema)); } -export async function deleteDatasource (id: number): Promise { - await fetch(requestUrl(`/api/v1/admin/datasources/${id}`), { +export async function deleteDatasource (kbId: number, id: number): Promise { + await fetch(requestUrl(`/api/v1/admin/knowledge_bases/${kbId}/datasources/${id}`), { method: 'DELETE', headers: await authenticationHeaders(), }).then(handleErrors); } -export async function createDatasource (params: CreateDatasourceParams) { - return fetch(requestUrl(`/api/v1/admin/datasources`), { +export async function createDatasource (kbId: number, params: CreateDatasourceParams) { + return fetch(requestUrl(`/api/v1/admin/knowledge_bases/${kbId}/datasources`), { method: 'POST', headers: { ...await authenticationHeaders(), 'Content-Type': 'application/json', }, body: JSON.stringify(params), - }).then(handleResponse(deprecatedDatasourceSchema)); + }).then(handleResponse(datasourceSchema)); +} + +export async function updateDatasource (kbId: number, id: number, params: { name: string }) { + return fetch(requestUrl(`/api/v1/admin/knowledge_bases/${kbId}/datasources/${id}`), { + method: 'PUT', + headers: { + ...await authenticationHeaders(), + 'Content-Type': 'application/json', + }, + body: JSON.stringify(params), + }).then(handleResponse(datasourceSchema)); } export async function uploadFiles (files: File[]) { diff --git a/frontend/app/src/app/(main)/(admin)/datasources/[id]/documents/page.tsx b/frontend/app/src/app/(main)/(admin)/datasources/[id]/documents/page.tsx deleted file mode 100644 index 685e69e7b..000000000 --- a/frontend/app/src/app/(main)/(admin)/datasources/[id]/documents/page.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import { getDatasource } from '@/api/datasources'; -import { AdminPageHeading } from '@/components/admin-page-heading'; -import { DatasourceDeprecationAlert } from '@/components/datasource/DatasourceDeprecationAlert'; -import { DocumentsTable } from '@/components/documents/documents-table'; -import { isServerError } from '@/lib/request'; -import { notFound } from 'next/navigation'; -import { cache } from 'react'; - -export default async function ChatEnginesPage ({ params }: { params: { id: string } }) { - const datasource = await cachedGetDatasource(parseInt(params.id)); - - return ( - <> - - - - - ); -} - -const cachedGetDatasource = cache(async (id: number) => { - try { - return await getDatasource(id); - } catch (error) { - if (isServerError(error, [404])) { - notFound(); - } else { - return Promise.reject(error); - } - } -}); - diff --git a/frontend/app/src/app/(main)/(admin)/datasources/[id]/page.tsx b/frontend/app/src/app/(main)/(admin)/datasources/[id]/page.tsx deleted file mode 100644 index 1714c4aaa..000000000 --- a/frontend/app/src/app/(main)/(admin)/datasources/[id]/page.tsx +++ /dev/null @@ -1,21 +0,0 @@ -import { AdminPageHeading } from '@/components/admin-page-heading'; -import { DatasourceDeprecationAlert } from '@/components/datasource/DatasourceDeprecationAlert'; -import { DatasourceDetails } from '@/components/datasource/DatasourceDetails'; -import { DatasourceName } from '@/components/datasource/DatasourceName'; - -export default function DatasourcePage ({ params }: { params: { id: string } }) { - const id = parseInt(params.id); - - return ( -
- , url: `/datasources/${id}` }, - ]} - /> - - -
- ); -} \ No newline at end of file diff --git a/frontend/app/src/app/(main)/(admin)/datasources/page.tsx b/frontend/app/src/app/(main)/(admin)/datasources/page.tsx deleted file mode 100644 index 2b5083629..000000000 --- a/frontend/app/src/app/(main)/(admin)/datasources/page.tsx +++ /dev/null @@ -1,13 +0,0 @@ -import { AdminPageHeading } from '@/components/admin-page-heading'; -import { DatasourceDeprecationAlert } from '@/components/datasource/DatasourceDeprecationAlert'; -import { DatasourceTable } from '@/components/datasource/DatasourceTable'; - -export default function ChatEnginesPage () { - return ( - <> - - - - - ); -} diff --git a/frontend/app/src/app/(main)/(admin)/documents/page.tsx b/frontend/app/src/app/(main)/(admin)/documents/page.tsx deleted file mode 100644 index e15427939..000000000 --- a/frontend/app/src/app/(main)/(admin)/documents/page.tsx +++ /dev/null @@ -1,14 +0,0 @@ -import { AdminPageHeading } from '@/components/admin-page-heading'; -import { DocumentDeprecationAlert } from '@/components/documents/DocumentDeprecationAlert'; -import { DocumentsTable } from '@/components/documents/documents-table'; - -export default function DocumentsPage () { - return ( - <> - - - - - ); -} - diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(special)/data-sources/new/page.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(special)/data-sources/new/page.tsx new file mode 100644 index 000000000..cec0558d9 --- /dev/null +++ b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(special)/data-sources/new/page.tsx @@ -0,0 +1,33 @@ +'use client'; + +import { AdminPageHeading } from '@/components/admin-page-heading'; +import { CreateDatasourceForm } from '@/components/datasource/create-datasource-form'; +import { mutateKnowledgeBases, useKnowledgeBase } from '@/components/knowledge-base/hooks'; +import { Loader2Icon } from 'lucide-react'; +import { useRouter } from 'next/navigation'; + +export default function NewKnowledgeBaseDataSourcePage ({ params }: { params: { id: string } }) { + const id = parseInt(decodeURIComponent(params.id)); + const { knowledgeBase } = useKnowledgeBase(id); + const router = useRouter(); + + return ( + <> + , url: `/knowledge-bases/${id}` }, + { title: 'DataSources', url: `/knowledge-bases/${id}/data-sources` }, + { title: 'New' }, + ]} + /> + { + router.back(); + mutateKnowledgeBases(); + }} + /> + + ); +} diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/data-sources/page.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/data-sources/page.tsx new file mode 100644 index 000000000..20d3efa1b --- /dev/null +++ b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/data-sources/page.tsx @@ -0,0 +1,57 @@ +'use client'; + +import { DatasourceCard } from '@/components/datasource/datasource-card'; +import { DatasourceCreateOption } from '@/components/datasource/datasource-create-option'; +import { NoDatasourcePlaceholder } from '@/components/datasource/no-datasource-placeholder'; +import { useAllKnowledgeBaseDataSources } from '@/components/knowledge-base/hooks'; +import { Skeleton } from '@/components/ui/skeleton'; +import { FileDownIcon, GlobeIcon, PaperclipIcon } from 'lucide-react'; + +export default function KnowledgeBaseDataSourcesPage ({ params }: { params: { id: string } }) { + const id = parseInt(decodeURIComponent(params.id)); + const { data: dataSources, isLoading } = useAllKnowledgeBaseDataSources(id); + + return ( +
+
+

Create Data Source

+
+ } + title="Files" + > + Upload files + + } + title="Web Pages" + > + Select pages. + + } + title="Website by sitemap" + > + Select web sitemap. + +
+
+
+

Browse existing Data Sources

+ {isLoading && } + {dataSources?.map(datasource => ( + + ))} + {dataSources?.length === 0 && ( + + )} +
+
+ ); +} diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/index-progress/page.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/index-progress/page.tsx new file mode 100644 index 000000000..af956e9b8 --- /dev/null +++ b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/index-progress/page.tsx @@ -0,0 +1,12 @@ +import { KnowledgeBaseIndexProgress } from '@/components/knowledge-base/knowledge-base-index'; + +export default function KnowledgeBaseIndexProgressPage ({ params }: { params: { id: string } }) { + const id = parseInt(decodeURIComponent(params.id)); + + return ( +
+

Index Progress

+ +
+ ); +} \ No newline at end of file diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/knowledge-graph-explorer/create-synopsis-entity/page.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/knowledge-graph-explorer/create-synopsis-entity/page.tsx similarity index 96% rename from frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/knowledge-graph-explorer/create-synopsis-entity/page.tsx rename to frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/knowledge-graph-explorer/create-synopsis-entity/page.tsx index fb6149b4d..9923bc5c4 100644 --- a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/knowledge-graph-explorer/create-synopsis-entity/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/knowledge-graph-explorer/create-synopsis-entity/page.tsx @@ -15,7 +15,7 @@ export default function CreateSynopsisEntityPage ({ params }: { params: { id: st return ( <> - + Back diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/knowledge-graph-explorer/page.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/knowledge-graph-explorer/page.tsx new file mode 100644 index 000000000..ab8f4e7f7 --- /dev/null +++ b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/knowledge-graph-explorer/page.tsx @@ -0,0 +1,11 @@ +import { GraphEditor } from '@/components/graph/GraphEditor'; + +export default function KnowledgeGraphExplorerPage ({ params }: { params: { id: string } }) { + const id = parseInt(decodeURIComponent(params.id)); + + return ( +
+ +
+ ); +} \ No newline at end of file diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/layout.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/layout.tsx new file mode 100644 index 000000000..a7c529d6a --- /dev/null +++ b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/layout.tsx @@ -0,0 +1,59 @@ +'use client'; + +import { KnowledgeBaseTabs } from '@/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/tabs'; +import { AdminPageHeading } from '@/components/admin-page-heading'; +import { ArrowRightIcon } from '@/components/icons'; +import { useKnowledgeBase } from '@/components/knowledge-base/hooks'; +import { SecondaryNavigatorLayout, SecondaryNavigatorList, SecondaryNavigatorMain } from '@/components/secondary-navigator-list'; +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip'; +import { AlertTriangleIcon, Loader2Icon } from 'lucide-react'; +import Link from 'next/link'; +import type { ReactNode } from 'react'; + +export default function KnowledgeBaseLayout ({ params, children }: { params: { id: string }, children: ReactNode }) { + const id = parseInt(decodeURIComponent(params.id)); + const { knowledgeBase } = useKnowledgeBase(id); + + return ( + <> + + {knowledgeBase?.data_sources_total === 0 && ( + + + + + + +

This Knowledge Base has no datasource.

+ + Create Data Source + + +
+
+
+ )} + + {knowledgeBase?.name ?? } + + + ), + }, + ]} + /> + + + + + + {children} + + + + ); +} \ No newline at end of file diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/page.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/page.tsx similarity index 100% rename from frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/page.tsx rename to frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/page.tsx diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/settings/page.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/settings/page.tsx similarity index 100% rename from frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/settings/page.tsx rename to frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/settings/page.tsx diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/tabs.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/tabs.tsx new file mode 100644 index 000000000..75ab7952e --- /dev/null +++ b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/tabs.tsx @@ -0,0 +1,43 @@ +'use client'; + +import { useKnowledgeBase } from '@/components/knowledge-base/hooks'; +import { SecondaryNavigatorLink } from '@/components/secondary-navigator-list'; + +export function KnowledgeBaseTabs ({ knowledgeBaseId }: { knowledgeBaseId: number }) { + const { knowledgeBase } = useKnowledgeBase(knowledgeBaseId); + + return ( + <> + + Documents + + {knowledgeBase?.documents_total} + + + + Data Sources + + {knowledgeBase?.data_sources_total} + + + + Index Progress + + {/* startTransition(() => {*/} + {/* router.push(`/knowledge-bases/${knowledgeBase.id}/retrieval-tester`);*/} + {/* })}*/} + {/*>*/} + {/* Retrieval Tester*/} + {/**/} + + Graph Explorer + + + Settings + + + ); +} \ No newline at end of file diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/context.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/context.tsx index f8a331e6f..6658940c9 100644 --- a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/context.tsx +++ b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/context.tsx @@ -12,7 +12,3 @@ export function KBProvider ({ children, value }: { children: ReactNode, value: K ); } - -export function useKB () { - return useContext(KBContext); -} diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/data-sources/page.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/data-sources/page.tsx deleted file mode 100644 index b2c81b505..000000000 --- a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/data-sources/page.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import { cachedGetKnowledgeBaseById } from '@/app/(main)/(admin)/knowledge-bases/[id]/api'; -import { KBProvider } from '@/app/(main)/(admin)/knowledge-bases/[id]/context'; -import { KnowledgeBaseDatasourceDetails } from '@/components/knowledge-base/datasource-details'; - -export default async function KnowledgeBaseDataSourcesPage ({ params }: { params: { id: string } }) { - const id = parseInt(decodeURIComponent(params.id)); - const kb = await cachedGetKnowledgeBaseById(id); - - return ( - -
-

Data Sources

-
- {kb.data_sources.map(datasource => ( -
-

- {datasource.name} -

-
- -
-
- ))} -
-
-
- ); -} \ No newline at end of file diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/index-progress/page.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/index-progress/page.tsx deleted file mode 100644 index c2de24db9..000000000 --- a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/index-progress/page.tsx +++ /dev/null @@ -1,17 +0,0 @@ -import { cachedGetKnowledgeBaseById } from '@/app/(main)/(admin)/knowledge-bases/[id]/api'; -import { KBProvider } from '@/app/(main)/(admin)/knowledge-bases/[id]/context'; -import { KnowledgeBaseIndexProgress } from '@/components/knowledge-base/knowledge-base-index'; - -export default async function KnowledgeBaseIndexProgressPage ({ params }: { params: { id: string } }) { - const id = parseInt(decodeURIComponent(params.id)); - const kb = await cachedGetKnowledgeBaseById(id); - - return ( - -
-

Index Progress

- -
-
- ); -} \ No newline at end of file diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/knowledge-graph-explorer/page.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/knowledge-graph-explorer/page.tsx deleted file mode 100644 index 4c3b124a8..000000000 --- a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/knowledge-graph-explorer/page.tsx +++ /dev/null @@ -1,16 +0,0 @@ -import { cachedGetKnowledgeBaseById } from '@/app/(main)/(admin)/knowledge-bases/[id]/api'; -import { KBProvider } from '@/app/(main)/(admin)/knowledge-bases/[id]/context'; -import { GraphEditor } from '@/components/graph/GraphEditor'; - -export default async function KnowledgeGraphExplorerPage ({ params }: { params: { id: string } }) { - const id = parseInt(decodeURIComponent(params.id)); - const kb = await cachedGetKnowledgeBaseById(id); - - return ( - -
- -
-
- ); -} \ No newline at end of file diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/layout.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/layout.tsx deleted file mode 100644 index 827d6c653..000000000 --- a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/layout.tsx +++ /dev/null @@ -1,22 +0,0 @@ -import { cachedGetKnowledgeBaseById } from '@/app/(main)/(admin)/knowledge-bases/[id]/api'; -import { KnowledgeBaseTabs } from '@/app/(main)/(admin)/knowledge-bases/[id]/tabs'; -import { AdminPageHeading } from '@/components/admin-page-heading'; -import type { ReactNode } from 'react'; - -export default async function KnowledgeBaseLayout ({ params, children }: { params: { id: string }, children: ReactNode }) { - const id = parseInt(decodeURIComponent(params.id)); - const kb = await cachedGetKnowledgeBaseById(id); - - return ( - <> - - - {children} - - ); -} \ No newline at end of file diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/tabs.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/tabs.tsx deleted file mode 100644 index c0cb23ee7..000000000 --- a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/tabs.tsx +++ /dev/null @@ -1,74 +0,0 @@ -'use client'; - -import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs'; -import { useRouter, useSelectedLayoutSegments } from 'next/navigation'; -import { useTransition } from 'react'; - -export function KnowledgeBaseTabs ({ id }: { id: number }) { - const router = useRouter(); - const [transitioning, startTransition] = useTransition(); - const segments = useSelectedLayoutSegments(); - - const segment = segments?.[0] ?? ''; - - return ( - - - startTransition(() => { - router.push(`/knowledge-bases/${id}`); - })} - > - Documents - - startTransition(() => { - router.push(`/knowledge-bases/${id}/data-sources`); - })} - > - Data Sources - - startTransition(() => { - router.push(`/knowledge-bases/${id}/index-progress`); - })} - > - Index Progress - - startTransition(() => { - router.push(`/knowledge-bases/${id}/retrieval-tester`); - })} - > - Retrieval Tester - - startTransition(() => { - router.push(`/knowledge-bases/${id}/knowledge-graph-explorer`); - })} - > - Knowledge Graph Explorer - - startTransition(() => { - router.push(`/knowledge-bases/${id}/settings`); - })} - > - Settings - - - - ); -} \ No newline at end of file diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/page.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/page.tsx index 47476ca11..b27f22757 100644 --- a/frontend/app/src/app/(main)/(admin)/knowledge-bases/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/knowledge-bases/page.tsx @@ -2,15 +2,12 @@ import { AdminPageHeading } from '@/components/admin-page-heading'; import KnowledgeBaseEmptyState from '@/components/knowledge-base/empty-state'; -import { useKnowledgeBases } from '@/components/knowledge-base/hooks'; +import { useAllKnowledgeBases } from '@/components/knowledge-base/hooks'; import { KnowledgeBaseCard, KnowledgeBaseCardPlaceholder } from '@/components/knowledge-base/knowledge-base-card'; import { NextLink } from '@/components/nextjs/NextLink'; -import type { PaginationState } from '@tanstack/table-core'; -import { useState } from 'react'; export default function KnowledgeBasesPage () { - const [pagination, setPagination] = useState({ pageIndex: 0, pageSize: 10 }); - const { knowledgeBases, isLoading } = useKnowledgeBases(pagination.pageIndex, pagination.pageSize); + const { data: knowledgeBases, isLoading } = useAllKnowledgeBases(); return ( <> @@ -25,9 +22,9 @@ export default function KnowledgeBasesPage () { { isLoading ?
- : !!knowledgeBases?.items.length + : !!knowledgeBases?.length ?
- {knowledgeBases?.items.map(kb => ( + {knowledgeBases.map(kb => ( ))}
diff --git a/frontend/app/src/app/(main)/(admin)/site-settings/layout.tsx b/frontend/app/src/app/(main)/(admin)/site-settings/layout.tsx index 7876fa0f6..512ebebc0 100644 --- a/frontend/app/src/app/(main)/(admin)/site-settings/layout.tsx +++ b/frontend/app/src/app/(main)/(admin)/site-settings/layout.tsx @@ -24,7 +24,7 @@ export default function SiteSettingsLayout ({ children }: { children: ReactNode JS Widget
- + {children}
diff --git a/frontend/app/src/app/(main)/nav.tsx b/frontend/app/src/app/(main)/nav.tsx index f98e44e85..c9974ad3d 100644 --- a/frontend/app/src/app/(main)/nav.tsx +++ b/frontend/app/src/app/(main)/nav.tsx @@ -1,13 +1,13 @@ 'use client'; import { logout } from '@/api/auth'; -import { listChatEngines } from '@/api/chat-engines'; import type { PublicWebsiteSettings } from '@/api/site-settings'; import { useAuth } from '@/components/auth/AuthProvider'; import { Branding } from '@/components/branding'; +import { useAllChatEngines } from '@/components/chat-engine/hooks'; import { ChatNewDialog } from '@/components/chat/chat-new-dialog'; import { ChatsHistory } from '@/components/chat/chats-history'; -import { useKnowledgeBases } from '@/components/knowledge-base/hooks'; +import { useAllKnowledgeBases } from '@/components/knowledge-base/hooks'; import { type NavGroup, SiteNav } from '@/components/site-nav'; import { useBootstrapStatus } from '@/components/system/BootstrapStatusProvider'; import { Avatar, AvatarFallback } from '@/components/ui/avatar'; @@ -17,12 +17,11 @@ import { Sidebar, SidebarContent, SidebarFooter, SidebarHeader } from '@/compone import { Skeleton } from '@/components/ui/skeleton'; import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from '@/components/ui/tooltip'; import { useHref } from '@/components/use-href'; -import { ActivitySquareIcon, AlertTriangleIcon, BinaryIcon, BotMessageSquareIcon, BrainCircuitIcon, CogIcon, ComponentIcon, FilesIcon, HomeIcon, KeyRoundIcon, LibraryBigIcon, LibraryIcon, LogInIcon, MessageCircleQuestionIcon, MessagesSquareIcon, ShuffleIcon } from 'lucide-react'; +import { ActivitySquareIcon, AlertTriangleIcon, BinaryIcon, BotMessageSquareIcon, BrainCircuitIcon, CogIcon, ComponentIcon, HomeIcon, KeyRoundIcon, LibraryBigIcon, LogInIcon, MessageCircleQuestionIcon, MessagesSquareIcon, ShuffleIcon } from 'lucide-react'; import NextLink from 'next/link'; import Link from 'next/link'; import { useRouter } from 'next/navigation'; import type { ReactNode } from 'react'; -import useSWR from 'swr'; export function SiteSidebar ({ setting }: { setting: PublicWebsiteSettings }) { return ( @@ -84,15 +83,6 @@ function NavContent () { ], sectionProps: { className: 'mt-auto mb-0' }, }); - - groups.push({ - title: 'Legacy', - items: [ - { href: '/documents', title: 'Documents', icon: FilesIcon }, - { href: '/datasources', title: 'Datasources', icon: LibraryIcon, details: !required.datasource && You need to configure at least one Datasource. }, - ], - sectionProps: { className: 'mt-auto mb-0' }, - }); } if (user?.is_superuser) { @@ -176,21 +166,21 @@ function CountSpan ({ children }: { children?: ReactNode }) { } function KnowledgeBaseNavDetails () { - const { knowledgeBases, isLoading } = useKnowledgeBases(0, 10); + const { data: knowledgeBases, isLoading } = useAllKnowledgeBases(); if (isLoading) { return ; } - return {knowledgeBases?.total}; + return {knowledgeBases?.length}; } function ChatEnginesNavDetails () { - const { data, isLoading } = useSWR('api.chat-engines.list-all', () => listChatEngines({ page: 1, size: 100 })); + const { data, isLoading } = useAllChatEngines(); if (isLoading) { return ; } - return {data?.total}; + return {data?.length}; } diff --git a/frontend/app/src/components/cells/reference.tsx b/frontend/app/src/components/cells/reference.tsx index d3c2494c6..2fca0e8da 100644 --- a/frontend/app/src/components/cells/reference.tsx +++ b/frontend/app/src/components/cells/reference.tsx @@ -3,7 +3,7 @@ import Link from 'next/link'; export function DatasourceCell ({ id, name }: { id: number, name: string }) { - return {name}; + return {name}; } export function KnowledgeBaseCell ({ id, name }: { id?: number, name?: string }) { diff --git a/frontend/app/src/components/chat-engine/hooks.ts b/frontend/app/src/components/chat-engine/hooks.ts new file mode 100644 index 000000000..574087529 --- /dev/null +++ b/frontend/app/src/components/chat-engine/hooks.ts @@ -0,0 +1,7 @@ +import { listChatEngines } from '@/api/chat-engines'; +import { listAllHelper } from '@/lib/request'; +import useSWR from 'swr'; + +export function useAllChatEngines () { + return useSWR('api.chat-engines.list-all', () => listAllHelper(listChatEngines, 'id')); +} \ No newline at end of file diff --git a/frontend/app/src/components/chat/chat-controller.ts b/frontend/app/src/components/chat/chat-controller.ts index 5f9947b23..710d0672e 100644 --- a/frontend/app/src/components/chat/chat-controller.ts +++ b/frontend/app/src/components/chat/chat-controller.ts @@ -300,7 +300,7 @@ export class ChatController listChatEngines({ page: 1, size: 100 })); + const { data, isLoading } = useAllChatEngines(); return (
@@ -66,11 +67,11 @@ export function MessageInput ({ - {data?.items.map(item => ( + {data?.map(item => ( {item.is_default ? default : item.name} - {item.engine_options.external_engine_config + {!!item.engine_options.external_engine_config?.stream_chat_api_url ? External Engine (StackVM) : item.engine_options.knowledge_graph?.enabled !== false /* TODO: require default config */ ? Knowledge graph enabled diff --git a/frontend/app/src/components/datasource/DatasourceDeprecationAlert.tsx b/frontend/app/src/components/datasource/DatasourceDeprecationAlert.tsx deleted file mode 100644 index 0e287f330..000000000 --- a/frontend/app/src/components/datasource/DatasourceDeprecationAlert.tsx +++ /dev/null @@ -1,16 +0,0 @@ -import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert'; -import { AlertCircleIcon } from 'lucide-react'; - -export function DatasourceDeprecationAlert () { - return ( - - - - Bare Datasource management was deprecated. - - - TiDB.ai now uses Knowledge Base to manage multiple datasources. This page will be removed soon. - - - ); -} \ No newline at end of file diff --git a/frontend/app/src/components/datasource/DatasourceDetails.tsx b/frontend/app/src/components/datasource/DatasourceDetails.tsx deleted file mode 100644 index d8c2b6190..000000000 --- a/frontend/app/src/components/datasource/DatasourceDetails.tsx +++ /dev/null @@ -1,57 +0,0 @@ -'use client'; - -import { useDatasource } from '@/components/datasource/hooks'; -import { DateFormat } from '@/components/date-format'; -import { OptionDetail } from '@/components/option-detail'; -import { Badge } from '@/components/ui/badge'; - -export function DatasourceDetails ({ id }: { id: number }) { - return ( - <> - - - - ); -} - -function DatasourceFields ({ id }: { id: number }) { - const { datasource } = useDatasource(id); - return ( -
- - - } /> - } /> - {(datasource?.data_source_type === 'web_sitemap') && ( - - )} - {(datasource?.data_source_type === 'web_single_page') && ( - {datasource?.config.urls.map(url =>
  • {url}
  • )}} /> - )} - -
    - ); -} - -function DatasourceUploadFiles ({ id }: { id: number }) { - const { datasource } = useDatasource(id); - - if (datasource?.data_source_type !== 'file' || datasource.config.length === 0) { - return null; - } - return ( -
    -

    Files

    -
    - {datasource.config.map(file => ( - - - {file.file_name} - - #{file.file_id} - - ))} -
    -
    - ); -} diff --git a/frontend/app/src/components/datasource/DatasourceName.tsx b/frontend/app/src/components/datasource/DatasourceName.tsx deleted file mode 100644 index 6e6684c14..000000000 --- a/frontend/app/src/components/datasource/DatasourceName.tsx +++ /dev/null @@ -1,15 +0,0 @@ -'use client'; - -import { getDatasource } from '@/api/datasources'; -import { cn } from '@/lib/utils'; -import { Loader2Icon } from 'lucide-react'; -import useSWR from 'swr'; - -export function DatasourceName ({ id }: { id: number }) { - const { data: datasource, isLoading, isValidating } = useSWR(`api.datasources.${id}`, () => getDatasource(id)); - - if (isLoading) { - return ; - } - return {datasource?.name ?? '(Unknown Datasource)'}; -} diff --git a/frontend/app/src/components/datasource/DatasourceTable.tsx b/frontend/app/src/components/datasource/DatasourceTable.tsx deleted file mode 100644 index f84939f1f..000000000 --- a/frontend/app/src/components/datasource/DatasourceTable.tsx +++ /dev/null @@ -1,46 +0,0 @@ -'use client'; - -import { deleteDatasource, type DeprecatedDatasource, listDataSources } from '@/api/datasources'; -import { actions } from '@/components/cells/actions'; -import { link } from '@/components/cells/link'; -import { DataTableRemote } from '@/components/data-table-remote'; -import { LlmInfo } from '@/components/llm/LlmInfo'; -import type { ColumnDef } from '@tanstack/react-table'; -import { createColumnHelper } from '@tanstack/table-core'; -import { Trash2Icon } from 'lucide-react'; - -const helper = createColumnHelper(); - -const columns = [ - helper.accessor('name', { cell: link({ url: datasource => `/datasources/${datasource.id}` }) }), - helper.accessor('data_source_type', {}), - helper.accessor('llm_id', { cell: (ctx) => }), - helper.accessor('build_kg_index', {}), - helper.accessor('user_id', {}), - helper.display({ - header: 'Actions', - cell: actions(datasource => [ - { - key: 'delete', - title: 'Delete', - icon: , - dangerous: {}, - action: async ({ table }) => { - await deleteDatasource(datasource.id); - table.reload?.(); - }, - }, - ]), - }), -] as ColumnDef[]; - -export function DatasourceTable () { - return ( - - ); -} diff --git a/frontend/app/src/components/datasource/create-datasource-form.tsx b/frontend/app/src/components/datasource/create-datasource-form.tsx new file mode 100644 index 000000000..2caabb954 --- /dev/null +++ b/frontend/app/src/components/datasource/create-datasource-form.tsx @@ -0,0 +1,187 @@ +import { type BaseCreateDatasourceParams, createDatasource, type CreateDatasourceSpecParams, uploadFiles } from '@/api/datasources'; +import { FormInput } from '@/components/form/control-widget'; +import { FormFieldBasicLayout, FormPrimitiveArrayFieldBasicLayout } from '@/components/form/field-layout'; +import { handleSubmitHelper } from '@/components/form/utils'; +import { FilesInput } from '@/components/form/widgets/FilesInput'; +import { Button } from '@/components/ui/button'; +import { Form, FormField, FormItem, FormLabel } from '@/components/ui/form'; +import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group'; +import { zodFile } from '@/lib/zod'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { FileDownIcon, GlobeIcon, PaperclipIcon } from 'lucide-react'; +import { useSearchParams } from 'next/navigation'; +import { useForm, useFormContext } from 'react-hook-form'; +import { z } from 'zod'; + +const types = ['file', 'web_single_page', 'web_sitemap'] as const; + +const isType = (value: string | null): value is typeof types[number] => types.includes(value as any); + +export function CreateDatasourceForm ({ knowledgeBaseId, onCreated }: { knowledgeBaseId: number, onCreated?: () => void }) { + const usp = useSearchParams()!; + const uType = usp.get('type'); + + const form = useForm({ + resolver: zodResolver(createDatasourceSchema), + defaultValues: switchDatasource({ + data_source_type: 'file', + name: '', + files: [], + }, isType(uType) ? uType : 'file'), + }); + + const handleSubmit = handleSubmitHelper(form, async (ds) => { + const createParams = await preCreate(ds); + await createDatasource(knowledgeBaseId, createParams); + onCreated?.(); + }); + + return ( +
    + + ( + + + Data Source Type + + { + form.reset(switchDatasource(form.getValues(), value as never)); + })} + onBlur={field.onBlur} + > + + + File + + + + Web Single Page + + + + Web Sitemap + + + + )} + /> + + + + + + + + ); +} + +function SpecFields () { + const { watch: useWatch } = useFormContext(); + const type = useWatch('data_source_type'); + + return ( + <> + {type === 'file' && ( + + + + )} + {type === 'web_single_page' && ( + ''}> + + + )} + {type === 'web_sitemap' && ( + + + + )} + + ); +} + +type CreateDatasourceFormParams = z.infer; + +export const createDatasourceSchema = z.object({ + name: z.string().trim().min(1, 'Must not blank'), +}).and(z.discriminatedUnion('data_source_type', [ + z.object({ + data_source_type: z.literal('file'), + files: zodFile().array().min(1), + }), + z.object({ + data_source_type: z.literal('web_single_page'), + urls: z.string().url().array().min(1), + }), + z.object({ + data_source_type: z.literal('web_sitemap'), + url: z.string().url(), + }), +])); + +function switchDatasource (data: CreateDatasourceFormParams, type: CreateDatasourceSpecParams['data_source_type']): CreateDatasourceFormParams { + if (data.data_source_type === type) { + return data; + } + + switch (type) { + case 'file': + return { + name: data.name, + data_source_type: 'file', + files: [], + }; + case 'web_single_page': + return { + name: data.name, + data_source_type: 'web_single_page', + urls: [], + }; + case 'web_sitemap': + return { + name: data.name, + data_source_type: 'web_sitemap', + url: '', + }; + } +} + +async function preCreate (ds: CreateDatasourceFormParams): Promise { + switch (ds.data_source_type) { + case 'file': { + const { files, ...rest } = ds; + const uploadedFiles = await uploadFiles(ds.files); + return { + ...rest, + config: uploadedFiles.map(f => ({ + file_id: f.id, + file_name: f.name, + })), + }; + } + case 'web_single_page': { + const { urls, ...rest } = ds; + + return { + ...rest, + config: { urls }, + }; + } + + case 'web_sitemap': + const { url, ...rest } = ds; + + return { + ...rest, + config: { url }, + }; + } +} diff --git a/frontend/app/src/components/datasource/datasource-card.tsx b/frontend/app/src/components/datasource/datasource-card.tsx new file mode 100644 index 000000000..fe46d1a37 --- /dev/null +++ b/frontend/app/src/components/datasource/datasource-card.tsx @@ -0,0 +1,108 @@ +'use client'; + +import { type Datasource, deleteDatasource } from '@/api/datasources'; +import { DangerousActionButton } from '@/components/dangerous-action-button'; +import { UpdateDatasourceForm } from '@/components/datasource/update-datasource-form'; +import { ManagedDialog } from '@/components/managed-dialog'; +import { ManagedPanelContext } from '@/components/managed-panel'; +import { Button } from '@/components/ui/button'; +import { Card, CardDescription, CardFooter, CardHeader, CardTitle } from '@/components/ui/card'; +import { DialogContent, DialogDescription, DialogHeader, DialogTitle, DialogTrigger } from '@/components/ui/dialog'; +import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover'; +import { FileDownIcon, GlobeIcon, PaperclipIcon } from 'lucide-react'; +import { useRouter } from 'next/navigation'; + +export function DatasourceCard ({ knowledgeBaseId, datasource }: { knowledgeBaseId: number, datasource: Datasource }) { + const router = useRouter(); + + return ( + + + {datasource.name} + + + + + + + + + + + + Configure Datasource + + + + {({ setOpen }) => ( + { + router.refresh(); + setOpen(false); + }} + /> + )} + + + + { + await deleteDatasource(knowledgeBaseId, datasource.id); + }} + asChild + > + + + + + ); +} + +function DatasourceCardDetails ({ datasource }: { datasource: Datasource }) { + return ( + + {(() => { + switch (datasource.data_source_type) { + case 'web_sitemap': + return ; + case 'web_single_page': + return ; + case 'file': + return ; + } + })()} + + {(() => { + switch (datasource.data_source_type) { + case 'web_sitemap': + return datasource.config.url; + case 'web_single_page': + return datasource.config.urls.join(', '); + case 'file': + if (datasource.config.length === 1) { + return datasource.config[0].file_name; + } else { + return ( + <> + {datasource.config[0]?.file_name} + + + +{datasource.config.length - 1} files + + + {datasource.config.slice(1).map(file => ( + {file.file_name} + ))} + + + + ); + } + } + })()} + + + ); +} diff --git a/frontend/app/src/components/datasource/datasource-create-option.tsx b/frontend/app/src/components/datasource/datasource-create-option.tsx new file mode 100644 index 000000000..3da149aec --- /dev/null +++ b/frontend/app/src/components/datasource/datasource-create-option.tsx @@ -0,0 +1,36 @@ +'use client'; + +import { NextLink } from '@/components/nextjs/NextLink'; +import type { ReactNode } from 'react'; + +export function DatasourceCreateOption ({ + knowledgeBaseId, + type, + icon, + title, + children, +}: { + knowledgeBaseId: number + type: string + icon?: ReactNode + title: ReactNode + children?: ReactNode +}) { + return ( + +
    + + {icon} + + {title} +
    +
    + {children} +
    +
    + ); +} diff --git a/frontend/app/src/components/datasource/hooks.ts b/frontend/app/src/components/datasource/hooks.ts deleted file mode 100644 index ebe5b8975..000000000 --- a/frontend/app/src/components/datasource/hooks.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { getDatasource } from '@/api/datasources'; -import useSWR from 'swr'; - -export function useDatasource (id: number) { - const { data: datasource, ...rest } = useSWR(`api.datasource.${id}`, () => getDatasource(id)); - - return { datasource, ...rest }; -} diff --git a/frontend/app/src/components/datasource/no-datasource-placeholder.tsx b/frontend/app/src/components/datasource/no-datasource-placeholder.tsx new file mode 100644 index 000000000..b01f64f03 --- /dev/null +++ b/frontend/app/src/components/datasource/no-datasource-placeholder.tsx @@ -0,0 +1,9 @@ +export function NoDatasourcePlaceholder () { + return ( +
    + + Empty Data Sources list + +
    + ); +} diff --git a/frontend/app/src/components/datasource/schema.test.ts b/frontend/app/src/components/datasource/schema.test.ts deleted file mode 100644 index 3b5a2fd44..000000000 --- a/frontend/app/src/components/datasource/schema.test.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { createDatasourceBaseSchema } from '@/components/datasource/schema'; - -test('datasource name should not be empty', () => { - expect(createDatasourceBaseSchema.safeParse({ - name: '', - description: '', - build_kg_index: false, - llm_id: null, - }).success).toBe(false); - - expect(createDatasourceBaseSchema.safeParse({ - name: ' \t\n', - description: '', - build_kg_index: false, - llm_id: null, - }).success).toBe(false); - - expect(createDatasourceBaseSchema.safeParse({ - name: 'a', - description: '', - build_kg_index: false, - llm_id: null, - }).success).toBe(true); -}); diff --git a/frontend/app/src/components/datasource/schema.ts b/frontend/app/src/components/datasource/schema.ts deleted file mode 100644 index 850ac3616..000000000 --- a/frontend/app/src/components/datasource/schema.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { DeprecatedBaseCreateDatasourceParams } from '@/api/datasources'; -import { z, type ZodType } from 'zod'; - -export const createDatasourceBaseSchema = z.object({ - name: z.string().trim().min(1, 'Must not blank'), - description: z.string(), - build_kg_index: z.boolean(), - llm_id: z.number().nullable(), -}) satisfies ZodType; diff --git a/frontend/app/src/components/datasource/types.ts b/frontend/app/src/components/datasource/types.ts deleted file mode 100644 index da1b8f66b..000000000 --- a/frontend/app/src/components/datasource/types.ts +++ /dev/null @@ -1,6 +0,0 @@ -const types = ['file', 'web-sitemap', 'web-single-page'] as const; -export type DatasourceType = typeof types[number]; - -export function isDatasourceType (value: string): value is DatasourceType { - return types.includes(value as any); -} \ No newline at end of file diff --git a/frontend/app/src/components/datasource/update-datasource-form.tsx b/frontend/app/src/components/datasource/update-datasource-form.tsx new file mode 100644 index 000000000..a92b2e227 --- /dev/null +++ b/frontend/app/src/components/datasource/update-datasource-form.tsx @@ -0,0 +1,44 @@ +import { type Datasource, updateDatasource } from '@/api/datasources'; +import { FormInput } from '@/components/form/control-widget'; +import { FormFieldBasicLayout } from '@/components/form/field-layout'; +import { handleSubmitHelper } from '@/components/form/utils'; +import { Button } from '@/components/ui/button'; +import { Form } from '@/components/ui/form'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { useForm } from 'react-hook-form'; +import { z } from 'zod'; + +export function UpdateDatasourceForm ({ knowledgeBaseId, datasource, onUpdated }: { knowledgeBaseId: number, datasource: Datasource, onUpdated?: () => void }) { + const form = useForm({ + resolver: zodResolver(schema), + defaultValues: { + name: datasource.name, + }, + }); + + const handleSubmit = handleSubmitHelper(form, async data => { + await updateDatasource(knowledgeBaseId, datasource.id, data); + onUpdated?.(); + }); + + return ( +
    + + + + + +
    + + ); +} + +interface UpdateDatasourceFormParams { + name: string; +} + +const schema = z.object({ + name: z.string().min(1, 'Must not empty'), +}); \ No newline at end of file diff --git a/frontend/app/src/components/documents/DocumentDeprecationAlert.tsx b/frontend/app/src/components/documents/DocumentDeprecationAlert.tsx deleted file mode 100644 index f807e6607..000000000 --- a/frontend/app/src/components/documents/DocumentDeprecationAlert.tsx +++ /dev/null @@ -1,16 +0,0 @@ -import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert'; -import { AlertCircleIcon } from 'lucide-react'; - -export function DocumentDeprecationAlert () { - return ( - - - - Bare Documents management was deprecated. - - - TiDB.ai now uses Knowledge Base to manage documents. This page will be removed soon. - - - ); -} \ No newline at end of file diff --git a/frontend/app/src/components/documents/documents-table-filters.tsx b/frontend/app/src/components/documents/documents-table-filters.tsx index 1d8aedf22..a63c6298b 100644 --- a/frontend/app/src/components/documents/documents-table-filters.tsx +++ b/frontend/app/src/components/documents/documents-table-filters.tsx @@ -22,23 +22,25 @@ export function DocumentsTableFilters ({ onFilterChange }: { table: ReactTable -
    - ( - - - - - - - )} - /> - - - - Advanced Filters - + + +
    + ( + + + + + + + )} + /> + + Advanced Filters + + +
    - Select Index Status...} /> + Select Index Status...} /> {indexStatuses.map(indexStatus => ( @@ -170,9 +172,9 @@ export function DocumentsTableFilters ({ onFilterChange }: { table: ReactTable
    - - - + + + ); } \ No newline at end of file diff --git a/frontend/app/src/components/documents/documents-table.tsx b/frontend/app/src/components/documents/documents-table.tsx index ac1d60253..c62980003 100644 --- a/frontend/app/src/components/documents/documents-table.tsx +++ b/frontend/app/src/components/documents/documents-table.tsx @@ -8,10 +8,12 @@ import { DataTableRemote } from '@/components/data-table-remote'; import { DocumentPreviewDialog } from '@/components/document-viewer'; import { DocumentsTableFilters } from '@/components/documents/documents-table-filters'; import { DocumentChunksTable } from '@/components/knowledge-base/document-chunks-table'; +import { NextLink } from '@/components/nextjs/NextLink'; import { Button } from '@/components/ui/button'; import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle, DialogTrigger } from '@/components/ui/dialog'; import type { CellContext, ColumnDef } from '@tanstack/react-table'; import { createColumnHelper } from '@tanstack/table-core'; +import { UploadIcon } from 'lucide-react'; import { useMemo, useState } from 'react'; const helper = createColumnHelper(); @@ -28,13 +30,14 @@ const href = (cell: CellContext) => [ helper.accessor('id', { cell: mono }), helper.accessor('knowledge_base', { cell: ctx => }), - helper.display({ id: 'name', header: 'name', cell: ({ row }) => - + helper.display({ + id: 'name', header: 'name', cell: ({ row }) => + , }), helper.accessor('source_uri', { cell: href }), helper.accessor('mime_type', { cell: mono }), @@ -47,8 +50,8 @@ const getColumns = (kbId?: number) => [ header: 'action', cell: ({ row }) => (kbId ?? row.original.knowledge_base?.id) != null && ( - - @@ -70,23 +73,28 @@ const getColumns = (kbId?: number) => [ }), ] as ColumnDef[]; -export function DocumentsTable ({ knowledgeBaseId }: { knowledgeBaseId?: number }) { +export function DocumentsTable ({ knowledgeBaseId }: { knowledgeBaseId: number }) { const [filters, setFilters] = useState({}); const columns = useMemo(() => { - if (knowledgeBaseId != null) { - const columns = [...getColumns(knowledgeBaseId)]; - columns.splice(1, 1); - return columns; - } else { - return getColumns(knowledgeBaseId); - } + const columns = [...getColumns(knowledgeBaseId)]; + columns.splice(1, 1); + return columns; }, [knowledgeBaseId]); return ( ( - +
    + + + Upload documents + + +
    ))} columns={columns} apiKey={knowledgeBaseId != null ? `api.datasource.${knowledgeBaseId}.documents` : 'api.documents.list'} diff --git a/frontend/app/src/components/documents/hooks.ts b/frontend/app/src/components/documents/hooks.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/frontend/app/src/components/embedding-models/EmbeddingModelInfo.tsx b/frontend/app/src/components/embedding-models/EmbeddingModelInfo.tsx index 6d869f9f3..fb59f307d 100644 --- a/frontend/app/src/components/embedding-models/EmbeddingModelInfo.tsx +++ b/frontend/app/src/components/embedding-models/EmbeddingModelInfo.tsx @@ -3,7 +3,7 @@ import { useEmbeddingModel } from '@/components/embedding-models/hooks'; import { ModelComponentInfo } from '@/components/model-component-info'; -export function EmbeddingModelInfo ({ className, reverse = false, detailed = false, id }: { className?: string, reverse?: boolean, detailed?: boolean, id: number | undefined | null }) { +export function EmbeddingModelInfo ({ className, id }: { className?: string, id: number | undefined | null }) { const { embeddingModel, isLoading } = useEmbeddingModel(id); return `/embedding-models/${embeddingModel.id}`} isLoading={isLoading} - reverse={reverse} - detailed={detailed} defaultName="Default Embedding Model" />; } diff --git a/frontend/app/src/components/embedding-models/EmbeddingModelsTable.tsx b/frontend/app/src/components/embedding-models/EmbeddingModelsTable.tsx index 148d460a9..747f8efc0 100644 --- a/frontend/app/src/components/embedding-models/EmbeddingModelsTable.tsx +++ b/frontend/app/src/components/embedding-models/EmbeddingModelsTable.tsx @@ -50,10 +50,9 @@ const columns: ColumnDef[] = [ cell: ({ row }) => { const { model, provider } = row.original; return ( - - {provider} - {model} - + <> + {provider}:{model} + ); }, }), diff --git a/frontend/app/src/components/embedding-models/hooks.tsx b/frontend/app/src/components/embedding-models/hooks.tsx index 4ba7682cc..ca7d55e06 100644 --- a/frontend/app/src/components/embedding-models/hooks.tsx +++ b/frontend/app/src/components/embedding-models/hooks.tsx @@ -1,7 +1,16 @@ -import { getEmbeddingModelById } from '@/api/embedding-models'; +import { listEmbeddingModels } from '@/api/embedding-models'; +import { listAllHelper } from '@/lib/request'; import useSWR from 'swr'; +export function useAllEmbeddingModels (flag = true) { + return useSWR(flag && 'api.embedding-models.list-all', () => listAllHelper(listEmbeddingModels, 'id')); +} + export function useEmbeddingModel (id: number | null | undefined) { - const { data: embeddingModel, ...rest } = useSWR(id == null ? null : `api.embedding-model.get?id=${id}`, () => getEmbeddingModelById(id as number)); - return { embeddingModel, ...rest }; + const { data, mutate, ...rest } = useAllEmbeddingModels(id != null); + + return { + embeddingModel: data?.find(embeddingModel => embeddingModel.id === id), + ...rest, + }; } diff --git a/frontend/app/src/components/form/biz.tsx b/frontend/app/src/components/form/biz.tsx index 90e30eea6..066671458 100644 --- a/frontend/app/src/components/form/biz.tsx +++ b/frontend/app/src/components/form/biz.tsx @@ -1,35 +1,35 @@ -import { type EmbeddingModel, listEmbeddingModels } from '@/api/embedding-models'; -import { type KnowledgeBaseSummary, listKnowledgeBases } from '@/api/knowledge-base'; -import { listLlms, type LLM } from '@/api/llms'; +import { type EmbeddingModel } from '@/api/embedding-models'; +import { type KnowledgeBaseSummary } from '@/api/knowledge-base'; +import { type LLM } from '@/api/llms'; import type { ProviderOption } from '@/api/providers'; -import { listRerankers, type Reranker } from '@/api/rerankers'; +import { type Reranker } from '@/api/rerankers'; import { CreateEmbeddingModelForm } from '@/components/embedding-models/CreateEmbeddingModelForm'; -import { FormCombobox, type FormComboboxConfig, type FormComboboxProps, FormSelect, type FormSelectConfig, type FormSelectProps } from '@/components/form/control-widget'; -import { KBInfo } from '@/components/knowledge-base/KBInfo'; +import { useAllEmbeddingModels } from '@/components/embedding-models/hooks'; +import { FormCombobox, type FormComboboxConfig, type FormComboboxProps } from '@/components/form/control-widget'; +import { useAllKnowledgeBases } from '@/components/knowledge-base/hooks'; import { CreateLLMForm } from '@/components/llm/CreateLLMForm'; +import { useAllLlms } from '@/components/llm/hooks'; import { ManagedDialog } from '@/components/managed-dialog'; import { ManagedPanelContext } from '@/components/managed-panel'; import { CreateRerankerForm } from '@/components/reranker/CreateRerankerForm'; -import { RerankerInfo } from '@/components/reranker/RerankerInfo'; +import { useAllRerankers } from '@/components/reranker/hooks'; import { DialogContent, DialogDescription, DialogHeader, DialogTitle } from '@/components/ui/dialog'; -import { PlusIcon } from 'lucide-react'; +import { AlertTriangleIcon, DotIcon, PlusIcon } from 'lucide-react'; import { forwardRef } from 'react'; -import useSWR from 'swr'; export const EmbeddingModelSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { - // TODO - const { data: embeddingModels, isLoading, mutate, error } = useSWR('api.embedding-models.list-all', () => listEmbeddingModels({ size: 100 })); + const { data: embeddingModels, isLoading, mutate, error } = useAllEmbeddingModels(); return ( [option.name, option.provider, option.model], loading: isLoading, error, - renderValue: option => ({option.name} [{option.vector_dimension}]), + renderValue: option => ({option.name} [{option.vector_dimension}]), renderOption: option => (
    {option.name}
    @@ -78,14 +78,14 @@ export const EmbeddingModelSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { - const { data: llms, isLoading, mutate, error } = useSWR('api.llms.list-all', () => listLlms({ size: 100 })); + const { data: llms, isLoading, mutate, error } = useAllLlms(); return ( ({option.name}), @@ -138,14 +138,14 @@ export const LLMSelect = forwardRef & { r LLMSelect.displayName = 'LLMSelect'; export const RerankerSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { - const { data: rerankers, mutate, isLoading, error } = useSWR('api.rerankers.list-all', () => listRerankers({ size: 100 })); + const { data: rerankers, mutate, isLoading, error } = useAllRerankers(); return ( [option.name, option.provider, option.model], loading: isLoading, error, @@ -197,7 +197,7 @@ export const RerankerSelect = forwardRef RerankerSelect.displayName = 'RerankerSelect'; -export interface ProviderSelectProps extends Omit { +export interface ProviderSelectProps extends Omit { options: ProviderOption[] | undefined; isLoading: boolean; error: unknown; @@ -207,22 +207,24 @@ export const ProviderSelect = forwardRef(({ options, isLoading, error, ...props }, ref) => { return ( - [option.provider, option.provider_description ?? '', option.provider_display_name ?? ''], loading: isLoading, error, renderOption: option => ( - <> -
    {option.provider_display_name ?? option.provider}
    +
    +
    {option.provider_display_name ?? option.provider}
    {option.provider_description &&
    {option.provider_description}
    } - +
    ), itemClassName: 'space-y-1', renderValue: option => option.provider_display_name ?? option.provider, key: 'provider', - } satisfies FormSelectConfig} + } satisfies FormComboboxConfig} + contentWidth="anchor" {...props} /> ); @@ -230,22 +232,56 @@ export const ProviderSelect = forwardRef(({ ProviderSelect.displayName = 'ProviderSelect'; -export const KBSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { - const { data: kbs, isLoading, error } = useSWR('api.knowledge-bases.list-all', () => listKnowledgeBases({ size: 100 })); +export const KBSelect = forwardRef & { reverse?: boolean }>(({ reverse = true, ...props }, ref) => { + const { data: kbs, isLoading, error } = useAllKnowledgeBases(); return ( - [String(option.id), option.name, option.description], loading: isLoading, error, - renderValue: option => (), - renderOption: option => (), + renderValue: option => ( +
    + {option.name} +
    + + {(option.documents_total ?? 0) || <> no} documents + + + + {(option.data_sources_total ?? 0) || <> no} data sources + +
    +
    + ), + renderOption: option => ( +
    +
    + + {option.name} + +
    +
    + + {(option.documents_total ?? 0) || <> no} documents + + + + {(option.data_sources_total ?? 0) || <> no} data sources + +
    +
    + {option.description} +
    +
    + ), key: 'id', - } satisfies FormSelectConfig} + } satisfies FormComboboxConfig} /> ); }); diff --git a/frontend/app/src/components/form/control-widget.tsx b/frontend/app/src/components/form/control-widget.tsx index 232a6847f..e0d7ad679 100644 --- a/frontend/app/src/components/form/control-widget.tsx +++ b/frontend/app/src/components/form/control-widget.tsx @@ -140,9 +140,10 @@ export interface FormComboboxProps extends FormControlWidgetProps { children?: ReactElement; placeholder?: string; config: FormComboboxConfig; + contentWidth?: 'anchor'; } -export const FormCombobox = forwardRef(({ config, placeholder, value, onChange, name, disabled, children, ...props }, ref) => { +export const FormCombobox = forwardRef(({ config, placeholder, value, onChange, name, disabled, children, contentWidth, ...props }, ref) => { const [open, setOpen] = useState(false); const isConfigReady = !config.loading && !config.error; const current = config.options.find(option => option[config.key] === value); @@ -190,7 +191,7 @@ export const FormCombobox = forwardRef(({ config, placeh
    - + @@ -209,7 +210,7 @@ export const FormCombobox = forwardRef(({ config, placeh item.split(/\s+/))} className={cn('group', config.itemClassName)} onSelect={value => { const item = config.options.find(option => String(option[config.key]) === value); diff --git a/frontend/app/src/components/knowledge-base/KBInfo.tsx b/frontend/app/src/components/knowledge-base/KBInfo.tsx deleted file mode 100644 index e9bb2cf62..000000000 --- a/frontend/app/src/components/knowledge-base/KBInfo.tsx +++ /dev/null @@ -1,11 +0,0 @@ -'use client'; - -import { useKnowledgeBase } from '@/components/knowledge-base/hooks'; - -export function KBInfo ({ className, detailed = false, id }: { className?: string, detailed?: boolean, id: number | undefined | null }) { - const { knowledgeBase, isLoading } = useKnowledgeBase(id); - - return ( - {knowledgeBase?.name} - ); -} diff --git a/frontend/app/src/components/knowledge-base/create-knowledge-base-form.tsx b/frontend/app/src/components/knowledge-base/create-knowledge-base-form.tsx index 4a4455df9..12a5c1155 100644 --- a/frontend/app/src/components/knowledge-base/create-knowledge-base-form.tsx +++ b/frontend/app/src/components/knowledge-base/create-knowledge-base-form.tsx @@ -1,11 +1,10 @@ -import { uploadFiles } from '@/api/datasources'; import { createKnowledgeBase } from '@/api/knowledge-base'; import { EmbeddingModelSelect, LLMSelect } from '@/components/form/biz'; import { FormInput, FormTextarea } from '@/components/form/control-widget'; import { FormFieldBasicLayout } from '@/components/form/field-layout'; import { handleSubmitHelper } from '@/components/form/utils'; -import { createDatasourceSchema, FormCreateDataSources } from '@/components/knowledge-base/form-create-data-sources'; import { FormIndexMethods } from '@/components/knowledge-base/form-index-methods'; +import { mutateKnowledgeBases } from '@/components/knowledge-base/hooks'; import { Button } from '@/components/ui/button'; import { Form } from '@/components/ui/form'; import { zodResolver } from '@hookform/resolvers/zod'; @@ -26,58 +25,26 @@ export function CreateKnowledgeBaseForm ({}: {}) { name: '', description: '', index_methods: ['vector'], - data_sources: [ - { name: 'Default Datasource', description: '', data_source_type: 'file', files: [] }, - ], + data_sources: [], }, }); const handleSubmit = handleSubmitHelper(form, async (data) => { - const dataSources = await Promise.all(data.data_sources.map(async (ds) => { - switch (ds.data_source_type) { - case 'file': { - const { files, ...rest } = ds; - const uploadedFiles = await uploadFiles(ds.files); - return { - ...rest, - config: uploadedFiles.map(f => ({ - file_id: f.id, - file_name: f.name, - })), - }; - } - case 'web_single_page': { - const { urls, ...rest } = ds; - - return { - ...rest, - config: { urls }, - }; - } - - case 'web_sitemap': - const { url, ...rest } = ds; - - return { - ...rest, - config: { url }, - }; - } - })); const kb = await createKnowledgeBase({ ...data, - data_sources: dataSources, + data_sources: [], }); startTransition(() => { - router.push(`/knowledge-bases/${kb.id}`); + router.push(`/knowledge-bases/${kb.id}/data-sources`); + mutateKnowledgeBases(); }); }); return (
    - + @@ -90,7 +57,6 @@ export function CreateKnowledgeBaseForm ({}: {}) { - @@ -106,5 +72,5 @@ const createKnowledgeBaseParamsSchema = z.object({ index_methods: z.enum(['knowledge_graph', 'vector']).array(), llm_id: z.number().nullable().optional(), embedding_model_id: z.number().nullable().optional(), - data_sources: createDatasourceSchema.array(), // use external form + data_sources: z.never().array().length(0), // use external form }); diff --git a/frontend/app/src/components/knowledge-base/datasource-details.tsx b/frontend/app/src/components/knowledge-base/datasource-details.tsx deleted file mode 100644 index 4b94735ec..000000000 --- a/frontend/app/src/components/knowledge-base/datasource-details.tsx +++ /dev/null @@ -1,54 +0,0 @@ -'use client'; - -import { useKnowledgeBaseDatasource } from '@/components/knowledge-base/hooks'; -import { OptionDetail } from '@/components/option-detail'; -import { Badge } from '@/components/ui/badge'; - -export function KnowledgeBaseDatasourceDetails ({ id }: { id: number }) { - return ( - <> - - - - ); -} - -function KnowledgeBaseDatasourceFields ({ id }: { id: number }) { - const datasource = useKnowledgeBaseDatasource(id); - return ( - - ); -} - -function KnowledgeBaseDatasourceUploadFiles ({ id }: { id: number }) { - const datasource = useKnowledgeBaseDatasource(id); - - if (datasource?.data_source_type !== 'file' || datasource.config.length === 0) { - return null; - } - return ( -
    -
    Files
    -
    - {datasource.config.map(file => ( - - - {file.file_name} - - #{file.file_id} - - ))} -
    -
    - ); -} diff --git a/frontend/app/src/components/knowledge-base/form-create-data-sources.tsx b/frontend/app/src/components/knowledge-base/form-create-data-sources.tsx deleted file mode 100644 index 479f259a9..000000000 --- a/frontend/app/src/components/knowledge-base/form-create-data-sources.tsx +++ /dev/null @@ -1,139 +0,0 @@ -import type { BaseCreateDatasourceParams, CreateDatasourceSpecParams } from '@/api/datasources'; -import type { CreateKnowledgeBaseParams } from '@/api/knowledge-base'; -import { FormInput } from '@/components/form/control-widget'; -import { FormFieldBasicLayout, FormPrimitiveArrayFieldBasicLayout } from '@/components/form/field-layout'; -import { FilesInput } from '@/components/form/widgets/FilesInput'; -import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@/components/ui/accordion'; -import { Button } from '@/components/ui/button'; -import { FormItem, FormLabel, FormMessage } from '@/components/ui/form'; -import { FormArrayField } from '@/components/ui/form.ext'; -import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs'; -import { cn } from '@/lib/utils'; -import { zodFile } from '@/lib/zod'; -import { PlusIcon } from 'lucide-react'; -import { useId } from 'react'; -import { useFormContext } from 'react-hook-form'; -import { z } from 'zod'; - -export const createDatasourceSchema = z.object({ - name: z.string().trim().min(1, 'Must not blank'), - description: z.string(), -}).and(z.discriminatedUnion('data_source_type', [ - z.object({ - data_source_type: z.literal('file'), - files: zodFile().array().min(1), - }), - z.object({ - data_source_type: z.literal('web_single_page'), - urls: z.string().url().array().min(1), - }), - z.object({ - data_source_type: z.literal('web_sitemap'), - url: z.string().url(), - }), -])); - -export function FormCreateDataSources () { - return ( - - name="data_sources" - render={({ field: { fields, update, append, remove } }) => ( - - Datasource - - {(fields).map((field, index) => ( - - - - - - update(index, { ...switchDatasource(field, value as never) }))} - > - - - File - - - Web Single Page - - - Web Sitemap - - - - {field.data_source_type === 'file' && ( - - - - )} - {field.data_source_type === 'web_single_page' && ( - ''}> - - - )} - {field.data_source_type === 'web_sitemap' && ( - - - - )} - - - - - {fields.length > 1 && } - - - ))} - - - - - )} - />) - ; -} - -function DatasourceName ({ index }: { index: number }) { - const { watch, formState } = useFormContext(); - const errors = formState.errors.data_sources?.[index]; - const name = watch(`data_sources.${index}.name`); - - return ( - - {index + 1}. - {name || Unnamed} - - ); -} - -function switchDatasource (data: BaseCreateDatasourceParams & CreateDatasourceSpecParams, type: CreateDatasourceSpecParams['data_source_type']): BaseCreateDatasourceParams & CreateDatasourceSpecParams { - switch (type) { - case 'file': - return { - name: data.name, - data_source_type: 'file', - config: [], - }; - case 'web_single_page': - return { - name: data.name, - data_source_type: 'web_single_page', - config: { urls: [] }, - }; - case 'web_sitemap': - return { - name: data.name, - data_source_type: 'web_sitemap', - config: { url: '' }, - }; - } -} diff --git a/frontend/app/src/components/knowledge-base/hooks.ts b/frontend/app/src/components/knowledge-base/hooks.ts index 5c1d3597f..6d67e077f 100644 --- a/frontend/app/src/components/knowledge-base/hooks.ts +++ b/frontend/app/src/components/knowledge-base/hooks.ts @@ -1,33 +1,35 @@ -import { getKnowledgeBaseById, getKnowledgeGraphIndexProgress, listKnowledgeBases } from '@/api/knowledge-base'; -import { useKB } from '@/app/(main)/(admin)/knowledge-bases/[id]/context'; -import useSWR from 'swr'; - -export function useKnowledgeBase (id: number | null | undefined) { - const { data: knowledgeBase, ...rest } = useSWR(id != null && `api.knowledge-bases.get?id=${id}`, () => getKnowledgeBaseById(id!)) - return { knowledgeBase, ...rest } -} - -export function useKnowledgeBaseDatasource (id: number) { - const { data_sources } = useKB(); - - return data_sources.find(ds => ds.id === id); -} +import { listDataSources } from '@/api/datasources'; +import { getKnowledgeGraphIndexProgress, listKnowledgeBases } from '@/api/knowledge-base'; +import { listAllHelper } from '@/lib/request'; +import useSWR, { mutate } from 'swr'; export function useKnowledgeBaseIndexProgress (id: number) { const { data: progress, ...rest } = useSWR(`api.knowledge-base.${id}.index-progress`, () => getKnowledgeGraphIndexProgress(id)); return { progress, ...rest }; } -export function useKnowledgeBases (pageIndex: number, pageSize: number) { - const { data: knowledgeBases, ...rest } = useSWR(`api.knowledge-bases.list?page=${pageIndex}&size=${pageSize}`, () => listKnowledgeBases({ page: pageIndex + 1, size: pageSize }), { - revalidateOnReconnect: false, - revalidateOnFocus: false, - focusThrottleInterval: 1000, - keepPreviousData: true, - }); +export function useAllKnowledgeBases (flag = true) { + return useSWR(flag && `api.knowledge-bases.list-all`, () => listAllHelper(listKnowledgeBases, 'id')); +} + +export function useKnowledgeBase (id: number | null | undefined) { + const { data, mutate, ...rest } = useAllKnowledgeBases(id != null); return { - knowledgeBases, + knowledgeBase: data?.find(llm => llm.id === id), ...rest, }; } + +export function useAllKnowledgeBaseDataSources (kbId: number, flag = true) { + return useSWR(flag && `api.knowledge-bases.${kbId}.data-sources.list-all`, () => listAllHelper((params) => listDataSources(kbId, params), 'id')); +} + +export function mutateKnowledgeBases () { + return mutate(key => { + if (typeof key === 'string') { + return key.startsWith(`api.knowledge-bases.`); + } + return false; + }); +} \ No newline at end of file diff --git a/frontend/app/src/components/knowledge-base/knowledge-base-card.tsx b/frontend/app/src/components/knowledge-base/knowledge-base-card.tsx index d4e740a6e..a0c14995c 100644 --- a/frontend/app/src/components/knowledge-base/knowledge-base-card.tsx +++ b/frontend/app/src/components/knowledge-base/knowledge-base-card.tsx @@ -6,7 +6,8 @@ import { Card, CardContent, CardDescription, CardFooter, CardHeader } from '@/co import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuSeparator, DropdownMenuTrigger } from '@/components/ui/dropdown-menu'; import { Separator } from '@/components/ui/separator'; import { Skeleton } from '@/components/ui/skeleton'; -import { Book, Ellipsis } from 'lucide-react'; +import { cn } from '@/lib/utils'; +import { AlertTriangleIcon, Book, Ellipsis } from 'lucide-react'; import { useRouter } from 'next/navigation'; import { ReactNode, startTransition, useState } from 'react'; import { mutate } from 'swr'; @@ -40,7 +41,7 @@ export function KnowledgeBaseCard ({ knowledgeBase, children }: { knowledgeBase: }; return ( - +
    @@ -51,7 +52,7 @@ export function KnowledgeBaseCard ({ knowledgeBase, children }: { knowledgeBase:
    {knowledgeBase.documents_total ?? 0} documents · - {knowledgeBase.data_sources_total ?? 0} data sources + {(knowledgeBase.data_sources_total ?? 0) || <> No} data sources
    @@ -73,7 +74,7 @@ export function KnowledgeBaseCard ({ knowledgeBase, children }: { knowledgeBase: - event.stopPropagation()}> + event.stopPropagation()}> Settings diff --git a/frontend/app/src/components/knowledge-base/knowledge-base-index.tsx b/frontend/app/src/components/knowledge-base/knowledge-base-index.tsx index bc55c2439..c22a3224e 100644 --- a/frontend/app/src/components/knowledge-base/knowledge-base-index.tsx +++ b/frontend/app/src/components/knowledge-base/knowledge-base-index.tsx @@ -2,7 +2,6 @@ import { type DatasourceKgIndexError, type DatasourceVectorIndexError } from '@/api/datasources'; import { listKnowledgeBaseKgIndexErrors, listKnowledgeBaseVectorIndexErrors, retryKnowledgeBaseAllFailedTasks } from '@/api/knowledge-base'; -import { useKB } from '@/app/(main)/(admin)/knowledge-bases/[id]/context'; import { link } from '@/components/cells/link'; import { IndexProgressChart, IndexProgressChartPlaceholder } from '@/components/charts/IndexProgressChart'; import { TotalCard } from '@/components/charts/TotalCard'; @@ -17,7 +16,6 @@ import { ArrowRightIcon, FileTextIcon, PuzzleIcon, RouteIcon } from 'lucide-reac import Link from 'next/link'; export function KnowledgeBaseIndexProgress ({ id }: { id: number }) { - const { index_methods } = useKB(); const { progress, isLoading } = useKnowledgeBaseIndexProgress(id); return ( diff --git a/frontend/app/src/components/knowledge-base/knowledge-base-settings-form.tsx b/frontend/app/src/components/knowledge-base/knowledge-base-settings-form.tsx index a1ef24724..75109a913 100644 --- a/frontend/app/src/components/knowledge-base/knowledge-base-settings-form.tsx +++ b/frontend/app/src/components/knowledge-base/knowledge-base-settings-form.tsx @@ -4,6 +4,7 @@ import { type KnowledgeBase, type KnowledgeBaseIndexMethod, updateKnowledgeBase import { EmbeddingModelSelect, LLMSelect } from '@/components/form/biz'; import { FormInput, FormSwitch, FormTextarea } from '@/components/form/control-widget'; import { FormFieldBasicLayout, FormFieldContainedLayout } from '@/components/form/field-layout'; +import { mutateKnowledgeBases } from '@/components/knowledge-base/hooks'; import { fieldAccessor, GeneralSettingsField, type GeneralSettingsFieldAccessor, GeneralSettingsForm, shallowPick } from '@/components/settings-form'; import type { KeyOfType } from '@/lib/typing-utils'; import { format } from 'date-fns'; @@ -26,6 +27,7 @@ export function KnowledgeBaseSettingsForm ({ knowledgeBase }: { knowledgeBase: K await updateKnowledgeBase(knowledgeBase.id, partial); startTransition(() => { router.refresh(); + mutateKnowledgeBases(); }); } else { throw new Error(`${path.map(p => String(p)).join('.')} is not updatable currently.`); diff --git a/frontend/app/src/components/llm/LLMsTable.tsx b/frontend/app/src/components/llm/LLMsTable.tsx index 4d6bd03ab..67f916d6a 100644 --- a/frontend/app/src/components/llm/LLMsTable.tsx +++ b/frontend/app/src/components/llm/LLMsTable.tsx @@ -49,10 +49,9 @@ const columns: ColumnDef[] = [ cell: ({ row }) => { const { model, provider } = row.original; return ( - - {provider} - {model} - + <> + {provider}:{model} + ); }, }), diff --git a/frontend/app/src/components/llm/LlmInfo.tsx b/frontend/app/src/components/llm/LlmInfo.tsx index ec15b4c1b..37b385858 100644 --- a/frontend/app/src/components/llm/LlmInfo.tsx +++ b/frontend/app/src/components/llm/LlmInfo.tsx @@ -3,7 +3,7 @@ import { useLlm } from '@/components/llm/hooks'; import { ModelComponentInfo } from '@/components/model-component-info'; -export function LlmInfo ({ className, reverse = false, detailed = false, id }: { className?: string, reverse?: boolean, detailed?: boolean, id: number | undefined | null }) { +export function LlmInfo ({ className, id }: { className?: string, id: number | undefined | null }) { const { llm, isLoading } = useLlm(id); return `/llms/${llm.id}`} isLoading={isLoading} - reverse={reverse} - detailed={detailed} defaultName="Default LLM" />; } diff --git a/frontend/app/src/components/llm/hooks.ts b/frontend/app/src/components/llm/hooks.ts index fa1a8d03b..9fe596bb2 100644 --- a/frontend/app/src/components/llm/hooks.ts +++ b/frontend/app/src/components/llm/hooks.ts @@ -1,7 +1,16 @@ -import { getLlm } from '@/api/llms'; +import { listLlms } from '@/api/llms'; +import { listAllHelper } from '@/lib/request'; import useSWR from 'swr'; +export function useAllLlms (flag: boolean = true) { + return useSWR(flag && 'api.llms.list-all', () => listAllHelper(listLlms, 'id')); +} + export function useLlm (id: number | null | undefined) { - const { data: llm, ...rest } = useSWR(id == null ? null : `api.llms.get?id=${id}`, () => getLlm(id as number)); - return { llm, ...rest }; + const { data, mutate, ...rest } = useAllLlms(id != null); + + return { + llm: data?.find(llm => llm.id === id), + ...rest, + }; } diff --git a/frontend/app/src/components/model-component-info.tsx b/frontend/app/src/components/model-component-info.tsx index d489cb0e5..9b2d57ebe 100644 --- a/frontend/app/src/components/model-component-info.tsx +++ b/frontend/app/src/components/model-component-info.tsx @@ -1,4 +1,3 @@ -import { Badge, badgeVariants } from '@/components/ui/badge'; import { cn } from '@/lib/utils'; import { Loader2Icon } from 'lucide-react'; import Link from 'next/link'; @@ -14,26 +13,25 @@ export interface ModelComponentInfoProps { className?: string; isLoading?: boolean; model: Model | null | undefined; - detailed?: boolean; - reverse?: boolean; url: (model: Model) => string; defaultName?: string; } -export function ModelComponentInfo ({ className, reverse = false, detailed = false, isLoading = false, model, url, defaultName }: ModelComponentInfoProps) { - +export function ModelComponentInfo ({ className, isLoading = false, model, url, defaultName }: ModelComponentInfoProps) { if (isLoading) { return ; } if (!model) { - return defaultName && {defaultName}; + return defaultName && {defaultName}; } return ( - - {detailed && {model.provider ?? 'unknown-provider'}:{model.model}} - {model.name} + + {model.name} + + {model.provider}:{model.model} + ); } diff --git a/frontend/app/src/components/reranker/RerankerInfo.tsx b/frontend/app/src/components/reranker/RerankerInfo.tsx index f054e2a9e..36e80a70c 100644 --- a/frontend/app/src/components/reranker/RerankerInfo.tsx +++ b/frontend/app/src/components/reranker/RerankerInfo.tsx @@ -3,7 +3,7 @@ import { ModelComponentInfo } from '@/components/model-component-info'; import { useReranker } from '@/components/reranker/hooks'; -export function RerankerInfo ({ className, reverse = false, detailed = false, id }: { className?: string, reverse?: boolean, detailed?: boolean, id: number | undefined | null }) { +export function RerankerInfo ({ className, id }: { className?: string, id: number | undefined | null }) { const { reranker, isLoading } = useReranker(id); return `/reranker-models/${reranker.id}`} isLoading={isLoading} - reverse={reverse} - detailed={detailed} defaultName="Default Reranker Model" />; } diff --git a/frontend/app/src/components/reranker/RerankerModelsTable.tsx b/frontend/app/src/components/reranker/RerankerModelsTable.tsx index 419ce44d8..57e0bd773 100644 --- a/frontend/app/src/components/reranker/RerankerModelsTable.tsx +++ b/frontend/app/src/components/reranker/RerankerModelsTable.tsx @@ -48,10 +48,9 @@ const columns: ColumnDef[] = [ cell: ({ row }) => { const { model, provider } = row.original; return ( - - {provider} - {model} - + <> + {provider}:{model} + ); }, }), diff --git a/frontend/app/src/components/reranker/hooks.ts b/frontend/app/src/components/reranker/hooks.ts index 19fa497de..96a956f21 100644 --- a/frontend/app/src/components/reranker/hooks.ts +++ b/frontend/app/src/components/reranker/hooks.ts @@ -1,7 +1,16 @@ -import { getReranker } from '@/api/rerankers'; +import { listRerankers } from '@/api/rerankers'; +import { listAllHelper } from '@/lib/request'; import useSWR from 'swr'; +export function useAllRerankers (flag = true) { + return useSWR(flag && 'api.rerankers.list-all', () => listAllHelper(listRerankers, 'id')); +} + export function useReranker (id: number | null | undefined) { - const { data: reranker, ...rest } = useSWR(id == null ? null : `api.rerankers.get?id=${id}`, () => getReranker(id as number)); - return { reranker, ...rest }; + const { data, mutate, ...rest } = useAllRerankers(id != null); + + return { + reranker: data?.find(reranker => reranker.id === id), + ...rest, + }; } diff --git a/frontend/app/src/lib/request/index.ts b/frontend/app/src/lib/request/index.ts index 3e04acaef..7457e4ba5 100644 --- a/frontend/app/src/lib/request/index.ts +++ b/frontend/app/src/lib/request/index.ts @@ -5,3 +5,4 @@ export * from './errors'; export * from './params'; export { type Page, type PageParams, zodPage } from '../zod'; export * from './url'; +export * from './list-all-helper'; diff --git a/frontend/app/src/lib/request/list-all-helper.ts b/frontend/app/src/lib/request/list-all-helper.ts new file mode 100644 index 000000000..d0a3f4178 --- /dev/null +++ b/frontend/app/src/lib/request/list-all-helper.ts @@ -0,0 +1,30 @@ +import type { Page, PageParams } from '@/lib/zod'; + +export async function listAllHelper (api: (params: PageParams) => Promise>, idField: keyof T) { + let page = 1; + const chunks: Page[] = []; + + while (true) { + const current = await api({ page, size: 100 }); + chunks.push(current); + if (page < current.pages) { + page += 1; + } else { + break; + } + } + + const idSet = new Set(); + const result: T[] = []; + + for (const chunk of chunks) { + for (const item of chunk.items) { + if (!idSet.has(item[idField])) { + idSet.add(item[idField]); + result.push(item); + } + } + } + + return result; +} \ No newline at end of file From f88781924cdefc25c17efafa97f9b2a1ff4e8408 Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Wed, 27 Nov 2024 14:07:58 +0800 Subject: [PATCH 020/114] fix(frontend): update knowledge graph url --- .../src/components/chat/knowledge-graph-debug-info.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/frontend/app/src/components/chat/knowledge-graph-debug-info.tsx b/frontend/app/src/components/chat/knowledge-graph-debug-info.tsx index 770a0bbb4..e29f4aa94 100644 --- a/frontend/app/src/components/chat/knowledge-graph-debug-info.tsx +++ b/frontend/app/src/components/chat/knowledge-graph-debug-info.tsx @@ -1,6 +1,6 @@ import { getChatMessageSubgraph } from '@/api/chats'; import { useAuth } from '@/components/auth/AuthProvider'; -import { type ChatMessageGroup, useChatMessageStreamState } from '@/components/chat/chat-hooks'; +import { type ChatMessageGroup, useChatInfo, useChatMessageStreamState, useCurrentChatController } from '@/components/chat/chat-hooks'; import type { OngoingState } from '@/components/chat/chat-message-controller'; import { AppChatStreamState, type StackVMState } from '@/components/chat/chat-stream-state'; import { NetworkViewer } from '@/components/graph/components/NetworkViewer'; @@ -11,9 +11,11 @@ import { useEffect } from 'react'; import useSWR from 'swr'; export function KnowledgeGraphDebugInfo ({ group }: { group: ChatMessageGroup }) { + const { engine_options } = useChatInfo(useCurrentChatController()) ?? {}; const auth = useAuth(); const ongoing = useChatMessageStreamState(group.assistant); - const canEdit = !!auth.me?.is_superuser; + const kbId = engine_options?.knowledge_base?.linked_knowledge_base?.id; + const canEdit = !!auth.me?.is_superuser && kbId != null; const shouldFetch = (!ongoing || ongoing.finished || couldFetchKnowledgeGraphDebugInfo(ongoing)); const { data: span, isLoading, mutate } = useSWR( @@ -42,7 +44,7 @@ export function KnowledgeGraphDebugInfo ({ group }: { group: ChatMessageGroup }) network={network} Details={() => canEdit ? ( - + Edit graph From aa68dbadde5d767f66499c817ed59521d27cf0e4 Mon Sep 17 00:00:00 2001 From: Mini256 Date: Wed, 27 Nov 2024 15:09:17 +0800 Subject: [PATCH 021/114] fix: get message subgraph adapt kb (#417) using kb graph editor if the chat message is completed by a chat engine with kb. --- backend/app/api/routes/chat.py | 8 ++++++-- backend/app/rag/chat.py | 11 ++++++----- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/backend/app/api/routes/chat.py b/backend/app/api/routes/chat.py index dd1680e17..657e2cfe0 100644 --- a/backend/app/api/routes/chat.py +++ b/backend/app/api/routes/chat.py @@ -14,6 +14,8 @@ from app.api.deps import SessionDep, OptionalUserDep, CurrentUserDep from app.rag.chat_config import get_default_embedding_model, ChatEngineConfig from app.rag.knowledge_base.config import get_kb_embed_model +from app.rag.knowledge_base.index_store import get_kb_tidb_graph_editor +from app.rag.knowledge_graph.graph_store.tidb_graph_editor import legacy_tidb_graph_editor from app.repositories import chat_repo, knowledge_base_repo from app.models import Chat, ChatUpdate from app.rag.chat import ( @@ -196,15 +198,17 @@ def get_chat_subgraph(session: SessionDep, user: OptionalUserDep, chat_message_i raise HTTPException(status_code=HTTPStatus.FORBIDDEN, detail="Access denied") engine_options = chat_message.chat.engine_options - chat_engine_config = ChatEngineConfig.validate(engine_options) + chat_engine_config = ChatEngineConfig.model_validate(engine_options) if chat_engine_config.knowledge_base: kb = knowledge_base_repo.must_get(session, chat_engine_config.knowledge_base.linked_knowledge_base.id) embed_model = get_kb_embed_model(session, kb) + graph_editor = get_kb_tidb_graph_editor(session, kb) else: embed_model = get_default_embedding_model(session) + graph_editor = legacy_tidb_graph_editor - entities, relations = get_chat_message_subgraph(session, chat_message, embed_model) + entities, relations = get_chat_message_subgraph(graph_editor, session, chat_message, embed_model) return SubgraphResponse(entities=entities, relationships=relations) diff --git a/backend/app/rag/chat.py b/backend/app/rag/chat.py index fd8502596..9ded9201f 100644 --- a/backend/app/rag/chat.py +++ b/backend/app/rag/chat.py @@ -55,7 +55,7 @@ from app.rag.knowledge_base.config import get_kb_embed_model from app.rag.knowledge_graph.graph_store import TiDBGraphStore from app.rag.vector_store.tidb_vector_store import TiDBVectorStore -from app.rag.knowledge_graph.graph_store.tidb_graph_editor import legacy_tidb_graph_editor +from app.rag.knowledge_graph.graph_store.tidb_graph_editor import TiDBGraphEditor from app.rag.knowledge_graph import KnowledgeGraphIndex from app.rag.chat_config import ChatEngineConfig, get_default_embedding_model, KnowledgeGraphOption @@ -995,6 +995,7 @@ def get_graph_data_from_langfuse(trace_url: str): def get_chat_message_subgraph( + graph_editor: TiDBGraphEditor, session: Session, chat_message: DBChatMessage, embed_model: BaseEmbedding, @@ -1007,12 +1008,12 @@ def get_chat_message_subgraph( # try to get subgraph from chat_message.graph_data try: if ( - chat_message.graph_data - and "relationships" in chat_message.graph_data - and len(chat_message.graph_data["relationships"]) > 0 + chat_message.graph_data + and "relationships" in chat_message.graph_data + and len(chat_message.graph_data["relationships"]) > 0 ): relationship_ids = chat_message.graph_data["relationships"] - all_entities, all_relationships = legacy_tidb_graph_editor.get_relationship_by_ids( + all_entities, all_relationships = graph_editor.get_relationship_by_ids( session, relationship_ids ) entities = [ From 7f184af0a048e26d625868386554efdd1cf0ddb5 Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Wed, 27 Nov 2024 15:39:36 +0800 Subject: [PATCH 022/114] ui(frontend): update delete datasource alerting --- frontend/app/src/components/datasource/datasource-card.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/app/src/components/datasource/datasource-card.tsx b/frontend/app/src/components/datasource/datasource-card.tsx index fe46d1a37..678bf097c 100644 --- a/frontend/app/src/components/datasource/datasource-card.tsx +++ b/frontend/app/src/components/datasource/datasource-card.tsx @@ -52,6 +52,8 @@ export function DatasourceCard ({ knowledgeBaseId, datasource }: { knowledgeBase await deleteDatasource(knowledgeBaseId, datasource.id); }} asChild + dialogTitle={`Confirm to delete the datasource ${datasource.name} #${datasource.id}`} + dialogDescription={<>All documents, chunks, entities and relationships related to this datasource will be deleted. This action cannot be undone.} > From b7cc40980f6ad2d8190dc48e8914c048bd143261 Mon Sep 17 00:00:00 2001 From: Cheese Date: Wed, 27 Nov 2024 16:46:26 +0800 Subject: [PATCH 023/114] feat: added evaluation checkpoint (#406) Added checkpoint feature for evaluation, in case cannot resume for the big batch. --- backend/.gitignore | 4 ++- backend/app/evaluation/evals.py | 58 ++++++++++++++++++++++++++------- 2 files changed, 50 insertions(+), 12 deletions(-) diff --git a/backend/.gitignore b/backend/.gitignore index c2e5176cf..ae4830d43 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -166,4 +166,6 @@ cython_debug/ .DS_Store # VSCode -.vscode/ \ No newline at end of file +.vscode/ + +checkpoint.json \ No newline at end of file diff --git a/backend/app/evaluation/evals.py b/backend/app/evaluation/evals.py index 22f4a327f..92ef1cba1 100644 --- a/backend/app/evaluation/evals.py +++ b/backend/app/evaluation/evals.py @@ -87,7 +87,7 @@ def __init__( "e2e_rag": E2ERagEvaluator(model="gpt-4o"), } - def runeval_dataset(self, csv_dataset: str, run_size: int = 30) -> None: + def runeval_dataset(self, csv_dataset: str, run_size: int = 30, checkpoint_file: str = "checkpoint.json", error_file: str = "eval_error.csv") -> None: if not os.path.exists(csv_dataset): raise FileNotFoundError(f"File not found: {csv_dataset}") @@ -96,19 +96,55 @@ def runeval_dataset(self, csv_dataset: str, run_size: int = 30) -> None: random.shuffle(eval_list) eval_list = eval_list[:run_size] + # checkpoint info ragas_list = [] + completed_queries = set() + if os.path.exists(checkpoint_file): + with open(checkpoint_file, "r") as f: + checkpoint_data = json.load(f) + completed_queries = set(checkpoint_data["completed_queries"]) + ragas_list = checkpoint_data["ragas_list"] + + # error info + error_list = [] + errored_queries = set() + if os.path.exists(error_file): + error_df = pd.read_csv(error_file) + error_list = error_df.to_dict(orient='records') + errored_queries = set(item["query"] for item in error_list) + for item in tqdm(eval_list): + if item['query'] in completed_queries or item['query'] in errored_queries: + continue # skip completed or errored queries + messages = [{"role": "user", "content": item['query']}] - response, _ = self._generate_answer_by_tidb_ai(messages) - user_input = json.dumps(messages) - - ragas_list.append({ - "user_input": user_input, - "reference": item["reference"], - "response": response, - # TODO: we cannot get retrieved_contexts now, due to the external engine - # "retrieved_contexts": [], - }) + try: + response, _ = self._generate_answer_by_tidb_ai(messages) + user_input = json.dumps(messages) + + ragas_list.append({ + "user_input": user_input, + "reference": item["reference"], + "response": response, + # TODO: we cannot get retrieved_contexts now, due to the external engine + # "retrieved_contexts": [], + }) + + # save the checkpoint file + completed_queries.add(item['query']) + checkpoint_data = { + "completed_queries": list(completed_queries), + "ragas_list": ragas_list + } + with open(checkpoint_file, "w") as f: + json.dump(checkpoint_data, f) + except Exception as e: + print(f"Error processing query: {item['query']}, error: {e}") + item["error_message"] = str(e) + error_list.append(item) # Add the item to the error list + + # Save the errors to the error file + pd.DataFrame(error_list).to_csv(error_file, index=False) ragas_dataset = EvaluationDataset.from_list(ragas_list) evaluator_llm = LangchainLLMWrapper(ChatOpenAI(model="gpt-4o")) From b8e0d15cc6c458edcdafe86fb1d3b1ca1500c6dc Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Wed, 27 Nov 2024 20:02:40 +0800 Subject: [PATCH 024/114] chore(frontend): cleanup codes --- frontend/app/src/components/divider.tsx | 12 ---------- .../src/components/managed-collapsible.tsx | 21 ---------------- frontend/app/src/components/managed-panel.tsx | 20 +--------------- frontend/app/src/components/option-detail.tsx | 24 +------------------ 4 files changed, 2 insertions(+), 75 deletions(-) delete mode 100644 frontend/app/src/components/divider.tsx delete mode 100644 frontend/app/src/components/managed-collapsible.tsx diff --git a/frontend/app/src/components/divider.tsx b/frontend/app/src/components/divider.tsx deleted file mode 100644 index 657e22b13..000000000 --- a/frontend/app/src/components/divider.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import { cn } from '@/lib/utils'; -import type { ReactNode } from 'react'; - -export function Divider ({ className, children }: { className: string, children: ReactNode }) { - return ( -
    - - {children} - -
    - ); -} \ No newline at end of file diff --git a/frontend/app/src/components/managed-collapsible.tsx b/frontend/app/src/components/managed-collapsible.tsx deleted file mode 100644 index 857a021a4..000000000 --- a/frontend/app/src/components/managed-collapsible.tsx +++ /dev/null @@ -1,21 +0,0 @@ -'use client'; - -import { ManagedPanelContext } from '@/components/managed-panel'; -import { Collapsible } from '@/components/ui/collapsible'; -import { type ComponentProps, useState } from 'react'; - -export interface ManagedDialogProps extends Omit, 'open' | 'onOpenChange'> { -} - -export function ManagedCollapsible (props: ManagedDialogProps) { - const [open, setOpen] = useState(false); - - return ( - - - - ); -} - -export { useManagedPanel as useManagedCollapsible } from './managed-panel'; - diff --git a/frontend/app/src/components/managed-panel.tsx b/frontend/app/src/components/managed-panel.tsx index 40341bb46..6ab3ff467 100644 --- a/frontend/app/src/components/managed-panel.tsx +++ b/frontend/app/src/components/managed-panel.tsx @@ -1,8 +1,6 @@ 'use client'; -import { CollapsibleTrigger } from '@/components/ui/collapsible'; -import type { CollapsibleTriggerProps } from '@radix-ui/react-collapsible'; -import { createContext, type Dispatch, type ReactNode, type SetStateAction, useContext } from 'react'; +import { createContext, type Dispatch, type SetStateAction, useContext } from 'react'; export const ManagedPanelContext = createContext<{ open: boolean, setOpen: Dispatch> }>({ open: false, @@ -12,19 +10,3 @@ export const ManagedPanelContext = createContext<{ open: boolean, setOpen: Dispa export function useManagedPanel () { return useContext(ManagedPanelContext); } - -export function ManagedPanelTrigger ({ on, off, ...props }: { on: ReactNode, off: ReactNode } & Omit) { - const { open } = useContext(ManagedPanelContext); - - return ( - - {open ? on : off} - - ); -} - -export function ManagedPanelStatic ({ on, off }: { on: ReactNode, off: ReactNode }) { - const { open } = useContext(ManagedPanelContext); - - return open ? on : off; -} diff --git a/frontend/app/src/components/option-detail.tsx b/frontend/app/src/components/option-detail.tsx index 8a310e31f..b76b2fe57 100644 --- a/frontend/app/src/components/option-detail.tsx +++ b/frontend/app/src/components/option-detail.tsx @@ -1,41 +1,19 @@ -import { ManagedCollapsible } from '@/components/managed-collapsible'; -import { ManagedPanelStatic, ManagedPanelTrigger } from '@/components/managed-panel'; -import { CollapsibleContent } from '@/components/ui/collapsible'; import { cn } from '@/lib/utils'; -import { PencilIcon, PencilOffIcon } from 'lucide-react'; import type { ReactNode } from 'react'; export function OptionDetail ({ valueClassName, title, value, - editPanel, }: { valueClassName?: string title: string value: ReactNode - editPanel?: ReactNode }) { return (
    {title}
    - {editPanel ? ( -
    - - } - off={{value}} - /> - - {editPanel} - - } - off={} - /> - -
    - ) :
    {value}
    } +
    {value}
    ); } From 944d206ba5437d1d021277bf1b091e2b8f549326 Mon Sep 17 00:00:00 2001 From: Jagger <634750802@qq.com> Date: Wed, 27 Nov 2024 21:50:26 +0800 Subject: [PATCH 025/114] style(frontend): refine theme colors --- frontend/app/.storybook/main.ts | 3 +- .../knowledge-bases/[id]/(tabs)/layout.tsx | 2 +- .../src/app/(main)/(admin)/llms/[id]/page.tsx | 2 +- .../(admin)/reranker-models/[id]/page.tsx | 2 +- frontend/app/src/app/(main)/nav.tsx | 4 +- frontend/app/src/app/globals.css | 26 +++- frontend/app/src/components/VersionStatus.tsx | 50 ------- frontend/app/src/components/cells/boolean.tsx | 2 +- .../app/src/components/cells/metadata.tsx | 2 +- .../chat-engine/chat-engines-table.tsx | 2 +- .../chat-engine/create-chat-engine-form.tsx | 8 +- .../src/components/chat/message-feedback.tsx | 70 +--------- .../app/src/components/chat/message-input.tsx | 2 +- .../components/chat/message-operations.tsx | 4 +- frontend/app/src/components/copy-button.tsx | 2 +- .../CreateEmbeddingModelForm.tsx | 2 +- .../components/feedbacks/feedbacks-table.tsx | 4 +- frontend/app/src/components/form/biz.tsx | 4 +- .../app/src/components/llm/CreateLLMForm.tsx | 2 +- .../reranker/CreateRerankerForm.tsx | 2 +- .../src/components/settings/SettingsField.tsx | 2 +- .../components/system/SystemWizardBanner.tsx | 2 +- frontend/app/src/components/theme.stories.tsx | 123 ++++++++++++++++++ frontend/app/src/components/ui/alert.tsx | 8 +- frontend/app/src/components/ui/sonner.tsx | 6 +- .../message-verify-result-markdown.tsx | 4 +- .../chat-verify-service/message-verify.tsx | 4 +- frontend/app/src/lib/ui-error.tsx | 15 +-- frontend/app/tailwind.config.ts | 12 ++ frontend/pnpm-lock.yaml | 18 +-- 30 files changed, 208 insertions(+), 181 deletions(-) delete mode 100644 frontend/app/src/components/VersionStatus.tsx create mode 100644 frontend/app/src/components/theme.stories.tsx diff --git a/frontend/app/.storybook/main.ts b/frontend/app/.storybook/main.ts index 4123d182a..c24a6a9d3 100644 --- a/frontend/app/.storybook/main.ts +++ b/frontend/app/.storybook/main.ts @@ -2,8 +2,7 @@ import type { StorybookConfig } from '@storybook/nextjs'; const config: StorybookConfig = { stories: [ - '../src/!(pages)/**/*.mdx', - '../src/!(pages)/**/*.stories.@(js|jsx|mjs|ts|tsx)', + '../src/!(pages)/**/*.stories.@(js|jsx|mjs|ts|tsx|mdx)', ], addons: [ '@storybook/addon-onboarding', diff --git a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/layout.tsx b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/layout.tsx index a7c529d6a..dff916f6d 100644 --- a/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/layout.tsx +++ b/frontend/app/src/app/(main)/(admin)/knowledge-bases/[id]/(tabs)/layout.tsx @@ -26,7 +26,7 @@ export default function KnowledgeBaseLayout ({ params, children }: { params: { i - +

    This Knowledge Base has no datasource.

    diff --git a/frontend/app/src/app/(main)/(admin)/llms/[id]/page.tsx b/frontend/app/src/app/(main)/(admin)/llms/[id]/page.tsx index 6c628174a..80eef8850 100644 --- a/frontend/app/src/app/(main)/(admin)/llms/[id]/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/llms/[id]/page.tsx @@ -32,7 +32,7 @@ export default function Page ({ params }: { params: { id: string } }) { } /> - + } /> } />
    diff --git a/frontend/app/src/app/(main)/(admin)/reranker-models/[id]/page.tsx b/frontend/app/src/app/(main)/(admin)/reranker-models/[id]/page.tsx index 8f49be176..a5f582c55 100644 --- a/frontend/app/src/app/(main)/(admin)/reranker-models/[id]/page.tsx +++ b/frontend/app/src/app/(main)/(admin)/reranker-models/[id]/page.tsx @@ -33,7 +33,7 @@ export default function Page ({ params }: { params: { id: string } }) { } /> - + } /> } />
    diff --git a/frontend/app/src/app/(main)/nav.tsx b/frontend/app/src/app/(main)/nav.tsx index c9974ad3d..6e71b9ee6 100644 --- a/frontend/app/src/app/(main)/nav.tsx +++ b/frontend/app/src/app/(main)/nav.tsx @@ -145,13 +145,13 @@ function NavFooter () { function NavWarningDetails ({ children }: { children?: ReactNode }) { if (!children) { - return ; + return ; } return ( - + {children} diff --git a/frontend/app/src/app/globals.css b/frontend/app/src/app/globals.css index 7d6f9b6f5..7dac9d9a1 100644 --- a/frontend/app/src/app/globals.css +++ b/frontend/app/src/app/globals.css @@ -75,7 +75,7 @@ --accent: 240 3.7% 15.9%; --accent-foreground: 0 0% 98%; - --destructive: 0 62.8% 30.6%; + --destructive: 0 72.2% 50.6%; --destructive-foreground: 0 0% 98%; --border: 240 3.7% 15.9%; @@ -114,4 +114,26 @@ } } -.grecaptcha-badge { visibility: hidden; } +@layer base { + :root { + --warning: 37.7 92.1% 50.2%; + --warning-foreground: 0 0% 98%; + --info: 198.6 88.7% 48.4%; + --info-foreground: 0 0% 98%; + --success: 142.1 70.6% 45.3%; + --success-foreground: 0 0% 98%; + } + + .dark { + --warning: 32.1 94.6% 43.7%; + --warning-foreground: 0 0% 98%; + --info: 200.4 98% 39.4%; + --info-foreground: 0 0% 98%; + --success: 142.1 76.2% 36.3%; + --success-foreground: 0 0% 98%; + } +} + +.grecaptcha-badge { + visibility: hidden; +} diff --git a/frontend/app/src/components/VersionStatus.tsx b/frontend/app/src/components/VersionStatus.tsx deleted file mode 100644 index 5886db825..000000000 --- a/frontend/app/src/components/VersionStatus.tsx +++ /dev/null @@ -1,50 +0,0 @@ -import { handleErrors, ServerError } from '@/lib/request'; -import { cn } from '@/lib/utils'; - -type Status = { - html_url: string - status: 'diverged' | 'identical' | 'ahead' | 'behind' - ahead_by: number - behind_by: number -} - -export interface VersionStatusProps { - gitCommitHash?: string | null; -} - -export async function VersionStatus ({ gitCommitHash }: VersionStatusProps) { - if (!gitCommitHash) { - return No git revision info; - } - try { - const response = await fetch(`https://api.github.com/repos/pingcap/tidb.ai/compare/HEAD...${gitCommitHash}`, { - headers: { - ...(process.env.GITHUB_TOKEN ? { Authorization: `Bearer ${process.env.GITHUB_TOKEN}` } : {}), - 'X-GitHub-Api-Version': '2022-11-28', - }, - }).then(handleErrors); - const data: Status = await response.json(); - switch (data.status) { - case 'identical': - return Updated; - case 'ahead': - return {data.ahead_by} commits ahead; - case 'behind': - return {data.behind_by} commits behind; - case 'diverged': - return {data.ahead_by} commits ahead, {data.behind_by} commits behind.; - } - } catch (e) { - if (e instanceof ServerError) { - if (e.response.status === 404) { - return Revision not found; - } - } - console.error(e); - return Failed to get revision info; - } -} - -function StatusDot ({ className }: { className: string }) { - return ; -} \ No newline at end of file diff --git a/frontend/app/src/components/cells/boolean.tsx b/frontend/app/src/components/cells/boolean.tsx index 5145d3bad..2685e9836 100644 --- a/frontend/app/src/components/cells/boolean.tsx +++ b/frontend/app/src/components/cells/boolean.tsx @@ -10,7 +10,7 @@ export function boolean (props: CellContext) { if (bool) { return ( - + Yes diff --git a/frontend/app/src/components/cells/metadata.tsx b/frontend/app/src/components/cells/metadata.tsx index e0f05979a..e71b9b4bf 100644 --- a/frontend/app/src/components/cells/metadata.tsx +++ b/frontend/app/src/components/cells/metadata.tsx @@ -19,7 +19,7 @@ export const metadataCell = (props: CellContext) => { warningEl = ( - + {warnings.length} diff --git a/frontend/app/src/components/chat-engine/chat-engines-table.tsx b/frontend/app/src/components/chat-engine/chat-engines-table.tsx index 110219ba8..0f1069246 100644 --- a/frontend/app/src/components/chat-engine/chat-engines-table.tsx +++ b/frontend/app/src/components/chat-engine/chat-engines-table.tsx @@ -32,7 +32,7 @@ const columns = [ name: `${name} Copy`, llm_id, fast_llm_id, engine_options, }) .then(newEngine => { - toast('Chat Engine successfully cloned.'); + toast.success('Chat Engine successfully cloned.'); startTransition(() => { router.push(`/chat-engines/${newEngine.id}`); }); diff --git a/frontend/app/src/components/chat-engine/create-chat-engine-form.tsx b/frontend/app/src/components/chat-engine/create-chat-engine-form.tsx index 90048d627..3a9b861d9 100644 --- a/frontend/app/src/components/chat-engine/create-chat-engine-form.tsx +++ b/frontend/app/src/components/chat-engine/create-chat-engine-form.tsx @@ -60,13 +60,7 @@ export function CreateChatEngineForm ({ defaultChatEngineOptions }: { defaultCha router.push(`/chat-engines/${ce.id}`); }); }, () => { - toast.error('Validation failed', { - description: 'Please check your chat engine configurations.', - classNames: { - toast: 'group-[.toaster]:bg-destructive group-[.toaster]:text-destructive-foreground', - description: 'group-[.toast]:text-destructive-foreground/70', - }, - }); + toast.error('Validation failed', { description: 'Please check your chat engine configurations.' }); }); return ( diff --git a/frontend/app/src/components/chat/message-feedback.tsx b/frontend/app/src/components/chat/message-feedback.tsx index 41a51254b..a4198f3a4 100644 --- a/frontend/app/src/components/chat/message-feedback.tsx +++ b/frontend/app/src/components/chat/message-feedback.tsx @@ -10,7 +10,6 @@ import { type ReactNode, useEffect, useState } from 'react'; export function MessageFeedback ({ initial, onFeedback, defaultAction, children }: { initial?: FeedbackParams, defaultAction?: 'like' | 'dislike', onFeedback: (action: 'like' | 'dislike', comment: string) => Promise, children: ReactNode }) { const [open, setOpen] = useState(false); const [action, setAction] = useState<'like' | 'dislike'>(initial?.feedback_type ?? defaultAction ?? 'like'); - // const [detail, setDetail] = useState>(() => (initial ?? {})); const [comment, setComment] = useState(initial?.comment ?? ''); const [running, setRunning] = useState(false); const [deleting, setDeleting] = useState(false); @@ -24,13 +23,11 @@ export function MessageFeedback ({ initial, onFeedback, defaultAction, children useEffect(() => { if (initial) { setAction(initial.feedback_type); - // setDetail(initial.knowledge_graph_detail); setComment(initial.comment); } }, [initial]); const disabled = running || deleting || !!initial; - const deleteDisabled = running || deleting || !initial; const container = usePortalContainer(); @@ -46,49 +43,16 @@ export function MessageFeedback ({ initial, onFeedback, defaultAction, children
    Do you like this answer
    setAction(value as any)}> - + Like - + Dislike
    - {/*
    */} - {/*
    Sources from Knowledge Graph
    */} - {/* {!source && sourceLoading &&
    Loading...
    }*/} - {/* {source && (*/} - {/*
      */} - {/* {source.markdownSources.kgRelationshipUrls.map(url => (*/} - {/*
    • */} - {/*
      */} - {/* */} - {/* */} - {/* {url}*/} - {/* */} - {/* */} - {/*

      {source.kgSources[url].description}

      */} - {/*
      */} - {/* setDetail(detail => {*/} - {/* if (action) {*/} - {/* return { ...detail, [url]: action };*/} - {/* } else {*/} - {/* detail = { ...detail };*/} - {/* delete detail[url];*/} - {/* return detail;*/} - {/* }*/} - {/* })}*/} - {/* />*/} - {/*
    • */} - {/* ))}*/} - {/*
    */} - {/* )}*/} - {/*
    */}