feat: tool call support interactive node

packages/global/common/string/markdown.ts

@@ -70,10 +70,10 @@ export const uploadMarkdownBase64 = async ({
   }
 
   // Remove white space on both sides of the picture
-  const trimReg = /(!\[.*\]\(.*\))\s*/g;
-  if (trimReg.test(rawText)) {
-    rawText = rawText.replace(trimReg, '$1');
-  }
+  // const trimReg = /(!\[.*\]\(.*\))\s*/g;
+  // if (trimReg.test(rawText)) {
+  //   rawText = rawText.replace(trimReg, '$1');
+  // }
 
   return rawText;
 };

packages/global/core/ai/type.d.ts

@@ -4,12 +4,14 @@ import type {
   ChatCompletionChunk,
   ChatCompletionMessageParam as SdkChatCompletionMessageParam,
   ChatCompletionToolMessageParam,
-  ChatCompletionAssistantMessageParam,
   ChatCompletionContentPart as SdkChatCompletionContentPart,
-  ChatCompletionUserMessageParam as SdkChatCompletionUserMessageParam
+  ChatCompletionUserMessageParam as SdkChatCompletionUserMessageParam,
+  ChatCompletionToolMessageParam as SdkChatCompletionToolMessageParam,
+  ChatCompletionAssistantMessageParam as SdkChatCompletionAssistantMessageParam,
+  ChatCompletionContentPartText
 } from 'openai/resources';
 import { ChatMessageTypeEnum } from './constants';
-import { InteractiveNodeResponseItemType } from '../workflow/template/system/interactive/type';
+import { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type';
 export * from 'openai/resources';
 
 // Extension of ChatCompletionMessageParam, Add file url type
@@ -22,18 +24,31 @@ export type ChatCompletionContentPartFile = {
 export type ChatCompletionContentPart =
   | SdkChatCompletionContentPart
   | ChatCompletionContentPartFile;
-type CustomChatCompletionUserMessageParam = {
-  content: string | Array<ChatCompletionContentPart>;
+type CustomChatCompletionUserMessageParam = Omit<ChatCompletionUserMessageParam, 'content'> & {
   role: 'user';
+  content: string | Array<ChatCompletionContentPart>;
+};
+type CustomChatCompletionToolMessageParam = SdkChatCompletionToolMessageParam & {
+  role: 'tool';
   name?: string;
 };
+type CustomChatCompletionAssistantMessageParam = SdkChatCompletionAssistantMessageParam & {
+  role: 'assistant';
+  interactive?: WorkflowInteractiveResponseType;
+};
 
 export type ChatCompletionMessageParam = (
-  | Exclude<SdkChatCompletionMessageParam, SdkChatCompletionUserMessageParam>
+  | Exclude<
+      SdkChatCompletionMessageParam,
+      | SdkChatCompletionUserMessageParam
+      | SdkChatCompletionToolMessageParam
+      | SdkChatCompletionAssistantMessageParam
+    >
   | CustomChatCompletionUserMessageParam
+  | CustomChatCompletionToolMessageParam
+  | CustomChatCompletionAssistantMessageParam
 ) & {
   dataId?: string;
-  interactive?: InteractiveNodeResponseItemType;
 };
 export type SdkChatCompletionMessageParam = SdkChatCompletionMessageParam;
 
@@ -47,11 +62,12 @@ export type ChatCompletionMessageToolCall = ChatCompletionMessageToolCall & {
   toolName?: string;
   toolAvatar?: string;
 };
-export type ChatCompletionMessageFunctionCall = ChatCompletionAssistantMessageParam.FunctionCall & {
-  id?: string;
-  toolName?: string;
-  toolAvatar?: string;
-};
+export type ChatCompletionMessageFunctionCall =
+  SdkChatCompletionAssistantMessageParam.FunctionCall & {
+    id?: string;
+    toolName?: string;
+    toolAvatar?: string;
+  };
 
 // Stream response
 export type StreamChatType = Stream<ChatCompletionChunk>;

packages/global/core/chat/type.d.ts

@@ -15,7 +15,7 @@ import type { AppSchema as AppType } from '@fastgpt/global/core/app/type.d';
 import { DatasetSearchModeEnum } from '../dataset/constants';
 import { DispatchNodeResponseType } from '../workflow/runtime/type.d';
 import { ChatBoxInputType } from '../../../../projects/app/src/components/core/chat/ChatContainer/ChatBox/type';
-import { InteractiveNodeResponseItemType } from '../workflow/template/system/interactive/type';
+import { WorkflowInteractiveResponseType } from '../workflow/template/system/interactive/type';
 
 export type ChatSchema = {
   _id: string;
@@ -73,7 +73,7 @@ export type AIChatItemValueItemType = {
     content: string;
   };
   tools?: ToolModuleResponseItemType[];
-  interactive?: InteractiveNodeResponseItemType;
+  interactive?: WorkflowInteractiveResponseType;
 };
 export type AIChatItemType = {
   obj: ChatRoleEnum.AI;

packages/global/core/chat/utils.ts

@@ -143,3 +143,33 @@ export const getChatSourceByPublishChannel = (publishChannel: PublishChannelEnum
       return ChatSourceEnum.online;
   }
 };
+
+/* 
+  Merge chat responseData
+  1. Same tool mergeSignId (Interactive tool node)
+*/
+export const mergeChatResponseData = (responseDataList: ChatHistoryItemResType[]) => {
+  let lastResponse: ChatHistoryItemResType | undefined = undefined;
+
+  return responseDataList.reduce<ChatHistoryItemResType[]>((acc, curr) => {
+    if (
+      lastResponse &&
+      lastResponse.toolMergeSignId &&
+      curr.toolMergeSignId === lastResponse.toolMergeSignId
+    ) {
+      // 替换 lastResponse
+      const concatResponse: ChatHistoryItemResType = {
+        ...curr,
+        runningTime: +((lastResponse.runningTime || 0) + (curr.runningTime || 0)).toFixed(2),
+        totalPoints: (lastResponse.totalPoints || 0) + (curr.totalPoints || 0),
+        childTotalPoints: (lastResponse.childTotalPoints || 0) + (curr.childTotalPoints || 0),
+        toolCallTokens: (lastResponse.toolCallTokens || 0) + (curr.toolCallTokens || 0),
+        toolDetail: [...(lastResponse.toolDetail || []), ...(curr.toolDetail || [])]
+      };
+      return [...acc.slice(0, -1), concatResponse];
+    } else {
+      lastResponse = curr;
+      return [...acc, curr];
+    }
+  }, []);
+};

packages/global/core/workflow/runtime/type.d.ts

@@ -73,7 +73,7 @@ export type RuntimeNodeItemType = {
   intro?: StoreNodeItemType['intro'];
   flowNodeType: StoreNodeItemType['flowNodeType'];
   showStatus?: StoreNodeItemType['showStatus'];
-  isEntry?: StoreNodeItemType['isEntry'];
+  isEntry?: boolean;
 
   inputs: FlowNodeInputItemType[];
   outputs: FlowNodeOutputItemType[];
@@ -114,6 +114,7 @@ export type DispatchNodeResponseType = {
   model?: string;
   contextTotalLen?: number;
   totalPoints?: number;
+  childTotalPoints?: number;
 
   // chat
   temperature?: number;
@@ -158,6 +159,7 @@ export type DispatchNodeResponseType = {
   toolCallTokens?: number;
   toolDetail?: ChatHistoryItemResType[];
   toolStop?: boolean;
+  toolMergeSignId?: string;
 
   // code
   codeLog?: string;

packages/global/core/workflow/runtime/utils.ts

@@ -69,7 +69,7 @@ export const initWorkflowEdgeStatus = (
   histories?: ChatItemType[]
 ): RuntimeEdgeItemType[] => {
   // If there is a history, use the last interactive value
-  if (!!histories) {
+  if (histories && histories.length > 0) {
     const memoryEdges = getLastInteractiveValue(histories)?.memoryEdges;
 
     if (memoryEdges && memoryEdges.length > 0) {
@@ -90,7 +90,7 @@ export const getWorkflowEntryNodeIds = (
   histories?: ChatItemType[]
 ) => {
   // If there is a history, use the last interactive entry node
-  if (!!histories) {
+  if (histories && histories.length > 0) {
     const entryNodeIds = getLastInteractiveValue(histories)?.entryNodeIds;
 
     if (Array.isArray(entryNodeIds) && entryNodeIds.length > 0) {

packages/global/core/workflow/template/system/interactive/type.d.ts

@@ -1,8 +1,9 @@
-import { NodeOutputItemType } from '../../../../chat/type';
-import { FlowNodeOutputItemType } from '../../../type/io';
-import { RuntimeEdgeItemType } from '../../../runtime/type';
+import type { NodeOutputItemType } from '../../../../chat/type';
+import type { FlowNodeOutputItemType } from '../../../type/io';
+import type { RuntimeEdgeItemType } from '../../../runtime/type';
 import { FlowNodeInputTypeEnum } from 'core/workflow/node/constant';
 import { WorkflowIOValueTypeEnum } from 'core/workflow/constants';
+import type { ChatCompletionMessageParam } from '../../../../ai/type';
 
 export type UserSelectOptionItemType = {
   key: string;
@@ -32,6 +33,12 @@ type InteractiveBasicType = {
   entryNodeIds: string[];
   memoryEdges: RuntimeEdgeItemType[];
   nodeOutputs: NodeOutputItemType[];
+
+  toolParams?: {
+    entryNodeIds: string[]; // 记录工具中，交互节点的 Id，而不是起始工作流的入口
+    memoryMessages: ChatCompletionMessageParam[]; // 这轮工具中，产生的新的 messages
+    toolCallId: string; // 记录对应 tool 的id，用于后续交互节点可以替换掉 tool 的 response
+  };
 };
 
 type UserSelectInteractive = {
@@ -52,5 +59,5 @@ type UserInputInteractive = {
   };
 };
 
-export type InteractiveNodeResponseItemType = InteractiveBasicType &
-  (UserSelectInteractive | UserInputInteractive);
+export type InteractiveNodeResponseType = UserSelectInteractive | UserInputInteractive;
+export type WorkflowInteractiveResponseType = InteractiveBasicType & InteractiveNodeResponseType;

packages/service/core/chat/saveChat.ts

@@ -12,6 +12,7 @@ import { mongoSessionRun } from '../../common/mongo/sessionRun';
 import { StoreNodeItemType } from '@fastgpt/global/core/workflow/type/node';
 import { getAppChatConfig, getGuideModule } from '@fastgpt/global/core/workflow/utils';
 import { AppChatConfigType } from '@fastgpt/global/core/app/type';
+import { mergeChatResponseData } from '@fastgpt/global/core/chat/utils';
 
 type Props = {
   chatId: string;
@@ -143,6 +144,7 @@ export const updateInteractiveChat = async ({
 
   if (!chatItem || chatItem.obj !== ChatRoleEnum.AI) return;
 
+  // Update interactive value
   const interactiveValue = chatItem.value[chatItem.value.length - 1];
 
   if (
@@ -194,7 +196,7 @@ export const updateInteractiveChat = async ({
 
   if (aiResponse.responseData) {
     chatItem.responseData = chatItem.responseData
-      ? [...chatItem.responseData, ...aiResponse.responseData]
+      ? mergeChatResponseData([...chatItem.responseData, ...aiResponse.responseData])
       : aiResponse.responseData;
   }