From b58249fc3a1ef9bca876c47675967ba8d12a67eb Mon Sep 17 00:00:00 2001 From: Archer <545436317@qq.com> Date: Fri, 8 Dec 2023 15:01:11 +0800 Subject: [PATCH] 4.6.4-alpha (#582) --- files/deploy/fastgpt/docker-compose.yml | 3 +- packages/global/common/error/code/common.ts | 24 +++ packages/global/common/error/code/dataset.ts | 10 +- packages/global/common/error/errorCode.ts | 4 +- packages/global/common/file/api.d.ts | 7 + packages/global/common/file/tools.ts | 9 +- packages/global/common/string/markdown.ts | 8 +- packages/global/common/string/textSplitter.ts | 66 ++++--- packages/global/core/app/utils.ts | 2 +- packages/global/core/chat/constants.ts | 2 + .../module/template/system/datasetSearch.ts | 13 +- .../service/common/file/gridfs/controller.ts | 60 ++++++- .../service/common/file/image/controller.ts | 17 +- packages/service/common/file/image/schema.ts | 8 +- packages/service/common/response/index.ts | 2 +- .../service/core/dataset/data/controller.ts | 27 +-- .../service/core/dataset/file/controller.ts | 9 - .../service/support/permission/auth/common.ts | 2 +- .../support/permission/auth/dataset.ts | 5 + projects/app/public/locales/en/common.json | 24 ++- projects/app/public/locales/zh/common.json | 26 ++- .../src/components/ChatBox/MessageInput.tsx | 139 ++++++++------- .../components/ChatBox/WholeResponseModal.tsx | 44 ++++- projects/app/src/components/ChatBox/index.tsx | 75 ++++---- .../Icon/icons/core/chat/quoteSign.svg | 11 ++ .../Icon/icons/core/chat/sendLight.svg | 8 + projects/app/src/components/Icon/index.tsx | 2 + .../app/src/components/Markdown/CodeLight.tsx | 2 +- .../src/components/Markdown/chat/Guide.tsx | 4 +- .../Markdown/chat/QuestionGuide.tsx | 92 ++++++++++ .../src/components/Markdown/chat/Quote.tsx | 64 ------- .../app/src/components/Markdown/img/Image.tsx | 2 +- .../app/src/components/Markdown/index.tsx | 73 +++++++- .../src/components/common/MyRadio/index.tsx | 2 +- .../core/module/DatasetParamsModal.tsx | 54 +++--- .../Flow/components/render/RenderInput.tsx | 24 ++- projects/app/src/constants/app.ts | 2 +- projects/app/src/global/core/chat/api.d.ts | 2 +- projects/app/src/global/core/prompt/AIChat.ts | 106 +++++++---- .../timeTasks/checkUnValidDatasetFiles.ts | 7 +- .../app/src/pages/api/common/file/read.ts | 5 + .../src/pages/api/common/file/uploadImage.ts | 13 +- .../api/core/ai/agent/createQuestionGuide.ts | 5 +- projects/app/src/pages/api/core/app/update.ts | 33 ++++ .../src/pages/api/core/chat/item/delete.ts | 4 + .../src/pages/api/core/chat/item/getSpeech.ts | 4 +- .../api/core/dataset/collection/delete.ts | 4 +- .../api/core/dataset/collection/sync/link.ts | 1 - .../app/src/pages/api/core/dataset/update.ts | 7 +- .../detail/components/SimpleEdit/index.tsx | 14 +- projects/app/src/pages/app/detail/index.tsx | 164 +++++++++--------- projects/app/src/pages/chat/index.tsx | 2 +- projects/app/src/pages/chat/share.tsx | 18 +- .../detail/components/Import/FileSelect.tsx | 4 +- .../detail/components/InputDataModal.tsx | 2 +- projects/app/src/pages/login/provider.tsx | 1 - .../app/src/service/core/dataset/data/pg.ts | 62 +++++-- .../src/service/moduleDispatch/chat/oneapi.ts | 4 + .../app/src/utils/service/core/chat/index.ts | 1 - projects/app/src/web/common/file/api.ts | 4 +- .../app/src/web/common/file/controller.ts | 34 ++-- projects/app/src/web/common/file/utils.ts | 17 +- projects/app/src/web/common/hooks/useToast.ts | 2 +- projects/app/src/web/common/utils/eventbus.ts | 3 +- projects/app/src/web/core/app/templates.ts | 34 +--- projects/app/src/web/core/chat/storeChat.ts | 10 +- 66 files changed, 964 insertions(+), 529 deletions(-) create mode 100644 packages/global/common/error/code/common.ts delete mode 100644 packages/service/core/dataset/file/controller.ts create mode 100644 projects/app/src/components/Icon/icons/core/chat/quoteSign.svg create mode 100644 projects/app/src/components/Icon/icons/core/chat/sendLight.svg create mode 100644 projects/app/src/components/Markdown/chat/QuestionGuide.tsx delete mode 100644 projects/app/src/components/Markdown/chat/Quote.tsx diff --git a/files/deploy/fastgpt/docker-compose.yml b/files/deploy/fastgpt/docker-compose.yml index 2f54448d01a..896c0bcd560 100644 --- a/files/deploy/fastgpt/docker-compose.yml +++ b/files/deploy/fastgpt/docker-compose.yml @@ -1,4 +1,5 @@ # 非 host 版本, 不使用本机代理 +# (不懂 Docker 的，只需要关心 OPENAI_BASE_URL 和 CHAT_API_KEY 即可！) version: '3.3' services: pg: @@ -47,7 +48,7 @@ services: environment: # root 密码，用户名为: root - DEFAULT_ROOT_PSW=1234 - # 中转地址，如果是用官方号，不需要管 + # 中转地址，如果是用官方号，不需要管。务必加 /v1 - OPENAI_BASE_URL=https://api.openai.com/v1 - CHAT_API_KEY=sk-xxxx - DB_MAX_LINK=5 # database max link diff --git a/packages/global/common/error/code/common.ts b/packages/global/common/error/code/common.ts new file mode 100644 index 00000000000..323ed8ad797 --- /dev/null +++ b/packages/global/common/error/code/common.ts @@ -0,0 +1,24 @@ +import { ErrType } from '../errorCode'; + +/* dataset: 507000 */ +const startCode = 507000; +export enum CommonErrEnum { + fileNotFound = 'fileNotFound' +} +const datasetErr = [ + { + statusText: CommonErrEnum.fileNotFound, + message: 'error.fileNotFound' + } +]; +export default datasetErr.reduce((acc, cur, index) => { + return { + ...acc, + [cur.statusText]: { + code: startCode + index, + statusText: cur.statusText, + message: cur.message, + data: null + } + }; +}, {} as ErrType<`${CommonErrEnum}`>); diff --git a/packages/global/common/error/code/dataset.ts b/packages/global/common/error/code/dataset.ts index bc995507640..ae1ea556dab 100644 --- a/packages/global/common/error/code/dataset.ts +++ b/packages/global/common/error/code/dataset.ts @@ -13,23 +13,23 @@ export enum DatasetErrEnum { const datasetErr = [ { statusText: DatasetErrEnum.unAuthDataset, - message: '无权操作该知识库' + message: 'core.dataset.error.unAuthDataset' }, { statusText: DatasetErrEnum.unAuthDatasetCollection, - message: '无权操作该数据集' + message: 'core.dataset.error.unAuthDatasetCollection' }, { statusText: DatasetErrEnum.unAuthDatasetData, - message: '无权操作该数据' + message: 'core.dataset.error.unAuthDatasetData' }, { statusText: DatasetErrEnum.unAuthDatasetFile, - message: '无权操作该文件' + message: 'core.dataset.error.unAuthDatasetFile' }, { statusText: DatasetErrEnum.unCreateCollection, - message: '无权创建数据集' + message: 'core.dataset.error.unCreateCollection' }, { statusText: DatasetErrEnum.unLinkCollection, diff --git a/packages/global/common/error/errorCode.ts b/packages/global/common/error/errorCode.ts index 380c3b33b39..4fbc28ae5a7 100644 --- a/packages/global/common/error/errorCode.ts +++ b/packages/global/common/error/errorCode.ts @@ -6,6 +6,7 @@ import pluginErr from './code/plugin'; import outLinkErr from './code/outLink'; import teamErr from './code/team'; import userErr from './code/user'; +import commonErr from './code/common'; export const ERROR_CODE: { [key: number]: string } = { 400: '请求失败', @@ -96,5 +97,6 @@ export const ERROR_RESPONSE: Record< ...outLinkErr, ...teamErr, ...userErr, - ...pluginErr + ...pluginErr, + ...commonErr }; diff --git a/packages/global/common/file/api.d.ts b/packages/global/common/file/api.d.ts index 40b2078e671..d086baf64bf 100644 --- a/packages/global/common/file/api.d.ts +++ b/packages/global/common/file/api.d.ts @@ -1,3 +1,10 @@ +export type UploadImgProps = { + base64Img: string; + expiredTime?: Date; + metadata?: Record; + shareId?: string; +}; + export type UrlFetchParams = { urlList: string[]; selector?: string; diff --git a/packages/global/common/file/tools.ts b/packages/global/common/file/tools.ts index b7c9e5d86b3..36aa5087db8 100644 --- a/packages/global/common/file/tools.ts +++ b/packages/global/common/file/tools.ts @@ -49,7 +49,14 @@ export const cheerioToHtml = ({ } }); - return $(selector || 'body').html(); + const html = $(selector || 'body') + .map((item, dom) => { + return $(dom).html(); + }) + .get() + .join('\n'); + + return html; }; export const urlsFetch = async ({ urlList, diff --git a/packages/global/common/string/markdown.ts b/packages/global/common/string/markdown.ts index 5eb3c551841..e1e10042e3a 100644 --- a/packages/global/common/string/markdown.ts +++ b/packages/global/common/string/markdown.ts @@ -26,10 +26,14 @@ export const simpleMarkdownText = (rawText: string) => { rawText = rawText.replace(/\\\\n/g, '\\n'); // Remove headings and code blocks front spaces - ['####', '###', '##', '#', '```', '~~~'].forEach((item) => { + ['####', '###', '##', '#', '```', '~~~'].forEach((item, i) => { + const isMarkdown = i <= 3; const reg = new RegExp(`\\n\\s*${item}`, 'g'); if (reg.test(rawText)) { - rawText = rawText.replace(new RegExp(`\\n\\s*(${item})`, 'g'), '\n$1'); + rawText = rawText.replace( + new RegExp(`(\\n)\\s*(${item})`, 'g'), + isMarkdown ? '\n$1$2' : '$1$2' + ); } }); diff --git a/packages/global/common/string/textSplitter.ts b/packages/global/common/string/textSplitter.ts index 59e7b5f9131..6291fbe7950 100644 --- a/packages/global/common/string/textSplitter.ts +++ b/packages/global/common/string/textSplitter.ts @@ -12,12 +12,13 @@ export const splitText2Chunks = (props: { text: string; chunkLen: number; overlapRatio?: number; + customReg?: string[]; }): { chunks: string[]; tokens: number; overlapRatio?: number; } => { - let { text = '', chunkLen, overlapRatio = 0.2 } = props; + let { text = '', chunkLen, overlapRatio = 0.2, customReg = [] } = props; const splitMarker = 'SPLIT_HERE_SPLIT_HERE'; const codeBlockMarker = 'CODE_BLOCK_LINE_MARKER'; const overlapLen = Math.round(chunkLen * overlapRatio); @@ -29,22 +30,29 @@ export const splitText2Chunks = (props: { // The larger maxLen is, the next sentence is less likely to trigger splitting const stepReges: { reg: RegExp; maxLen: number }[] = [ - { reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 }, - { reg: /^(##\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 }, - { reg: /^(###\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 }, - { reg: /^(####\s[^\n]+)\n/gm, maxLen: chunkLen * 1.4 }, - - { reg: /([\n](`))/g, maxLen: chunkLen * 4 }, // code block - { reg: /([\n](?![\*\-|>0-9]))/g, maxLen: chunkLen * 1.8 }, // (?![\*\-|>`0-9]): markdown special char - { reg: /([\n])/g, maxLen: chunkLen * 1.4 }, - - { reg: /([。]|([a-zA-Z])\.\s)/g, maxLen: chunkLen * 1.4 }, - { reg: /([！]|!\s)/g, maxLen: chunkLen * 1.4 }, - { reg: /([？]|\?\s)/g, maxLen: chunkLen * 1.6 }, - { reg: /([；]|;\s)/g, maxLen: chunkLen * 1.8 }, + ...customReg.map((text) => ({ reg: new RegExp(`([${text}])`, 'g'), maxLen: chunkLen * 1.4 })), + { reg: /^(#\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 }, + { reg: /^(##\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 }, + { reg: /^(###\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 }, + { reg: /^(####\s[^\n]+)\n/gm, maxLen: chunkLen * 1.2 }, + + { reg: /([\n]([`~]))/g, maxLen: chunkLen * 4 }, // code block + { reg: /([\n](?!\s*[\*\-|>0-9]))/g, maxLen: chunkLen * 2 }, // (?![\*\-|>`0-9]): markdown special char + { reg: /([\n])/g, maxLen: chunkLen * 1.2 }, + + { reg: /([。]|([a-zA-Z])\.\s)/g, maxLen: chunkLen * 1.2 }, + { reg: /([！]|!\s)/g, maxLen: chunkLen * 1.2 }, + { reg: /([？]|\?\s)/g, maxLen: chunkLen * 1.4 }, + { reg: /([；]|;\s)/g, maxLen: chunkLen * 1.6 }, { reg: /([，]|,\s)/g, maxLen: chunkLen * 2 } ]; + const customRegLen = customReg.length; + const checkIsCustomStep = (step: number) => step < customRegLen; + const checkIsMarkdownSplit = (step: number) => step >= customRegLen && step <= 3 + customRegLen; + const checkIndependentChunk = (step: number) => step >= customRegLen && step <= 4 + customRegLen; + const checkForbidOverlap = (step: number) => step <= 6 + customRegLen; + // if use markdown title split, Separate record title title const getSplitTexts = ({ text, step }: { text: string; step: number }) => { if (step >= stepReges.length) { @@ -55,11 +63,13 @@ export const splitText2Chunks = (props: { } ]; } - const isMarkdownSplit = step <= 3; + const isMarkdownSplit = checkIsMarkdownSplit(step); + const independentChunk = checkIndependentChunk(step); + const { reg } = stepReges[step]; const splitTexts = text - .replace(reg, isMarkdownSplit ? `${splitMarker}$1` : `$1${splitMarker}`) + .replace(reg, independentChunk ? `${splitMarker}$1` : `$1${splitMarker}`) .split(`${splitMarker}`) .filter((part) => part.trim()); @@ -76,7 +86,7 @@ export const splitText2Chunks = (props: { }; const getOneTextOverlapText = ({ text, step }: { text: string; step: number }): string => { - const forbidOverlap = step <= 6; + const forbidOverlap = checkForbidOverlap(step); const maxOverlapLen = chunkLen * 0.4; // step >= stepReges.length: Do not overlap incomplete sentences @@ -114,7 +124,8 @@ export const splitText2Chunks = (props: { lastText: string; mdTitle: string; }): string[] => { - const isMarkdownSplit = step <= 3; + const independentChunk = checkIndependentChunk(step); + const isCustomStep = checkIsCustomStep(step); // mini text if (text.length <= chunkLen) { @@ -134,12 +145,13 @@ export const splitText2Chunks = (props: { return chunks; } - const { maxLen } = stepReges[step]; - const minChunkLen = chunkLen * 0.7; - // split text by special char const splitTexts = getSplitTexts({ text, step }); + const maxLen = splitTexts.length > 1 ? stepReges[step].maxLen : chunkLen; + const minChunkLen = chunkLen * 0.7; + const miniChunkLen = 30; + const chunks: string[] = []; for (let i = 0; i < splitTexts.length; i++) { const item = splitTexts[i]; @@ -170,8 +182,8 @@ export const splitText2Chunks = (props: { mdTitle: currentTitle }); const lastChunk = innerChunks[innerChunks.length - 1]; - // last chunk is too small, concat it to lastText - if (!isMarkdownSplit && lastChunk.length < minChunkLen) { + // last chunk is too small, concat it to lastText(next chunk start) + if (!independentChunk && lastChunk.length < minChunkLen) { chunks.push(...innerChunks.slice(0, -1)); lastText = lastChunk; } else { @@ -189,10 +201,14 @@ export const splitText2Chunks = (props: { lastText = newText; // markdown paragraph block: Direct addition; If the chunk size reaches, add a chunk - if (isMarkdownSplit || newTextLen >= chunkLen) { + if ( + isCustomStep || + (independentChunk && newTextLen > miniChunkLen) || + newTextLen >= chunkLen + ) { chunks.push(`${currentTitle}${lastText}`); - lastText = isMarkdownSplit ? '' : getOneTextOverlapText({ text: lastText, step }); + lastText = getOneTextOverlapText({ text: lastText, step }); } } diff --git a/packages/global/core/app/utils.ts b/packages/global/core/app/utils.ts index dbe46aab8f6..1ab676443b1 100644 --- a/packages/global/core/app/utils.ts +++ b/packages/global/core/app/utils.ts @@ -24,7 +24,7 @@ export const getDefaultAppForm = (templateId = 'fastgpt-universal'): AppSimpleEd dataset: { datasets: [], similarity: 0.4, - limit: 5, + limit: 1500, searchEmptyText: '', searchMode: DatasetSearchModeEnum.embedding }, diff --git a/packages/global/core/chat/constants.ts b/packages/global/core/chat/constants.ts index 000884eeca0..8b5c322b43c 100644 --- a/packages/global/core/chat/constants.ts +++ b/packages/global/core/chat/constants.ts @@ -55,3 +55,5 @@ export const LOGO_ICON = `/icon/logo.svg`; export const IMG_BLOCK_KEY = 'img-block'; export const FILE_BLOCK_KEY = 'file-block'; + +export const MARKDOWN_QUOTE_SIGN = 'QUOTE SIGN'; diff --git a/packages/global/core/module/template/system/datasetSearch.ts b/packages/global/core/module/template/system/datasetSearch.ts index c876b2c1eb6..b72b3309329 100644 --- a/packages/global/core/module/template/system/datasetSearch.ts +++ b/packages/global/core/module/template/system/datasetSearch.ts @@ -54,17 +54,10 @@ export const DatasetSearchModule: FlowModuleTemplateType = { { key: ModuleInputKeyEnum.datasetLimit, type: FlowNodeInputTypeEnum.hidden, - label: '单次搜索上限', - description: '最多取 n 条记录作为本次问题引用', - value: 5, + label: '引用上限', + description: '单次搜索最大的 Tokens 数量，中文约1字=1.7Tokens，英文约1字=1Tokens', + value: 1500, valueType: ModuleDataTypeEnum.number, - min: 1, - max: 20, - step: 1, - markList: [ - { label: '1', value: 1 }, - { label: '20', value: 20 } - ], showTargetInApp: false, showTargetInPlugin: false }, diff --git a/packages/service/common/file/gridfs/controller.ts b/packages/service/common/file/gridfs/controller.ts index 7174917929c..84a05d631b0 100644 --- a/packages/service/common/file/gridfs/controller.ts +++ b/packages/service/common/file/gridfs/controller.ts @@ -3,6 +3,7 @@ import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; import fsp from 'fs/promises'; import fs from 'fs'; import { DatasetFileSchema } from '@fastgpt/global/core/dataset/type'; +import { delImgByFileIdList } from '../image/controller'; export function getGFSCollection(bucket: `${BucketNameEnum}`) { return connectionMongo.connection.db.collection(`${bucket}.files`); @@ -69,24 +70,65 @@ export async function getFileById({ _id: new Types.ObjectId(fileId) }); - if (!file) { - return Promise.reject('File not found'); - } + // if (!file) { + // return Promise.reject('File not found'); + // } - return file; + return file || undefined; } -export async function delFileById({ +export async function delFileByFileIdList({ bucketName, - fileId + fileIdList, + retry = 3 }: { bucketName: `${BucketNameEnum}`; - fileId: string; + fileIdList: string[]; + retry?: number; +}): Promise { + try { + const bucket = getGridBucket(bucketName); + + await Promise.all(fileIdList.map((id) => bucket.delete(new Types.ObjectId(id)))); + } catch (error) { + if (retry > 0) { + return delFileByFileIdList({ bucketName, fileIdList, retry: retry - 1 }); + } + } +} +// delete file by metadata(datasetId) +export async function delFileByMetadata({ + bucketName, + datasetId +}: { + bucketName: `${BucketNameEnum}`; + datasetId?: string; }) { const bucket = getGridBucket(bucketName); - await bucket.delete(new Types.ObjectId(fileId)); - return true; + const files = await bucket + .find( + { + ...(datasetId && { 'metadata.datasetId': datasetId }) + }, + { + projection: { + _id: 1 + } + } + ) + .toArray(); + + const idList = files.map((item) => String(item._id)); + + // delete img + await delImgByFileIdList(idList); + + // delete file + await delFileByFileIdList({ + bucketName, + fileIdList: idList + }); } export async function getDownloadStream({ diff --git a/packages/service/common/file/image/controller.ts b/packages/service/common/file/image/controller.ts index e8c1c4b97bd..7cfe448286e 100644 --- a/packages/service/common/file/image/controller.ts +++ b/packages/service/common/file/image/controller.ts @@ -1,3 +1,4 @@ +import { UploadImgProps } from '@fastgpt/global/common/file/api'; import { imageBaseUrl } from './constant'; import { MongoImage } from './schema'; @@ -9,11 +10,10 @@ export const maxImgSize = 1024 * 1024 * 12; export async function uploadMongoImg({ base64Img, teamId, - expiredTime -}: { - base64Img: string; + expiredTime, + metadata +}: UploadImgProps & { teamId: string; - expiredTime?: Date; }) { if (base64Img.length > maxImgSize) { return Promise.reject('Image too large'); @@ -24,7 +24,8 @@ export async function uploadMongoImg({ const { _id } = await MongoImage.create({ teamId, binary: Buffer.from(base64Data, 'base64'), - expiredTime + expiredTime: expiredTime, + metadata }); return getMongoImgUrl(String(_id)); @@ -37,3 +38,9 @@ export async function readMongoImg({ id }: { id: string }) { } return data?.binary; } + +export async function delImgByFileIdList(fileIds: string[]) { + return MongoImage.deleteMany({ + 'metadata.fileId': { $in: fileIds.map((item) => String(item)) } + }); +} diff --git a/packages/service/common/file/image/schema.ts b/packages/service/common/file/image/schema.ts index fbb484c002c..466518b9e59 100644 --- a/packages/service/common/file/image/schema.ts +++ b/packages/service/common/file/image/schema.ts @@ -5,13 +5,17 @@ const { Schema, model, models } = connectionMongo; const ImageSchema = new Schema({ teamId: { type: Schema.Types.ObjectId, - ref: TeamCollectionName + ref: TeamCollectionName, + required: true }, binary: { type: Buffer }, expiredTime: { type: Date + }, + metadata: { + type: Object } }); @@ -21,7 +25,7 @@ try { console.log(error); } -export const MongoImage: Model<{ teamId: string; binary: Buffer }> = +export const MongoImage: Model<{ teamId: string; binary: Buffer; metadata?: Record }> = models['image'] || model('image', ImageSchema); MongoImage.syncIndexes(); diff --git a/packages/service/common/response/index.ts b/packages/service/common/response/index.ts index 04a6b451d67..a82555abede 100644 --- a/packages/service/common/response/index.ts +++ b/packages/service/common/response/index.ts @@ -82,7 +82,7 @@ export const sseErrRes = (res: NextApiResponse, error: any) => { } else if (error?.response?.data?.error?.message) { msg = error?.response?.data?.error?.message; } else if (error?.error?.message) { - msg = error?.error?.message; + msg = `${error?.error?.code} ${error?.error?.message}`; } addLog.error(`sse error: ${msg}`, error); diff --git a/packages/service/core/dataset/data/controller.ts b/packages/service/core/dataset/data/controller.ts index 76a82fe8a78..c7ca8e592c0 100644 --- a/packages/service/core/dataset/data/controller.ts +++ b/packages/service/core/dataset/data/controller.ts @@ -1,11 +1,11 @@ import { MongoDatasetData } from './schema'; import { deletePgDataById } from './pg'; import { MongoDatasetTraining } from '../training/schema'; -import { delFileById } from '../../../common/file/gridfs/controller'; +import { delFileByFileIdList, delFileByMetadata } from '../../../common/file/gridfs/controller'; import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; import { MongoDatasetCollection } from '../collection/schema'; -import { delDatasetFiles } from '../file/controller'; import { delay } from '@fastgpt/global/common/system/utils'; +import { delImgByFileIdList } from '../../../common/file/image/controller'; /* delete all data by datasetIds */ export async function delDatasetRelevantData({ datasetIds }: { datasetIds: string[] }) { @@ -17,9 +17,11 @@ export async function delDatasetRelevantData({ datasetIds }: { datasetIds: strin }); // delete related files - await Promise.all(datasetIds.map((id) => delDatasetFiles({ datasetId: id }))); + await Promise.all( + datasetIds.map((id) => delFileByMetadata({ bucketName: BucketNameEnum.dataset, datasetId: id })) + ); - await delay(1000); + await delay(500); // delete pg data await deletePgDataById(`dataset_id IN ('${datasetIds.join("','")}')`); @@ -49,17 +51,16 @@ export async function delCollectionRelevantData({ collectionId: { $in: collectionIds } }); - // delete file - await Promise.all( - filterFileIds.map((fileId) => { - return delFileById({ - bucketName: BucketNameEnum.dataset, - fileId - }); + // delete file and imgs + await Promise.all([ + delImgByFileIdList(filterFileIds), + delFileByFileIdList({ + bucketName: BucketNameEnum.dataset, + fileIdList: filterFileIds }) - ); + ]); - await delay(1000); + await delay(500); // delete pg data await deletePgDataById(`collection_id IN ('${collectionIds.join("','")}')`); diff --git a/packages/service/core/dataset/file/controller.ts b/packages/service/core/dataset/file/controller.ts deleted file mode 100644 index bd3c38b0994..00000000000 --- a/packages/service/core/dataset/file/controller.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; -import { getGFSCollection } from '../../../common/file/gridfs/controller'; - -export async function delDatasetFiles({ datasetId }: { datasetId: string }) { - const db = getGFSCollection(BucketNameEnum.dataset); - await db.deleteMany({ - 'metadata.datasetId': String(datasetId) - }); -} diff --git a/packages/service/support/permission/auth/common.ts b/packages/service/support/permission/auth/common.ts index 682a8128781..65c269b963e 100644 --- a/packages/service/support/permission/auth/common.ts +++ b/packages/service/support/permission/auth/common.ts @@ -12,7 +12,7 @@ export const authCert = async (props: AuthModeType) => { canWrite: true }; }; -export async function authCertAndShareId({ +export async function authCertOrShareId({ shareId, ...props }: AuthModeType & { shareId?: string }) { diff --git a/packages/service/support/permission/auth/dataset.ts b/packages/service/support/permission/auth/dataset.ts index 8b867df1fce..aa3bbe90605 100644 --- a/packages/service/support/permission/auth/dataset.ts +++ b/packages/service/support/permission/auth/dataset.ts @@ -14,6 +14,7 @@ import { import { getFileById } from '../../../common/file/gridfs/controller'; import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; import { getTeamInfoByTmbId } from '../../user/team/controller'; +import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; export async function authDatasetByTmbId({ teamId, @@ -167,6 +168,10 @@ export async function authDatasetFile({ const file = await getFileById({ bucketName: BucketNameEnum.dataset, fileId }); + if (!file) { + return Promise.reject(CommonErrEnum.fileNotFound); + } + if (file.metadata.teamId !== teamId) { return Promise.reject(DatasetErrEnum.unAuthDatasetFile); } diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json index 4a98f6424d5..f4d3f2b8882 100644 --- a/projects/app/public/locales/en/common.json +++ b/projects/app/public/locales/en/common.json @@ -283,6 +283,11 @@ "Speaking": "I'm listening...", "Stop Speak": "Stop Speak", "Type a message": "Input problem", + "markdown": { + "Edit Question": "Edit Question", + "Quick Question": "Ask the question immediately", + "Send Question": "Send Question" + }, "quote": { "Quote Tip": "Only the actual reference content is displayed here. If the data is updated, it will not be updated in real time", "Read Quote": "Read Quote", @@ -290,6 +295,9 @@ }, "tts": { "Stop Speech": "Stop" + }, + "error": { + "Messages empty": "Interface content is empty, maybe the text is too long ~" } }, "dataset": { @@ -313,7 +321,6 @@ "Name": "Name", "Quote Length": "Quote Length", "Read Dataset": "Read Dataset", - "Search Top K": "Top K", "Set Empty Result Tip": ",Response empty text", "Set Website Config": "Configuring Website", "Similarity": "Similarity", @@ -372,6 +379,12 @@ "id": "Data ID" }, "error": { + "unAuthDataset": "No access to this knowledge base ", + "unAuthDatasetCollection": "Not authorized to manipulate this data set ", + "unAuthDatasetData": "Not authorized to manipulate this data ", + "unAuthDatasetFile": "No permission to manipulate this file ", + "unCreateCollection": "No permission to manipulate this data ", + "unLinkCollection": "not a network link collection ", "Start Sync Failed": "Start Sync Failed" }, "file": "File", @@ -403,8 +416,11 @@ }, "link": "Link", "search": { + "Dataset Search Params": "Dataset Search Params", "Empty result response": "Empty Response", "Empty result response Tips": "If you fill in the content, if no suitable content is found, you will directly reply to the content.", + "Max Tokens": "Max Tokens", + "Max Tokens Tips": "The maximum number of Tokens in a single search, about 1 word in Chinese =1.7Tokens, about 1 word in English =1 tokens", "Min Similarity": "Min Similarity", "Min Similarity Tips": "The similarity of different index models is different, please use the search test to select the appropriate value", "Params Setting": "Params Setting", @@ -517,7 +533,6 @@ } }, "dataset": { - "Chunk Length": "Chunk Length", "Confirm move the folder": "Confirm Move", "Confirm to delete the data": "Confirm to delete the data?", "Confirm to delete the file": "Are you sure to delete the file and all its data?", @@ -585,13 +600,10 @@ "import csv tip": "Ensure that the CSV is in UTF-8 format; otherwise, garbled characters will be displayed", "test": { "noResult": "Search results are empty" - }, - "website": { - "Base Url": "BaseUrl", - "Selector": "Selector" } }, "error": { + "fileNotFound": "File not found ~", "team": { "overSize": "Team member exceeds limit" } diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json index 48912a7658c..103ea1336d5 100644 --- a/projects/app/public/locales/zh/common.json +++ b/projects/app/public/locales/zh/common.json @@ -283,6 +283,11 @@ "Speaking": "我在听，请说...", "Stop Speak": "停止录音", "Type a message": "输入问题", + "markdown": { + "Edit Question": "编辑问题", + "Quick Question": "点我立即提问", + "Send Question": "发送问题" + }, "quote": { "Quote Tip": "此处仅显示实际引用内容，若数据有更新，此处不会实时更新", "Read Quote": "查看引用", @@ -290,6 +295,9 @@ }, "tts": { "Stop Speech": "停止" + }, + "error": { + "Messages empty": "接口内容为空，可能文本超长了~" } }, "dataset": { @@ -313,7 +321,6 @@ "Name": "知识库名称", "Quote Length": "引用内容长度", "Read Dataset": "查看知识库详情", - "Search Top K": "单次搜索数量", "Set Empty Result Tip": ",未搜索到内容时回复指定内容", "Set Website Config": "开始配置网站信息", "Similarity": "相关度", @@ -372,7 +379,13 @@ "id": "数据ID" }, "error": { - "Start Sync Failed": "开始同步失败" + "Start Sync Failed": "开始同步失败", + "unAuthDataset": "无权操作该知识库", + "unAuthDatasetCollection": "无权操作该数据集", + "unAuthDatasetData": "无权操作该数据", + "unAuthDatasetFile": "无权操作该文件", + "unCreateCollection": "无权操作该数据", + "unLinkCollection": "不是网络链接集合" }, "file": "文件", "folder": "目录", @@ -403,8 +416,11 @@ }, "link": "链接", "search": { + "Dataset Search Params": "搜索参数", "Empty result response": "空搜索回复", "Empty result response Tips": "若填写该内容，没有搜索到合适内容时，将直接回复填写的内容。", + "Max Tokens": "引用上限", + "Max Tokens Tips": "单次搜索最大的 Tokens 数量，中文约1字=1.7Tokens，英文约1字=1Tokens", "Min Similarity": "最低相关度", "Min Similarity Tips": "不同索引模型的相关度有区别，请通过搜索测试来选择合适的数值，使用 ReRank 时，相关度可能会很低。", "Params Setting": "搜索参数设置", @@ -517,7 +533,6 @@ } }, "dataset": { - "Chunk Length": "数据总量", "Confirm move the folder": "确认移动到该目录", "Confirm to delete the data": "确认删除该数据？", "Confirm to delete the file": "确认删除该文件及其所有数据？", @@ -585,13 +600,10 @@ "import csv tip": "请确保CSV为UTF-8格式，否则会乱码", "test": { "noResult": "搜索结果为空" - }, - "website": { - "Base Url": "", - "Selector": "" } }, "error": { + "fileNotFound": "文件找不到了~", "team": { "overSize": "团队成员超出上限" } diff --git a/projects/app/src/components/ChatBox/MessageInput.tsx b/projects/app/src/components/ChatBox/MessageInput.tsx index 22da4b3ddee..8555b4216ee 100644 --- a/projects/app/src/components/ChatBox/MessageInput.tsx +++ b/projects/app/src/components/ChatBox/MessageInput.tsx @@ -69,78 +69,85 @@ const MessageInput = ({ maxCount: 10 }); - const uploadFile = async (file: FileItemType) => { - if (file.type === FileTypeEnum.image) { - try { - const src = await compressImgFileAndUpload({ - file: file.rawFile, - maxW: 4329, - maxH: 4329, - maxSize: 1024 * 1024 * 5, - // 30 day expired. - expiredTime: addDays(new Date(), 30) - }); - setFileList((state) => - state.map((item) => - item.id === file.id - ? { - ...item, - src: `${location.origin}${src}` - } - : item - ) - ); - } catch (error) { - setFileList((state) => state.filter((item) => item.id !== file.id)); - console.log(error); + const uploadFile = useCallback( + async (file: FileItemType) => { + if (file.type === FileTypeEnum.image) { + try { + const src = await compressImgFileAndUpload({ + file: file.rawFile, + maxW: 4329, + maxH: 4329, + maxSize: 1024 * 1024 * 5, + // 30 day expired. + expiredTime: addDays(new Date(), 30), + shareId + }); + setFileList((state) => + state.map((item) => + item.id === file.id + ? { + ...item, + src: `${location.origin}${src}` + } + : item + ) + ); + } catch (error) { + setFileList((state) => state.filter((item) => item.id !== file.id)); + console.log(error); - toast({ - status: 'error', - title: t('common.Upload File Failed') - }); + toast({ + status: 'error', + title: t('common.Upload File Failed') + }); + } } - } - }; - const onSelectFile = useCallback(async (files: File[]) => { - if (!files || files.length === 0) { - return; - } - const loadFiles = await Promise.all( - files.map( - (file) => - new Promise((resolve, reject) => { - if (file.type.includes('image')) { - const reader = new FileReader(); - reader.readAsDataURL(file); - reader.onload = () => { - const item = { + }, + [shareId, t, toast] + ); + const onSelectFile = useCallback( + async (files: File[]) => { + if (!files || files.length === 0) { + return; + } + const loadFiles = await Promise.all( + files.map( + (file) => + new Promise((resolve, reject) => { + if (file.type.includes('image')) { + const reader = new FileReader(); + reader.readAsDataURL(file); + reader.onload = () => { + const item = { + id: nanoid(), + rawFile: file, + type: FileTypeEnum.image, + name: file.name, + icon: reader.result as string + }; + uploadFile(item); + resolve(item); + }; + reader.onerror = () => { + reject(reader.error); + }; + } else { + resolve({ id: nanoid(), rawFile: file, - type: FileTypeEnum.image, + type: FileTypeEnum.file, name: file.name, - icon: reader.result as string - }; - uploadFile(item); - resolve(item); - }; - reader.onerror = () => { - reject(reader.error); - }; - } else { - resolve({ - id: nanoid(), - rawFile: file, - type: FileTypeEnum.file, - name: file.name, - icon: 'pdf' - }); - } - }) - ) - ); + icon: 'pdf' + }); + } + }) + ) + ); - setFileList((state) => [...state, ...loadFiles]); - }, []); + setFileList((state) => [...state, ...loadFiles]); + }, + [uploadFile] + ); const handleSend = useCallback(async () => { const textareaValue = TextareaDom.current?.value || ''; diff --git a/projects/app/src/components/ChatBox/WholeResponseModal.tsx b/projects/app/src/components/ChatBox/WholeResponseModal.tsx index 6707f4ef24f..46e193467df 100644 --- a/projects/app/src/components/ChatBox/WholeResponseModal.tsx +++ b/projects/app/src/components/ChatBox/WholeResponseModal.tsx @@ -12,12 +12,21 @@ import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools'; import Markdown from '../Markdown'; import { DatasetSearchModeMap } from '@fastgpt/global/core/dataset/constant'; -function Row({ label, value }: { label: string; value?: string | number }) { +function Row({ + label, + value, + rawDom +}: { + label: string; + value?: string | number; + rawDom?: React.ReactNode; +}) { const theme = useTheme(); + const val = value || rawDom; const strValue = `${value}`; const isCodeBlock = strValue.startsWith('~~~json'); - return value !== undefined && value !== '' && value !== 'undefined' ? ( + return val !== undefined && val !== '' && val !== 'undefined' ? ( {label}: @@ -29,7 +38,8 @@ function Row({ label, value }: { label: string; value?: string | number }) { ? { transform: 'translateY(-3px)' } : { px: 3, py: 1, border: theme.borders.base })} > - + {value && } + {rawDom} ) : null; @@ -113,12 +123,28 @@ const WholeResponseModal = ({ { - if (!activeModule?.historyPreview) return ''; - return activeModule.historyPreview - .map((item, i) => `**${item.obj}**\n${item.value}`) - .join('\n\n---\n\n'); - })()} + rawDom={ + activeModule.historyPreview ? ( + <> + {activeModule.historyPreview?.map((item, i) => ( + + {item.obj} + {item.value} + + ))} + + ) : ( + '' + ) + } /> {activeModule.quoteList && activeModule.quoteList.length > 0 && ( (null); const theme = useTheme(); const router = useRouter(); + const { shareId } = router.query as { shareId?: string }; const { t } = useTranslation(); const { toast } = useToast(); const { isPc, setLoading } = useSystemStore(); @@ -258,7 +260,7 @@ const ChatBox = ( const result = await postQuestionGuide( { messages: adaptChat2GptMessages({ messages: history, reserveId: false }).slice(-6), - shareId: router.query.shareId as string + shareId }, abortSignal ); @@ -270,7 +272,7 @@ const ChatBox = ( } } catch (error) {} }, - [questionGuide, scrollToBottom, router.query.shareId] + [questionGuide, scrollToBottom, shareId] ); /** @@ -323,7 +325,6 @@ const ChatBox = ( setTimeout(() => { scrollToBottom(); }, 100); - try { // create abort obj const abortSignal = new AbortController(); @@ -518,16 +519,22 @@ const ChatBox = ( } }; window.addEventListener('message', windowMessage); - eventBus.on('guideClick', ({ text }: { text: string }) => { + + eventBus.on(EventNameEnum.sendQuestion, ({ text }: { text: string }) => { if (!text) return; handleSubmit((data) => sendPrompt(data, text))(); }); + eventBus.on(EventNameEnum.editQuestion, ({ text }: { text: string }) => { + if (!text) return; + resetInputVal(text); + }); return () => { - eventBus.off('guideClick'); + eventBus.off(EventNameEnum.sendQuestion); + eventBus.off(EventNameEnum.editQuestion); window.removeEventListener('message', windowMessage); }; - }, [handleSubmit, sendPrompt]); + }, [handleSubmit, resetInputVal, sendPrompt]); return ( @@ -757,40 +764,30 @@ const ChatBox = ( { + const text = item.value as string; + + // replace quote tag: [source1] 标识第一个来源，需要提取数字1，从而去数组里查找来源 + const quoteReg = /\[source:(.+)\]/g; + const replaceText = text.replace(quoteReg, `[QUOTE SIGN]($1)`); + + // question guide + if ( + index === chatHistory.length - 1 && + !isChatting && + questionGuides.length > 0 + ) { + return `${replaceText}\n\`\`\`${ + CodeClassName.questionGuide + }\n${JSON.stringify(questionGuides)}`; + } + return replaceText; + })()} isChatting={index === chatHistory.length - 1 && isChatting} /> + - {/* question guide */} - {index === chatHistory.length - 1 && - !isChatting && - questionGuides.length > 0 && ( - - - - {questionGuides.map((item) => ( - - ))} - - - )} + {/* admin mark content */} {showMarkIcon && item.adminFeedback && ( diff --git a/projects/app/src/components/Icon/icons/core/chat/quoteSign.svg b/projects/app/src/components/Icon/icons/core/chat/quoteSign.svg new file mode 100644 index 00000000000..81623535277 --- /dev/null +++ b/projects/app/src/components/Icon/icons/core/chat/quoteSign.svg @@ -0,0 +1,11 @@ + + + + + \ No newline at end of file diff --git a/projects/app/src/components/Icon/icons/core/chat/sendLight.svg b/projects/app/src/components/Icon/icons/core/chat/sendLight.svg new file mode 100644 index 00000000000..e5a86c65261 --- /dev/null +++ b/projects/app/src/components/Icon/icons/core/chat/sendLight.svg @@ -0,0 +1,8 @@ + + + + \ No newline at end of file diff --git a/projects/app/src/components/Icon/index.tsx b/projects/app/src/components/Icon/index.tsx index aa6b758502e..6335732f93f 100644 --- a/projects/app/src/components/Icon/index.tsx +++ b/projects/app/src/components/Icon/index.tsx @@ -103,6 +103,8 @@ const iconPaths = { 'core/app/tts': () => import('./icons/core/app/tts.svg'), 'core/app/headphones': () => import('./icons/core/app/headphones.svg'), 'common/playLight': () => import('./icons/common/playLight.svg'), + 'core/chat/quoteSign': () => import('./icons/core/chat/quoteSign.svg'), + 'core/chat/sendLight': () => import('./icons/core/chat/sendLight.svg'), 'core/chat/sendFill': () => import('./icons/core/chat/sendFill.svg'), 'core/chat/recordFill': () => import('./icons/core/chat/recordFill.svg'), 'core/chat/stopSpeechFill': () => import('./icons/core/chat/stopSpeechFill.svg'), diff --git a/projects/app/src/components/Markdown/CodeLight.tsx b/projects/app/src/components/Markdown/CodeLight.tsx index 78b0f88d63f..e9848fc917b 100644 --- a/projects/app/src/components/Markdown/CodeLight.tsx +++ b/projects/app/src/components/Markdown/CodeLight.tsx @@ -315,7 +315,7 @@ const CodeLight = ({ - {String(children)} + {String(children).replace(/ /g, ' ')} ); diff --git a/projects/app/src/components/Markdown/chat/Guide.tsx b/projects/app/src/components/Markdown/chat/Guide.tsx index c89fb189372..06b015c6126 100644 --- a/projects/app/src/components/Markdown/chat/Guide.tsx +++ b/projects/app/src/components/Markdown/chat/Guide.tsx @@ -5,7 +5,7 @@ import RemarkGfm from 'remark-gfm'; import RemarkMath from 'remark-math'; import RehypeKatex from 'rehype-katex'; import RemarkBreaks from 'remark-breaks'; -import { eventBus } from '@/web/common/utils/eventbus'; +import { EventNameEnum, eventBus } from '@/web/common/utils/eventbus'; import 'katex/dist/katex.min.css'; import styles from '../index.module.scss'; @@ -27,7 +27,7 @@ function MyLink(e: any) { textDecoration={'underline'} cursor={'pointer'} onClick={() => { - eventBus.emit('guideClick', { text }); + eventBus.emit(EventNameEnum.sendQuestion, { text }); }} > {text} diff --git a/projects/app/src/components/Markdown/chat/QuestionGuide.tsx b/projects/app/src/components/Markdown/chat/QuestionGuide.tsx new file mode 100644 index 00000000000..c1240ca3566 --- /dev/null +++ b/projects/app/src/components/Markdown/chat/QuestionGuide.tsx @@ -0,0 +1,92 @@ +import React, { useMemo } from 'react'; +import { Box, Flex, useTheme } from '@chakra-ui/react'; +import 'katex/dist/katex.min.css'; +import ChatBoxDivider from '@/components/core/chat/Divider'; +import { useTranslation } from 'next-i18next'; +import { EventNameEnum, eventBus } from '@/web/common/utils/eventbus'; +import MyTooltip from '@/components/MyTooltip'; +import MyIcon from '@/components/Icon'; + +const QuestionGuide = ({ text }: { text: string }) => { + const theme = useTheme(); + const { t } = useTranslation(); + const questionGuides = useMemo(() => { + try { + const json = JSON.parse(text); + if (Array.isArray(json) && !json.find((item) => typeof item !== 'string')) { + return json as string[]; + } + return []; + } catch (error) { + return []; + } + }, [text]); + + return questionGuides.length > 0 ? ( + + + + {questionGuides.map((text) => ( + + + {text} + + + + eventBus.emit(EventNameEnum.editQuestion, { text })} + /> + + + eventBus.emit(EventNameEnum.sendQuestion, { text })} + /> + + + + ))} + + + ) : null; +}; + +export default React.memo(QuestionGuide); diff --git a/projects/app/src/components/Markdown/chat/Quote.tsx b/projects/app/src/components/Markdown/chat/Quote.tsx deleted file mode 100644 index e0721bd37f6..00000000000 --- a/projects/app/src/components/Markdown/chat/Quote.tsx +++ /dev/null @@ -1,64 +0,0 @@ -import React, { useMemo } from 'react'; -import { Box, useTheme } from '@chakra-ui/react'; -import { getFileAndOpen } from '@/web/core/dataset/utils'; -import { useToast } from '@/web/common/hooks/useToast'; -import { getErrText } from '@fastgpt/global/common/error/utils'; - -type QuoteItemType = { - file_id?: string; - filename: string; -}; - -const QuoteBlock = ({ code }: { code: string }) => { - const theme = useTheme(); - const { toast } = useToast(); - const quoteList = useMemo(() => { - try { - return JSON.parse(code) as QuoteItemType[]; - } catch (error) { - return []; - } - }, [code]); - - return ( - - {quoteList.length > 0 ? ( - <> - 本次回答的引用: - - {quoteList.map((item, i) => ( - { - if (!item.file_id) return; - try { - await getFileAndOpen(item.file_id); - } catch (error) { - toast({ - status: 'warning', - title: getErrText(error, '打开文件失败') - }); - } - }} - > - {item.filename} - - ))} - - - ) : ( - 正在生成引用…… - )} - - ); -}; - -export default QuoteBlock; diff --git a/projects/app/src/components/Markdown/img/Image.tsx b/projects/app/src/components/Markdown/img/Image.tsx index 0c607f6cd2c..ef2c898f055 100644 --- a/projects/app/src/components/Markdown/img/Image.tsx +++ b/projects/app/src/components/Markdown/img/Image.tsx @@ -46,7 +46,7 @@ const MdImage = ({ src }: { src?: string }) => { /> - + import('./CodeLight')); const MermaidCodeBlock = dynamic(() => import('./img/MermaidCodeBlock')); const MdImage = dynamic(() => import('./img/Image')); -const ChatGuide = dynamic(() => import('./chat/Guide')); const EChartsCodeBlock = dynamic(() => import('./img/EChartsCodeBlock')); -const QuoteBlock = dynamic(() => import('./chat/Quote')); + +const ChatGuide = dynamic(() => import('./chat/Guide')); +const QuestionGuide = dynamic(() => import('./chat/QuestionGuide')); const ImageBlock = dynamic(() => import('./chat/Image')); export enum CodeClassName { guide = 'guide', + questionGuide = 'questionGuide', mermaid = 'mermaid', echarts = 'echarts', quote = 'quote', @@ -37,12 +46,12 @@ function Code({ inline, className, children }: any) { if (codeType === CodeClassName.guide) { return ; } + if (codeType === CodeClassName.questionGuide) { + return ; + } if (codeType === CodeClassName.echarts) { return ; } - if (codeType === CodeClassName.quote) { - return ; - } if (codeType === CodeClassName.img) { return ; } @@ -55,6 +64,52 @@ function Code({ inline, className, children }: any) { function Image({ src }: { src?: string }) { return ; } +function A({ children, ...props }: any) { + const { t } = useTranslation(); + + // empty href link + if (!props.href && typeof children?.[0] === 'string') { + const text = useMemo(() => String(children), [children]); + + return ( + + + + ); + } + + // quote link + if (children?.length === 1 && typeof children?.[0] === 'string') { + const text = String(children); + if (text === MARKDOWN_QUOTE_SIGN && props.href) { + return ( + + getFileAndOpen(props.href)} + /> + + ); + } + } + + return {children}; +} const Markdown = ({ source, isChatting = false }: { source: string; isChatting?: boolean }) => { const components = useMemo( @@ -62,14 +117,16 @@ const Markdown = ({ source, isChatting = false }: { source: string; isChatting?: img: Image, pre: 'div', p: 'div', - code: Code + code: Code, + a: A }), [] ); const formatSource = source .replace(/\\n/g, '\n ') - .replace(/(http[s]?:\/\/[^\s，。]+)([。，])/g, '$1 $2'); + .replace(/(http[s]?:\/\/[^\s，。]+)([。，])/g, '$1 $2') + .replace(/\n*(\[QUOTE SIGN\]\(.*\))/g, '$1'); return ( {t(item.title)} {!!item.desc && ( - + {t(item.desc)} )} diff --git a/projects/app/src/components/core/module/DatasetParamsModal.tsx b/projects/app/src/components/core/module/DatasetParamsModal.tsx index 7d63690fbb0..03a7f116292 100644 --- a/projects/app/src/components/core/module/DatasetParamsModal.tsx +++ b/projects/app/src/components/core/module/DatasetParamsModal.tsx @@ -18,6 +18,7 @@ type DatasetParamsProps = { limit?: number; searchMode: `${DatasetSearchModeEnum}`; searchEmptyText?: string; + maxTokens?: number; }; const DatasetParamsModal = ({ @@ -25,6 +26,7 @@ const DatasetParamsModal = ({ limit, similarity, searchMode = DatasetSearchModeEnum.embedding, + maxTokens = 3000, onClose, onSuccess }: DatasetParamsProps & { onClose: () => void; onSuccess: (e: DatasetParamsProps) => void }) => { @@ -52,8 +54,8 @@ const DatasetParamsModal = ({ isOpen={true} onClose={onClose} iconSrc="/imgs/modal/params.svg" - title={'搜索参数调整'} - minW={['90vw', '500px']} + title={t('core.dataset.search.Dataset Search Params')} + w={['90vw', '550px']} h={['90vh', 'auto']} overflow={'unset'} isCentered={searchEmptyText !== undefined} @@ -78,36 +80,42 @@ const DatasetParamsModal = ({ - { - setValue(ModuleInputKeyEnum.datasetSimilarity, val); - setRefresh(!refresh); - }} - /> + + { + setValue(ModuleInputKeyEnum.datasetSimilarity, val); + setRefresh(!refresh); + }} + /> + )} {limit !== undefined && ( - {t('core.dataset.search.Top K')} + {t('core.dataset.search.Max Tokens')} + + + - + { setValue(ModuleInputKeyEnum.datasetLimit, val); setRefresh(!refresh); diff --git a/projects/app/src/components/core/module/Flow/components/render/RenderInput.tsx b/projects/app/src/components/core/module/Flow/components/render/RenderInput.tsx index c485764ce97..aaf1d6f404e 100644 --- a/projects/app/src/components/core/module/Flow/components/render/RenderInput.tsx +++ b/projects/app/src/components/core/module/Flow/components/render/RenderInput.tsx @@ -17,7 +17,7 @@ import { Grid, Switch } from '@chakra-ui/react'; -import { FlowNodeInputTypeEnum } from '@fastgpt/global/core/module/node/constant'; +import { FlowNodeInputTypeEnum, FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant'; import { QuestionOutlineIcon } from '@chakra-ui/icons'; import dynamic from 'next/dynamic'; import { onChangeNode, useFlowProviderStore } from '../../FlowProvider'; @@ -37,6 +37,7 @@ import { useQuery } from '@tanstack/react-query'; import type { EditFieldModeType, EditFieldType } from '../modules/FieldEditModal'; import { feConfigs } from '@/web/common/system/staticData'; import { DatasetSearchModeEnum } from '@fastgpt/global/core/dataset/constant'; +import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; const FieldEditModal = dynamic(() => import('../modules/FieldEditModal')); const SelectAppModal = dynamic(() => import('../../SelectAppModal')); @@ -635,9 +636,12 @@ const SelectAppRender = React.memo(function SelectAppRender({ item, moduleId }: }); const SelectDatasetParamsRender = React.memo(function SelectDatasetParamsRender({ + item, inputs = [], moduleId }: RenderProps) { + const { nodes } = useFlowProviderStore(); + const { t } = useTranslation(); const [data, setData] = useState({ searchMode: DatasetSearchModeEnum.embedding, @@ -645,6 +649,23 @@ const SelectDatasetParamsRender = React.memo(function SelectDatasetParamsRender( similarity: 0.5 }); + const tokenLimit = useMemo(() => { + let maxTokens = 3000; + + nodes.forEach((item) => { + if (item.type === FlowNodeTypeEnum.chatNode) { + const model = + item.data.inputs.find((item) => item.key === ModuleInputKeyEnum.aiModel)?.value || ''; + const quoteMaxToken = + chatModelList.find((item) => item.model === model)?.quoteMaxToken || 3000; + + maxTokens = Math.max(maxTokens, quoteMaxToken); + } + }); + + return maxTokens; + }, [nodes]); + const { isOpen, onOpen, onClose } = useDisclosure(); useEffect(() => { @@ -671,6 +692,7 @@ const SelectDatasetParamsRender = React.memo(function SelectDatasetParamsRender( {isOpen && ( { for (let key in e) { diff --git a/projects/app/src/constants/app.ts b/projects/app/src/constants/app.ts index 5f67ed3c8be..53651df4311 100644 --- a/projects/app/src/constants/app.ts +++ b/projects/app/src/constants/app.ts @@ -4,7 +4,7 @@ import type { OutLinkEditType } from '@fastgpt/global/support/outLink/type.d'; export const defaultApp: AppDetailType = { _id: '', userId: 'userId', - name: '模型加载中', + name: '应用加载中', type: 'simple', simpleTemplateId: 'fastgpt-universal', avatar: '/icon/logo.svg', diff --git a/projects/app/src/global/core/chat/api.d.ts b/projects/app/src/global/core/chat/api.d.ts index b4c54929a6a..647dc22b181 100644 --- a/projects/app/src/global/core/chat/api.d.ts +++ b/projects/app/src/global/core/chat/api.d.ts @@ -65,7 +65,7 @@ export type ClearHistoriesProps = { /* -------- chat item ---------- */ export type DeleteChatItemProps = { chatId: string; - contentId: string; + contentId?: string; shareId?: string; outLinkUid?: string; }; diff --git a/projects/app/src/global/core/prompt/AIChat.ts b/projects/app/src/global/core/prompt/AIChat.ts index a44c132fa39..2965df48d72 100644 --- a/projects/app/src/global/core/prompt/AIChat.ts +++ b/projects/app/src/global/core/prompt/AIChat.ts @@ -4,22 +4,42 @@ export const Prompt_QuoteTemplateList: PromptTemplateItem[] = [ { title: '标准模板', desc: '标准提示词，用于结构不固定的知识库。', - value: `{{q}}\n{{a}}` + value: ` +{{q}} +{{a}} +` }, { title: '问答模板', - desc: '适合 QA 问答结构的知识库，或大部分核心介绍位于 a 的知识库。', - value: `{instruction:"{{q}}",output:"{{a}}"}` + desc: '适合 QA 问答结构的知识库，可以让AI较为严格的按预设内容回答', + value: ` +<问题> +{{q}} + +<答案> +{{a}} + +` }, { title: '标准严格模板', desc: '在标准模板基础上，对模型的回答做更严格的要求。', - value: `{{q}}\n{{a}}` + value: ` +{{q}} +{{a}} +` }, { title: '严格问答模板', desc: '在问答模板基础上，对模型的回答做更严格的要求。', - value: `{question:"{{q}}",answer:"{{a}}"}` + value: ` +<问题> +{{q}} + +<答案> +{{a}} + +` } ]; @@ -27,54 +47,70 @@ export const Prompt_QuotePromptList: PromptTemplateItem[] = [ { title: '标准模板', desc: '', - value: `你的知识库: -""" + value: `使用 标记中的内容作为你的知识: + {{quote}} -""" + 回答要求： -1. 优先使用知识库内容回答问题。 -2. 不要提及你是从知识库获取的知识。 -3. 知识库包含 markdown 内容时，按 markdown 格式返回。 -我的问题是:"{{question}}"` +- 如果你不清楚答案，你需要澄清。 +- 避免提及你是从 data 获取的知识。 +- 保持答案与 data 中描述的一致。 +- 使用 Markdown 语法优化回答格式。 +- 使用与问题相同的语言回答。 + +问题:"{{question}}"` }, { title: '问答模板', desc: '', - value: `你的知识库: -""" + value: `使用 标记中的问答对进行回答。 + {{quote}} -""" + 回答要求： -1. 优先使用知识库内容回答问题，其中 instruction 是相关介绍，output 是预期回答或补充。 -2. 不要提及你是从知识库获取的知识。 -3. 知识库包含 markdown 内容时，按 markdown 格式返回。 -我的问题是:"{{question}}"` +- 选择其中一个或多个问答对进行回答。 +- 回答的内容应尽可能与 <答案> 中的内容一致。 +- 如果没有相关的问答对，你需要澄清。 +- 避免提及你是从 QA 获取的知识，只需要回复答案。 + +问题:"{{question}}"` }, { title: '标准严格模板', desc: '', - value: `你的知识库: -""" + value: `忘记你已有的知识，仅使用 标记中的内容作为你的知识: + {{quote}} -""" + +思考流程： +1. 判断问题是否与 标记中的内容有关。 +2. 如果有关，你按下面的要求回答。 +3. 如果无关，你直接拒绝回答本次问题。 + 回答要求： -1. 仅使用知识库内容回答问题。 -2. 与知识库无关的问题，你直接回答我不知道。 -3. 不要提及你是从知识库获取的知识。 -4. 知识库包含 markdown 内容时，按 markdown 格式返回。 -我的问题是:"{{question}}"` +- 避免提及你是从 data 获取的知识。 +- 保持答案与 data 中描述的一致。 +- 使用 Markdown 语法优化回答格式。 +- 使用与问题相同的语言回答。 + +问题:"{{question}}"` }, { title: '严格问答模板', desc: '', - value: `你的知识库: -""" + value: `忘记你已有的知识，仅使用 标记中的问答对进行回答。 + {{quote}} -""" -回答要求： -1. 从知识库中选择一个合适的答案进行回答，其中 instruction 是相关问题，answer 是已知答案。 -2. 与知识库无关的问题，你直接回答我不知道。 -3. 不要提及你是从知识库获取的知识。 -我的问题是:"{{question}}"` + +思考流程： +1. 判断问题是否与 标记中的内容有关。 +2. 如果无关，你直接拒绝回答本次问题。 +3. 判断是否有相近或相同的问题。 +4. 如果有相同的问题，直接输出对应答案。 +5. 如果只有相近的问题，请把相近的问题和答案一起输出。 + +最后，避免提及你是从 QA 获取的知识，只需要回复答案。 + +问题:"{{question}}"` } ]; diff --git a/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts b/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts index ed256c154bb..b2209d643f0 100644 --- a/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts +++ b/projects/app/src/pages/api/admin/timeTasks/checkUnValidDatasetFiles.ts @@ -2,7 +2,10 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { delFileById, getGFSCollection } from '@fastgpt/service/common/file/gridfs/controller'; +import { + delFileByFileIdList, + getGFSCollection +} from '@fastgpt/service/common/file/gridfs/controller'; import { addLog } from '@fastgpt/service/common/mongo/controller'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { delay } from '@fastgpt/global/common/system/utils'; @@ -77,7 +80,7 @@ export async function checkFiles(start: Date, end: Date, limit: number) { // 3. if not found, delete file if (hasCollection === 0) { - await delFileById({ bucketName: 'dataset', fileId: String(_id) }); + await delFileByFileIdList({ bucketName: 'dataset', fileIdList: [String(_id)] }); console.log('delete file', _id); deleteFileAmount++; } diff --git a/projects/app/src/pages/api/common/file/read.ts b/projects/app/src/pages/api/common/file/read.ts index 576e3ead855..d1d366c3646 100644 --- a/projects/app/src/pages/api/common/file/read.ts +++ b/projects/app/src/pages/api/common/file/read.ts @@ -4,6 +4,7 @@ import { connectToDatabase } from '@/service/mongo'; import { authFileToken } from '@fastgpt/service/support/permission/controller'; import { detect } from 'jschardet'; import { getDownloadStream, getFileById } from '@fastgpt/service/common/file/gridfs/controller'; +import { CommonErrEnum } from '@fastgpt/global/common/error/code/common'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -22,6 +23,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< getDownloadStream({ bucketName, fileId }) ]); + if (!file) { + return Promise.reject(CommonErrEnum.fileNotFound); + } + // get encoding let buffers: Buffer = Buffer.from([]); for await (const chunk of encodeStream) { diff --git a/projects/app/src/pages/api/common/file/uploadImage.ts b/projects/app/src/pages/api/common/file/uploadImage.ts index bf1a42f46c4..65bc8a57381 100644 --- a/projects/app/src/pages/api/common/file/uploadImage.ts +++ b/projects/app/src/pages/api/common/file/uploadImage.ts @@ -1,21 +1,22 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { authCertOrShareId } from '@fastgpt/service/support/permission/auth/common'; import { uploadMongoImg } from '@fastgpt/service/common/file/image/controller'; - -type Props = { base64Img: string; expiredTime?: Date }; +import { UploadImgProps } from '@fastgpt/global/common/file/api'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { teamId } = await authCert({ req, authToken: true }); - const { base64Img, expiredTime } = req.body as Props; + const { base64Img, expiredTime, metadata, shareId } = req.body as UploadImgProps; + + const { teamId } = await authCertOrShareId({ req, shareId, authToken: true }); const data = await uploadMongoImg({ teamId, base64Img, - expiredTime + expiredTime, + metadata }); jsonRes(res, { data }); diff --git a/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts b/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts index 2036f14a9a8..494842473f6 100644 --- a/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts +++ b/projects/app/src/pages/api/core/ai/agent/createQuestionGuide.ts @@ -4,13 +4,14 @@ import { connectToDatabase } from '@/service/mongo'; import type { CreateQuestionGuideParams } from '@/global/core/ai/api.d'; import { pushQuestionGuideBill } from '@/service/support/wallet/bill/push'; import { createQuestionGuide } from '@fastgpt/service/core/ai/functions/createQuestionGuide'; -import { authCertAndShareId } from '@fastgpt/service/support/permission/auth/common'; +import { authCertOrShareId } from '@fastgpt/service/support/permission/auth/common'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); const { messages, shareId } = req.body as CreateQuestionGuideParams; - const { tmbId, teamId } = await authCertAndShareId({ + + const { tmbId, teamId } = await authCertOrShareId({ req, authToken: true, shareId diff --git a/projects/app/src/pages/api/core/app/update.ts b/projects/app/src/pages/api/core/app/update.ts index c2526305ac2..adde147608a 100644 --- a/projects/app/src/pages/api/core/app/update.ts +++ b/projects/app/src/pages/api/core/app/update.ts @@ -4,6 +4,9 @@ import { connectToDatabase } from '@/service/mongo'; import { MongoApp } from '@fastgpt/service/core/app/schema'; import type { AppUpdateParams } from '@fastgpt/global/core/app/api'; import { authApp } from '@fastgpt/service/support/permission/auth/app'; +import { FlowNodeTypeEnum } from '@fastgpt/global/core/module/node/constant'; +import { ModuleInputKeyEnum } from '@fastgpt/global/core/module/constants'; +import { getChatModel } from '@/service/core/ai/model'; /* 获取我的模型 */ export default async function handler(req: NextApiRequest, res: NextApiResponse) { @@ -20,6 +23,36 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< // 凭证校验 await authApp({ req, authToken: true, appId, per: permission ? 'owner' : 'w' }); + // check modules + // 1. dataset search limit, less than model quoteMaxToken + if (modules) { + let maxTokens = 3000; + + modules.forEach((item) => { + if (item.flowType === FlowNodeTypeEnum.chatNode) { + const model = + item.inputs.find((item) => item.key === ModuleInputKeyEnum.aiModel)?.value || ''; + const chatModel = getChatModel(model); + const quoteMaxToken = chatModel.quoteMaxToken || 3000; + + maxTokens = Math.max(maxTokens, quoteMaxToken); + } + }); + + modules.forEach((item) => { + if (item.flowType === FlowNodeTypeEnum.datasetSearchNode) { + item.inputs.forEach((input) => { + if (input.key === ModuleInputKeyEnum.datasetLimit) { + const val = input.value as number; + if (val > maxTokens) { + input.value = maxTokens; + } + } + }); + } + }); + } + // 更新模型 await MongoApp.updateOne( { diff --git a/projects/app/src/pages/api/core/chat/item/delete.ts b/projects/app/src/pages/api/core/chat/item/delete.ts index f9ec4263be2..41c3a5fc58e 100644 --- a/projects/app/src/pages/api/core/chat/item/delete.ts +++ b/projects/app/src/pages/api/core/chat/item/delete.ts @@ -10,6 +10,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) await connectToDatabase(); const { chatId, contentId, shareId, outLinkUid } = req.query as DeleteChatItemProps; + if (!contentId || !chatId) { + return jsonRes(res); + } + await autChatCrud({ req, authToken: true, diff --git a/projects/app/src/pages/api/core/chat/item/getSpeech.ts b/projects/app/src/pages/api/core/chat/item/getSpeech.ts index ba3369fafd3..d157a2ece03 100644 --- a/projects/app/src/pages/api/core/chat/item/getSpeech.ts +++ b/projects/app/src/pages/api/core/chat/item/getSpeech.ts @@ -4,7 +4,7 @@ import { connectToDatabase } from '@/service/mongo'; import { GetChatSpeechProps } from '@/global/core/chat/api.d'; import { text2Speech } from '@fastgpt/service/core/ai/audio/speech'; import { pushAudioSpeechBill } from '@/service/support/wallet/bill/push'; -import { authCertAndShareId } from '@fastgpt/service/support/permission/auth/common'; +import { authCertOrShareId } from '@fastgpt/service/support/permission/auth/common'; import { authType2BillSource } from '@/service/support/wallet/bill/utils'; import { getAudioSpeechModel } from '@/service/core/ai/model'; import { MongoTTSBuffer } from '@fastgpt/service/common/buffer/tts/schema'; @@ -25,7 +25,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) throw new Error('model or voice not found'); } - const { teamId, tmbId, authType } = await authCertAndShareId({ req, authToken: true, shareId }); + const { teamId, tmbId, authType } = await authCertOrShareId({ req, authToken: true, shareId }); const ttsModel = getAudioSpeechModel(ttsConfig.model); const voiceData = ttsModel.voices?.find((item) => item.value === ttsConfig.voice); diff --git a/projects/app/src/pages/api/core/dataset/collection/delete.ts b/projects/app/src/pages/api/core/dataset/collection/delete.ts index 415d3df7a65..50dbf119680 100644 --- a/projects/app/src/pages/api/core/dataset/collection/delete.ts +++ b/projects/app/src/pages/api/core/dataset/collection/delete.ts @@ -24,13 +24,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }); // find all delete id - const collections = await findCollectionAndChild(collectionId, '_id metadata'); + const collections = await findCollectionAndChild(collectionId, '_id fileId'); const delIdList = collections.map((item) => item._id); // delete await delCollectionRelevantData({ collectionIds: delIdList, - fileIds: collections.map((item) => item.metadata?.fileId).filter(Boolean) + fileIds: collections.map((item) => item?.fileId || '').filter(Boolean) }); // delete collection diff --git a/projects/app/src/pages/api/core/dataset/collection/sync/link.ts b/projects/app/src/pages/api/core/dataset/collection/sync/link.ts index 67d07991fc1..0ff92518442 100644 --- a/projects/app/src/pages/api/core/dataset/collection/sync/link.ts +++ b/projects/app/src/pages/api/core/dataset/collection/sync/link.ts @@ -4,7 +4,6 @@ import { connectToDatabase } from '@/service/mongo'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; import { loadingOneChunkCollection } from '@fastgpt/service/core/dataset/collection/utils'; import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller'; -import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; import { DatasetErrEnum } from '@fastgpt/global/common/error/code/dataset'; diff --git a/projects/app/src/pages/api/core/dataset/update.ts b/projects/app/src/pages/api/core/dataset/update.ts index 83b93a70b93..9d4b5ae4f89 100644 --- a/projects/app/src/pages/api/core/dataset/update.ts +++ b/projects/app/src/pages/api/core/dataset/update.ts @@ -15,8 +15,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< throw new Error('缺少参数'); } - // 凭证校验 - await authDataset({ req, authToken: true, datasetId: id, per: 'owner' }); + if (permission) { + await authDataset({ req, authToken: true, datasetId: id, per: 'owner' }); + } else { + await authDataset({ req, authToken: true, datasetId: id, per: 'w' }); + } await MongoDataset.findOneAndUpdate( { diff --git a/projects/app/src/pages/app/detail/components/SimpleEdit/index.tsx b/projects/app/src/pages/app/detail/components/SimpleEdit/index.tsx index 09442bffc34..6b47ad4dff6 100644 --- a/projects/app/src/pages/app/detail/components/SimpleEdit/index.tsx +++ b/projects/app/src/pages/app/detail/components/SimpleEdit/index.tsx @@ -21,7 +21,6 @@ import type { AppSimpleEditFormType } from '@fastgpt/global/core/app/type.d'; import { chatModelList, simpleModeTemplates } from '@/web/common/system/staticData'; import { formatPrice } from '@fastgpt/global/support/wallet/bill/tools'; import { chatNodeSystemPromptTip, welcomeTextTip } from '@fastgpt/global/core/module/template/tip'; -import type { VariableItemType } from '@fastgpt/global/core/module/type.d'; import type { ModuleItemType } from '@fastgpt/global/core/module/type'; import { useRequest } from '@/web/common/hooks/useRequest'; import { useConfirm } from '@/web/common/hooks/useConfirm'; @@ -67,7 +66,6 @@ function ConfigForm({ }) { const theme = useTheme(); const router = useRouter(); - const { toast } = useToast(); const { t } = useTranslation(); const { appDetail, updateAppDetail } = useAppStore(); const { loadAllDatasets, allDatasets } = useDatasetStore(); @@ -124,6 +122,13 @@ function ConfigForm({ [getValues, refresh] ); + const tokenLimit = useMemo(() => { + return ( + chatModelList.find((item) => item.model === getValues('aiSettings.model'))?.quoteMaxToken || + 3000 + ); + }, [getValues, refresh]); + const { mutate: onSubmitSave, isLoading: isSaving } = useRequest({ mutationFn: async (data: AppSimpleEditFormType) => { const modules = await postForm2Modules(data, data.templateId); @@ -361,8 +366,8 @@ function ConfigForm({ )} - {t('core.dataset.Similarity')}: {getValues('dataset.similarity')},{' '} - {t('core.dataset.Search Top K')}: {getValues('dataset.limit')} + {t('core.dataset.search.Min Similarity')}: {getValues('dataset.similarity')},{' '} + {t('core.dataset.search.Max Tokens')}: {getValues('dataset.limit')} {getValues('dataset.searchEmptyText') === '' ? '' : t('core.dataset.Set Empty Result Tip')} @@ -458,6 +463,7 @@ function ConfigForm({ {isOpenDatasetParams && ( { setValue('dataset', { diff --git a/projects/app/src/pages/app/detail/index.tsx b/projects/app/src/pages/app/detail/index.tsx index c9cfb6f8028..f3bb8d98d9e 100644 --- a/projects/app/src/pages/app/detail/index.tsx +++ b/projects/app/src/pages/app/detail/index.tsx @@ -15,6 +15,7 @@ import Loading from '@/components/Loading'; import SimpleEdit from './components/SimpleEdit'; import { serviceSideProps } from '@/web/common/utils/i18n'; import { useAppStore } from '@/web/core/app/store/useAppStore'; +import Head from 'next/head'; const AdEdit = dynamic(() => import('./components/AdEdit'), { loading: () => @@ -92,90 +93,95 @@ const AppDetail = ({ currentTab }: { currentTab: `${TabEnum}` }) => { }); return ( - - - {/* pc tab */} - - - - + <> + + {appDetail.name} + + + + {/* pc tab */} + + + + + {appDetail.name} + + + { + if (e === 'startChat') { + router.push(`/chat?appId=${appId}`); + } else { + setCurrentTab(e); + } + }} + /> + router.replace('/app/list')} + > + } + bg={'white'} + boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'} + h={'28px'} + size={'sm'} + borderRadius={'50%'} + aria-label={''} + /> + 我的应用 + + + {/* phone tab */} + + {appDetail.name} - - { - if (e === 'startChat') { - router.push(`/chat?appId=${appId}`); - } else { - setCurrentTab(e); - } - }} - /> - router.replace('/app/list')} - > - } - bg={'white'} - boxShadow={'1px 1px 9px rgba(0,0,0,0.15)'} - h={'28px'} + { + if (e === 'startChat') { + router.push(`/chat?appId=${appId}`); + } else { + setCurrentTab(e); + } + }} /> - 我的应用 - - - {/* phone tab */} - - - {appDetail.name} - { - if (e === 'startChat') { - router.push(`/chat?appId=${appId}`); - } else { - setCurrentTab(e); - } - }} - /> - - - {currentTab === TabEnum.simpleEdit && } - {currentTab === TabEnum.adEdit && appDetail && ( - setCurrentTab(TabEnum.simpleEdit)} /> - )} - {currentTab === TabEnum.logs && } - {currentTab === TabEnum.outLink && } - - - + + {currentTab === TabEnum.simpleEdit && } + {currentTab === TabEnum.adEdit && appDetail && ( + setCurrentTab(TabEnum.simpleEdit)} /> + )} + {currentTab === TabEnum.logs && } + {currentTab === TabEnum.outLink && } + + + + ); }; diff --git a/projects/app/src/pages/chat/index.tsx b/projects/app/src/pages/chat/index.tsx index a7f8cb0f45f..17341598b53 100644 --- a/projects/app/src/pages/chat/index.tsx +++ b/projects/app/src/pages/chat/index.tsx @@ -1,7 +1,7 @@ import React, { useCallback, useRef } from 'react'; import Head from 'next/head'; import { useRouter } from 'next/router'; -import { getInitChatInfo, putChatHistory } from '@/web/core/chat/api'; +import { getInitChatInfo } from '@/web/core/chat/api'; import { Box, Flex, diff --git a/projects/app/src/pages/chat/share.tsx b/projects/app/src/pages/chat/share.tsx index cdaa20a92b5..be5bcb5247e 100644 --- a/projects/app/src/pages/chat/share.tsx +++ b/projects/app/src/pages/chat/share.tsx @@ -116,7 +116,9 @@ const OutLink = ({ updateHistory({ ...currentChat, updateTime: new Date(), - title: newTitle + title: newTitle, + shareId, + outLinkUid }); } @@ -148,7 +150,7 @@ const OutLink = ({ return { responseText, responseData, isNewChat: forbidRefresh.current }; }, - [chatId, shareId, outLinkUid, setChatData, appId, updateHistory, router, histories] + [chatId, shareId, outLinkUid, setChatData, appId, pushHistory, router, histories, updateHistory] ); const loadChatInfo = useCallback( @@ -309,13 +311,19 @@ const OutLink = ({ }); }} onSetHistoryTop={(e) => { - updateHistory(e); + updateHistory({ + ...e, + shareId, + outLinkUid + }); }} onSetCustomTitle={async (e) => { updateHistory({ chatId: e.chatId, title: e.title, - customTitle: e.title + customTitle: e.title, + shareId, + outLinkUid }); }} /> @@ -349,7 +357,7 @@ const OutLink = ({ feedbackType={'user'} onUpdateVariable={(e) => {}} onStartChat={startChat} - onDelMessage={(e) => delOneHistoryItem({ ...e, chatId })} + onDelMessage={(e) => delOneHistoryItem({ ...e, chatId, shareId, outLinkUid })} /> diff --git a/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx b/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx index 5325cc60630..4b1e6fef724 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx @@ -173,7 +173,9 @@ const FileSelect = ({ case 'pdf': return readPdfContent(file); case 'docx': - return readDocContent(file); + return readDocContent(file, { + fileId + }); } return ''; })(); diff --git a/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx b/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx index 7068a9c4cfb..18a913f9a68 100644 --- a/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx +++ b/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx @@ -408,7 +408,7 @@ export function RawSourceText({ await getFileAndOpen(sourceId as string); } catch (error) { toast({ - title: getErrText(error, '获取文件地址失败'), + title: t(getErrText(error, 'error.fileNotFound')), status: 'error' }); } diff --git a/projects/app/src/pages/login/provider.tsx b/projects/app/src/pages/login/provider.tsx index ee8c4096e74..10856835121 100644 --- a/projects/app/src/pages/login/provider.tsx +++ b/projects/app/src/pages/login/provider.tsx @@ -9,7 +9,6 @@ import { oauthLogin } from '@/web/support/user/api'; import { useToast } from '@/web/common/hooks/useToast'; import Loading from '@/components/Loading'; import { serviceSideProps } from '@/web/common/utils/i18n'; -import { useQuery } from '@tanstack/react-query'; import { getErrText } from '@fastgpt/global/common/error/utils'; const provider = ({ code, state }: { code: string; state: string }) => { diff --git a/projects/app/src/service/core/dataset/data/pg.ts b/projects/app/src/service/core/dataset/data/pg.ts index e2c02cae592..06d6732f9ab 100644 --- a/projects/app/src/service/core/dataset/data/pg.ts +++ b/projects/app/src/service/core/dataset/data/pg.ts @@ -11,6 +11,7 @@ import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection import { MongoDatasetData } from '@fastgpt/service/core/dataset/data/schema'; import { jiebaSplit } from '../utils'; import { reRankRecall } from '../../ai/rerank'; +import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; export async function insertData2Pg({ mongoDataId, @@ -108,38 +109,51 @@ type SearchProps = { text: string; model: string; similarity?: number; // min distance - limit: number; + limit: number; // max Token limit datasetIds: string[]; searchMode?: `${DatasetSearchModeEnum}`; }; export async function searchDatasetData(props: SearchProps) { - let { text, similarity = 0, limit, searchMode = DatasetSearchModeEnum.embedding } = props; + let { + text, + similarity = 0, + limit: maxTokens, + searchMode = DatasetSearchModeEnum.embedding + } = props; searchMode = global.systemEnv.pluginBaseUrl ? searchMode : DatasetSearchModeEnum.embedding; + // Compatible with topk limit + if (maxTokens < 50) { + maxTokens = 1500; + } + const rerank = searchMode === DatasetSearchModeEnum.embeddingReRank || searchMode === DatasetSearchModeEnum.embFullTextReRank; + const oneChunkToken = 50; const { embeddingLimit, fullTextLimit } = (() => { - // Increase search range, reduce hnsw loss + const estimatedLen = Math.max(20, Math.ceil(maxTokens / oneChunkToken)); + + // Increase search range, reduce hnsw loss. 20 ~ 100 if (searchMode === DatasetSearchModeEnum.embedding) { return { - embeddingLimit: limit * 2, + embeddingLimit: Math.min(estimatedLen, 100), fullTextLimit: 0 }; } // 50 < 2*limit < value < 100 if (searchMode === DatasetSearchModeEnum.embeddingReRank) { return { - embeddingLimit: Math.min(100, Math.max(50, limit * 2)), + embeddingLimit: Math.min(100, Math.max(50, estimatedLen * 2)), fullTextLimit: 0 }; } - // 50 < 3*limit < embedding < 80 + // 50 < 2*limit < embedding < 80 // 20 < limit < fullTextLimit < 40 return { - embeddingLimit: Math.min(80, Math.max(50, limit * 2)), - fullTextLimit: Math.min(40, Math.max(20, limit)) + embeddingLimit: Math.min(80, Math.max(50, estimatedLen * 2)), + fullTextLimit: Math.min(40, Math.max(20, estimatedLen)) }; })(); @@ -174,9 +188,14 @@ export async function searchDatasetData(props: SearchProps) { return true; }); + // token slice + if (!rerank) { return { - searchRes: filterSameDataResults.filter((item) => item.score >= similarity).slice(0, limit), + searchRes: filterResultsByMaxTokens( + filterSameDataResults.filter((item) => item.score >= similarity), + maxTokens + ), tokenLen }; } @@ -190,7 +209,10 @@ export async function searchDatasetData(props: SearchProps) { ).filter((item) => item.score > similarity); return { - searchRes: reRankResults.slice(0, limit), + searchRes: filterResultsByMaxTokens( + reRankResults.filter((item) => item.score >= similarity), + maxTokens + ), tokenLen }; } @@ -357,6 +379,8 @@ export async function reRankSearchResult({ })) }); + if (!Array.isArray(results)) return data; + // add new score to data const mergeResult = results .map((item) => { @@ -376,4 +400,22 @@ export async function reRankSearchResult({ return data; } } +export function filterResultsByMaxTokens(list: SearchDataResponseItemType[], maxTokens: number) { + const results: SearchDataResponseItemType[] = []; + let totalTokens = 0; + + for (let i = 0; i < list.length; i++) { + const item = list[i]; + totalTokens += countPromptTokens(item.q + item.a); + if (totalTokens > maxTokens + 200) { + break; + } + results.push(item); + if (totalTokens > maxTokens) { + break; + } + } + + return results; +} // ------------------ search end ------------------ diff --git a/projects/app/src/service/moduleDispatch/chat/oneapi.ts b/projects/app/src/service/moduleDispatch/chat/oneapi.ts index c19ae739f11..5ad09ef6f52 100644 --- a/projects/app/src/service/moduleDispatch/chat/oneapi.ts +++ b/projects/app/src/service/moduleDispatch/chat/oneapi.ts @@ -124,6 +124,10 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise - POST('/common/file/uploadImage', { base64Img, expiredTime }); +export const postUploadImg = (e: UploadImgProps) => POST('/common/file/uploadImage', e); export const postUploadFiles = ( data: FormData, diff --git a/projects/app/src/web/common/file/controller.ts b/projects/app/src/web/common/file/controller.ts index ad144e6e625..f67e029b3d3 100644 --- a/projects/app/src/web/common/file/controller.ts +++ b/projects/app/src/web/common/file/controller.ts @@ -1,4 +1,5 @@ import { postUploadImg, postUploadFiles } from '@/web/common/file/api'; +import { UploadImgProps } from '@fastgpt/global/common/file/api'; import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; /** @@ -34,23 +35,24 @@ export const uploadFiles = ({ * @param maxSize The max size of the compressed image */ export const compressBase64ImgAndUpload = ({ - base64, + base64Img, maxW = 1080, maxH = 1080, maxSize = 1024 * 500, // 300kb - expiredTime -}: { - base64: string; + expiredTime, + metadata, + shareId +}: UploadImgProps & { maxW?: number; maxH?: number; maxSize?: number; - expiredTime?: Date; }) => { return new Promise((resolve, reject) => { - const fileType = /^data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,/.exec(base64)?.[1] || 'image/jpeg'; + const fileType = + /^data:([a-zA-Z0-9]+\/[a-zA-Z0-9-.+]+).*,/.exec(base64Img)?.[1] || 'image/jpeg'; const img = new Image(); - img.src = base64; + img.src = base64Img; img.onload = async () => { let width = img.width; let height = img.height; @@ -86,7 +88,12 @@ export const compressBase64ImgAndUpload = ({ } try { - const src = await postUploadImg(compressedDataUrl, expiredTime); + const src = await postUploadImg({ + shareId, + base64Img: compressedDataUrl, + expiredTime, + metadata + }); resolve(src); } catch (error) { reject(error); @@ -100,18 +107,20 @@ export const compressImgFileAndUpload = async ({ maxW, maxH, maxSize, - expiredTime + expiredTime, + shareId }: { file: File; maxW?: number; maxH?: number; maxSize?: number; expiredTime?: Date; + shareId?: string; }) => { const reader = new FileReader(); reader.readAsDataURL(file); - const base64 = await new Promise((resolve, reject) => { + const base64Img = await new Promise((resolve, reject) => { reader.onload = async () => { resolve(reader.result as string); }; @@ -122,10 +131,11 @@ export const compressImgFileAndUpload = async ({ }); return compressBase64ImgAndUpload({ - base64, + base64Img, maxW, maxH, maxSize, - expiredTime + expiredTime, + shareId }); }; diff --git a/projects/app/src/web/common/file/utils.ts b/projects/app/src/web/common/file/utils.ts index 4a4ddb7e0fe..68e4db62637 100644 --- a/projects/app/src/web/common/file/utils.ts +++ b/projects/app/src/web/common/file/utils.ts @@ -107,7 +107,7 @@ export const readPdfContent = (file: File) => /** * read docx to markdown */ -export const readDocContent = (file: File) => +export const readDocContent = (file: File, metadata: Record) => new Promise((resolve, reject) => { try { const reader = new FileReader(); @@ -120,7 +120,7 @@ export const readDocContent = (file: File) => arrayBuffer: target.result as ArrayBuffer }); - const rawText = await formatMarkdown(res?.value); + const rawText = await formatMarkdown(res?.value, metadata); resolve(rawText); } catch (error) { @@ -173,24 +173,25 @@ export const readCsvContent = async (file: File) => { * 1. upload base64 * 2. replace \ */ -export const formatMarkdown = async (rawText: string = '') => { +export const formatMarkdown = async (rawText: string = '', metadata: Record) => { // match base64, upload and replace it const base64Regex = /data:image\/.*;base64,([^\)]+)/g; const base64Arr = rawText.match(base64Regex) || []; // upload base64 and replace it await Promise.all( - base64Arr.map(async (base64) => { + base64Arr.map(async (base64Img) => { try { const str = await compressBase64ImgAndUpload({ - base64, + base64Img, maxW: 4329, maxH: 4329, - maxSize: 1024 * 1024 * 5 + maxSize: 1024 * 1024 * 5, + metadata }); - rawText = rawText.replace(base64, str); + rawText = rawText.replace(base64Img, str); } catch (error) { - rawText = rawText.replace(base64, ''); + rawText = rawText.replace(base64Img, ''); rawText = rawText.replace(/!\[.*\]\(\)/g, ''); } }) diff --git a/projects/app/src/web/common/hooks/useToast.ts b/projects/app/src/web/common/hooks/useToast.ts index e2555d1597a..217321c9759 100644 --- a/projects/app/src/web/common/hooks/useToast.ts +++ b/projects/app/src/web/common/hooks/useToast.ts @@ -4,7 +4,7 @@ export const useToast = (props?: UseToastOptions) => { const toast = uToast({ position: 'top', duration: 2000, - ...props + ...(props && props) }); return { diff --git a/projects/app/src/web/common/utils/eventbus.ts b/projects/app/src/web/common/utils/eventbus.ts index 8fc2711df3d..ab36ddfb593 100644 --- a/projects/app/src/web/common/utils/eventbus.ts +++ b/projects/app/src/web/common/utils/eventbus.ts @@ -1,5 +1,6 @@ export enum EventNameEnum { - guideClick = 'guideClick', + sendQuestion = 'sendQuestion', + editQuestion = 'editQuestion', updaterNode = 'updaterNode' } type EventNameType = `${EventNameEnum}`; diff --git a/projects/app/src/web/core/app/templates.ts b/projects/app/src/web/core/app/templates.ts index f4b40820079..91065d1a8b1 100644 --- a/projects/app/src/web/core/app/templates.ts +++ b/projects/app/src/web/core/app/templates.ts @@ -368,20 +368,7 @@ export const appTemplates: (AppItemType & { type: 'slider', label: '单次搜索上限', description: '最多取 n 条记录作为本次问题引用', - value: 5, - min: 1, - max: 20, - step: 1, - markList: [ - { - label: '1', - value: 1 - }, - { - label: '20', - value: 20 - } - ], + value: 1500, connected: true }, { @@ -1418,22 +1405,9 @@ export const appTemplates: (AppItemType & { { key: 'limit', type: 'slider', - label: '单次搜索上限', - description: '最多取 n 条记录作为本次问题引用', - value: 5, - min: 1, - max: 20, - step: 1, - markList: [ - { - label: '1', - value: 1 - }, - { - label: '20', - value: 20 - } - ], + label: '引用上限', + description: '单次搜索最大的 Tokens 数量，中文约1字=1.7Tokens，英文约1字=1Tokens', + value: 1500, connected: true }, { diff --git a/projects/app/src/web/core/chat/storeChat.ts b/projects/app/src/web/core/chat/storeChat.ts index 620b09538b4..d721e754fc7 100644 --- a/projects/app/src/web/core/chat/storeChat.ts +++ b/projects/app/src/web/core/chat/storeChat.ts @@ -7,7 +7,8 @@ import type { getHistoriesProps, ClearHistoriesProps, DelHistoryProps, - UpdateHistoryProps + UpdateHistoryProps, + DeleteChatItemProps } from '@/global/core/chat/api'; import { delChatHistoryById, @@ -31,7 +32,7 @@ type State = { setLastChatAppId: (id: string) => void; lastChatId: string; setLastChatId: (id: string) => void; - delOneHistoryItem: (e: { chatId: string; contentId?: string; index: number }) => Promise; + delOneHistoryItem: (e: DeleteChatItemProps & { index: number }) => Promise; }; export const useChatStore = create()( @@ -119,7 +120,8 @@ export const useChatStore = create()( }); } }, - async delOneHistoryItem({ chatId, contentId, index }) { + async delOneHistoryItem({ index, ...props }) { + const { chatId, contentId } = props; if (!chatId || !contentId) return; try { @@ -127,7 +129,7 @@ export const useChatStore = create()( ...state, history: state.history.filter((_, i) => i !== index) })); - await delChatRecordById({ chatId, contentId }); + await delChatRecordById(props); } catch (err) { console.log(err); }