diff --git a/.gitignore b/.gitignore index 1497f1f3d88..4d05069b9c9 100644 --- a/.gitignore +++ b/.gitignore @@ -39,5 +39,4 @@ docSite/.vercel *.local.* -# jetbrains .idea/ diff --git a/dev.md b/dev.md new file mode 100644 index 00000000000..c400cc6cee8 --- /dev/null +++ b/dev.md @@ -0,0 +1,17 @@ +# 打包命令 + +```sh +# Build image, not proxy +docker build -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.4.7 --build-arg name=app . + +# build image with proxy +docker build -t registry.cn-hangzhou.aliyuncs.com/fastgpt/fastgpt:v4.4.7 --build-arg name=app --build-arg proxy=taobao . +``` + +# Pg 常用索引 + +```sql +CREATE INDEX IF NOT EXISTS modelData_dataset_id_index ON modeldata (dataset_id); +CREATE INDEX IF NOT EXISTS modelData_collection_id_index ON modeldata (collection_id); +CREATE INDEX IF NOT EXISTS modelData_teamId_index ON modeldata (team_id); +``` \ No newline at end of file diff --git a/files/deploy/fastgpt/clash/Country.mmdb b/files/deploy/fastgpt/clash/Country.mmdb deleted file mode 100644 index c668d31db6c..00000000000 Binary files a/files/deploy/fastgpt/clash/Country.mmdb and /dev/null differ diff --git a/files/deploy/fastgpt/clash/clash-linux-amd64-v3 b/files/deploy/fastgpt/clash/clash-linux-amd64-v3 deleted file mode 100644 index c1dd3a33e93..00000000000 Binary files a/files/deploy/fastgpt/clash/clash-linux-amd64-v3 and /dev/null differ diff --git a/files/deploy/fastgpt/clash/config.yaml b/files/deploy/fastgpt/clash/config.yaml deleted file mode 100644 index 5a8a90c9084..00000000000 --- a/files/deploy/fastgpt/clash/config.yaml +++ /dev/null @@ -1,43 +0,0 @@ -mixed-port: 7890 -allow-lan: false -bind-address: '*' -mode: rule -log-level: warning -dns: - enable: true - ipv6: false - nameserver: - - 8.8.8.8 - - 8.8.4.4 - cache-size: 400 -proxies: - -proxy-groups: - - { - name: '♻️ 自动选择', - type: url-test, - proxies: - [ - 香港V02×1.5, - ABC, - 印度01, - 台湾03, - 新加坡02, - 新加坡03, - 日本01, - 日本02, - 新加坡01, - 美国01, - 美国02, - 台湾01, - 台湾02 - ], - url: 'https://api.openai.com', - interval: 3600 - } -rules: - - 'DOMAIN-SUFFIX,google.com,♻️ 自动选择' - - 'DOMAIN-SUFFIX,ai.fastgpt.in,♻️ 自动选择' - - 'DOMAIN-SUFFIX,openai.com,♻️ 自动选择' - - 'DOMAIN-SUFFIX,api.openai.com,♻️ 自动选择' - - 'MATCH,DIRECT' diff --git a/files/deploy/fastgpt/clash/proxy.sh b/files/deploy/fastgpt/clash/proxy.sh deleted file mode 100644 index fcfa4b1a189..00000000000 --- a/files/deploy/fastgpt/clash/proxy.sh +++ /dev/null @@ -1,18 +0,0 @@ -export ALL_PROXY=socks5://127.0.0.1:7891 -export http_proxy=http://127.0.0.1:7890 -export https_proxy=http://127.0.0.1:7890 -export HTTP_PROXY=http://127.0.0.1:7890 -export HTTPS_PROXY=http://127.0.0.1:7890 - -OLD_PROCESS=$(pgrep clash) -if [ ! -z "$OLD_PROCESS" ]; then - echo "Killing old process: $OLD_PROCESS" - kill $OLD_PROCESS -fi -sleep 2 - -cd /root/fastgpt/clash/fast -rm -f ./nohup.out || true -rm -f ./cache.db || true -nohup ./clash-linux-amd64-v3 -d ./ & -echo "Restart clash fast" diff --git a/files/deploy/fastgpt/clash/stop.sh b/files/deploy/fastgpt/clash/stop.sh deleted file mode 100644 index 989e5a2bbd5..00000000000 --- a/files/deploy/fastgpt/clash/stop.sh +++ /dev/null @@ -1,10 +0,0 @@ -export ALL_PROXY='' -export http_proxy='' -export https_proxy='' -export HTTP_PROXY='' -export HTTPS_PROXY='' -OLD_PROCESS=$(pgrep clash) -if [ ! -z "$OLD_PROCESS" ]; then - echo "Killing old process: $OLD_PROCESS" - kill $OLD_PROCESS -fi diff --git a/packages/global/common/file/api.d.ts b/packages/global/common/file/api.d.ts index b1ae998dc59..7e05c5f93cb 100644 --- a/packages/global/common/file/api.d.ts +++ b/packages/global/common/file/api.d.ts @@ -1,9 +1,15 @@ -export type UploadImgProps = { - base64Img: string; +import { MongoImageTypeEnum } from './image/constants'; + +export type preUploadImgProps = { + type: `${MongoImageTypeEnum}`; + expiredTime?: Date; metadata?: Record; shareId?: string; }; +export type UploadImgProps = preUploadImgProps & { + base64Img: string; +}; export type UrlFetchParams = { urlList: string[]; @@ -11,6 +17,7 @@ export type UrlFetchParams = { }; export type UrlFetchResponse = { url: string; + title: string; content: string; selector?: string; }[]; diff --git a/packages/global/common/file/image/constants.ts b/packages/global/common/file/image/constants.ts new file mode 100644 index 00000000000..7136f8da403 --- /dev/null +++ b/packages/global/common/file/image/constants.ts @@ -0,0 +1,52 @@ +export const imageBaseUrl = '/api/system/img/'; + +export enum MongoImageTypeEnum { + systemAvatar = 'systemAvatar', + appAvatar = 'appAvatar', + pluginAvatar = 'pluginAvatar', + datasetAvatar = 'datasetAvatar', + userAvatar = 'userAvatar', + teamAvatar = 'teamAvatar', + + chatImage = 'chatImage', + docImage = 'docImage' +} +export const mongoImageTypeMap = { + [MongoImageTypeEnum.systemAvatar]: { + label: 'common.file.type.appAvatar', + unique: true + }, + [MongoImageTypeEnum.appAvatar]: { + label: 'common.file.type.appAvatar', + unique: true + }, + [MongoImageTypeEnum.pluginAvatar]: { + label: 'common.file.type.pluginAvatar', + unique: true + }, + [MongoImageTypeEnum.datasetAvatar]: { + label: 'common.file.type.datasetAvatar', + unique: true + }, + [MongoImageTypeEnum.userAvatar]: { + label: 'common.file.type.userAvatar', + unique: true + }, + [MongoImageTypeEnum.teamAvatar]: { + label: 'common.file.type.teamAvatar', + unique: true + }, + + [MongoImageTypeEnum.chatImage]: { + label: 'common.file.type.chatImage', + unique: false + }, + [MongoImageTypeEnum.docImage]: { + label: 'common.file.type.docImage', + unique: false + } +}; + +export const uniqueImageTypeList = Object.entries(mongoImageTypeMap) + .filter(([key, value]) => value.unique) + .map(([key]) => key as `${MongoImageTypeEnum}`); diff --git a/packages/global/common/file/image/type.d.ts b/packages/global/common/file/image/type.d.ts new file mode 100644 index 00000000000..79228d3d548 --- /dev/null +++ b/packages/global/common/file/image/type.d.ts @@ -0,0 +1,11 @@ +import { MongoImageTypeEnum } from './constants'; + +export type MongoImageSchemaType = { + teamId: string; + binary: Buffer; + createTime: Date; + expiredTime?: Date; + type: `${MongoImageTypeEnum}`; + + metadata?: { fileId?: string }; +}; diff --git a/packages/global/common/math/date.ts b/packages/global/common/math/date.ts new file mode 100644 index 00000000000..86985b5157b --- /dev/null +++ b/packages/global/common/math/date.ts @@ -0,0 +1,10 @@ +// The number of days left in the month is calculated as 30 days per month, and less than 1 day is calculated as 1 day +export const getMonthRemainingDays = () => { + const now = new Date(); + const year = now.getFullYear(); + const month = now.getMonth(); + const date = now.getDate(); + const days = new Date(year, month + 1, 0).getDate(); + const remainingDays = days - date; + return remainingDays + 1; +}; diff --git a/packages/global/common/string/markdown.ts b/packages/global/common/string/markdown.ts index b52a76b26bb..32299f2ddfc 100644 --- a/packages/global/common/string/markdown.ts +++ b/packages/global/common/string/markdown.ts @@ -15,10 +15,10 @@ export const simpleMarkdownText = (rawText: string) => { return `[${cleanedLinkText}](${url})`; }); - // replace special \.* …… - const reg1 = /\\([-.!`_(){}\[\]])/g; + // replace special #\.* …… + const reg1 = /\\([#`!*()+-_\[\]{}\\.])/g; if (reg1.test(rawText)) { - rawText = rawText.replace(/\\([`!*()+-_\[\]{}\\.])/g, '$1'); + rawText = rawText.replace(reg1, '$1'); } // replace \\n @@ -45,24 +45,26 @@ export const uploadMarkdownBase64 = async ({ uploadImgController }: { rawText: string; - uploadImgController: (base64: string) => Promise; + uploadImgController?: (base64: string) => Promise; }) => { - // match base64, upload and replace it - const base64Regex = /data:image\/.*;base64,([^\)]+)/g; - const base64Arr = rawText.match(base64Regex) || []; - // upload base64 and replace it - await Promise.all( - base64Arr.map(async (base64Img) => { - try { - const str = await uploadImgController(base64Img); + if (uploadImgController) { + // match base64, upload and replace it + const base64Regex = /data:image\/.*;base64,([^\)]+)/g; + const base64Arr = rawText.match(base64Regex) || []; + // upload base64 and replace it + await Promise.all( + base64Arr.map(async (base64Img) => { + try { + const str = await uploadImgController(base64Img); - rawText = rawText.replace(base64Img, str); - } catch (error) { - rawText = rawText.replace(base64Img, ''); - rawText = rawText.replace(/!\[.*\]\(\)/g, ''); - } - }) - ); + rawText = rawText.replace(base64Img, str); + } catch (error) { + rawText = rawText.replace(base64Img, ''); + rawText = rawText.replace(/!\[.*\]\(\)/g, ''); + } + }) + ); + } // Remove white space on both sides of the picture const trimReg = /(!\[.*\]\(.*\))\s*/g; @@ -70,5 +72,20 @@ export const uploadMarkdownBase64 = async ({ rawText = rawText.replace(trimReg, '$1'); } - return simpleMarkdownText(rawText); + return rawText; +}; + +export const markdownProcess = async ({ + rawText, + uploadImgController +}: { + rawText: string; + uploadImgController?: (base64: string) => Promise; +}) => { + const imageProcess = await uploadMarkdownBase64({ + rawText, + uploadImgController + }); + + return simpleMarkdownText(imageProcess); }; diff --git a/packages/global/common/string/tiktoken/index.ts b/packages/global/common/string/tiktoken/index.ts index 4770ba26865..2f6a3e4d386 100644 --- a/packages/global/common/string/tiktoken/index.ts +++ b/packages/global/common/string/tiktoken/index.ts @@ -33,6 +33,12 @@ export function countPromptTokens( ) { const enc = getTikTokenEnc(); const text = `${role}\n${prompt}`; + + // too large a text will block the thread + if (text.length > 15000) { + return text.length * 1.7; + } + try { const encodeText = enc.encode(text); return encodeText.length + role.length; // 补充 role 估算值 diff --git a/packages/global/common/string/time.ts b/packages/global/common/string/time.ts index fc80c93e549..5f9bd137a3d 100644 --- a/packages/global/common/string/time.ts +++ b/packages/global/common/string/time.ts @@ -1,3 +1,4 @@ import dayjs from 'dayjs'; -export const formatTime2YMDHM = (time: Date) => dayjs(time).format('YYYY-MM-DD HH:mm'); +export const formatTime2YMDHM = (time?: Date) => + time ? dayjs(time).format('YYYY-MM-DD HH:mm') : ''; diff --git a/packages/global/common/string/tools.ts b/packages/global/common/string/tools.ts index 485062b2d11..2fa177ac1b6 100644 --- a/packages/global/common/string/tools.ts +++ b/packages/global/common/string/tools.ts @@ -1,4 +1,5 @@ import crypto from 'crypto'; +import { customAlphabet } from 'nanoid'; /* check string is a web link */ export function strIsLink(str?: string) { @@ -36,3 +37,7 @@ export function replaceVariable(text: string, obj: Record { + return customAlphabet('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890', size)(); +}; diff --git a/packages/global/common/system/types/index.d.ts b/packages/global/common/system/types/index.d.ts index 2b78c82fd0c..2dd489f0aaa 100644 --- a/packages/global/common/system/types/index.d.ts +++ b/packages/global/common/system/types/index.d.ts @@ -51,6 +51,10 @@ export type FastGPTFeConfigsType = { favicon?: string; customApiDomain?: string; customSharePageDomain?: string; + subscription?: { + datasetStoreFreeSize?: number; + datasetStorePrice?: number; + }; }; export type SystemEnvType = { @@ -63,4 +67,5 @@ export type SystemEnvType = { declare global { var feConfigs: FastGPTFeConfigsType; var systemEnv: SystemEnvType; + var systemInitd: boolean; } diff --git a/packages/global/core/chat/constants.ts b/packages/global/core/chat/constants.ts index 6474e74025b..43f650dc85f 100644 --- a/packages/global/core/chat/constants.ts +++ b/packages/global/core/chat/constants.ts @@ -31,16 +31,16 @@ export enum ChatSourceEnum { } export const ChatSourceMap = { [ChatSourceEnum.test]: { - name: 'chat.logs.test' + name: 'core.chat.logs.test' }, [ChatSourceEnum.online]: { - name: 'chat.logs.online' + name: 'core.chat.logs.online' }, [ChatSourceEnum.share]: { - name: 'chat.logs.share' + name: 'core.chat.logs.share' }, [ChatSourceEnum.api]: { - name: 'chat.logs.api' + name: 'core.chat.logs.api' } }; diff --git a/packages/global/core/dataset/api.d.ts b/packages/global/core/dataset/api.d.ts index 0a482b54366..0b3f1869760 100644 --- a/packages/global/core/dataset/api.d.ts +++ b/packages/global/core/dataset/api.d.ts @@ -1,5 +1,5 @@ import { DatasetDataIndexItemType, DatasetSchemaType } from './type'; -import { DatasetCollectionTrainingModeEnum, DatasetCollectionTypeEnum } from './constant'; +import { TrainingModeEnum, DatasetCollectionTypeEnum } from './constant'; import type { LLMModelItemType } from '../ai/model.d'; /* ================= dataset ===================== */ @@ -16,21 +16,38 @@ export type DatasetUpdateBody = { }; /* ================= collection ===================== */ -export type CreateDatasetCollectionParams = { +export type DatasetCollectionChunkMetadataType = { + trainingType?: `${TrainingModeEnum}`; + chunkSize?: number; + chunkSplitter?: string; + qaPrompt?: string; +}; +export type CreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & { datasetId: string; parentId?: string; name: string; type: `${DatasetCollectionTypeEnum}`; - trainingType?: `${DatasetCollectionTrainingModeEnum}`; - chunkSize?: number; fileId?: string; rawLink?: string; - qaPrompt?: string; rawTextLength?: number; hashRawText?: string; metadata?: Record; }; +export type ApiCreateDatasetCollectionParams = DatasetCollectionChunkMetadataType & { + datasetId: string; + parentId?: string; + metadata?: Record; +}; +export type TextCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { + name: string; + text: string; +}; +export type LinkCreateDatasetCollectionParams = ApiCreateDatasetCollectionParams & { + link: string; + chunkSplitter?: string; +}; + /* ================= data ===================== */ export type PgSearchRawType = { id: string; diff --git a/packages/global/core/dataset/constant.ts b/packages/global/core/dataset/constant.ts index fb451b0e2c7..8c6aaf38d8e 100644 --- a/packages/global/core/dataset/constant.ts +++ b/packages/global/core/dataset/constant.ts @@ -53,23 +53,7 @@ export const DatasetCollectionTypeMap = { name: 'core.dataset.link' }, [DatasetCollectionTypeEnum.virtual]: { - name: 'core.dataset.Virtual File' - } -}; -export enum DatasetCollectionTrainingModeEnum { - manual = 'manual', - chunk = 'chunk', - qa = 'qa' -} -export const DatasetCollectionTrainingTypeMap = { - [DatasetCollectionTrainingModeEnum.manual]: { - label: 'core.dataset.collection.training.type manual' - }, - [DatasetCollectionTrainingModeEnum.chunk]: { - label: 'core.dataset.collection.training.type chunk' - }, - [DatasetCollectionTrainingModeEnum.qa]: { - label: 'core.dataset.collection.training.type qa' + name: 'core.dataset.Manual collection' } }; diff --git a/packages/global/core/dataset/type.d.ts b/packages/global/core/dataset/type.d.ts index 0c05982cc54..8e99af6ddb7 100644 --- a/packages/global/core/dataset/type.d.ts +++ b/packages/global/core/dataset/type.d.ts @@ -42,11 +42,15 @@ export type DatasetCollectionSchemaType = { type: `${DatasetCollectionTypeEnum}`; createTime: Date; updateTime: Date; + trainingType: `${TrainingModeEnum}`; chunkSize: number; + chunkSplitter?: string; + qaPrompt?: string; + fileId?: string; rawLink?: string; - qaPrompt?: string; + rawTextLength?: number; hashRawText?: string; metadata?: { diff --git a/packages/global/core/dataset/utils.ts b/packages/global/core/dataset/utils.ts index da848ad5a1f..577d009b478 100644 --- a/packages/global/core/dataset/utils.ts +++ b/packages/global/core/dataset/utils.ts @@ -1,4 +1,4 @@ -import { DatasetCollectionTypeEnum, DatasetDataIndexTypeEnum } from './constant'; +import { TrainingModeEnum, DatasetCollectionTypeEnum, DatasetDataIndexTypeEnum } from './constant'; import { getFileIcon } from '../../common/file/icon'; import { strIsLink } from '../../common/string/tools'; @@ -55,3 +55,8 @@ export function getDefaultIndex(props?: { q?: string; a?: string; dataId?: strin dataId }; } + +export const predictDataLimitLength = (mode: `${TrainingModeEnum}`, data: any[]) => { + if (mode === TrainingModeEnum.qa) return data.length * 20; + return data.length; +}; diff --git a/packages/global/package.json b/packages/global/package.json index 7e57ff0343f..cce0af6d6af 100644 --- a/packages/global/package.json +++ b/packages/global/package.json @@ -7,7 +7,7 @@ "encoding": "^0.1.13", "js-tiktoken": "^1.0.7", "openai": "4.23.0", - "pdfjs-dist": "^4.0.269", + "nanoid": "^4.0.1", "timezones-list": "^3.0.2" }, "devDependencies": { diff --git a/packages/global/support/user/team/type.d.ts b/packages/global/support/user/team/type.d.ts index 7a937d8cfc0..4fff5930a3e 100644 --- a/packages/global/support/user/team/type.d.ts +++ b/packages/global/support/user/team/type.d.ts @@ -9,7 +9,6 @@ export type TeamSchema = { createTime: Date; balance: number; maxSize: number; - lastDatasetBillTime: Date; limit: { lastExportDatasetTime: Date; lastWebsiteSyncTime: Date; diff --git a/packages/global/support/wallet/bill/constants.ts b/packages/global/support/wallet/bill/constants.ts index 319bd5ff8f0..6d30230bb45 100644 --- a/packages/global/support/wallet/bill/constants.ts +++ b/packages/global/support/wallet/bill/constants.ts @@ -7,7 +7,7 @@ export enum BillSourceEnum { api = 'api', shareLink = 'shareLink', training = 'training', - datasetStore = 'datasetStore' + datasetExpand = 'datasetExpand' } export const BillSourceMap: Record<`${BillSourceEnum}`, string> = { @@ -15,5 +15,5 @@ export const BillSourceMap: Record<`${BillSourceEnum}`, string> = { [BillSourceEnum.api]: 'Api', [BillSourceEnum.shareLink]: '免登录链接', [BillSourceEnum.training]: '数据训练', - [BillSourceEnum.datasetStore]: '知识库存储' + [BillSourceEnum.datasetExpand]: '知识库扩容' }; diff --git a/packages/global/support/wallet/sub/api.d.ts b/packages/global/support/wallet/sub/api.d.ts new file mode 100644 index 00000000000..ce0b9e68115 --- /dev/null +++ b/packages/global/support/wallet/sub/api.d.ts @@ -0,0 +1,4 @@ +export type SubDatasetSizeParams = { + size: number; + renew: boolean; +}; diff --git a/packages/global/support/wallet/sub/constants.ts b/packages/global/support/wallet/sub/constants.ts new file mode 100644 index 00000000000..6e03e56caf0 --- /dev/null +++ b/packages/global/support/wallet/sub/constants.ts @@ -0,0 +1,37 @@ +export enum SubTypeEnum { + datasetStore = 'datasetStore' +} + +export const subTypeMap = { + [SubTypeEnum.datasetStore]: { + label: 'support.user.team.subscription.type.datasetStore' + } +}; + +export enum SubModeEnum { + month = 'month', + year = 'year' +} + +export const subModeMap = { + [SubModeEnum.month]: { + label: 'support.user.team.subscription.mode.month' + }, + [SubModeEnum.year]: { + label: 'support.user.team.subscription.mode.year' + } +}; + +export enum SubStatusEnum { + active = 'active', + expired = 'expired' +} + +export const subStatusMap = { + [SubStatusEnum.active]: { + label: 'support.user.team.subscription.status.active' + }, + [SubStatusEnum.expired]: { + label: 'support.user.team.subscription.status.expired' + } +}; diff --git a/packages/global/support/wallet/sub/type.d.ts b/packages/global/support/wallet/sub/type.d.ts new file mode 100644 index 00000000000..31e174a7e68 --- /dev/null +++ b/packages/global/support/wallet/sub/type.d.ts @@ -0,0 +1,12 @@ +import { SubModeEnum, SubStatusEnum, SubTypeEnum } from './constants'; + +export type TeamSubSchema = { + teamId: string; + type: `${SubTypeEnum}`; + mode: `${SubModeEnum}`; + status: `${SubStatusEnum}`; + renew: boolean; + startTime: Date; + expiredTime: Date; + datasetStoreAmount?: number; +}; diff --git a/packages/service/common/file/constants.ts b/packages/service/common/file/constants.ts new file mode 100644 index 00000000000..a3449c4beb5 --- /dev/null +++ b/packages/service/common/file/constants.ts @@ -0,0 +1,6 @@ +import path from 'path'; + +export const tmpFileDirPath = + process.env.NODE_ENV === 'production' ? '/app/tmp' : path.join(process.cwd(), 'tmp'); + +export const previewMaxCharCount = 3000; diff --git a/packages/service/common/file/image/constant.ts b/packages/service/common/file/image/constant.ts deleted file mode 100644 index 518900c302f..00000000000 --- a/packages/service/common/file/image/constant.ts +++ /dev/null @@ -1 +0,0 @@ -export const imageBaseUrl = '/api/system/img/'; diff --git a/packages/service/common/file/image/controller.ts b/packages/service/common/file/image/controller.ts index 7cfe448286e..c8da371c8b1 100644 --- a/packages/service/common/file/image/controller.ts +++ b/packages/service/common/file/image/controller.ts @@ -1,5 +1,5 @@ import { UploadImgProps } from '@fastgpt/global/common/file/api'; -import { imageBaseUrl } from './constant'; +import { imageBaseUrl } from '@fastgpt/global/common/file/image/constants'; import { MongoImage } from './schema'; export function getMongoImgUrl(id: string) { @@ -8,10 +8,13 @@ export function getMongoImgUrl(id: string) { export const maxImgSize = 1024 * 1024 * 12; export async function uploadMongoImg({ + type, base64Img, teamId, expiredTime, - metadata + metadata, + + shareId }: UploadImgProps & { teamId: string; }) { @@ -20,12 +23,16 @@ export async function uploadMongoImg({ } const base64Data = base64Img.split(',')[1]; + const binary = Buffer.from(base64Data, 'base64'); const { _id } = await MongoImage.create({ + type, teamId, - binary: Buffer.from(base64Data, 'base64'), + binary, expiredTime: expiredTime, - metadata + metadata, + + shareId }); return getMongoImgUrl(String(_id)); diff --git a/packages/service/common/file/image/schema.ts b/packages/service/common/file/image/schema.ts index b401be0e210..0b4ff832a3e 100644 --- a/packages/service/common/file/image/schema.ts +++ b/packages/service/common/file/image/schema.ts @@ -1,5 +1,7 @@ import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; import { connectionMongo, type Model } from '../../mongo'; +import { MongoImageSchemaType } from '@fastgpt/global/common/file/image/type.d'; +import { mongoImageTypeMap } from '@fastgpt/global/common/file/image/constants'; const { Schema, model, models } = connectionMongo; const ImageSchema = new Schema({ @@ -12,12 +14,18 @@ const ImageSchema = new Schema({ type: Date, default: () => new Date() }, + expiredTime: { + type: Date + }, binary: { type: Buffer }, - expiredTime: { - type: Date + type: { + type: String, + enum: Object.keys(mongoImageTypeMap), + required: true }, + metadata: { type: Object } @@ -25,14 +33,13 @@ const ImageSchema = new Schema({ try { ImageSchema.index({ expiredTime: 1 }, { expireAfterSeconds: 60 }); + ImageSchema.index({ type: 1 }); + ImageSchema.index({ teamId: 1 }); } catch (error) { console.log(error); } -export const MongoImage: Model<{ - teamId: string; - binary: Buffer; - metadata?: { fileId?: string }; -}> = models['image'] || model('image', ImageSchema); +export const MongoImage: Model = + models['image'] || model('image', ImageSchema); MongoImage.syncIndexes(); diff --git a/packages/service/common/file/load/pdf.ts b/packages/service/common/file/load/pdf.ts new file mode 100644 index 00000000000..4cbb4673d1d --- /dev/null +++ b/packages/service/common/file/load/pdf.ts @@ -0,0 +1,68 @@ +import * as pdfjs from 'pdfjs-dist/legacy/build/pdf.mjs'; +// @ts-ignore +import('pdfjs-dist/legacy/build/pdf.worker.min.mjs'); +import { ReadFileParams } from './type'; + +type TokenType = { + str: string; + dir: string; + width: number; + height: number; + transform: number[]; + fontName: string; + hasEOL: boolean; +}; + +export const readPdfFile = async ({ path }: ReadFileParams) => { + const readPDFPage = async (doc: any, pageNo: number) => { + const page = await doc.getPage(pageNo); + const tokenizedText = await page.getTextContent(); + + const viewport = page.getViewport({ scale: 1 }); + const pageHeight = viewport.height; + const headerThreshold = pageHeight * 0.95; + const footerThreshold = pageHeight * 0.05; + + const pageTexts: TokenType[] = tokenizedText.items.filter((token: TokenType) => { + return ( + !token.transform || + (token.transform[5] < headerThreshold && token.transform[5] > footerThreshold) + ); + }); + + // concat empty string 'hasEOL' + for (let i = 0; i < pageTexts.length; i++) { + const item = pageTexts[i]; + if (item.str === '' && pageTexts[i - 1]) { + pageTexts[i - 1].hasEOL = item.hasEOL; + pageTexts.splice(i, 1); + i--; + } + } + + page.cleanup(); + + return pageTexts + .map((token) => { + const paragraphEnd = token.hasEOL && /([。？！.?!\n\r]|(\r\n))$/.test(token.str); + + return paragraphEnd ? `${token.str}\n` : token.str; + }) + .join(''); + }; + + const loadingTask = pdfjs.getDocument(path); + const doc = await loadingTask.promise; + + const pageTextPromises = []; + for (let pageNo = 1; pageNo <= doc.numPages; pageNo++) { + pageTextPromises.push(readPDFPage(doc, pageNo)); + } + const pageTexts = await Promise.all(pageTextPromises); + + loadingTask.destroy(); + + return { + rawText: pageTexts.join('') + }; +}; diff --git a/packages/service/common/file/load/type.d.ts b/packages/service/common/file/load/type.d.ts new file mode 100644 index 00000000000..ffb9f3ee0a1 --- /dev/null +++ b/packages/service/common/file/load/type.d.ts @@ -0,0 +1,18 @@ +export type ReadFileParams = { + preview: boolean; + teamId: string; + path: string; + metadata?: Record; +}; + +export type ReadFileResponse = { + rawText: string; +}; + +export type ReadFileBufferItemType = ReadFileParams & { + rawText: string; +}; + +declare global { + var readFileBuffers: ReadFileBufferItemType[]; +} diff --git a/packages/service/common/file/load/utils.ts b/packages/service/common/file/load/utils.ts new file mode 100644 index 00000000000..760bc610eb8 --- /dev/null +++ b/packages/service/common/file/load/utils.ts @@ -0,0 +1,50 @@ +import { readPdfFile } from './pdf'; +import { readDocFle } from './word'; +import { ReadFileBufferItemType, ReadFileParams } from './type'; + +global.readFileBuffers = global.readFileBuffers || []; + +const bufferMaxSize = 200; + +export const pushFileReadBuffer = (params: ReadFileBufferItemType) => { + global.readFileBuffers.push(params); + + if (global.readFileBuffers.length > bufferMaxSize) { + global.readFileBuffers.shift(); + } +}; +export const getReadFileBuffer = ({ path, teamId }: ReadFileParams) => + global.readFileBuffers.find((item) => item.path === path && item.teamId === teamId); + +export const readFileContent = async (params: ReadFileParams) => { + const { path } = params; + + const buffer = getReadFileBuffer(params); + + if (buffer) { + return buffer; + } + + const extension = path?.split('.')?.pop()?.toLowerCase() || ''; + + const { rawText } = await (async () => { + switch (extension) { + case 'pdf': + return readPdfFile(params); + case 'docx': + return readDocFle(params); + default: + return Promise.reject('Only support .pdf, .docx'); + } + })(); + + pushFileReadBuffer({ + ...params, + rawText + }); + + return { + ...params, + rawText + }; +}; diff --git a/packages/service/common/file/load/word.ts b/packages/service/common/file/load/word.ts new file mode 100644 index 00000000000..8842a6fd918 --- /dev/null +++ b/packages/service/common/file/load/word.ts @@ -0,0 +1,22 @@ +import mammoth from 'mammoth'; +import { htmlToMarkdown } from '../../string/markdown'; +import { ReadFileParams } from './type'; +/** + * read docx to markdown + */ +export const readDocFle = async ({ path, metadata = {} }: ReadFileParams) => { + try { + const { value: html } = await mammoth.convertToHtml({ + path + }); + + const md = await htmlToMarkdown(html); + + return { + rawText: md + }; + } catch (error) { + console.log('error doc read:', error); + return Promise.reject('Can not read doc file, please convert to PDF'); + } +}; diff --git a/packages/service/common/file/upload/multer.ts b/packages/service/common/file/multer.ts similarity index 80% rename from packages/service/common/file/upload/multer.ts rename to packages/service/common/file/multer.ts index e74303cb43f..62ff533135d 100644 --- a/packages/service/common/file/upload/multer.ts +++ b/packages/service/common/file/multer.ts @@ -1,11 +1,9 @@ import type { NextApiRequest, NextApiResponse } from 'next'; -import { customAlphabet } from 'nanoid'; import multer from 'multer'; import path from 'path'; import { BucketNameEnum, bucketNameMap } from '@fastgpt/global/common/file/constants'; -import fs from 'fs'; - -const nanoid = customAlphabet('1234567890abcdef', 12); +import { getNanoid } from '@fastgpt/global/common/string/tools'; +import { tmpFileDirPath } from './constants'; type FileType = { fieldname: string; @@ -17,7 +15,9 @@ type FileType = { size: number; }; -export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) { +const expiredTime = 30 * 60 * 1000; + +export const getUploadModel = ({ maxSize = 500 }: { maxSize?: number }) => { maxSize *= 1024 * 1024; class UploadModel { uploader = multer({ @@ -26,9 +26,12 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) { }, preservePath: true, storage: multer.diskStorage({ - filename: (_req, file, cb) => { + // destination: (_req, _file, cb) => { + // cb(null, tmpFileDirPath); + // }, + filename: async (req, file, cb) => { const { ext } = path.parse(decodeURIComponent(file.originalname)); - cb(null, nanoid() + ext); + cb(null, `${Date.now() + expiredTime}-${getNanoid(32)}${ext}`); } }) }).any(); @@ -75,14 +78,4 @@ export function getUploadModel({ maxSize = 500 }: { maxSize?: number }) { } return new UploadModel(); -} - -export const removeFilesByPaths = (paths: string[]) => { - paths.forEach((path) => { - fs.unlink(path, (err) => { - if (err) { - console.error(err); - } - }); - }); }; diff --git a/packages/service/common/file/utils.ts b/packages/service/common/file/utils.ts new file mode 100644 index 00000000000..641aed8f9d8 --- /dev/null +++ b/packages/service/common/file/utils.ts @@ -0,0 +1,33 @@ +import fs from 'fs'; +import { tmpFileDirPath } from './constants'; + +export const removeFilesByPaths = (paths: string[]) => { + paths.forEach((path) => { + fs.unlink(path, (err) => { + if (err) { + console.error(err); + } + }); + }); +}; + +/* cron job. check expired tmp files */ +export const checkExpiredTmpFiles = () => { + // get all file name + const files = fs.readdirSync(tmpFileDirPath).map((name) => { + const timestampStr = name.split('-')[0]; + const expiredTimestamp = timestampStr ? Number(timestampStr) : 0; + + return { + filename: name, + expiredTimestamp, + path: `${tmpFileDirPath}/${name}` + }; + }); + + // count expiredFiles + const expiredFiles = files.filter((item) => item.expiredTimestamp < Date.now()); + + // remove expiredFiles + removeFilesByPaths(expiredFiles.map((item) => item.path)); +}; diff --git a/packages/service/common/string/cheerio.ts b/packages/service/common/string/cheerio.ts index 44a6c134479..722e77c460c 100644 --- a/packages/service/common/string/cheerio.ts +++ b/packages/service/common/string/cheerio.ts @@ -50,8 +50,11 @@ export const cheerioToHtml = ({ .get() .join('\n'); + const title = $('head title').text() || $('h1:first').text() || fetchUrl; + return { html, + title, usedSelector }; }; @@ -70,7 +73,7 @@ export const urlsFetch = async ({ }); const $ = cheerio.load(fetchRes.data); - const { html, usedSelector } = cheerioToHtml({ + const { title, html, usedSelector } = cheerioToHtml({ fetchUrl: url, $, selector @@ -79,6 +82,7 @@ export const urlsFetch = async ({ return { url, + title, content: md, selector: usedSelector }; @@ -87,6 +91,7 @@ export const urlsFetch = async ({ return { url, + title: '', content: '', selector: '' }; diff --git a/packages/service/common/string/markdown.ts b/packages/service/common/string/markdown.ts index fd292d86120..21c0987cdad 100644 --- a/packages/service/common/string/markdown.ts +++ b/packages/service/common/string/markdown.ts @@ -15,7 +15,9 @@ export const htmlToMarkdown = (html?: string | null) => worker.on('message', (md: string) => { worker.terminate(); - resolve(simpleMarkdownText(md)); + let rawText = simpleMarkdownText(md); + + resolve(rawText); }); worker.on('error', (err) => { worker.terminate(); diff --git a/packages/service/common/system/cron.ts b/packages/service/common/system/cron.ts new file mode 100644 index 00000000000..7b5e32513e9 --- /dev/null +++ b/packages/service/common/system/cron.ts @@ -0,0 +1,6 @@ +import nodeCron from 'node-cron'; + +export const setCron = (time: string, cb: () => void) => { + // second minute hour day month week + return nodeCron.schedule(time, cb); +}; diff --git a/packages/service/common/system/log.ts b/packages/service/common/system/log.ts index 4035c4b190c..ed7a2fd8a8b 100644 --- a/packages/service/common/system/log.ts +++ b/packages/service/common/system/log.ts @@ -49,6 +49,7 @@ export const addLog = { }, error(msg: string, error?: any) { this.log('error', msg, { + message: error?.message, stack: error?.stack, ...(error?.config && { config: { diff --git a/packages/service/common/vectorStore/controller.d.ts b/packages/service/common/vectorStore/controller.d.ts index 3ddd01fa05c..6712302996e 100644 --- a/packages/service/common/vectorStore/controller.d.ts +++ b/packages/service/common/vectorStore/controller.d.ts @@ -2,6 +2,8 @@ export type DeleteDatasetVectorProps = { id?: string; datasetIds?: string[]; collectionIds?: string[]; + + collectionId?: string; dataIds?: string[]; }; diff --git a/packages/service/common/vectorStore/pg/controller.ts b/packages/service/common/vectorStore/pg/controller.ts index 0828c31f7e6..a2e044d6f70 100644 --- a/packages/service/common/vectorStore/pg/controller.ts +++ b/packages/service/common/vectorStore/pg/controller.ts @@ -101,14 +101,19 @@ export const deleteDatasetDataVector = async ( retry?: number; } ): Promise => { - const { id, datasetIds, collectionIds, dataIds, retry = 2 } = props; + const { id, datasetIds, collectionIds, collectionId, dataIds, retry = 2 } = props; const where = await (() => { if (id) return `id=${id}`; if (datasetIds) return `dataset_id IN (${datasetIds.map((id) => `'${String(id)}'`).join(',')})`; - if (collectionIds) + if (collectionIds) { return `collection_id IN (${collectionIds.map((id) => `'${String(id)}'`).join(',')})`; - if (dataIds) return `data_id IN (${dataIds.map((id) => `'${String(id)}'`).join(',')})`; + } + if (collectionId && dataIds) { + return `collection_id='${String(collectionId)}' and data_id IN (${dataIds + .map((id) => `'${String(id)}'`) + .join(',')})`; + } return Promise.reject('deleteDatasetData: no where'); })(); diff --git a/packages/service/core/ai/embedding/index.ts b/packages/service/core/ai/embedding/index.ts index 1b617179af3..2b785ab0a08 100644 --- a/packages/service/core/ai/embedding/index.ts +++ b/packages/service/core/ai/embedding/index.ts @@ -32,7 +32,7 @@ export async function getVectorsByText({ return Promise.reject('Embedding API 404'); } if (!res?.data?.[0]?.embedding) { - console.log(res?.data); + console.log(res); // @ts-ignore return Promise.reject(res.data?.err?.message || 'Embedding API Error'); } diff --git a/packages/service/core/chat/chatItemSchema.ts b/packages/service/core/chat/chatItemSchema.ts index beed3af23aa..aefbd7d0194 100644 --- a/packages/service/core/chat/chatItemSchema.ts +++ b/packages/service/core/chat/chatItemSchema.ts @@ -2,8 +2,7 @@ import { connectionMongo, type Model } from '../../common/mongo'; const { Schema, model, models } = connectionMongo; import { ChatItemSchema as ChatItemType } from '@fastgpt/global/core/chat/type'; import { ChatRoleMap } from '@fastgpt/global/core/chat/constants'; -import { customAlphabet } from 'nanoid'; -const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 24); +import { getNanoid } from '@fastgpt/global/common/string/tools'; import { TeamCollectionName, TeamMemberCollectionName @@ -13,32 +12,32 @@ import { userCollectionName } from '../../support/user/schema'; import { ModuleOutputKeyEnum } from '@fastgpt/global/core/module/constants'; const ChatItemSchema = new Schema({ - dataId: { - type: String, - require: true, - default: () => nanoid() - }, - appId: { + teamId: { type: Schema.Types.ObjectId, - ref: appCollectionName, + ref: TeamCollectionName, required: true }, - chatId: { - type: String, - require: true + tmbId: { + type: Schema.Types.ObjectId, + ref: TeamMemberCollectionName, + required: true }, userId: { type: Schema.Types.ObjectId, ref: userCollectionName }, - teamId: { - type: Schema.Types.ObjectId, - ref: TeamCollectionName, - required: true + chatId: { + type: String, + require: true }, - tmbId: { + dataId: { + type: String, + require: true, + default: () => getNanoid(22) + }, + appId: { type: Schema.Types.ObjectId, - ref: TeamMemberCollectionName, + ref: appCollectionName, required: true }, time: { @@ -80,10 +79,11 @@ const ChatItemSchema = new Schema({ }); try { - ChatItemSchema.index({ dataId: -1 }); + ChatItemSchema.index({ teamId: 1 }); ChatItemSchema.index({ time: -1 }); ChatItemSchema.index({ appId: 1 }); ChatItemSchema.index({ chatId: 1 }); + ChatItemSchema.index({ obj: 1 }); ChatItemSchema.index({ userGoodFeedback: 1 }); ChatItemSchema.index({ userBadFeedback: 1 }); ChatItemSchema.index({ customFeedbacks: 1 }); diff --git a/packages/service/core/dataset/collection/controller.ts b/packages/service/core/dataset/collection/controller.ts index 95434451729..d0e7c14d241 100644 --- a/packages/service/core/dataset/collection/controller.ts +++ b/packages/service/core/dataset/collection/controller.ts @@ -1,7 +1,4 @@ -import { - DatasetCollectionTrainingModeEnum, - DatasetCollectionTypeEnum -} from '@fastgpt/global/core/dataset/constant'; +import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; import { MongoDatasetCollection } from './schema'; @@ -12,11 +9,15 @@ export async function createOneCollection({ parentId, datasetId, type, - trainingType = DatasetCollectionTrainingModeEnum.manual, + + trainingType = TrainingModeEnum.chunk, chunkSize = 0, + chunkSplitter, + qaPrompt, + fileId, rawLink, - qaPrompt, + hashRawText, rawTextLength, metadata = {}, @@ -30,11 +31,15 @@ export async function createOneCollection({ datasetId, name, type, + trainingType, chunkSize, + chunkSplitter, + qaPrompt, + fileId, rawLink, - qaPrompt, + rawTextLength, hashRawText, metadata @@ -74,7 +79,7 @@ export function createDefaultCollection({ datasetId, parentId, type: DatasetCollectionTypeEnum.virtual, - trainingType: DatasetCollectionTrainingModeEnum.manual, + trainingType: TrainingModeEnum.chunk, chunkSize: 0, updateTime: new Date('2099') }); diff --git a/packages/service/core/dataset/collection/schema.ts b/packages/service/core/dataset/collection/schema.ts index 02ba9a5764a..f2b5f8defe6 100644 --- a/packages/service/core/dataset/collection/schema.ts +++ b/packages/service/core/dataset/collection/schema.ts @@ -1,10 +1,7 @@ import { connectionMongo, type Model } from '../../../common/mongo'; const { Schema, model, models } = connectionMongo; import { DatasetCollectionSchemaType } from '@fastgpt/global/core/dataset/type.d'; -import { - DatasetCollectionTrainingTypeMap, - DatasetCollectionTypeMap -} from '@fastgpt/global/core/dataset/constant'; +import { TrainingTypeMap, DatasetCollectionTypeMap } from '@fastgpt/global/core/dataset/constant'; import { DatasetCollectionName } from '../schema'; import { TeamCollectionName, @@ -56,15 +53,23 @@ const DatasetCollectionSchema = new Schema({ type: Date, default: () => new Date() }, + trainingType: { type: String, - enum: Object.keys(DatasetCollectionTrainingTypeMap), + enum: Object.keys(TrainingTypeMap), required: true }, chunkSize: { type: Number, required: true }, + chunkSplitter: { + type: String + }, + qaPrompt: { + type: String + }, + fileId: { type: Schema.Types.ObjectId, ref: 'dataset.files' @@ -72,9 +77,6 @@ const DatasetCollectionSchema = new Schema({ rawLink: { type: String }, - qaPrompt: { - type: String - }, rawTextLength: { type: Number @@ -89,8 +91,9 @@ const DatasetCollectionSchema = new Schema({ }); try { + DatasetCollectionSchema.index({ teamId: 1 }); DatasetCollectionSchema.index({ datasetId: 1 }); - DatasetCollectionSchema.index({ datasetId: 1, parentId: 1 }); + DatasetCollectionSchema.index({ teamId: 1, datasetId: 1, parentId: 1 }); DatasetCollectionSchema.index({ updateTime: -1 }); DatasetCollectionSchema.index({ hashRawText: -1 }); } catch (error) { diff --git a/packages/service/core/dataset/collection/utils.ts b/packages/service/core/dataset/collection/utils.ts index 6918f037643..225e01b22cd 100644 --- a/packages/service/core/dataset/collection/utils.ts +++ b/packages/service/core/dataset/collection/utils.ts @@ -4,7 +4,7 @@ import type { ParentTreePathItemType } from '@fastgpt/global/common/parentFolder import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { MongoDatasetTraining } from '../training/schema'; import { urlsFetch } from '../../../common/string/cheerio'; -import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { DatasetCollectionTypeEnum, TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; import { hashStr } from '@fastgpt/global/common/string/tools'; /** @@ -92,8 +92,12 @@ export const getCollectionAndRawText = async ({ return Promise.reject('Collection not found'); } - const rawText = await (async () => { - if (newRawText) return newRawText; + const { title, rawText } = await (async () => { + if (newRawText) + return { + title: '', + rawText: newRawText + }; // link if (col.type === DatasetCollectionTypeEnum.link && col.rawLink) { // crawl new data @@ -102,12 +106,18 @@ export const getCollectionAndRawText = async ({ selector: col.datasetId?.websiteConfig?.selector || col?.metadata?.webPageSelector }); - return result[0].content; + return { + title: result[0].title, + rawText: result[0].content + }; } // file - return ''; + return { + title: '', + rawText: '' + }; })(); const hashRawText = hashStr(rawText); @@ -115,6 +125,7 @@ export const getCollectionAndRawText = async ({ return { collection: col, + title, rawText, isSameRawText }; @@ -135,6 +146,7 @@ export const reloadCollectionChunks = async ({ rawText?: string; }) => { const { + title, rawText: newRawText, collection: col, isSameRawText @@ -154,6 +166,11 @@ export const reloadCollectionChunks = async ({ }); // insert to training queue + const model = await (() => { + if (col.trainingType === TrainingModeEnum.chunk) return col.datasetId.vectorModel; + if (col.trainingType === TrainingModeEnum.qa) return col.datasetId.agentModel; + return Promise.reject('Training model error'); + })(); await MongoDatasetTraining.insertMany( chunks.map((item, i) => ({ teamId: col.teamId, @@ -163,7 +180,7 @@ export const reloadCollectionChunks = async ({ billId, mode: col.trainingType, prompt: '', - model: col.datasetId.vectorModel, + model, q: item, a: '', chunkIndex: i @@ -172,6 +189,7 @@ export const reloadCollectionChunks = async ({ // update raw text await MongoDatasetCollection.findByIdAndUpdate(col._id, { + ...(title && { name: title }), rawTextLength: newRawText.length, hashRawText: hashStr(newRawText) }); diff --git a/packages/service/core/dataset/data/controller.ts b/packages/service/core/dataset/data/controller.ts index bc0f84daa7b..e1af0ad0b7e 100644 --- a/packages/service/core/dataset/data/controller.ts +++ b/packages/service/core/dataset/data/controller.ts @@ -75,7 +75,13 @@ export async function delCollectionRelevantData({ /** * delete one data by mongoDataId */ -export async function delDatasetDataByDataId(mongoDataId: string) { - await deleteDatasetDataVector({ dataIds: [mongoDataId] }); +export async function delDatasetDataByDataId({ + collectionId, + mongoDataId +}: { + collectionId: string; + mongoDataId: string; +}) { + await deleteDatasetDataVector({ collectionId, dataIds: [mongoDataId] }); await MongoDatasetData.findByIdAndDelete(mongoDataId); } diff --git a/packages/service/core/dataset/data/schema.ts b/packages/service/core/dataset/data/schema.ts index f79b730b5be..8681cc9fc08 100644 --- a/packages/service/core/dataset/data/schema.ts +++ b/packages/service/core/dataset/data/schema.ts @@ -85,12 +85,13 @@ const DatasetDataSchema = new Schema({ }); try { + DatasetDataSchema.index({ teamId: 1 }); DatasetDataSchema.index({ datasetId: 1 }); DatasetDataSchema.index({ collectionId: 1 }); DatasetDataSchema.index({ updateTime: -1 }); + DatasetDataSchema.index({ collectionId: 1, q: 1, a: 1 }); // full text index DatasetDataSchema.index({ datasetId: 1, fullTextToken: 'text' }); - DatasetDataSchema.index({ inited: 1 }); } catch (error) { console.log(error); } diff --git a/packages/service/core/dataset/schema.ts b/packages/service/core/dataset/schema.ts index 039ead527ce..827487a23b2 100644 --- a/packages/service/core/dataset/schema.ts +++ b/packages/service/core/dataset/schema.ts @@ -92,7 +92,7 @@ const DatasetSchema = new Schema({ }); try { - DatasetSchema.index({ userId: 1 }); + DatasetSchema.index({ teamId: 1 }); } catch (error) { console.log(error); } diff --git a/packages/service/core/dataset/training/schema.ts b/packages/service/core/dataset/training/schema.ts index ab184038234..8d7742763ae 100644 --- a/packages/service/core/dataset/training/schema.ts +++ b/packages/service/core/dataset/training/schema.ts @@ -102,6 +102,7 @@ const TrainingDataSchema = new Schema({ }); try { + TrainingDataSchema.index({ teamId: 1 }); TrainingDataSchema.index({ weight: -1 }); TrainingDataSchema.index({ lockTime: 1 }); TrainingDataSchema.index({ datasetId: 1 }); diff --git a/packages/service/package.json b/packages/service/package.json index d1405aa3199..baa926fb520 100644 --- a/packages/service/package.json +++ b/packages/service/package.json @@ -3,17 +3,19 @@ "version": "1.0.0", "dependencies": { "@fastgpt/global": "workspace:*", + "axios": "^1.5.1", + "cheerio": "1.0.0-rc.12", "cookie": "^0.5.0", + "dayjs": "^1.11.7", "encoding": "^0.1.13", "jsonwebtoken": "^9.0.2", + "mammoth": "^1.6.0", "mongoose": "^7.0.2", - "nanoid": "^4.0.1", - "dayjs": "^1.11.7", - "next": "13.5.2", "multer": "1.4.5-lts.1", - "axios": "^1.5.1", - "cheerio": "1.0.0-rc.12", + "next": "13.5.2", "nextjs-cors": "^2.1.2", + "node-cron": "^3.0.3", + "pdfjs-dist": "^4.0.269", "pg": "^8.10.0", "tunnel": "^0.0.6" }, @@ -21,6 +23,7 @@ "@types/cookie": "^0.5.2", "@types/jsonwebtoken": "^9.0.3", "@types/multer": "^1.4.10", + "@types/node-cron": "^3.0.11", "@types/pg": "^8.6.6", "@types/tunnel": "^0.0.4" } diff --git a/packages/service/support/openapi/tools.ts b/packages/service/support/openapi/tools.ts index 852e1734251..b6bedee1ea2 100644 --- a/packages/service/support/openapi/tools.ts +++ b/packages/service/support/openapi/tools.ts @@ -1,18 +1,22 @@ import { MongoOpenApi } from './schema'; -export async function updateApiKeyUsedTime(id: string) { - await MongoOpenApi.findByIdAndUpdate(id, { +export function updateApiKeyUsedTime(id: string) { + MongoOpenApi.findByIdAndUpdate(id, { lastUsedTime: new Date() + }).catch((err) => { + console.log('update apiKey used time error', err); }); } -export async function updateApiKeyUsage({ apikey, usage }: { apikey: string; usage: number }) { - await MongoOpenApi.findOneAndUpdate( +export function updateApiKeyUsage({ apikey, usage }: { apikey: string; usage: number }) { + MongoOpenApi.findOneAndUpdate( { apiKey: apikey }, { $inc: { usage } } - ); + ).catch((err) => { + console.log('update apiKey usage error', err); + }); } diff --git a/packages/service/support/outLink/tools.ts b/packages/service/support/outLink/tools.ts index c4e7e737095..3b9afa32d6b 100644 --- a/packages/service/support/outLink/tools.ts +++ b/packages/service/support/outLink/tools.ts @@ -9,17 +9,15 @@ export const updateOutLinkUsage = async ({ shareId: string; total: number; }) => { - try { - await MongoOutLink.findOneAndUpdate( - { shareId }, - { - $inc: { total }, - lastTime: new Date() - } - ); - } catch (err) { + MongoOutLink.findOneAndUpdate( + { shareId }, + { + $inc: { total }, + lastTime: new Date() + } + ).catch((err) => { console.log('update shareChat error', err); - } + }); }; export const pushResult2Remote = async ({ diff --git a/packages/service/support/permission/limit/dataset.ts b/packages/service/support/permission/limit/dataset.ts new file mode 100644 index 00000000000..a214465e2e5 --- /dev/null +++ b/packages/service/support/permission/limit/dataset.ts @@ -0,0 +1,20 @@ +import { getVectorCountByTeamId } from '../../../common/vectorStore/controller'; +import { getTeamDatasetValidSub } from '../../wallet/sub/utils'; + +export const checkDatasetLimit = async ({ + teamId, + freeSize = Infinity, + insertLen = 0 +}: { + teamId: string; + freeSize?: number; + insertLen?: number; +}) => { + const { maxSize } = await getTeamDatasetValidSub({ teamId, freeSize }); + const usedSize = await getVectorCountByTeamId(teamId); + + if (usedSize + insertLen >= maxSize) { + return Promise.reject(`数据库容量已满，无法继续添加。可以在账号页面进行扩容。`); + } + return; +}; diff --git a/packages/service/support/user/team/teamSchema.ts b/packages/service/support/user/team/teamSchema.ts index 3001854abbb..eaeb9fba90e 100644 --- a/packages/service/support/user/team/teamSchema.ts +++ b/packages/service/support/user/team/teamSchema.ts @@ -30,9 +30,6 @@ const TeamSchema = new Schema({ type: Number, default: 5 }, - lastDatasetBillTime: { - type: Date - }, limit: { lastExportDatasetTime: { type: Date diff --git a/packages/service/support/wallet/bill/schema.ts b/packages/service/support/wallet/bill/schema.ts index e5822efc7ac..25c8b00c1dc 100644 --- a/packages/service/support/wallet/bill/schema.ts +++ b/packages/service/support/wallet/bill/schema.ts @@ -54,6 +54,7 @@ const BillSchema = new Schema({ try { BillSchema.index({ teamId: 1 }); BillSchema.index({ tmbId: 1 }); + BillSchema.index({ tmbId: 1, time: 1 }); BillSchema.index({ time: 1 }, { expireAfterSeconds: 90 * 24 * 60 * 60 }); } catch (error) { console.log(error); diff --git a/packages/service/support/wallet/sub/schema.ts b/packages/service/support/wallet/sub/schema.ts new file mode 100644 index 00000000000..2f2cc335792 --- /dev/null +++ b/packages/service/support/wallet/sub/schema.ts @@ -0,0 +1,55 @@ +import { connectionMongo, type Model } from '../../../common/mongo'; +const { Schema, model, models } = connectionMongo; +import { TeamCollectionName } from '@fastgpt/global/support/user/team/constant'; +import { subModeMap, subStatusMap, subTypeMap } from '@fastgpt/global/support/wallet/sub/constants'; +import type { TeamSubSchema } from '@fastgpt/global/support/wallet/sub/type'; + +export const subCollectionName = 'team.subscription'; + +const SubSchema = new Schema({ + teamId: { + type: Schema.Types.ObjectId, + ref: TeamCollectionName, + required: true + }, + type: { + type: String, + enum: Object.keys(subTypeMap), + required: true + }, + mode: { + type: String, + enum: Object.keys(subModeMap), + required: true + }, + status: { + type: String, + enum: Object.keys(subStatusMap), + required: true + }, + renew: { + type: Boolean, + default: true + }, + startTime: { + type: Date + }, + expiredTime: { + type: Date + }, + datasetStoreAmount: { + type: Number + } +}); + +try { + SubSchema.index({ teamId: 1 }); + SubSchema.index({ status: 1 }); + SubSchema.index({ type: 1 }); + SubSchema.index({ expiredTime: -1 }); +} catch (error) { + console.log(error); +} + +export const MongoTeamSub: Model = + models[subCollectionName] || model(subCollectionName, SubSchema); diff --git a/packages/service/support/wallet/sub/utils.ts b/packages/service/support/wallet/sub/utils.ts new file mode 100644 index 00000000000..8abd4834da6 --- /dev/null +++ b/packages/service/support/wallet/sub/utils.ts @@ -0,0 +1,31 @@ +import { SubStatusEnum } from '@fastgpt/global/support/wallet/sub/constants'; +import { MongoTeamSub } from './schema'; + +/* get team dataset size */ +export const getTeamDatasetValidSub = async ({ + teamId, + freeSize = Infinity +}: { + teamId: string; + freeSize?: number; +}) => { + const sub = await MongoTeamSub.findOne({ + teamId, + status: SubStatusEnum.active + }) + .sort({ + expiredTime: -1 + }) + .lean(); + + const maxSize = (() => { + if (!sub || !sub.datasetStoreAmount) return freeSize; + + return sub.datasetStoreAmount + freeSize; + })(); + + return { + maxSize, + sub + }; +}; diff --git a/packages/service/type.d.ts b/packages/service/type.d.ts new file mode 100644 index 00000000000..0f943512518 --- /dev/null +++ b/packages/service/type.d.ts @@ -0,0 +1,3 @@ +declare global { + var defaultTeamDatasetLimit: number; +} diff --git a/packages/web/common/file/img.ts b/packages/web/common/file/img.ts index 8f994eb72ae..cdffbc3bd7b 100644 --- a/packages/web/common/file/img.ts +++ b/packages/web/common/file/img.ts @@ -4,15 +4,13 @@ export type CompressImgProps = { maxSize?: number; }; -export const compressBase64ImgAndUpload = ({ +export const compressBase64Img = ({ base64Img, maxW = 1080, maxH = 1080, - maxSize = 1024 * 500, // 300kb - uploadController + maxSize = 1024 * 500 // 500kb }: CompressImgProps & { base64Img: string; - uploadController: (base64: string) => Promise; }) => { return new Promise((resolve, reject) => { const fileType = @@ -54,12 +52,7 @@ export const compressBase64ImgAndUpload = ({ return reject('图片太大了'); } - try { - const src = await uploadController(compressedDataUrl); - resolve(src); - } catch (error) { - reject(error); - } + resolve(compressedDataUrl); }; img.onerror = reject; }); diff --git a/packages/web/common/file/read.ts b/packages/web/common/file/read.ts deleted file mode 100644 index 5069be7b96b..00000000000 --- a/packages/web/common/file/read.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { uploadMarkdownBase64 } from '@fastgpt/global/common/string/markdown'; -import { htmlStr2Md } from '../string/markdown'; -/** - * read file raw text - */ -export const readFileRawText = (file: File) => { - return new Promise((resolve: (_: string) => void, reject) => { - try { - const reader = new FileReader(); - reader.onload = () => { - resolve(reader.result as string); - }; - reader.onerror = (err) => { - console.log('error txt read:', err); - reject('Read file error'); - }; - reader.readAsText(file); - } catch (error) { - reject(error); - } - }); -}; - -export const readMdFile = async ({ - file, - uploadImgController -}: { - file: File; - uploadImgController: (base64: string) => Promise; -}) => { - const md = await readFileRawText(file); - const rawText = await uploadMarkdownBase64({ - rawText: md, - uploadImgController - }); - return rawText; -}; - -export const readHtmlFile = async ({ - file, - uploadImgController -}: { - file: File; - uploadImgController: (base64: string) => Promise; -}) => { - const md = htmlStr2Md(await readFileRawText(file)); - const rawText = await uploadMarkdownBase64({ - rawText: md, - uploadImgController - }); - - return rawText; -}; diff --git a/packages/web/common/file/read/html.ts b/packages/web/common/file/read/html.ts new file mode 100644 index 00000000000..04eb911abcb --- /dev/null +++ b/packages/web/common/file/read/html.ts @@ -0,0 +1,21 @@ +import { htmlStr2Md } from '../../string/markdown'; +import { readFileRawText } from './rawText'; +import { markdownProcess } from '@fastgpt/global/common/string/markdown'; + +export const readHtmlFile = async ({ + file, + uploadImgController +}: { + file: File; + uploadImgController?: (base64: string) => Promise; +}) => { + const { rawText } = await readFileRawText(file); + const md = htmlStr2Md(rawText); + + const simpleMd = await markdownProcess({ + rawText: md, + uploadImgController + }); + + return { rawText: rawText }; +}; diff --git a/packages/web/common/file/read/index.ts b/packages/web/common/file/read/index.ts new file mode 100644 index 00000000000..d5e4c6ad7d4 --- /dev/null +++ b/packages/web/common/file/read/index.ts @@ -0,0 +1,46 @@ +import { loadFile2Buffer } from '../utils'; +import { readHtmlFile } from './html'; +import { readMdFile } from './md'; +import { readPdfFile } from './pdf'; +import { readFileRawText } from './rawText'; +import { readWordFile } from './word'; + +export const readFileRawContent = async ({ + file, + uploadBase64Controller +}: { + file: File; + uploadBase64Controller?: (base64: string) => Promise; +}): Promise<{ + rawText: string; +}> => { + const extension = file?.name?.split('.')?.pop()?.toLowerCase(); + + switch (extension) { + case 'txt': + return readFileRawText(file); + case 'md': + return readMdFile({ + file, + uploadImgController: uploadBase64Controller + }); + case 'html': + return readHtmlFile({ + file, + uploadImgController: uploadBase64Controller + }); + case 'pdf': + const pdf = await loadFile2Buffer({ file }); + return readPdfFile({ pdf }); + case 'docx': + return readWordFile({ + file, + uploadImgController: uploadBase64Controller + }); + + default: + return { + rawText: '' + }; + } +}; diff --git a/packages/web/common/file/read/md.ts b/packages/web/common/file/read/md.ts new file mode 100644 index 00000000000..5df750c92c5 --- /dev/null +++ b/packages/web/common/file/read/md.ts @@ -0,0 +1,17 @@ +import { markdownProcess } from '@fastgpt/global/common/string/markdown'; +import { readFileRawText } from './rawText'; + +export const readMdFile = async ({ + file, + uploadImgController +}: { + file: File; + uploadImgController?: (base64: string) => Promise; +}) => { + const { rawText: md } = await readFileRawText(file); + const simpleMd = await markdownProcess({ + rawText: md, + uploadImgController + }); + return { rawText: simpleMd }; +}; diff --git a/packages/global/common/file/read/index.ts b/packages/web/common/file/read/pdf.ts similarity index 84% rename from packages/global/common/file/read/index.ts rename to packages/web/common/file/read/pdf.ts index 3c4e149d652..2e7ac97eb3f 100644 --- a/packages/global/common/file/read/index.ts +++ b/packages/web/common/file/read/pdf.ts @@ -1,18 +1,18 @@ /* read file to txt */ import * as pdfjsLib from 'pdfjs-dist'; -export const readPdfFile = async ({ pdf }: { pdf: string | URL | ArrayBuffer }) => { - pdfjsLib.GlobalWorkerOptions.workerSrc = '/js/pdf.worker.js'; +type TokenType = { + str: string; + dir: string; + width: number; + height: number; + transform: number[]; + fontName: string; + hasEOL: boolean; +}; - type TokenType = { - str: string; - dir: string; - width: number; - height: number; - transform: number[]; - fontName: string; - hasEOL: boolean; - }; +export const readPdfFile = async ({ pdf }: { pdf: ArrayBuffer }) => { + pdfjsLib.GlobalWorkerOptions.workerSrc = '/js/pdf.worker.js'; const readPDFPage = async (doc: any, pageNo: number) => { const page = await doc.getPage(pageNo); @@ -58,5 +58,7 @@ export const readPdfFile = async ({ pdf }: { pdf: string | URL | ArrayBuffer }) } const pageTexts = await Promise.all(pageTextPromises); - return pageTexts.join(''); + return { + rawText: pageTexts.join('') + }; }; diff --git a/packages/web/common/file/read/rawText.ts b/packages/web/common/file/read/rawText.ts new file mode 100644 index 00000000000..6a9e4faeba0 --- /dev/null +++ b/packages/web/common/file/read/rawText.ts @@ -0,0 +1,22 @@ +/** + * read file raw text + */ +export const readFileRawText = (file: File) => { + return new Promise<{ rawText: string }>((resolve, reject) => { + try { + const reader = new FileReader(); + reader.onload = () => { + resolve({ + rawText: reader.result as string + }); + }; + reader.onerror = (err) => { + console.log('error txt read:', err); + reject('Read file error'); + }; + reader.readAsText(file); + } catch (error) { + reject(error); + } + }); +}; diff --git a/packages/web/common/file/read/word.ts b/packages/web/common/file/read/word.ts new file mode 100644 index 00000000000..24f93c789a5 --- /dev/null +++ b/packages/web/common/file/read/word.ts @@ -0,0 +1,28 @@ +import { markdownProcess } from '@fastgpt/global/common/string/markdown'; +import { htmlStr2Md } from '../../string/markdown'; +import { loadFile2Buffer } from '../utils'; +import mammoth from 'mammoth'; + +export const readWordFile = async ({ + file, + uploadImgController +}: { + file: File; + uploadImgController?: (base64: string) => Promise; +}) => { + const buffer = await loadFile2Buffer({ file }); + + const { value: html } = await mammoth.convertToHtml({ + arrayBuffer: buffer + }); + const md = htmlStr2Md(html); + + const rawText = await markdownProcess({ + rawText: md, + uploadImgController: uploadImgController + }); + + return { + rawText + }; +}; diff --git a/packages/web/common/file/utils.ts b/packages/web/common/file/utils.ts new file mode 100644 index 00000000000..74d16a22673 --- /dev/null +++ b/packages/web/common/file/utils.ts @@ -0,0 +1,31 @@ +import { getErrText } from '@fastgpt/global/common/error/utils'; + +export const loadFile2Buffer = ({ file, onError }: { file: File; onError?: (err: any) => void }) => + new Promise((resolve, reject) => { + try { + let reader = new FileReader(); + reader.readAsArrayBuffer(file); + reader.onload = async ({ target }) => { + if (!target?.result) { + onError?.('Load file error'); + return reject('Load file error'); + } + try { + resolve(target.result as ArrayBuffer); + } catch (err) { + console.log(err, 'Load file error'); + onError?.(err); + + reject(getErrText(err, 'Load file error')); + } + }; + reader.onerror = (err) => { + console.log(err, 'Load file error'); + onError?.(err); + + reject(getErrText(err, 'Load file error')); + }; + } catch (error) { + reject('The browser does not support file content reading'); + } + }); diff --git a/packages/web/components/common/Icon/constants.ts b/packages/web/components/common/Icon/constants.ts index eead911dc6b..15a935dd6a8 100644 --- a/packages/web/components/common/Icon/constants.ts +++ b/packages/web/components/common/Icon/constants.ts @@ -1,7 +1,6 @@ // @ts-nocheck export const iconPaths = { - chat: () => import('./icons/chat.svg'), chatSend: () => import('./icons/chatSend.svg'), closeSolid: () => import('./icons/closeSolid.svg'), collectionLight: () => import('./icons/collectionLight.svg'), diff --git a/packages/web/components/common/Icon/icons/chat.svg b/packages/web/components/common/Icon/icons/chat.svg deleted file mode 100644 index c83e51a52e5..00000000000 --- a/packages/web/components/common/Icon/icons/chat.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/courseLight.svg b/packages/web/components/common/Icon/icons/common/courseLight.svg index d8da2e2cbb1..f2499c88255 100644 --- a/packages/web/components/common/Icon/icons/common/courseLight.svg +++ b/packages/web/components/common/Icon/icons/common/courseLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/gitLight.svg b/packages/web/components/common/Icon/icons/common/gitLight.svg index dc921e04d5b..58970483313 100644 --- a/packages/web/components/common/Icon/icons/common/gitLight.svg +++ b/packages/web/components/common/Icon/icons/common/gitLight.svg @@ -1 +1,6 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/navbar/pluginFill.svg b/packages/web/components/common/Icon/icons/common/navbar/pluginFill.svg index 516bf45ba14..58780e79a5e 100644 --- a/packages/web/components/common/Icon/icons/common/navbar/pluginFill.svg +++ b/packages/web/components/common/Icon/icons/common/navbar/pluginFill.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/common/navbar/pluginLight.svg b/packages/web/components/common/Icon/icons/common/navbar/pluginLight.svg index e53209d8003..dc9d8da06a3 100644 --- a/packages/web/components/common/Icon/icons/common/navbar/pluginLight.svg +++ b/packages/web/components/common/Icon/icons/common/navbar/pluginLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/aiFill.svg b/packages/web/components/common/Icon/icons/core/app/aiFill.svg index cc478d8d145..e1c47f1935a 100644 --- a/packages/web/components/common/Icon/icons/core/app/aiFill.svg +++ b/packages/web/components/common/Icon/icons/core/app/aiFill.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/app/aiLight.svg b/packages/web/components/common/Icon/icons/core/app/aiLight.svg index 016b34de4c9..ac7dd776d75 100644 --- a/packages/web/components/common/Icon/icons/core/app/aiLight.svg +++ b/packages/web/components/common/Icon/icons/core/app/aiLight.svg @@ -1 +1,10 @@ - \ No newline at end of file + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/chat/chatFill.svg b/packages/web/components/common/Icon/icons/core/chat/chatFill.svg index 08f301463c6..dc84253cb26 100644 --- a/packages/web/components/common/Icon/icons/core/chat/chatFill.svg +++ b/packages/web/components/common/Icon/icons/core/chat/chatFill.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/chat/chatLight.svg b/packages/web/components/common/Icon/icons/core/chat/chatLight.svg index aa20dd8d388..17bfab514e0 100644 --- a/packages/web/components/common/Icon/icons/core/chat/chatLight.svg +++ b/packages/web/components/common/Icon/icons/core/chat/chatLight.svg @@ -1 +1,10 @@ - \ No newline at end of file + + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/commonDataset.svg b/packages/web/components/common/Icon/icons/core/dataset/commonDataset.svg index c495183842a..a4cddf7881f 100644 --- a/packages/web/components/common/Icon/icons/core/dataset/commonDataset.svg +++ b/packages/web/components/common/Icon/icons/core/dataset/commonDataset.svg @@ -1 +1,8 @@ - \ No newline at end of file + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/datasetFill.svg b/packages/web/components/common/Icon/icons/core/dataset/datasetFill.svg index 0cba56612a5..e8554594a72 100644 --- a/packages/web/components/common/Icon/icons/core/dataset/datasetFill.svg +++ b/packages/web/components/common/Icon/icons/core/dataset/datasetFill.svg @@ -1 +1,8 @@ - \ No newline at end of file + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/datasetLight.svg b/packages/web/components/common/Icon/icons/core/dataset/datasetLight.svg index b784f35dd5d..4114aec8ff1 100644 --- a/packages/web/components/common/Icon/icons/core/dataset/datasetLight.svg +++ b/packages/web/components/common/Icon/icons/core/dataset/datasetLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/core/dataset/websiteDataset.svg b/packages/web/components/common/Icon/icons/core/dataset/websiteDataset.svg index cd6efe52957..570bde86c74 100644 --- a/packages/web/components/common/Icon/icons/core/dataset/websiteDataset.svg +++ b/packages/web/components/common/Icon/icons/core/dataset/websiteDataset.svg @@ -1,14 +1,5 @@ - - - - + - + d="M9.85672 2.01367H4.14326C3.70589 2.01366 3.3426 2.01366 3.04618 2.03788C2.73757 2.06309 2.45044 2.11744 2.1797 2.25539C1.76241 2.468 1.42315 2.80727 1.21053 3.22455C1.07258 3.49529 1.01824 3.78242 0.993023 4.09103C0.968804 4.38745 0.968811 4.75074 0.968819 5.18812V7.63287C0.968811 8.07025 0.968804 8.43354 0.993023 8.72996C1.01824 9.03857 1.07258 9.3257 1.21053 9.59644C1.42315 10.0137 1.76241 10.353 2.1797 10.5656C2.45044 10.7036 2.73757 10.7579 3.04618 10.7831C3.34257 10.8073 3.70583 10.8073 4.14316 10.8073H6.41666V11.8198H4.82086C4.49869 11.8198 4.23752 12.081 4.23752 12.4031C4.23752 12.7253 4.49869 12.9865 4.82086 12.9865H9.17913C9.5013 12.9865 9.76246 12.7253 9.76246 12.4031C9.76246 12.081 9.5013 11.8198 9.17913 11.8198H7.58333V10.8073H9.85671C10.294 10.8073 10.6574 10.8073 10.9538 10.7831C11.2624 10.7579 11.5495 10.7036 11.8203 10.5656C12.2376 10.353 12.5768 10.0137 12.7895 9.59644C12.9274 9.3257 12.9818 9.03857 13.007 8.72996C13.0312 8.43355 13.0312 8.07027 13.0312 7.63292V5.1881C13.0312 4.75075 13.0312 4.38744 13.007 4.09103C12.9818 3.78242 12.9274 3.49529 12.7895 3.22455C12.5768 2.80727 12.2376 2.468 11.8203 2.25539C11.5495 2.11744 11.2624 2.06309 10.9538 2.03788C10.6574 2.01366 10.2941 2.01366 9.85672 2.01367Z" + fill="#8A95A7" /> \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/support/user/userFill.svg b/packages/web/components/common/Icon/icons/support/user/userFill.svg index e9f4226c535..8c17a63107a 100644 --- a/packages/web/components/common/Icon/icons/support/user/userFill.svg +++ b/packages/web/components/common/Icon/icons/support/user/userFill.svg @@ -1 +1,8 @@ - \ No newline at end of file + + + + + \ No newline at end of file diff --git a/packages/web/components/common/Icon/icons/support/user/userLight.svg b/packages/web/components/common/Icon/icons/support/user/userLight.svg index 9147dba3749..4f31779139d 100644 --- a/packages/web/components/common/Icon/icons/support/user/userLight.svg +++ b/packages/web/components/common/Icon/icons/support/user/userLight.svg @@ -1 +1,4 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/web/package.json b/packages/web/package.json index be7f3c7596e..119b245f6a0 100644 --- a/packages/web/package.json +++ b/packages/web/package.json @@ -13,9 +13,11 @@ "@fastgpt/global": "workspace:*", "@fingerprintjs/fingerprintjs": "^4.2.1", "@monaco-editor/react": "^4.6.0", + "mammoth": "^1.6.0", "i18next": "^22.5.1", "joplin-turndown-plugin-gfm": "^1.0.12", "next-i18next": "^13.3.0", + "pdfjs-dist": "^4.0.269", "react": "18.2.0", "react-dom": "18.2.0", "react-i18next": "^12.3.1", diff --git a/packages/web/tsconfig.json b/packages/web/tsconfig.json index d57aec33fb5..78e8afa6717 100644 --- a/packages/web/tsconfig.json +++ b/packages/web/tsconfig.json @@ -15,9 +15,6 @@ "jsx": "preserve", "incremental": true, "baseUrl": ".", - "paths": { - "@/*": ["./*"] - } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", "**/*.d.ts", "../**/*.d.ts"], "exclude": ["node_modules","./components/common/Icon/constants.ts"] diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 10aa86a18ce..eaa3df1c3c3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,12 +47,12 @@ importers: js-tiktoken: specifier: ^1.0.7 version: registry.npmmirror.com/js-tiktoken@1.0.7 + nanoid: + specifier: ^4.0.1 + version: registry.npmmirror.com/nanoid@4.0.1 openai: specifier: 4.23.0 version: registry.npmmirror.com/openai@4.23.0(encoding@0.1.13) - pdfjs-dist: - specifier: ^4.0.269 - version: registry.npmmirror.com/pdfjs-dist@4.0.269(encoding@0.1.13) timezones-list: specifier: ^3.0.2 version: registry.npmmirror.com/timezones-list@3.0.2 @@ -96,21 +96,27 @@ importers: jsonwebtoken: specifier: ^9.0.2 version: registry.npmmirror.com/jsonwebtoken@9.0.2 + mammoth: + specifier: ^1.6.0 + version: registry.npmmirror.com/mammoth@1.6.0 mongoose: specifier: ^7.0.2 version: registry.npmmirror.com/mongoose@7.0.2 multer: specifier: 1.4.5-lts.1 version: registry.npmmirror.com/multer@1.4.5-lts.1 - nanoid: - specifier: ^4.0.1 - version: registry.npmmirror.com/nanoid@4.0.1 next: specifier: 13.5.2 version: registry.npmmirror.com/next@13.5.2(@babel/core@7.23.7)(react-dom@18.2.0)(react@18.2.0)(sass@1.58.3) nextjs-cors: specifier: ^2.1.2 version: registry.npmmirror.com/nextjs-cors@2.1.2(next@13.5.2) + node-cron: + specifier: ^3.0.3 + version: registry.npmmirror.com/node-cron@3.0.3 + pdfjs-dist: + specifier: ^4.0.269 + version: registry.npmmirror.com/pdfjs-dist@4.0.269(encoding@0.1.13) pg: specifier: ^8.10.0 version: registry.npmmirror.com/pg@8.10.0 @@ -127,6 +133,9 @@ importers: '@types/multer': specifier: ^1.4.10 version: registry.npmmirror.com/@types/multer@1.4.10 + '@types/node-cron': + specifier: ^3.0.11 + version: registry.npmmirror.com/@types/node-cron@3.0.11 '@types/pg': specifier: ^8.6.6 version: registry.npmmirror.com/@types/pg@8.6.6 @@ -175,9 +184,15 @@ importers: joplin-turndown-plugin-gfm: specifier: ^1.0.12 version: registry.npmmirror.com/joplin-turndown-plugin-gfm@1.0.12 + mammoth: + specifier: ^1.6.0 + version: registry.npmmirror.com/mammoth@1.6.0 next-i18next: specifier: ^13.3.0 version: registry.npmmirror.com/next-i18next@13.3.0(i18next@22.5.1)(next@13.5.2)(react-i18next@12.3.1)(react@18.2.0) + pdfjs-dist: + specifier: ^4.0.269 + version: registry.npmmirror.com/pdfjs-dist@4.0.269(encoding@0.1.13) react: specifier: 18.2.0 version: registry.npmmirror.com/react@18.2.0 @@ -287,9 +302,6 @@ importers: lodash: specifier: ^4.17.21 version: registry.npmmirror.com/lodash@4.17.21 - mammoth: - specifier: ^1.6.0 - version: registry.npmmirror.com/mammoth@1.6.0 mermaid: specifier: ^10.2.3 version: registry.npmmirror.com/mermaid@10.2.3 @@ -5107,6 +5119,12 @@ packages: '@types/express': registry.npmmirror.com/@types/express@4.17.21 dev: true + registry.npmmirror.com/@types/node-cron@3.0.11: + resolution: {integrity: sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node-cron/-/node-cron-3.0.11.tgz} + name: '@types/node-cron' + version: 3.0.11 + dev: true + registry.npmmirror.com/@types/node-fetch@2.6.10: resolution: {integrity: sha512-PPpPK6F9ALFTn59Ka3BaL+qGuipRfxNE8qVgkp0bVixeiR2c2/L+IVOiBdu9JhhT22sWnQEp6YyHGI2b2+CMcA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/@types/node-fetch/-/node-fetch-2.6.10.tgz} name: '@types/node-fetch' @@ -11087,6 +11105,15 @@ packages: next: registry.npmmirror.com/next@13.5.2(@babel/core@7.23.7)(react-dom@18.2.0)(react@18.2.0)(sass@1.58.3) dev: false + registry.npmmirror.com/node-cron@3.0.3: + resolution: {integrity: sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-cron/-/node-cron-3.0.3.tgz} + name: node-cron + version: 3.0.3 + engines: {node: '>=6.0.0'} + dependencies: + uuid: registry.npmmirror.com/uuid@8.3.2 + dev: false + registry.npmmirror.com/node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/node-domexception/-/node-domexception-1.0.0.tgz} name: node-domexception @@ -13768,6 +13795,13 @@ packages: engines: {node: '>= 0.4.0'} dev: false + registry.npmmirror.com/uuid@8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/uuid/-/uuid-8.3.2.tgz} + name: uuid + version: 8.3.2 + hasBin: true + dev: false + registry.npmmirror.com/uuid@9.0.1: resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==, registry: https://registry.npm.taobao.org/, tarball: https://registry.npmmirror.com/uuid/-/uuid-9.0.1.tgz} name: uuid diff --git a/projects/app/.gitignore b/projects/app/.gitignore deleted file mode 100644 index 4b3390ce7a9..00000000000 --- a/projects/app/.gitignore +++ /dev/null @@ -1,32 +0,0 @@ -# dependencies -node_modules/ -# next.js -.next/ -out/ -# production -build/ - -# misc -.DS_Store -*.pem - -# debug -npm-debug.log* -yarn-debug.log* -yarn-error.log* -.pnpm-debug.log* - -# local env files -.env*.local - -# vercel -.vercel - -# typescript -*.tsbuildinfo -next-env.d.ts -platform.json -testApi/ -local/ -.husky/ -data/*.local.* \ No newline at end of file diff --git a/projects/app/package.json b/projects/app/package.json index a16b526961e..baaaa052cc2 100644 --- a/projects/app/package.json +++ b/projects/app/package.json @@ -37,7 +37,6 @@ "jschardet": "^3.0.0", "jsonwebtoken": "^9.0.2", "lodash": "^4.17.21", - "mammoth": "^1.6.0", "mermaid": "^10.2.3", "nanoid": "^4.0.1", "next": "13.5.2", diff --git a/projects/app/public/locales/en/common.json b/projects/app/public/locales/en/common.json index 940dbc8b437..3b3ba828e22 100644 --- a/projects/app/public/locales/en/common.json +++ b/projects/app/public/locales/en/common.json @@ -57,45 +57,6 @@ "Title is required": "Title is required" } }, - "chat": { - "Admin Mark Content": "Corrected response", - "Confirm to clear history": "Confirm to clear history?", - "Confirm to clear share chat history": " Are you sure to delete all chats?", - "Converting to text": "Converting to text...", - "Exit Chat": "Exit", - "Feedback Failed": "Feedback Failed", - "Feedback Mark": "Mark", - "Feedback Modal": "Feedback", - "Feedback Modal Tip": "Enter what you find unsatisfactory", - "Feedback Submit": "Submit", - "Feedback Success": "Feedback Success", - "Feedback Update Failed": "Feedback Update Failed", - "History": "History", - "Mark": "Mark", - "Mark Description": "The annotation feature is currently in beta. \n\n After clicking Add annotation, you need to select a knowledge base in order to store annotation data. You can use this feature to quickly annotate questions and expected answers to guide the model to the next answer. At present, the annotation function, like other data in the knowledge base, is affected by the model, which does not mean that the annotation meets 100% expectations. The \n\n annotation data is only unidirectional synchronization with the knowledge base. If the knowledge base modifies the annotation data, the annotation data displayed in the log cannot be synchronized", - "Mark Description Title": "Mark Description", - "New Chat": "New Chat", - "Fresh Chat": "New Chat", - "Question Guide Tips": "I guess what you're asking is", - "Quote": "Quote", - "Pin": "Pin", - "Unpin": "Unpin", - "Read Mark Description": "Read mark description", - "Select Mark Kb": "Select Dataset", - "Select Mark Kb Desc": "Select a dataset to store the expected answers", - "You need to a chat app": "You need to a chat app", - "Failed to initialize chat": "Failed to initialize chat", - "Custom History Title": "Custom history title", - "Custom History Title Description": "If set to empty, chat history will be followed automatically.", - "History Amount": "{{amount}} records", - "logs": { - "api": "API", - "online": "Online Chat", - "share": "Share", - "test": "Test Chat " - }, - "retry": "Retry" - }, "common": { "Add": "Add", "Add New": "Add", @@ -161,6 +122,7 @@ "Set Avatar": "Set Avatar", "Set Name": "Make a nice name", "Status": "Status", + "Submit success": "Update Success", "Team": "Team", "Test": "Test", "Time": "Time", @@ -170,6 +132,7 @@ "Update Success": "Update Success", "Update Successful": "Update Successful", "Update Time": "Update Time", + "Update success": "Update success", "Upload File Failed": "Upload File Failed", "Username": "UserName", "Website": "Website", @@ -188,6 +151,7 @@ "Common Tip": "No data" }, "error": { + "Update error": "Update error", "unKnow": "There was an accident" }, "export": "", @@ -298,17 +262,46 @@ } }, "chat": { + "Admin Mark Content": "Corrected response", "Audio Speech Error": "Audio Speech Error", + "Confirm to clear history": "Confirm to clear history?", + "Confirm to clear share chat history": " Are you sure to delete all chats?", + "Converting to text": "Converting to text...", + "Custom History Title": "Custom history title", + "Custom History Title Description": "If set to empty, chat history will be followed automatically.", + "Exit Chat": "Exit", + "Failed to initialize chat": "Failed to initialize chat", + "Feedback Failed": "Feedback Failed", + "Feedback Mark": "Mark", + "Feedback Modal": "Feedback", + "Feedback Modal Tip": "Enter what you find unsatisfactory", + "Feedback Submit": "Submit", + "Feedback Success": "Feedback Success", + "Feedback Update Failed": "Feedback Update Failed", + "History": "History", + "History Amount": "{{amount}} records", + "Mark": "Mark", + "Mark Description": "The annotation feature is currently in beta. \n\n After clicking Add annotation, you need to select a knowledge base in order to store annotation data. You can use this feature to quickly annotate questions and expected answers to guide the model to the next answer. At present, the annotation function, like other data in the knowledge base, is affected by the model, which does not mean that the annotation meets 100% expectations. The \n\n annotation data is only unidirectional synchronization with the knowledge base. If the knowledge base modifies the annotation data, the annotation data displayed in the log cannot be synchronized", + "Mark Description Title": "Mark Description", + "New Chat": "New Chat", + "Pin": "Pin", + "Question Guide Tips": "I guess what you're asking is", + "Quote": "Quote", "Quote Amount": "Dataset Quote:{{amount}}", + "Read Mark Description": "Read mark description", "Record": "Speech", "Restart": "Restart", "Select File": "Select file", "Select Image": "Select Image", + "Select Mark Kb": "Select Dataset", + "Select Mark Kb Desc": "Select a dataset to store the expected answers", "Send Message": "Send Message", "Speaking": "I'm listening...", "Start Chat": "Start Chat", "Stop Speak": "Stop Speak", "Type a message": "Input problem", + "Unpin": "Unpin", + "You need to a chat app": "You need to a chat app", "error": { "Chat error": "Chat error", "Messages empty": "Interface content is empty, maybe the text is too long ~", @@ -322,6 +315,12 @@ "No Content": "The user did not fill in the specific feedback content", "Read User dislike": "User dislike\nClick to view content" }, + "logs": { + "api": "API", + "online": "Online Chat", + "share": "Share", + "test": "Test Chat " + }, "markdown": { "Edit Question": "Edit Question", "Quick Question": "Ask the question immediately", @@ -365,6 +364,7 @@ "search using reRank": "ReRank", "text output": "Text Output" }, + "retry": "Retry", "tts": { "Stop Speech": "Stop" } @@ -383,9 +383,11 @@ "Delete Website Tips": "Confirm to delete the website", "Empty Dataset": "", "Empty Dataset Tips": "There is no knowledge base yet, go create one!", + "File collection": "File collection", "Folder Dataset": "Folder", "Go Dataset": "To Dataset", "Intro Placeholder": "This dataset has not yet been introduced~", + "Manual collection": "Manual collection", "My Dataset": "My Dataset", "Name": "Name", "Quote Length": "Quote Length", @@ -395,7 +397,6 @@ "Set Website Config": "Configuring Website", "Similarity": "Similarity", "Sync Time": "Update Time", - "Virtual File": "Virtual File", "Website Dataset": "Website Sync", "Website Dataset Desc": "Web site synchronization allows you to build a knowledge base directly from a web link", "collection": { @@ -455,7 +456,8 @@ "Total Amount": "{{total}} Chunks", "data is deleted": "Data is deleted", "get data error": "Get data error", - "id": "Data ID" + "id": "Data ID", + "unit": "pieces" }, "error": { "Start Sync Failed": "Start Sync Failed", @@ -704,7 +706,7 @@ "Confirm to delete the data": "Confirm to delete the data?", "Confirm to delete the file": "Are you sure to delete the file and all its data?", "Create Folder": "Create Folder", - "Create Virtual File": "Virtual File", + "Create manual collection": "Manual collection", "Delete Dataset Error": "Delete dataset failed", "Edit Folder": "Edit Folder", "Export": "Export", @@ -715,6 +717,7 @@ "Files": "{{total}} Files", "Folder Name": "Input folder name", "Insert Data": "Insert", + "Manual collection Tip": "Manual Collections allow you to create a custom container to hold data", "Manual Data": "Manual Data", "Manual Input": "Manual Input", "Manual Mark": "Manual Mark", @@ -727,7 +730,6 @@ "System Data Queue": "Data Queue", "Training Name": "Dataset Training", "Upload Time": "Upload Time", - "Virtual File Tip": "Virtual files allow you to create a custom container to hold data", "collections": { "Click to view file": "View File Data", "Click to view folder": "To Folder", @@ -735,7 +737,7 @@ "Confirm to delete the folder": "Are you sure to delete this folder and all its contents?", "Create And Import": "Create/Import", "Create Training Data": "Training-{{filename}}", - "Create Virtual File Success": "Create Virtual File Success", + "Create manual collection Success": "Create manual collection Success", "Data Amount": "Data Amount", "Select Collection": "Select Collection", "Select One Collection To Store": "Select the collection to store" @@ -918,6 +920,9 @@ "Response Quote tips": "The referenced content is returned in the share link, but the user is not allowed to download the original document." } }, + "subscription": { + "Cancel subscription": "Cancel" + }, "user": { "Price": "Price", "auth": { @@ -927,6 +932,21 @@ "Github": "Github", "Google": "Google", "Provider error": "Login exception, please try again" + }, + "team": { + "Dataset usage": "Dataset usage" + } + }, + "wallet": { + "Buy more": "Buy more", + "Confirm pay": "Confirm pay", + "Pay error": "Pay error", + "Pay success": "Pay success", + "subscription": { + "Current dataset store": "Current dataset store subscription", + "Dataset store": "Dataset store size", + "Dataset store price tip": "Deduct it from the account balance on the 1st of each month", + "Expand size": "Expand size" } } }, @@ -1069,4 +1089,4 @@ "qa": "QA Generation" } } -} \ No newline at end of file +} diff --git a/projects/app/public/locales/zh/common.json b/projects/app/public/locales/zh/common.json index a8d68971cc7..9b051284488 100644 --- a/projects/app/public/locales/zh/common.json +++ b/projects/app/public/locales/zh/common.json @@ -57,45 +57,6 @@ "Title is required": "模块名不能为空" } }, - "chat": { - "Admin Mark Content": "纠正后的回复", - "Confirm to clear history": "确认清空该应用的在线聊天记录？分享和 API 调用的记录不会被清空。", - "Confirm to clear share chat history": "确认删除所有聊天记录？", - "Converting to text": "正在转换为文本...", - "Exit Chat": "退出聊天", - "Feedback Failed": "提交反馈异常", - "Feedback Mark": "标注", - "Feedback Modal": "结果反馈", - "Feedback Modal Tip": "输入你觉得回答不满意的地方", - "Feedback Submit": "提交反馈", - "Feedback Success": "反馈成功!", - "Feedback Update Failed": "更新反馈状态失败", - "History": "记录", - "Mark": "标注预期回答", - "Mark Description": "当前标注功能为测试版。\n\n点击添加标注后，需要选择一个知识库，以便存储标注数据。你可以通过该功能快速的标注问题和预期回答，以便引导模型下次的回答。\n\n目前，标注功能同知识库其他数据一样，受模型的影响，不代表标注后 100% 符合预期。\n\n标注数据仅单向与知识库同步，如果知识库修改了该标注数据，日志展示的标注数据无法同步", - "Mark Description Title": "标注功能介绍", - "New Chat": "新对话", - "Fresh Chat": "新的对话", - "Question Guide Tips": "猜你想问", - "Quote": "引用", - "Pin": "置顶", - "Unpin": "取消置顶", - "Read Mark Description": "查看标注功能介绍", - "Select Mark Kb": "选择知识库", - "Select Mark Kb Desc": "选择一个知识库存储预期答案", - "You need to a chat app": "你需要创建一个应用", - "Failed to initialize chat": "初始化聊天失败", - "Custom History Title": "自定义历史记录标题", - "Custom History Title Description": "如果设置为空，会自动跟随聊天记录。", - "History Amount": "{{amount}}条记录", - "logs": { - "api": "API 调用", - "online": "在线使用", - "share": "外部链接调用", - "test": "测试" - }, - "retry": "重新生成" - }, "common": { "Add": "添加", "Add New": "新增", @@ -161,6 +122,7 @@ "Set Avatar": "点击设置头像", "Set Name": "取个名字", "Status": "状态", + "Submit success": "提交成功", "Team": "团队", "Test": "测试", "Time": "时间", @@ -170,6 +132,7 @@ "Update Success": "更新成功", "Update Successful": "更新成功", "Update Time": "更新时间", + "Update success": "更新成功", "Upload File Failed": "上传文件失败", "Username": "用户名", "Website": "网站", @@ -188,6 +151,7 @@ "Common Tip": "没有什么数据噢~" }, "error": { + "Update error": "更新失败", "unKnow": "出现了点意外~" }, "export": "", @@ -298,17 +262,46 @@ } }, "chat": { + "Admin Mark Content": "纠正后的回复", "Audio Speech Error": "语音播报异常", + "Confirm to clear history": "确认清空该应用的在线聊天记录？分享和 API 调用的记录不会被清空。", + "Confirm to clear share chat history": "确认删除所有聊天记录？", + "Converting to text": "正在转换为文本...", + "Custom History Title": "自定义历史记录标题", + "Custom History Title Description": "如果设置为空，会自动跟随聊天记录。", + "Exit Chat": "退出聊天", + "Failed to initialize chat": "初始化聊天失败", + "Feedback Failed": "提交反馈异常", + "Feedback Mark": "标注", + "Feedback Modal": "结果反馈", + "Feedback Modal Tip": "输入你觉得回答不满意的地方", + "Feedback Submit": "提交反馈", + "Feedback Success": "反馈成功!", + "Feedback Update Failed": "更新反馈状态失败", + "History": "记录", + "History Amount": "{{amount}}条记录", + "Mark": "标注预期回答", + "Mark Description": "当前标注功能为测试版。\n\n点击添加标注后，需要选择一个知识库，以便存储标注数据。你可以通过该功能快速的标注问题和预期回答，以便引导模型下次的回答。\n\n目前，标注功能同知识库其他数据一样，受模型的影响，不代表标注后 100% 符合预期。\n\n标注数据仅单向与知识库同步，如果知识库修改了该标注数据，日志展示的标注数据无法同步", + "Mark Description Title": "标注功能介绍", + "New Chat": "新对话", + "Pin": "置顶", + "Question Guide Tips": "猜你想问", + "Quote": "引用", "Quote Amount": "知识库引用({{amount}}条)", + "Read Mark Description": "查看标注功能介绍", "Record": "语音输入", "Restart": "重开对话", "Select File": "选择文件", "Select Image": "选择图片", + "Select Mark Kb": "选择知识库", + "Select Mark Kb Desc": "选择一个知识库存储预期答案", "Send Message": "发送", "Speaking": "我在听，请说...", "Start Chat": "开始对话", "Stop Speak": "停止录音", "Type a message": "输入问题", + "Unpin": "取消置顶", + "You need to a chat app": "你需要创建一个应用", "error": { "Chat error": "对话出现异常", "Messages empty": "接口内容为空，可能文本超长了~", @@ -322,6 +315,12 @@ "No Content": "用户没有填写具体反馈内容", "Read User dislike": "用户表示反对\n点击查看内容" }, + "logs": { + "api": "API 调用", + "online": "在线使用", + "share": "外部链接调用", + "test": "测试" + }, "markdown": { "Edit Question": "编辑问题", "Quick Question": "点我立即提问", @@ -365,6 +364,7 @@ "search using reRank": "结果重排", "text output": "文本输出" }, + "retry": "重新生成", "tts": { "Stop Speech": "停止" } @@ -383,9 +383,11 @@ "Delete Website Tips": "确认删除该站点？", "Empty Dataset": "", "Empty Dataset Tips": "还没有知识库，快去创建一个吧！", + "File collection": "文件数据集", "Folder Dataset": "文件夹", "Go Dataset": "前往知识库", "Intro Placeholder": "这个知识库还没有介绍~", + "Manual collection": "手动数据集", "My Dataset": "我的知识库", "Name": "知识库名称", "Quote Length": "引用内容长度", @@ -395,7 +397,6 @@ "Set Website Config": "开始配置网站信息", "Similarity": "相关度", "Sync Time": "最后更新时间", - "Virtual File": "虚拟文件", "Website Dataset": "Web 站点同步", "Website Dataset Desc": "Web 站点同步允许你直接使用一个网页链接构建知识库", "collection": { @@ -455,7 +456,8 @@ "Total Amount": "{{total}} 组", "data is deleted": "该数据已被删除", "get data error": "获取数据异常", - "id": "数据ID" + "id": "数据ID", + "unit": "条" }, "error": { "Start Sync Failed": "开始同步失败", @@ -704,7 +706,7 @@ "Confirm to delete the data": "确认删除该数据？", "Confirm to delete the file": "确认删除该文件及其所有数据？", "Create Folder": "创建文件夹", - "Create Virtual File": "创建虚拟文件", + "Create manual collection": "创建手动数据集", "Delete Dataset Error": "删除知识库异常", "Edit Folder": "编辑文件夹", "Export": "导出", @@ -715,6 +717,7 @@ "Files": "文件: {{total}}个", "Folder Name": "输入文件夹名称", "Insert Data": "插入", + "Manual collection Tip": "手动数据集允许创建一个空的容器装入数据", "Manual Data": "手动录入", "Manual Input": "手动录入", "Manual Mark": "手动标注", @@ -727,7 +730,6 @@ "System Data Queue": "排队长度", "Training Name": "数据训练", "Upload Time": "上传时间", - "Virtual File Tip": "虚拟文件允许创建一个自定义的容器装入数据", "collections": { "Click to view file": "点击查看文件详情", "Click to view folder": "进入目录", @@ -735,7 +737,7 @@ "Confirm to delete the folder": "确认删除该文件夹及里面所有内容？", "Create And Import": "新建/导入", "Create Training Data": "文件训练-{{filename}}", - "Create Virtual File Success": "创建虚拟文件成功", + "Create manual collection Success": "创建手动数据集成功", "Data Amount": "数据总量", "Select Collection": "选择文件", "Select One Collection To Store": "选择一个文件进行存储" @@ -918,6 +920,9 @@ "Response Quote tips": "在分享链接中返回引用内容，但不会允许用户下载原文档" } }, + "subscription": { + "Cancel subscription": "取消订阅" + }, "user": { "Price": "计费标准", "auth": { @@ -927,6 +932,21 @@ "Github": "Github 登录", "Google": "Google 登录", "Provider error": "登录异常，请重试" + }, + "team": { + "Dataset usage": "知识库容量" + } + }, + "wallet": { + "Buy more": "扩容", + "Confirm pay": "支付确认", + "Pay error": "支付失败", + "Pay success": "支付成功", + "subscription": { + "Current dataset store": "当前额外容量", + "Dataset store": "知识库容量", + "Dataset store price tip": "每月1号从账号余额里扣除", + "Expand size": "扩大容量" } } }, @@ -1069,4 +1089,4 @@ "qa": "QA 拆分" } } -} \ No newline at end of file +} diff --git a/projects/app/src/components/ChatBox/FeedbackModal.tsx b/projects/app/src/components/ChatBox/FeedbackModal.tsx index 163b010929d..a29cfba5acf 100644 --- a/projects/app/src/components/ChatBox/FeedbackModal.tsx +++ b/projects/app/src/components/ChatBox/FeedbackModal.tsx @@ -40,8 +40,8 @@ const FeedbackModal = ({ onSuccess() { onSuccess(ref.current?.value || t('core.chat.feedback.No Content')); }, - successToast: t('chat.Feedback Success'), - errorToast: t('chat.Feedback Failed') + successToast: t('core.chat.Feedback Success'), + errorToast: t('core.chat.Feedback Failed') }); return ( @@ -49,17 +49,17 @@ const FeedbackModal = ({ isOpen={true} onClose={onClose} iconSrc="/imgs/modal/badAnswer.svg" - title={t('chat.Feedback Modal')} + title={t('core.chat.Feedback Modal')} > - + diff --git a/projects/app/src/components/ChatBox/MessageInput.tsx b/projects/app/src/components/ChatBox/MessageInput.tsx index 35115679009..39a727435ad 100644 --- a/projects/app/src/components/ChatBox/MessageInput.tsx +++ b/projects/app/src/components/ChatBox/MessageInput.tsx @@ -8,10 +8,11 @@ import MyIcon from '@fastgpt/web/components/common/Icon'; import { useRouter } from 'next/router'; import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { compressImgFileAndUpload } from '@/web/common/file/controller'; -import { useToast } from '@/web/common/hooks/useToast'; import { customAlphabet } from 'nanoid'; import { IMG_BLOCK_KEY } from '@fastgpt/global/core/chat/constants'; import { addDays } from 'date-fns'; +import { useRequest } from '@/web/common/hooks/useRequest'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 6); enum FileTypeEnum { @@ -45,7 +46,6 @@ const MessageInput = ({ resetInputVal: (val: string) => void; }) => { const { shareId } = useRouter().query as { shareId?: string }; - const { toast } = useToast(); const { isSpeaking, isTransCription, @@ -68,17 +68,18 @@ const MessageInput = ({ maxCount: 10 }); - const uploadFile = useCallback( - async (file: FileItemType) => { + const { mutate: uploadFile } = useRequest({ + mutationFn: async (file: FileItemType) => { if (file.type === FileTypeEnum.image) { try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.chatImage, file: file.rawFile, maxW: 4329, maxH: 4329, maxSize: 1024 * 1024 * 5, // 30 day expired. - expiredTime: addDays(new Date(), 30), + expiredTime: addDays(new Date(), 7), shareId }); setFileList((state) => @@ -94,16 +95,13 @@ const MessageInput = ({ } catch (error) { setFileList((state) => state.filter((item) => item.id !== file.id)); console.log(error); - - toast({ - status: 'error', - title: t('common.Upload File Failed') - }); + return Promise.reject(error); } } }, - [shareId, t, toast] - ); + errorToast: t('common.Upload File Failed') + }); + const onSelectFile = useCallback( async (files: File[]) => { if (!files || files.length === 0) { @@ -219,7 +217,7 @@ ${images.map((img) => JSON.stringify({ src: img.src })).join('\n')} visibility={isSpeaking && isTransCription ? 'visible' : 'hidden'} > - {t('chat.Converting to text')} + {t('core.chat.Converting to text')} {/* file preview */} diff --git a/projects/app/src/components/ChatBox/QuoteModal.tsx b/projects/app/src/components/ChatBox/QuoteModal.tsx index b058943e81a..3299b607bb0 100644 --- a/projects/app/src/components/ChatBox/QuoteModal.tsx +++ b/projects/app/src/components/ChatBox/QuoteModal.tsx @@ -5,7 +5,7 @@ import MyModal from '../MyModal'; import { useTranslation } from 'next-i18next'; import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type'; import QuoteItem from '../core/dataset/QuoteItem'; -import { RawSourceText } from '@/pages/dataset/detail/components/InputDataModal'; +import RawSourceBox from '../core/dataset/RawSourceBox'; const QuoteModal = ({ rawSearch = [], @@ -46,7 +46,7 @@ const QuoteModal = ({ title={ {metadata ? ( - + ) : ( <>{t('core.chat.Quote Amount', { amount: rawSearch.length })} )} diff --git a/projects/app/src/components/ChatBox/ReadFeedbackModal.tsx b/projects/app/src/components/ChatBox/ReadFeedbackModal.tsx index b9218e08436..7ff0ee33352 100644 --- a/projects/app/src/components/ChatBox/ReadFeedbackModal.tsx +++ b/projects/app/src/components/ChatBox/ReadFeedbackModal.tsx @@ -19,7 +19,7 @@ const ReadFeedbackModal = ({ isOpen={true} onClose={onClose} iconSrc="/imgs/modal/readFeedback.svg" - title={t('chat.Feedback Modal')} + title={t('core.chat.Feedback Modal')} > {content} diff --git a/projects/app/src/components/ChatBox/ResponseTags.tsx b/projects/app/src/components/ChatBox/ResponseTags.tsx index 8a1a1d6aaa0..6665f74e0f5 100644 --- a/projects/app/src/components/ChatBox/ResponseTags.tsx +++ b/projects/app/src/components/ChatBox/ResponseTags.tsx @@ -92,7 +92,7 @@ const ResponseTags = ({ <> {sourceList.length > 0 && ( <> - + {sourceList.map((item) => ( diff --git a/projects/app/src/components/ChatBox/SelectMarkCollection.tsx b/projects/app/src/components/ChatBox/SelectMarkCollection.tsx index db67d9b7b9f..425aa1af888 100644 --- a/projects/app/src/components/ChatBox/SelectMarkCollection.tsx +++ b/projects/app/src/components/ChatBox/SelectMarkCollection.tsx @@ -46,7 +46,7 @@ const SelectMarkCollection = ({ paths={paths} onClose={onClose} setParentId={setParentId} - tips={t('chat.Select Mark Kb Desc')} + tips={t('core.chat.Select Mark Kb Desc')} > { - if (!data.q || !adminMarkData.datasetId || !adminMarkData.collectionId || !data.id) { + if ( + !data.q || + !adminMarkData.datasetId || + !adminMarkData.collectionId || + !data.dataId + ) { return onClose(); } onSuccess({ - dataId: data.id, + dataId: data.dataId, datasetId: adminMarkData.datasetId, collectionId: adminMarkData.collectionId, q: data.q, diff --git a/projects/app/src/components/ChatBox/index.tsx b/projects/app/src/components/ChatBox/index.tsx index ef09f57249d..9c57b2f8876 100644 --- a/projects/app/src/components/ChatBox/index.tsx +++ b/projects/app/src/components/ChatBox/index.tsx @@ -910,14 +910,15 @@ const ChatBox = ( )} {/* admin mark content */} {showMarkIcon && item.adminFeedback && ( - + - {`${item.adminFeedback.q || ''}${ - item.adminFeedback.a ? `\n${item.adminFeedback.a}` : '' - }`} + + {item.adminFeedback.q} + {item.adminFeedback.a} + )} @@ -996,6 +997,7 @@ const ChatBox = ( setAdminMarkData={(e) => setAdminMarkData({ ...e, chatItemId: adminMarkData.chatItemId })} onClose={() => setAdminMarkData(undefined)} onSuccess={(adminFeedback) => { + console.log(adminMarkData); if (!appId || !chatId || !adminMarkData.chatItemId) return; updateChatAdminFeedback({ appId, @@ -1003,6 +1005,7 @@ const ChatBox = ( chatItemId: adminMarkData.chatItemId, ...adminFeedback }); + // update dom setChatHistory((state) => state.map((chatItem) => @@ -1234,7 +1237,7 @@ function ChatController({ {!!onDelete && ( <> {onRetry && ( - + ))} {!!onMark && ( - + { { label: t('navbar.Chat'), icon: 'core/chat/chatLight', - activeIcon: 'chatcore/dataset/chatFillFill', + activeIcon: 'core/chat/chatFill', link: `/chat?appId=${lastChatAppId}&chatId=${lastChatId}`, activeLink: ['/chat'] }, @@ -77,6 +77,12 @@ const Navbar = ({ unread }: { unread: number }) => { h: '58px', borderRadius: 'md' }; + const hoverStyle: LinkProps = { + _hover: { + bg: 'myGray.05', + color: 'primary.600' + } + }; return ( { @@ -161,10 +168,11 @@ const Navbar = ({ unread }: { unread: number }) => { @@ -177,8 +185,9 @@ const Navbar = ({ unread }: { unread: number }) => { href="https://github.com/labring/FastGPT" target={'_blank'} {...itemStyles} + {...hoverStyle} mt={0} - color={'#9096a5'} + color={'myGray.500'} > diff --git a/projects/app/src/components/Markdown/chat/QuestionGuide.tsx b/projects/app/src/components/Markdown/chat/QuestionGuide.tsx index 26a2f26509e..e187bb3d880 100644 --- a/projects/app/src/components/Markdown/chat/QuestionGuide.tsx +++ b/projects/app/src/components/Markdown/chat/QuestionGuide.tsx @@ -24,7 +24,7 @@ const QuestionGuide = ({ text }: { text: string }) => { return questionGuides.length > 0 ? ( - + {questionGuides.map((text) => ( { const [isLoading, setIsLoading] = useState(true); const [succeed, setSucceed] = useState(false); const { isOpen, onOpen, onClose } = useDisclosure(); + const [scale, setScale] = useState(1); + + const handleWheel: WheelEventHandler = (e) => { + setScale((prevScale) => { + const newScale = prevScale + e.deltaY * 0.5 * -0.01; + if (newScale < 0.5) return 0.5; + if (newScale > 10) return 10; + return newScale; + }); + }; + return ( { { fallbackSrc={'/imgs/errImg.png'} fallbackStrategy={'onError'} objectFit={'contain'} + onWheel={handleWheel} /> diff --git a/projects/app/src/components/Markdown/index.module.scss b/projects/app/src/components/Markdown/index.module.scss index 552151dec31..684e63fde79 100644 --- a/projects/app/src/components/Markdown/index.module.scss +++ b/projects/app/src/components/Markdown/index.module.scss @@ -343,7 +343,6 @@ margin: 10px 0; } .markdown { - text-align: justify; tab-size: 4; word-spacing: normal; width: 100%; diff --git a/projects/app/src/components/Markdown/index.tsx b/projects/app/src/components/Markdown/index.tsx index c413a9a8aed..2c51ac8e915 100644 --- a/projects/app/src/components/Markdown/index.tsx +++ b/projects/app/src/components/Markdown/index.tsx @@ -112,6 +112,17 @@ function A({ children, ...props }: any) { } const Markdown = ({ source, isChatting = false }: { source: string; isChatting?: boolean }) => { + const components = useMemo( + () => ({ + img: Image, + pre: 'div', + p: (pProps: any) =>

, + code: Code, + a: A + }), + [] + ); + const formatSource = source .replace(/\\n/g, '\n ') .replace(/(http[s]?:\/\/[^\s，。]+)([。，])/g, '$1 $2') @@ -124,13 +135,7 @@ const Markdown = ({ source, isChatting = false }: { source: string; isChatting?: `} remarkPlugins={[RemarkMath, RemarkGfm, RemarkBreaks]} rehypePlugins={[RehypeKatex]} - components={{ - img: Image, - pre: 'div', - p: (pProps) =>

, - code: Code, - a: A - }} + components={components} linkTarget={'_blank'} > {formatSource} diff --git a/projects/app/src/components/Select/SelectAiModel.tsx b/projects/app/src/components/Select/SelectAiModel.tsx index 28045e95f5e..3e7f1c947d5 100644 --- a/projects/app/src/components/Select/SelectAiModel.tsx +++ b/projects/app/src/components/Select/SelectAiModel.tsx @@ -4,18 +4,19 @@ import MySelect, { type SelectProps } from './index'; import { useTranslation } from 'next-i18next'; import dynamic from 'next/dynamic'; import { useDisclosure } from '@chakra-ui/react'; +import { feConfigs } from '@/web/common/system/staticData'; const PriceBox = dynamic(() => import('@/components/support/wallet/Price')); const SelectAiModel = ({ list, ...props }: SelectProps) => { const { t } = useTranslation(); - const expandList = useMemo( - () => - list.concat({ - label: t('support.user.Price'), - value: 'price' - }), - [list, t] - ); + const expandList = useMemo(() => { + return feConfigs.show_pay + ? list.concat({ + label: t('support.user.Price'), + value: 'price' + }) + : list; + }, [list, t]); const { isOpen: isOpenPriceBox, diff --git a/projects/app/src/components/core/dataset/DatasetTypeTag.tsx b/projects/app/src/components/core/dataset/DatasetTypeTag.tsx new file mode 100644 index 00000000000..12e8e00469a --- /dev/null +++ b/projects/app/src/components/core/dataset/DatasetTypeTag.tsx @@ -0,0 +1,30 @@ +import { Box, Flex, FlexProps } from '@chakra-ui/react'; +import { DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import MyIcon from '@fastgpt/web/components/common/Icon'; +import { useTranslation } from 'next-i18next'; +import React from 'react'; +import { DatasetTypeMap } from '@fastgpt/global/core/dataset/constant'; + +const DatasetTypeTag = ({ type, ...props }: { type: `${DatasetTypeEnum}` } & FlexProps) => { + const { t } = useTranslation(); + + const item = DatasetTypeMap[type]; + + return ( + + + {t(item.label)} + + ); +}; + +export default DatasetTypeTag; diff --git a/projects/app/src/components/core/dataset/QuoteItem.tsx b/projects/app/src/components/core/dataset/QuoteItem.tsx index bdc61b6433d..feb22152e69 100644 --- a/projects/app/src/components/core/dataset/QuoteItem.tsx +++ b/projects/app/src/components/core/dataset/QuoteItem.tsx @@ -1,9 +1,6 @@ import React, { useMemo, useState } from 'react'; import { Box, Flex, Link, Progress } from '@chakra-ui/react'; -import { - type InputDataType, - RawSourceText -} from '@/pages/dataset/detail/components/InputDataModal'; +import RawSourceBox from '@/components/core/dataset/RawSourceBox'; import type { SearchDataResponseItemType } from '@fastgpt/global/core/dataset/type.d'; import NextLink from 'next/link'; import MyIcon from '@fastgpt/web/components/common/Icon'; @@ -11,9 +8,6 @@ import { useTranslation } from 'next-i18next'; import MyTooltip from '@/components/MyTooltip'; import dynamic from 'next/dynamic'; import MyBox from '@/components/common/MyBox'; -import { getDatasetDataItemById } from '@/web/core/dataset/api'; -import { useRequest } from '@/web/common/hooks/useRequest'; -import { DatasetDataItemType } from '@fastgpt/global/core/dataset/type'; import { SearchScoreTypeEnum, SearchScoreTypeMap } from '@fastgpt/global/core/dataset/constant'; const InputDataModal = dynamic(() => import('@/pages/dataset/detail/components/InputDataModal')); @@ -58,17 +52,7 @@ const QuoteItem = ({ linkToDataset?: boolean; }) => { const { t } = useTranslation(); - const [editInputData, setEditInputData] = useState(); - - const { mutate: onclickEdit, isLoading } = useRequest({ - mutationFn: async (id: string) => { - return getDatasetDataItemById(id); - }, - onSuccess(data: DatasetDataItemType) { - setEditInputData(data); - }, - errorToast: t('core.dataset.data.get data error') - }); + const [editInputData, setEditInputData] = useState<{ dataId: string; collectionId: string }>(); const score = useMemo(() => { if (!Array.isArray(quoteItem.score)) { @@ -114,7 +98,6 @@ const QuoteItem = ({ return ( <> - + {score?.primaryScore && ( <> {canViewSource ? ( @@ -132,7 +115,6 @@ const QuoteItem = ({ ( - + #{item.index + 1} @@ -223,7 +205,7 @@ const QuoteItem = ({ {quoteItem.q.length + (quoteItem.a?.length || 0)} - onclickEdit(quoteItem.id)} + onClick={() => + setEditInputData({ + dataId: quoteItem.id, + collectionId: quoteItem.collectionId + }) + } /> @@ -271,7 +258,7 @@ const QuoteItem = ({ )} - {editInputData && editInputData.id && ( + {editInputData && ( setEditInputData(undefined)} onSuccess={() => { @@ -280,7 +267,7 @@ const QuoteItem = ({ onDelete={() => { console.log('删除引用成功'); }} - defaultValue={editInputData} + dataId={editInputData.dataId} collectionId={editInputData.collectionId} /> )} diff --git a/projects/app/src/components/core/dataset/RawSourceBox.tsx b/projects/app/src/components/core/dataset/RawSourceBox.tsx new file mode 100644 index 00000000000..3c88553851e --- /dev/null +++ b/projects/app/src/components/core/dataset/RawSourceBox.tsx @@ -0,0 +1,68 @@ +import React, { useMemo } from 'react'; +import { Box, BoxProps, Image } from '@chakra-ui/react'; +import { useToast } from '@/web/common/hooks/useToast'; +import { getErrText } from '@fastgpt/global/common/error/utils'; +import MyTooltip from '@/components/MyTooltip'; +import { useTranslation } from 'next-i18next'; +import { getFileAndOpen } from '@/web/core/dataset/utils'; +import { useSystemStore } from '@/web/common/system/useSystemStore'; +import { getSourceNameIcon } from '@fastgpt/global/core/dataset/utils'; + +type Props = BoxProps & { + sourceName?: string; + sourceId?: string; + canView?: boolean; +}; + +const RawSourceBox = ({ sourceId, sourceName = '', canView = true, ...props }: Props) => { + const { t } = useTranslation(); + const { toast } = useToast(); + const { setLoading } = useSystemStore(); + + const canPreview = useMemo(() => !!sourceId && canView, [canView, sourceId]); + + const icon = useMemo(() => getSourceNameIcon({ sourceId, sourceName }), [sourceId, sourceName]); + + return ( + + { + setLoading(true); + try { + await getFileAndOpen(sourceId as string); + } catch (error) { + toast({ + title: t(getErrText(error, 'error.fileNotFound')), + status: 'error' + }); + } + setLoading(false); + } + } + : {})} + {...props} + > + + + {sourceName || t('common.UnKnow Source')} + + + + ); +}; + +export default RawSourceBox; diff --git a/projects/app/src/components/core/dataset/SelectModal.tsx b/projects/app/src/components/core/dataset/SelectModal.tsx index ab333d3c805..e3557a8231c 100644 --- a/projects/app/src/components/core/dataset/SelectModal.tsx +++ b/projects/app/src/components/core/dataset/SelectModal.tsx @@ -38,7 +38,7 @@ const DatasetSelectContainer = ({ parentId: path.parentId, parentName: path.parentName }))} - FirstPathDom={t('chat.Select Mark Kb')} + FirstPathDom={t('core.chat.Select Mark Kb')} onClick={(e) => { setParentId(e); }} diff --git a/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx b/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx index de6407a1263..4583a07ee76 100644 --- a/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx +++ b/projects/app/src/components/support/user/team/TeamManageModal/EditModal.tsx @@ -12,6 +12,7 @@ import MyTooltip from '@/components/MyTooltip'; import Avatar from '@/components/Avatar'; import { postCreateTeam, putUpdateTeam } from '@/web/support/user/team/api'; import { CreateTeamProps } from '@fastgpt/global/support/user/team/controller.d'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; export type FormDataType = CreateTeamProps & { id?: string; @@ -50,6 +51,7 @@ function EditModal({ if (!file) return; try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.teamAvatar, file, maxW: 300, maxH: 300 diff --git a/projects/app/src/components/support/wallet/Price.tsx b/projects/app/src/components/support/wallet/Price.tsx index 34486924471..6182da4b1a7 100644 --- a/projects/app/src/components/support/wallet/Price.tsx +++ b/projects/app/src/components/support/wallet/Price.tsx @@ -17,14 +17,6 @@ import Markdown from '@/components/Markdown'; const Price = ({ onClose }: { onClose: () => void }) => { const list = [ - { - title: '知识库存储', - describe: '', - md: ` -| 计费项 | 价格(￥) | -| --- | --- | -| 知识库索引数量 | 0/1000条/天 |` - }, { title: '对话模型', describe: '', diff --git a/projects/app/src/components/support/wallet/SubDatasetModal.tsx b/projects/app/src/components/support/wallet/SubDatasetModal.tsx new file mode 100644 index 00000000000..de50f862d8b --- /dev/null +++ b/projects/app/src/components/support/wallet/SubDatasetModal.tsx @@ -0,0 +1,171 @@ +import React, { useState } from 'react'; + +import MyModal from '@/components/MyModal'; +import { useTranslation } from 'next-i18next'; +import { + Box, + Flex, + ModalBody, + NumberInput, + NumberInputField, + NumberInputStepper, + NumberIncrementStepper, + NumberDecrementStepper, + ModalFooter, + Button +} from '@chakra-ui/react'; +import { useQuery } from '@tanstack/react-query'; +import { getTeamDatasetValidSub, postExpandTeamDatasetSub } from '@/web/support/wallet/sub/api'; +import Markdown from '@/components/Markdown'; +import MyTooltip from '@/components/MyTooltip'; +import { QuestionOutlineIcon } from '@chakra-ui/icons'; +import { useConfirm } from '@/web/common/hooks/useConfirm'; +import { getMonthRemainingDays } from '@fastgpt/global/common/math/date'; +import { useRequest } from '@/web/common/hooks/useRequest'; +import { useRouter } from 'next/router'; +import { feConfigs } from '@/web/common/system/staticData'; +import { useToast } from '@/web/common/hooks/useToast'; +import { formatTime2YMDHM } from '@fastgpt/global/common/string/time'; +import MySelect from '@/components/Select'; + +const SubDatasetModal = ({ onClose }: { onClose: () => void }) => { + const datasetStoreFreeSize = feConfigs?.subscription?.datasetStoreFreeSize || 0; + const datasetStorePrice = feConfigs?.subscription?.datasetStorePrice || 0; + + const { t } = useTranslation(); + const { toast } = useToast(); + const router = useRouter(); + const { ConfirmModal, openConfirm } = useConfirm({}); + const [datasetSize, setDatasetSize] = useState(0); + const [isRenew, setIsRenew] = useState('false'); + + const isExpand = datasetSize > 0; + + const { data: datasetSub } = useQuery(['getTeamDatasetValidSub'], getTeamDatasetValidSub, { + onSuccess(res) { + setIsRenew(`${res?.sub?.renew}`); + } + }); + + const { mutate, isLoading } = useRequest({ + mutationFn: () => postExpandTeamDatasetSub({ size: datasetSize, renew: isRenew === 'true' }), + onSuccess(res) { + if (isExpand) { + router.reload(); + } else { + onClose(); + } + }, + successToast: isExpand ? t('support.wallet.Pay success') : t('common.Update success'), + errorToast: isExpand ? t('support.wallet.Pay error') : t('common.error.Update error') + }); + + return ( + + + <> + + {t('support.user.Price')} + + + + + + + + {t('support.wallet.subscription.Current dataset store')}: + + {datasetSub?.sub?.datasetStoreAmount || 0} + {t('core.dataset.data.unit')} + + + {datasetSub?.sub?.expiredTime && ( + + 到期时间: + {formatTime2YMDHM(datasetSub?.sub?.expiredTime)} + + )} + + + 是否续订: + + + + {t('support.wallet.subscription.Expand size')} + + { + setDatasetSize(Number(e)); + }} + > + + + + + + + 000{t('core.dataset.data.unit')} + + + + + + + + + + + ); +}; + +export default SubDatasetModal; diff --git a/projects/app/src/global/core/api/datasetReq.d.ts b/projects/app/src/global/core/api/datasetReq.d.ts index ed6c4105256..6cad6a2f89c 100644 --- a/projects/app/src/global/core/api/datasetReq.d.ts +++ b/projects/app/src/global/core/api/datasetReq.d.ts @@ -1,5 +1,5 @@ import { - DatasetCollectionTrainingModeEnum, + TrainingModeEnum, DatasetCollectionTypeEnum, DatasetTypeEnum } from '@fastgpt/global/core/dataset/constant'; diff --git a/projects/app/src/global/core/dataset/api.d.ts b/projects/app/src/global/core/dataset/api.d.ts index b4fd2a0420f..5ec3c01806e 100644 --- a/projects/app/src/global/core/dataset/api.d.ts +++ b/projects/app/src/global/core/dataset/api.d.ts @@ -30,7 +30,7 @@ export type InsertOneDatasetDataProps = PushDatasetDataChunkProps & { export type PushDatasetDataProps = { collectionId: string; data: PushDatasetDataChunkProps[]; - mode: `${TrainingModeEnum}`; + trainingMode: `${TrainingModeEnum}`; prompt?: string; billId?: string; }; diff --git a/projects/app/src/pages/account/components/Info.tsx b/projects/app/src/pages/account/components/Info.tsx index 7c4f4d4788b..b2f1743cec3 100644 --- a/projects/app/src/pages/account/components/Info.tsx +++ b/projects/app/src/pages/account/components/Info.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useRef } from 'react'; +import React, { useCallback, useMemo, useRef } from 'react'; import { Box, Flex, @@ -8,7 +8,8 @@ import { Divider, Select, Input, - Link + Link, + Progress } from '@chakra-ui/react'; import { useForm } from 'react-hook-form'; import { UserUpdateParams } from '@/types/user'; @@ -22,7 +23,6 @@ import { compressImgFileAndUpload } from '@/web/common/file/controller'; import { feConfigs, systemVersion } from '@/web/common/system/staticData'; import { useTranslation } from 'next-i18next'; import { timezoneList } from '@fastgpt/global/common/time/timezone'; -import Loading from '@/components/Loading'; import Avatar from '@/components/Avatar'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyTooltip from '@/components/MyTooltip'; @@ -32,20 +32,14 @@ import MySelect from '@/components/Select'; import { formatStorePrice2Read } from '@fastgpt/global/support/wallet/bill/tools'; import { putUpdateMemberName } from '@/web/support/user/team/api'; import { getDocPath } from '@/web/common/system/doc'; +import { getTeamDatasetValidSub } from '@/web/support/wallet/sub/api'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; const TeamMenu = dynamic(() => import('@/components/support/user/team/TeamMenu')); -const PayModal = dynamic(() => import('./PayModal'), { - loading: () => , - ssr: false -}); -const UpdatePswModal = dynamic(() => import('./UpdatePswModal'), { - loading: () => , - ssr: false -}); -const OpenAIAccountModal = dynamic(() => import('./OpenAIAccountModal'), { - loading: () => , - ssr: false -}); +const PayModal = dynamic(() => import('./PayModal')); +const UpdatePswModal = dynamic(() => import('./UpdatePswModal')); +const OpenAIAccountModal = dynamic(() => import('./OpenAIAccountModal')); +const SubDatasetModal = dynamic(() => import('@/components/support/wallet/SubDatasetModal')); const UserInfo = () => { const theme = useTheme(); @@ -69,6 +63,11 @@ const UserInfo = () => { onOpen: onOpenUpdatePsw } = useDisclosure(); const { isOpen: isOpenOpenai, onClose: onCloseOpenai, onOpen: onOpenOpenai } = useDisclosure(); + const { + isOpen: isOpenSubDatasetModal, + onClose: onCloseSubDatasetModal, + onOpen: onOpenSubDatasetModal + } = useDisclosure(); const { File, onOpen: onOpenSelectFile } = useSelectFile({ fileType: '.jpg,.png', @@ -97,6 +96,7 @@ const UserInfo = () => { if (!file || !userInfo) return; try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.userAvatar, file, maxW: 300, maxH: 300 @@ -122,6 +122,27 @@ const UserInfo = () => { } }); + const { data: datasetSub = { maxSize: 0, usedSize: 0 } } = useQuery( + ['getTeamDatasetValidSub'], + getTeamDatasetValidSub + ); + const datasetUsageMap = useMemo(() => { + const rate = datasetSub.usedSize / datasetSub.maxSize; + + const colorScheme = (() => { + if (rate < 0.5) return 'green'; + if (rate < 0.8) return 'yellow'; + return 'red'; + })(); + + return { + colorScheme, + value: rate * 100, + maxSize: datasetSub.maxSize, + usedSize: datasetSub.usedSize + }; + }, [datasetSub.maxSize, datasetSub.usedSize]); + return ( { {t('user.Change')} - - - - {t('user.team.Balance')}:  + {feConfigs.isPlus && ( + <> + + + + {t('user.team.Balance')}:  + + + {formatStorePrice2Read(userInfo?.team?.balance).toFixed(3)} 元 + + {feConfigs?.show_pay && userInfo?.team?.canWrite && ( + + )} + - - {formatStorePrice2Read(userInfo?.team?.balance).toFixed(3)} 元 + + + + {t('support.user.team.Dataset usage')}: {datasetUsageMap.usedSize}/ + {datasetSub.maxSize} + + + + + + - {feConfigs?.show_pay && userInfo?.team?.canWrite && ( - - )} - - + + )} + {feConfigs?.docUrl && ( { onClose={onCloseOpenai} /> )} + {isOpenSubDatasetModal && } ); }; -export default UserInfo; +export default React.memo(UserInfo); diff --git a/projects/app/src/pages/account/components/InformTable.tsx b/projects/app/src/pages/account/components/InformTable.tsx index c79964d1026..9602300535d 100644 --- a/projects/app/src/pages/account/components/InformTable.tsx +++ b/projects/app/src/pages/account/components/InformTable.tsx @@ -46,12 +46,12 @@ const BillTable = () => { }} > - {item.title} + {item.title} {formatTimeToChatTime(item.time)} - + {item.content} {!item.read && ( diff --git a/projects/app/src/pages/api/common/file/upload.ts b/projects/app/src/pages/api/common/file/upload.ts index d2876245174..76101589de7 100644 --- a/projects/app/src/pages/api/common/file/upload.ts +++ b/projects/app/src/pages/api/common/file/upload.ts @@ -3,7 +3,7 @@ import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { uploadFile } from '@fastgpt/service/common/file/gridfs/controller'; -import { getUploadModel, removeFilesByPaths } from '@fastgpt/service/common/file/upload/multer'; +import { getUploadModel } from '@fastgpt/service/common/file/multer'; /** * Creates the multer uploader @@ -16,12 +16,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< let filePaths: string[] = []; try { + const { userId, teamId, tmbId } = await authCert({ req, authToken: true }); + const { files, bucketName, metadata } = await upload.doUpload(req, res); filePaths = files.map((file) => file.path); await connectToDatabase(); - const { userId, teamId, tmbId } = await authCert({ req, authToken: true }); if (!bucketName) { throw new Error('bucketName is empty'); @@ -53,8 +54,6 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< error }); } - - removeFilesByPaths(filePaths); } export const config = { diff --git a/projects/app/src/pages/api/common/file/uploadImage.ts b/projects/app/src/pages/api/common/file/uploadImage.ts index 65bc8a57381..6e58af019bd 100644 --- a/projects/app/src/pages/api/common/file/uploadImage.ts +++ b/projects/app/src/pages/api/common/file/uploadImage.ts @@ -8,15 +8,13 @@ import { UploadImgProps } from '@fastgpt/global/common/file/api'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { base64Img, expiredTime, metadata, shareId } = req.body as UploadImgProps; + const { shareId, ...body } = req.body as UploadImgProps; const { teamId } = await authCertOrShareId({ req, shareId, authToken: true }); const data = await uploadMongoImg({ teamId, - base64Img, - expiredTime, - metadata + ...body }); jsonRes(res, { data }); diff --git a/projects/app/src/pages/api/common/system/getInitData.ts b/projects/app/src/pages/api/common/system/getInitData.ts index 4878e816556..8eb660f3570 100644 --- a/projects/app/src/pages/api/common/system/getInitData.ts +++ b/projects/app/src/pages/api/common/system/getInitData.ts @@ -59,39 +59,44 @@ const defaultFeConfigs: FastGPTFeConfigsType = { }; export async function getInitConfig() { + if (global.systemInitd) return; + global.systemInitd = true; + try { - if (global.feConfigs) return; await connectToDatabase(); - initGlobal(); - await initSystemConfig(); + await Promise.all([ + initGlobal(), + initSystemConfig(), + getSimpleModeTemplates(), + getSystemVersion(), + getSystemPlugin() + ]); + + console.log({ + simpleModeTemplates: global.simpleModeTemplates, + communityPlugins: global.communityPlugins + }); } catch (error) { console.error('Load init config error', error); + global.systemInitd = false; if (!global.feConfigs) { exit(1); } } - await getSimpleModeTemplates(); +} - getSystemVersion(); - getSystemPlugin(); +export function initGlobal() { + if (global.communityPlugins) return; - console.log({ - feConfigs: global.feConfigs, - systemEnv: global.systemEnv, - chatModels: global.chatModels, - qaModels: global.qaModels, - cqModels: global.cqModels, - extractModels: global.extractModels, - qgModels: global.qgModels, - vectorModels: global.vectorModels, - reRankModels: global.reRankModels, - audioSpeechModels: global.audioSpeechModels, - whisperModel: global.whisperModel, - simpleModeTemplates: global.simpleModeTemplates, - communityPlugins: global.communityPlugins - }); + global.communityPlugins = []; + global.simpleModeTemplates = []; + global.qaQueueLen = global.qaQueueLen ?? 0; + global.vectorQueueLen = global.vectorQueueLen ?? 0; + // init tikToken + getTikTokenEnc(); + initHttpAgent(); } export async function initSystemConfig() { @@ -137,19 +142,24 @@ export async function initSystemConfig() { global.reRankModels = config.reRankModels; global.audioSpeechModels = config.audioSpeechModels; global.whisperModel = config.whisperModel; -} -export function initGlobal() { - global.communityPlugins = []; - global.simpleModeTemplates = []; - global.qaQueueLen = global.qaQueueLen ?? 0; - global.vectorQueueLen = global.vectorQueueLen ?? 0; - // init tikToken - getTikTokenEnc(); - initHttpAgent(); + console.log({ + feConfigs: global.feConfigs, + systemEnv: global.systemEnv, + chatModels: global.chatModels, + qaModels: global.qaModels, + cqModels: global.cqModels, + extractModels: global.extractModels, + qgModels: global.qgModels, + vectorModels: global.vectorModels, + reRankModels: global.reRankModels, + audioSpeechModels: global.audioSpeechModels, + whisperModel: global.whisperModel + }); } export function getSystemVersion() { + if (global.systemVersion) return; try { if (process.env.NODE_ENV === 'development') { global.systemVersion = process.env.npm_package_version || '0.0.0'; diff --git a/projects/app/src/pages/api/common/system/refreshConfig.ts b/projects/app/src/pages/api/common/system/refreshConfig.ts deleted file mode 100644 index e877c27e295..00000000000 --- a/projects/app/src/pages/api/common/system/refreshConfig.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { NextApiRequest, NextApiResponse } from 'next'; -import { jsonRes } from '@fastgpt/service/common/response'; -import { connectToDatabase } from '@/service/mongo'; -import { authCert } from '@fastgpt/service/support/permission/auth/common'; -import { initSystemConfig } from './getInitData'; - -export default async function handler(req: NextApiRequest, res: NextApiResponse) { - try { - await connectToDatabase(); - await authCert({ req, authRoot: true }); - await initSystemConfig(); - - console.log(`refresh config`); - console.log({ - chatModels: global.chatModels, - qaModels: global.qaModels, - cqModels: global.cqModels, - extractModels: global.extractModels, - qgModels: global.qgModels, - vectorModels: global.vectorModels, - reRankModels: global.reRankModels, - audioSpeechModels: global.audioSpeechModels, - whisperModel: global.whisperModel, - feConfigs: global.feConfigs, - systemEnv: global.systemEnv - }); - } catch (error) { - console.log(error); - } - jsonRes(res); -} diff --git a/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts b/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts index 6d7c0749b26..d1655e738e2 100644 --- a/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts +++ b/projects/app/src/pages/api/core/chat/feedback/updateUserFeedback.ts @@ -29,6 +29,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) await MongoChatItem.findOneAndUpdate( { + chatId, dataId: chatItemId }, { diff --git a/projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts b/projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts new file mode 100644 index 00000000000..7a4f7480753 --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/collection/apiCreate/link.ts @@ -0,0 +1,88 @@ +/* + Create one dataset collection +*/ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import type { LinkCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; +import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; +import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; +import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; +import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; +import { createTrainingBill } from '@fastgpt/service/support/wallet/bill/controller'; +import { BillSourceEnum } from '@fastgpt/global/support/wallet/bill/constants'; +import { getQAModel, getVectorModel } from '@/service/core/ai/model'; +import { reloadCollectionChunks } from '@fastgpt/service/core/dataset/collection/utils'; +import { startQueue } from '@/service/utils/tools'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + const { + link, + trainingType = TrainingModeEnum.chunk, + chunkSize = 512, + chunkSplitter, + qaPrompt, + ...body + } = req.body as LinkCreateDatasetCollectionParams; + + const { teamId, tmbId, dataset } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId: body.datasetId, + per: 'w' + }); + + // 1. check dataset limit + await checkDatasetLimit({ + teamId, + freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize, + insertLen: predictDataLimitLength(trainingType, new Array(10)) + }); + + // 2. create collection + const collectionId = await createOneCollection({ + ...body, + name: link, + teamId, + tmbId, + type: DatasetCollectionTypeEnum.link, + + trainingType, + chunkSize, + chunkSplitter, + qaPrompt, + + rawLink: link + }); + + // 3. create bill and start sync + const { billId } = await createTrainingBill({ + teamId, + tmbId, + appName: 'core.dataset.collection.Sync Collection', + billSource: BillSourceEnum.training, + vectorModel: getVectorModel(dataset.vectorModel).name, + agentModel: getQAModel(dataset.agentModel).name + }); + await reloadCollectionChunks({ + collectionId, + tmbId, + billId + }); + + startQueue(); + + jsonRes(res, { + data: { collectionId } + }); + } catch (err) { + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts b/projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts new file mode 100644 index 00000000000..6e131c4dc23 --- /dev/null +++ b/projects/app/src/pages/api/core/dataset/collection/apiCreate/text.ts @@ -0,0 +1,90 @@ +/* + Create one dataset collection +*/ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import type { TextCreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; +import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; +import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; +import { TrainingModeEnum, DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; +import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; +import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; +import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; +import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller'; +import { hashStr } from '@fastgpt/global/common/string/tools'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + const { + text, + trainingType = TrainingModeEnum.chunk, + chunkSize = 512, + chunkSplitter, + qaPrompt, + ...body + } = req.body as TextCreateDatasetCollectionParams; + + const { teamId, tmbId } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId: body.datasetId, + per: 'w' + }); + + // 1. split text to chunks + const { chunks } = splitText2Chunks({ + text, + chunkLen: chunkSize, + overlapRatio: trainingType === TrainingModeEnum.chunk ? 0.2 : 0, + customReg: chunkSplitter ? [chunkSplitter] : [], + countTokens: false + }); + + // 2. check dataset limit + await checkDatasetLimit({ + teamId, + freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize, + insertLen: predictDataLimitLength(trainingType, chunks) + }); + + // 3. create collection + const collectionId = await createOneCollection({ + ...body, + teamId, + tmbId, + type: DatasetCollectionTypeEnum.virtual, + + trainingType, + chunkSize, + chunkSplitter, + qaPrompt, + + hashRawText: hashStr(text), + rawTextLength: text.length + }); + + // 4. push chunks to training queue + const insertResults = await pushDataToDatasetCollection({ + teamId, + tmbId, + collectionId, + trainingMode: trainingType, + data: chunks.map((text, index) => ({ + q: text, + chunkIndex: index + })) + }); + + jsonRes(res, { + data: { collectionId, results: insertResults } + }); + } catch (err) { + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/projects/app/src/pages/api/core/dataset/collection/create.ts b/projects/app/src/pages/api/core/dataset/collection/create.ts index 140d34bb8e5..a889f6c80d1 100644 --- a/projects/app/src/pages/api/core/dataset/collection/create.ts +++ b/projects/app/src/pages/api/core/dataset/collection/create.ts @@ -5,7 +5,6 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; import type { CreateDatasetCollectionParams } from '@fastgpt/global/core/dataset/api.d'; -import { authUserNotVisitor } from '@fastgpt/service/support/permission/auth/user'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { createOneCollection } from '@fastgpt/service/core/dataset/collection/controller'; @@ -14,13 +13,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< await connectToDatabase(); const body = req.body as CreateDatasetCollectionParams; - // auth. not visitor and dataset is public - const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true }); - await authDataset({ + const { teamId, tmbId } = await authDataset({ req, authToken: true, + authApiKey: true, datasetId: body.datasetId, - per: 'r' + per: 'w' }); jsonRes(res, { diff --git a/projects/app/src/pages/api/core/dataset/collection/delete.ts b/projects/app/src/pages/api/core/dataset/collection/delete.ts index 778b1b3da90..9544badecde 100644 --- a/projects/app/src/pages/api/core/dataset/collection/delete.ts +++ b/projects/app/src/pages/api/core/dataset/collection/delete.ts @@ -4,13 +4,12 @@ import { connectToDatabase } from '@/service/mongo'; import { findCollectionAndChild } from '@fastgpt/service/core/dataset/collection/utils'; import { delCollectionRelevantData } from '@fastgpt/service/core/dataset/data/controller'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; -import { MongoDatasetCollection } from '@fastgpt/service/core/dataset/collection/schema'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { collectionId } = req.query as { collectionId: string }; + const { id: collectionId } = req.query as { id: string }; if (!collectionId) { throw new Error('CollectionIdId is required'); @@ -19,6 +18,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< await authDatasetCollection({ req, authToken: true, + authApiKey: true, collectionId, per: 'w' }); diff --git a/projects/app/src/pages/api/core/dataset/collection/detail.ts b/projects/app/src/pages/api/core/dataset/collection/detail.ts index 737eb6b2413..8c3aa084aa2 100644 --- a/projects/app/src/pages/api/core/dataset/collection/detail.ts +++ b/projects/app/src/pages/api/core/dataset/collection/detail.ts @@ -22,6 +22,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const { collection, canWrite } = await authDatasetCollection({ req, authToken: true, + authApiKey: true, collectionId: id, per: 'r' }); diff --git a/projects/app/src/pages/api/core/dataset/collection/list.ts b/projects/app/src/pages/api/core/dataset/collection/list.ts index 153113184f3..4ffb1e94cd7 100644 --- a/projects/app/src/pages/api/core/dataset/collection/list.ts +++ b/projects/app/src/pages/api/core/dataset/collection/list.ts @@ -11,7 +11,6 @@ import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant import { startQueue } from '@/service/utils/tools'; import { authDataset } from '@fastgpt/service/support/permission/auth/dataset'; import { DatasetDataCollectionName } from '@fastgpt/service/core/dataset/data/schema'; -import { authUserRole } from '@fastgpt/service/support/permission/auth/user'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -27,12 +26,19 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< simple = false } = req.body as GetDatasetCollectionsProps; searchText = searchText?.replace(/'/g, ''); + pageSize = Math.min(pageSize, 30); // auth dataset and get my role - const { tmbId } = await authDataset({ req, authToken: true, datasetId, per: 'r' }); - const { canWrite } = await authUserRole({ req, authToken: true }); + const { teamId, tmbId, canWrite } = await authDataset({ + req, + authToken: true, + authApiKey: true, + datasetId, + per: 'r' + }); const match = { + teamId: new Types.ObjectId(teamId), datasetId: new Types.ObjectId(datasetId), parentId: parentId ? new Types.ObjectId(parentId) : null, ...(selectFolder ? { type: DatasetCollectionTypeEnum.folder } : {}), @@ -85,9 +91,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< } } }, - { $project: { _id: 1 } } + { $count: 'count' } ], - as: 'trainings' + as: 'trainingCount' } }, // count collection total data @@ -103,9 +109,9 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< } } }, - { $project: { _id: 1 } } + { $count: 'count' } ], - as: 'datas' + as: 'dataCount' } }, { @@ -117,10 +123,14 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< type: 1, status: 1, updateTime: 1, - dataAmount: { $size: '$datas' }, - trainingAmount: { $size: '$trainings' }, fileId: 1, - rawLink: 1 + rawLink: 1, + dataAmount: { + $ifNull: [{ $arrayElemAt: ['$dataCount.count', 0] }, 0] + }, + trainingAmount: { + $ifNull: [{ $arrayElemAt: ['$trainingCount.count', 0] }, 0] + } } }, { @@ -144,7 +154,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< ); if (data.find((item) => item.trainingAmount > 0)) { - startQueue(1); + startQueue(); } // count collections diff --git a/projects/app/src/pages/api/core/dataset/collection/sync/link.ts b/projects/app/src/pages/api/core/dataset/collection/sync/link.ts index 505e6d6b3ca..b7a1b9042ea 100644 --- a/projects/app/src/pages/api/core/dataset/collection/sync/link.ts +++ b/projects/app/src/pages/api/core/dataset/collection/sync/link.ts @@ -38,7 +38,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< return Promise.reject(DatasetErrEnum.unLinkCollection); } - const { rawText, isSameRawText } = await getCollectionAndRawText({ + const { title, rawText, isSameRawText } = await getCollectionAndRawText({ collection }); @@ -68,7 +68,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< tmbId: collection.tmbId, parentId: collection.parentId, datasetId: collection.datasetId._id, - name: collection.name, + name: title || collection.name, type: collection.type, trainingType: collection.trainingType, chunkSize: collection.chunkSize, diff --git a/projects/app/src/pages/api/core/dataset/collection/update.ts b/projects/app/src/pages/api/core/dataset/collection/update.ts index ac29bd7e35e..f45a3c16408 100644 --- a/projects/app/src/pages/api/core/dataset/collection/update.ts +++ b/projects/app/src/pages/api/core/dataset/collection/update.ts @@ -16,7 +16,13 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< } // 凭证校验 - await authDatasetCollection({ req, authToken: true, collectionId: id, per: 'w' }); + await authDatasetCollection({ + req, + authToken: true, + authApiKey: true, + collectionId: id, + per: 'w' + }); const updateFields: Record = { ...(parentId !== undefined && { parentId: parentId || null }), diff --git a/projects/app/src/pages/api/core/dataset/create.ts b/projects/app/src/pages/api/core/dataset/create.ts index cd37214ce52..d7533443d77 100644 --- a/projects/app/src/pages/api/core/dataset/create.ts +++ b/projects/app/src/pages/api/core/dataset/create.ts @@ -16,12 +16,28 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< type, avatar, vectorModel = global.vectorModels[0].model, - agentModel + agentModel = global.qaModels[0].model } = req.body as CreateDatasetParams; - // 凭证校验 + // auth const { teamId, tmbId } = await authUserNotVisitor({ req, authToken: true }); + // check model valid + const vectorModelStore = global.vectorModels.find((item) => item.model === vectorModel); + const agentModelStore = global.qaModels.find((item) => item.model === agentModel); + if (!vectorModelStore || !agentModelStore) { + throw new Error('vectorModel or qaModel is invalid'); + } + + // check limit + const authCount = await MongoDataset.countDocuments({ + teamId, + type: DatasetTypeEnum.dataset + }); + if (authCount >= 50) { + throw new Error('每个团队上限 50 个知识库'); + } + const { _id } = await MongoDataset.create({ name, teamId, diff --git a/projects/app/src/pages/api/core/dataset/data/delete.ts b/projects/app/src/pages/api/core/dataset/data/delete.ts index ab4a7d75ae3..5bf4df034ec 100644 --- a/projects/app/src/pages/api/core/dataset/data/delete.ts +++ b/projects/app/src/pages/api/core/dataset/data/delete.ts @@ -8,8 +8,8 @@ import { delDatasetDataByDataId } from '@fastgpt/service/core/dataset/data/contr export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { dataId } = req.query as { - dataId: string; + const { id: dataId } = req.query as { + id: string; }; if (!dataId) { @@ -17,9 +17,18 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex } // 凭证校验 - await authDatasetData({ req, authToken: true, dataId, per: 'w' }); + const { datasetData } = await authDatasetData({ + req, + authToken: true, + authApiKey: true, + dataId, + per: 'w' + }); - await delDatasetDataByDataId(dataId); + await delDatasetDataByDataId({ + collectionId: datasetData.collectionId, + mongoDataId: dataId + }); jsonRes(res, { data: 'success' diff --git a/projects/app/src/pages/api/core/dataset/data/detail.ts b/projects/app/src/pages/api/core/dataset/data/detail.ts index f734eb3c59a..57f93c29f36 100644 --- a/projects/app/src/pages/api/core/dataset/data/detail.ts +++ b/projects/app/src/pages/api/core/dataset/data/detail.ts @@ -13,12 +13,18 @@ export type Response = { export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { dataId } = req.query as { - dataId: string; + const { id: dataId } = req.query as { + id: string; }; // 凭证校验 - const { datasetData } = await authDatasetData({ req, authToken: true, dataId, per: 'r' }); + const { datasetData } = await authDatasetData({ + req, + authToken: true, + authApiKey: true, + dataId, + per: 'r' + }); jsonRes(res, { data: datasetData diff --git a/projects/app/src/pages/api/core/dataset/data/insertData.ts b/projects/app/src/pages/api/core/dataset/data/insertData.ts index fbf3f411e5e..0a794bfe359 100644 --- a/projects/app/src/pages/api/core/dataset/data/insertData.ts +++ b/projects/app/src/pages/api/core/dataset/data/insertData.ts @@ -16,6 +16,7 @@ import { authTeamBalance } from '@/service/support/permission/auth/bill'; import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push'; import { InsertOneDatasetDataProps } from '@/global/core/dataset/api'; import { simpleText } from '@fastgpt/global/common/string/tools'; +import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { @@ -39,6 +40,12 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex per: 'w' }); + await checkDatasetLimit({ + teamId, + freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize, + insertLen: 1 + }); + // auth collection and get dataset const [ { diff --git a/projects/app/src/pages/api/core/dataset/data/list.ts b/projects/app/src/pages/api/core/dataset/data/list.ts index b500fde74d0..e014ec7a4ff 100644 --- a/projects/app/src/pages/api/core/dataset/data/list.ts +++ b/projects/app/src/pages/api/core/dataset/data/list.ts @@ -17,8 +17,10 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< collectionId } = req.body as GetDatasetDataListProps; + pageSize = Math.min(pageSize, 30); + // 凭证校验 - await authDatasetCollection({ req, authToken: true, collectionId, per: 'r' }); + await authDatasetCollection({ req, authToken: true, authApiKey: true, collectionId, per: 'r' }); searchText = searchText.replace(/'/g, ''); @@ -32,7 +34,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< }; const [data, total] = await Promise.all([ - MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex indexes') + MongoDatasetData.find(match, '_id datasetId collectionId q a chunkIndex') .sort({ chunkIndex: 1, updateTime: -1 }) .skip((pageNum - 1) * pageSize) .limit(pageSize) diff --git a/projects/app/src/pages/api/core/dataset/data/pushData.ts b/projects/app/src/pages/api/core/dataset/data/pushData.ts index a43823dbf00..5e05a7fde2d 100644 --- a/projects/app/src/pages/api/core/dataset/data/pushData.ts +++ b/projects/app/src/pages/api/core/dataset/data/pushData.ts @@ -2,38 +2,30 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { connectToDatabase } from '@/service/mongo'; -import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; import { TrainingModeEnum, TrainingTypeMap } from '@fastgpt/global/core/dataset/constant'; -import { startQueue } from '@/service/utils/tools'; -import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; import type { PushDataResponse } from '@/global/core/api/datasetRes.d'; import type { PushDatasetDataProps } from '@/global/core/dataset/api.d'; -import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api'; -import { getQAModel, getVectorModel } from '@/service/core/ai/model'; import { authDatasetCollection } from '@fastgpt/service/support/permission/auth/dataset'; -import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller'; -import { simpleText } from '@fastgpt/global/common/string/tools'; +import { checkDatasetLimit } from '@fastgpt/service/support/permission/limit/dataset'; +import { predictDataLimitLength } from '@fastgpt/global/core/dataset/utils'; +import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { collectionId, data, mode = TrainingModeEnum.chunk } = req.body as PushDatasetDataProps; + const { collectionId, data } = req.body as PushDatasetDataProps; if (!collectionId || !Array.isArray(data)) { throw new Error('collectionId or data is empty'); } - if (!TrainingTypeMap[mode]) { - throw new Error(`Mode is not ${Object.keys(TrainingTypeMap).join(', ')}`); - } - if (data.length > 200) { throw new Error('Data is too long, max 200'); } // 凭证校验 - const { teamId, tmbId } = await authDatasetCollection({ + const { teamId, tmbId, collection } = await authDatasetCollection({ req, authToken: true, authApiKey: true, @@ -41,6 +33,13 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex per: 'w' }); + // auth dataset limit + await checkDatasetLimit({ + teamId, + freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize, + insertLen: predictDataLimitLength(collection.trainingType, data) + }); + jsonRes(res, { data: await pushDataToDatasetCollection({ ...req.body, @@ -56,141 +55,6 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex } }); -export async function pushDataToDatasetCollection({ - teamId, - tmbId, - collectionId, - data, - mode, - prompt, - billId -}: { - teamId: string; - tmbId: string; -} & PushDatasetDataProps): Promise { - const { datasetId, model, maxToken, weight } = await checkModelValid({ - mode, - collectionId - }); - - // format q and a, remove empty char - data.forEach((item) => { - item.q = simpleText(item.q); - item.a = simpleText(item.a); - - item.indexes = item.indexes - ?.map((index) => { - return { - ...index, - text: simpleText(index.text) - }; - }) - .filter(Boolean); - }); - - // filter repeat or equal content - const set = new Set(); - const filterResult: Record = { - success: [], - overToken: [], - repeat: [], - error: [] - }; - - data.forEach((item) => { - if (!item.q) { - filterResult.error.push(item); - return; - } - - const text = item.q + item.a; - - // count q token - const token = countPromptTokens(item.q); - - if (token > maxToken) { - filterResult.overToken.push(item); - return; - } - - if (set.has(text)) { - console.log('repeat', item); - filterResult.repeat.push(item); - } else { - filterResult.success.push(item); - set.add(text); - } - }); - - // 插入记录 - const insertRes = await MongoDatasetTraining.insertMany( - filterResult.success.map((item, i) => ({ - teamId, - tmbId, - datasetId, - collectionId, - billId, - mode, - prompt, - model, - q: item.q, - a: item.a, - chunkIndex: item.chunkIndex ?? i, - weight: weight ?? 0, - indexes: item.indexes - })) - ); - - insertRes.length > 0 && startQueue(); - delete filterResult.success; - - return { - insertLen: insertRes.length, - ...filterResult - }; -} - -export async function checkModelValid({ - mode, - collectionId -}: { - mode: `${TrainingModeEnum}`; - collectionId: string; -}) { - const { - datasetId: { _id: datasetId, vectorModel, agentModel } - } = await getCollectionWithDataset(collectionId); - - if (mode === TrainingModeEnum.chunk) { - if (!collectionId) return Promise.reject(`CollectionId is empty`); - const vectorModelData = getVectorModel(vectorModel); - if (!vectorModelData) { - return Promise.reject(`Model ${vectorModel} is inValid`); - } - - return { - datasetId, - maxToken: vectorModelData.maxToken * 1.5, - model: vectorModelData.model, - weight: vectorModelData.weight - }; - } - - if (mode === TrainingModeEnum.qa) { - const qaModelData = getQAModel(agentModel); - if (!qaModelData) { - return Promise.reject(`Model ${agentModel} is inValid`); - } - return { - datasetId, - maxToken: qaModelData.maxContext * 0.8, - model: qaModelData.model, - weight: 0 - }; - } - return Promise.reject(`Mode ${mode} is inValid`); -} - export const config = { api: { bodyParser: { diff --git a/projects/app/src/pages/api/core/dataset/data/update.ts b/projects/app/src/pages/api/core/dataset/data/update.ts index 05bdbbd6b5c..41279637a6d 100644 --- a/projects/app/src/pages/api/core/dataset/data/update.ts +++ b/projects/app/src/pages/api/core/dataset/data/update.ts @@ -11,7 +11,7 @@ import { UpdateDatasetDataProps } from '@/global/core/dataset/api'; export default withNextCors(async function handler(req: NextApiRequest, res: NextApiResponse) { try { await connectToDatabase(); - const { id, q = '', a, indexes } = req.body as UpdateDatasetDataProps; + const { id, q = '', a, indexes = [] } = req.body as UpdateDatasetDataProps; // auth data permission const { @@ -23,6 +23,7 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex } = await authDatasetData({ req, authToken: true, + authApiKey: true, dataId: id, per: 'w' }); diff --git a/projects/app/src/pages/api/core/dataset/detail.ts b/projects/app/src/pages/api/core/dataset/detail.ts index 399a887c02d..7e936a62208 100644 --- a/projects/app/src/pages/api/core/dataset/detail.ts +++ b/projects/app/src/pages/api/core/dataset/detail.ts @@ -20,6 +20,7 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< const { dataset, canWrite, isOwner } = await authDataset({ req, authToken: true, + authApiKey: true, datasetId, per: 'r' }); diff --git a/projects/app/src/pages/api/core/dataset/list.ts b/projects/app/src/pages/api/core/dataset/list.ts index d5a411f32bf..9ebea8e3383 100644 --- a/projects/app/src/pages/api/core/dataset/list.ts +++ b/projects/app/src/pages/api/core/dataset/list.ts @@ -15,7 +15,8 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse< // 凭证校验 const { teamId, tmbId, teamOwner, role, canWrite } = await authUserRole({ req, - authToken: true + authToken: true, + authApiKey: true }); const datasets = await MongoDataset.find({ diff --git a/projects/app/src/pages/api/plusApi/[...path].ts b/projects/app/src/pages/api/plusApi/[...path].ts index 58a275ead07..761fc3ba795 100644 --- a/projects/app/src/pages/api/plusApi/[...path].ts +++ b/projects/app/src/pages/api/plusApi/[...path].ts @@ -3,14 +3,11 @@ import { jsonRes } from '@fastgpt/service/common/response'; import { request } from '@fastgpt/service/common/api/plusRequest'; import type { Method } from 'axios'; import { setCookie } from '@fastgpt/service/support/permission/controller'; -import { getInitConfig } from '../common/system/getInitData'; -import { FastGPTProUrl } from '@fastgpt/service/common/system/constants'; +import { connectToDatabase } from '@/service/mongo'; export default async function handler(req: NextApiRequest, res: NextApiResponse) { try { - if (!FastGPTProUrl) { - await getInitConfig(); - } + await connectToDatabase(); const method = (req.method || 'POST') as Method; const { path = [], ...query } = req.query as any; diff --git a/projects/app/src/pages/api/support/wallet/sub/getDatasetSub.ts b/projects/app/src/pages/api/support/wallet/sub/getDatasetSub.ts new file mode 100644 index 00000000000..d55c9ddaa70 --- /dev/null +++ b/projects/app/src/pages/api/support/wallet/sub/getDatasetSub.ts @@ -0,0 +1,39 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { jsonRes } from '@fastgpt/service/common/response'; +import { connectToDatabase } from '@/service/mongo'; +import { authCert } from '@fastgpt/service/support/permission/auth/common'; +import { getTeamDatasetValidSub } from '@fastgpt/service/support/wallet/sub/utils'; +import { getVectorCountByTeamId } from '@fastgpt/service/common/vectorStore/controller'; + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + try { + await connectToDatabase(); + + // 凭证校验 + const { teamId } = await authCert({ + req, + authToken: true + }); + + const [{ sub, maxSize }, usedSize] = await Promise.all([ + getTeamDatasetValidSub({ + teamId, + freeSize: global.feConfigs?.subscription?.datasetStoreFreeSize + }), + getVectorCountByTeamId(teamId) + ]); + + jsonRes(res, { + data: { + sub, + maxSize, + usedSize + } + }); + } catch (err) { + jsonRes(res, { + code: 500, + error: err + }); + } +} diff --git a/projects/app/src/pages/api/v1/audio/transcriptions.ts b/projects/app/src/pages/api/v1/audio/transcriptions.ts index f7f86513e60..22fd91dce24 100644 --- a/projects/app/src/pages/api/v1/audio/transcriptions.ts +++ b/projects/app/src/pages/api/v1/audio/transcriptions.ts @@ -2,7 +2,8 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { jsonRes } from '@fastgpt/service/common/response'; import { authCert } from '@fastgpt/service/support/permission/auth/common'; import { withNextCors } from '@fastgpt/service/common/middle/cors'; -import { getUploadModel, removeFilesByPaths } from '@fastgpt/service/common/file/upload/multer'; +import { getUploadModel } from '@fastgpt/service/common/file/multer'; +import { removeFilesByPaths } from '@fastgpt/service/common/file/utils'; import fs from 'fs'; import { getAIApi } from '@fastgpt/service/core/ai/config'; import { pushWhisperBill } from '@/service/support/wallet/bill/push'; diff --git a/projects/app/src/pages/api/v1/embeddings.ts b/projects/app/src/pages/api/v1/embeddings.ts index 1bb57bbd1d2..4371c2609b4 100644 --- a/projects/app/src/pages/api/v1/embeddings.ts +++ b/projects/app/src/pages/api/v1/embeddings.ts @@ -35,19 +35,17 @@ export default withNextCors(async function handler(req: NextApiRequest, res: Nex const { tokens, vectors } = await getVectorsByText({ input: query, model }); - jsonRes(res, { - data: { - object: 'list', - data: vectors.map((item, index) => ({ - object: 'embedding', - index: index, - embedding: item - })), - model, - usage: { - prompt_tokens: tokens, - total_tokens: tokens - } + res.json({ + object: 'list', + data: vectors.map((item, index) => ({ + object: 'embedding', + index: index, + embedding: item + })), + model, + usage: { + prompt_tokens: tokens, + total_tokens: tokens } }); diff --git a/projects/app/src/pages/app/detail/components/InfoModal.tsx b/projects/app/src/pages/app/detail/components/InfoModal.tsx index c7a957a7b1d..6786ea4b88c 100644 --- a/projects/app/src/pages/app/detail/components/InfoModal.tsx +++ b/projects/app/src/pages/app/detail/components/InfoModal.tsx @@ -22,6 +22,7 @@ import MyModal from '@/components/MyModal'; import { useAppStore } from '@/web/core/app/store/useAppStore'; import PermissionRadio from '@/components/support/permission/Radio'; import { useTranslation } from 'next-i18next'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; const InfoModal = ({ defaultApp, @@ -45,7 +46,6 @@ const InfoModal = ({ setValue, getValues, formState: { errors }, - reset, handleSubmit } = useForm({ defaultValues: defaultApp @@ -102,6 +102,7 @@ const InfoModal = ({ if (!file) return; try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.appAvatar, file, maxW: 300, maxH: 300 @@ -187,4 +188,4 @@ const InfoModal = ({ ); }; -export default InfoModal; +export default React.memo(InfoModal); diff --git a/projects/app/src/pages/app/detail/components/Logs.tsx b/projects/app/src/pages/app/detail/components/Logs.tsx index 674190b9680..06e84306966 100644 --- a/projects/app/src/pages/app/detail/components/Logs.tsx +++ b/projects/app/src/pages/app/detail/components/Logs.tsx @@ -81,7 +81,7 @@ const Logs = ({ appId }: { appId: string }) => { cursor={'pointer'} onClick={onOpenMarkDesc} > - {t('chat.Read Mark Description')} + {t('core.chat.Read Mark Description')} @@ -202,9 +202,9 @@ const Logs = ({ appId }: { appId: string }) => { - {t('chat.Mark Description')} + {t('core.chat.Mark Description')} ); diff --git a/projects/app/src/pages/app/list/component/CreateModal.tsx b/projects/app/src/pages/app/list/component/CreateModal.tsx index 12a2e28c12e..2ccac40d281 100644 --- a/projects/app/src/pages/app/list/component/CreateModal.tsx +++ b/projects/app/src/pages/app/list/component/CreateModal.tsx @@ -26,6 +26,7 @@ import Avatar from '@/components/Avatar'; import MyTooltip from '@/components/MyTooltip'; import MyModal from '@/components/MyModal'; import { useTranslation } from 'next-i18next'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; type FormType = { avatar: string; @@ -59,6 +60,7 @@ const CreateModal = ({ onClose, onSuccess }: { onClose: () => void; onSuccess: ( if (!file) return; try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.appAvatar, file, maxW: 300, maxH: 300 diff --git a/projects/app/src/pages/chat/components/ChatHeader.tsx b/projects/app/src/pages/chat/components/ChatHeader.tsx index 3db3bdbc4c7..694da57673b 100644 --- a/projects/app/src/pages/chat/components/ChatHeader.tsx +++ b/projects/app/src/pages/chat/components/ChatHeader.tsx @@ -35,7 +35,7 @@ const ChatHeader = ({ () => chatContentReplaceBlock(history[history.length - 2]?.value)?.slice(0, 8) || appName || - t('chat.New Chat'), + t('core.chat.New Chat'), [appName, history] ); @@ -56,8 +56,8 @@ const ChatHeader = ({ {history.length === 0 - ? t('chat.Fresh Chat') - : t('chat.History Amount', { amount: history.length })} + ? t('core.chat.New Chat') + : t('core.chat.History Amount', { amount: history.length })} {!!chatModels && chatModels.length > 0 && ( diff --git a/projects/app/src/pages/chat/components/ChatHistorySlider.tsx b/projects/app/src/pages/chat/components/ChatHistorySlider.tsx index 96dc5281c64..ecc9f9efd56 100644 --- a/projects/app/src/pages/chat/components/ChatHistorySlider.tsx +++ b/projects/app/src/pages/chat/components/ChatHistorySlider.tsx @@ -74,18 +74,20 @@ const ChatHistorySlider = ({ // custom title edit const { onOpenModal, EditModal: EditTitleModal } = useEditTitle({ - title: t('chat.Custom History Title'), - placeholder: t('chat.Custom History Title Description') + title: t('core.chat.Custom History Title'), + placeholder: t('core.chat.Custom History Title Description') }); const { openConfirm, ConfirmModal } = useConfirm({ content: isShare - ? t('chat.Confirm to clear share chat history') - : t('chat.Confirm to clear history') + ? t('core.chat.Confirm to clear share chat history') + : t('core.chat.Confirm to clear history') }); const concatHistory = useMemo( () => - !activeChatId ? [{ id: activeChatId, title: t('chat.New Chat') }].concat(history) : history, + !activeChatId + ? [{ id: activeChatId, title: t('core.chat.New Chat') }].concat(history) + : history, [activeChatId, history, t] ); @@ -144,7 +146,7 @@ const ChatHistorySlider = ({ mr={2} list={[ { label: 'App', id: TabEnum.app }, - { label: 'chat.History', id: TabEnum.history } + { label: t('core.chat.History'), id: TabEnum.history } ]} activeId={currentTab} onChange={(e) => setCurrentTab(e as `${TabEnum}`)} @@ -160,7 +162,7 @@ const ChatHistorySlider = ({ overflow={'hidden'} onClick={() => onChangeChat()} > - {t('chat.New Chat')} + {t('core.chat.New Chat')} {(isPc || isShare) && ( @@ -240,7 +242,7 @@ const ChatHistorySlider = ({ }} > - {item.top ? t('chat.Unpin') : t('chat.Pin')} + {item.top ? t('core.chat.Unpin') : t('core.chat.Pin')} )} {onSetCustomTitle && ( @@ -336,7 +338,7 @@ const ChatHistorySlider = ({ borderRadius={'50%'} aria-label={''} /> - {t('chat.Exit Chat')} + {t('core.chat.Exit Chat')} )} diff --git a/projects/app/src/pages/chat/components/SliderApps.tsx b/projects/app/src/pages/chat/components/SliderApps.tsx index 5960a46f7f5..596be623fe4 100644 --- a/projects/app/src/pages/chat/components/SliderApps.tsx +++ b/projects/app/src/pages/chat/components/SliderApps.tsx @@ -35,7 +35,7 @@ const SliderApps = ({ appId }: { appId: string }) => { borderRadius={'50%'} aria-label={''} /> - {t('chat.Exit Chat')} + {t('core.chat.Exit Chat')} diff --git a/projects/app/src/pages/chat/components/ToolMenu.tsx b/projects/app/src/pages/chat/components/ToolMenu.tsx index d033fb766b1..d568e4bfe92 100644 --- a/projects/app/src/pages/chat/components/ToolMenu.tsx +++ b/projects/app/src/pages/chat/components/ToolMenu.tsx @@ -15,7 +15,7 @@ const ToolMenu = ({ history }: { history: ChatItemType[] }) => { () => [ { icon: 'core/chat/chatLight', - label: t('chat.New Chat'), + label: t('core.chat.New Chat'), onClick: () => { router.replace({ query: { diff --git a/projects/app/src/pages/chat/index.tsx b/projects/app/src/pages/chat/index.tsx index 897e5db287c..7f4b21f9b09 100644 --- a/projects/app/src/pages/chat/index.tsx +++ b/projects/app/src/pages/chat/index.tsx @@ -86,7 +86,7 @@ const Chat = ({ appId, chatId }: { appId: string; chatId: string }) => { const newTitle = chatContentReplaceBlock(prompts[0].content).slice(0, 20) || prompts[1]?.value?.slice(0, 20) || - t('chat.New Chat'); + t('core.chat.New Chat'); // new chat if (completionChatId !== chatId) { @@ -166,7 +166,7 @@ const Chat = ({ appId, chatId }: { appId: string; chatId: string }) => { setLastChatAppId(''); setLastChatId(''); toast({ - title: getErrText(e, t('chat.Failed to initialize chat')), + title: getErrText(e, t('core.chat.Failed to initialize chat')), status: 'error' }); if (e?.code === 501) { @@ -210,7 +210,7 @@ const Chat = ({ appId, chatId }: { appId: string; chatId: string }) => { if (apps.length === 0) { toast({ status: 'error', - title: t('chat.You need to a chat app') + title: t('core.chat.You need to a chat app') }); router.replace('/app/list'); } else { diff --git a/projects/app/src/pages/chat/share.tsx b/projects/app/src/pages/chat/share.tsx index 72ca51d20f6..0fd6c3939c4 100644 --- a/projects/app/src/pages/chat/share.tsx +++ b/projects/app/src/pages/chat/share.tsx @@ -88,7 +88,7 @@ const OutLink = ({ const newTitle = chatContentReplaceBlock(prompts[0].content).slice(0, 20) || prompts[1]?.value?.slice(0, 20) || - t('chat.New Chat'); + t('core.chat.New Chat'); // new chat if (completionChatId !== chatId) { diff --git a/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx b/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx index 7e635202290..6fe48d4e58d 100644 --- a/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx +++ b/projects/app/src/pages/dataset/detail/components/CollectionCard.tsx @@ -43,7 +43,7 @@ import EmptyTip from '@/components/EmptyTip'; import { FolderAvatarSrc, DatasetCollectionTypeEnum, - DatasetCollectionTrainingModeEnum, + TrainingModeEnum, DatasetTypeEnum, DatasetTypeMap, DatasetStatusEnum, @@ -63,6 +63,7 @@ import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; import { DatasetSchemaType } from '@fastgpt/global/core/dataset/type'; import { DatasetCollectionSyncResultEnum } from '../../../../../../../packages/global/core/dataset/constant'; +import MyBox from '@/components/common/MyBox'; const FileImportModal = dynamic(() => import('./Import/ImportModal'), {}); const WebSiteConfigModal = dynamic(() => import('./Import/WebsiteConfig'), {}); @@ -100,8 +101,8 @@ const CollectionCard = () => { } = useDisclosure(); const { onOpenModal: onOpenCreateVirtualFileModal, EditModal: EditCreateVirtualFileModal } = useEditTitle({ - title: t('dataset.Create Virtual File'), - tip: t('dataset.Virtual File Tip'), + title: t('dataset.Create manual collection'), + tip: t('dataset.Manual collection Tip'), canEmpty: false }); @@ -186,7 +187,7 @@ const CollectionCard = () => { name: string; type: `${DatasetCollectionTypeEnum}`; callback?: (id: string) => void; - trainingType?: `${DatasetCollectionTrainingModeEnum}`; + trainingType?: `${TrainingModeEnum}`; rawLink?: string; chunkSize?: number; }) => { @@ -224,7 +225,7 @@ const CollectionCard = () => { const { mutate: onDelCollection, isLoading: isDeleting } = useRequest({ mutationFn: (collectionId: string) => { return delDatasetCollectionById({ - collectionId + id: collectionId }); }, onSuccess() { @@ -296,501 +297,522 @@ const CollectionCard = () => { }, [parentId]); return ( - - - - ({ - parentId: path.parentId, - parentName: i === paths.length - 1 ? `${path.parentName}` : path.parentName - }))} - FirstPathDom={ - <> - - {t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel)}({total}) - - {datasetDetail?.websiteConfig?.url && ( - - {t('core.dataset.website.Base Url')}: - - {datasetDetail.websiteConfig.url} - - - )} - - } - onClick={(e) => { - router.replace({ - query: { - ...router.query, - parentId: e - } - }); - }} - /> - - - {isPc && ( - - + + + + + ({ + parentId: path.parentId, + parentName: i === paths.length - 1 ? `${path.parentName}` : path.parentName + }))} + FirstPathDom={ + <> + + {t(DatasetTypeMap[datasetDetail?.type]?.collectionLabel)}({total}) + + {datasetDetail?.websiteConfig?.url && ( + + {t('core.dataset.website.Base Url')}: + + {datasetDetail.websiteConfig.url} + + + )} + } - w={['100%', '250px']} - size={['sm', 'md']} - placeholder={t('common.Search') || ''} - value={searchText} - onChange={(e) => { - setSearchText(e.target.value); - debounceRefetch(); - }} - onBlur={() => { - if (searchText === lastSearch.current) return; - getData(1); - }} - onKeyDown={(e) => { - if (searchText === lastSearch.current) return; - if (e.key === 'Enter') { - getData(1); - } + onClick={(e) => { + router.replace({ + query: { + ...router.query, + parentId: e + } + }); }} /> - - )} - {datasetDetail?.type === DatasetTypeEnum.dataset && ( - <> - {userInfo?.team?.role !== TeamMemberRoleEnum.visitor && ( - - - - {t('dataset.collections.Create And Import')} - - - } - menuList={[ - { - child: ( - - {''} - {t('Folder')} - - ), - onClick: () => setEditFolderData({}) - }, - { - child: ( - - {''} - {t('dataset.Create Virtual File')} - - ), - onClick: () => { - onOpenCreateVirtualFileModal({ - defaultVal: '', - onSuccess: (name) => { - onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual }); - } - }); - } - }, - { - child: ( - - {''} - {t('dataset.File Input')} - - ), - onClick: onOpenFileImportModal - } - ]} - /> - )} - - )} - {datasetDetail?.type === DatasetTypeEnum.websiteDataset && ( - <> - {datasetDetail?.websiteConfig?.url ? ( - - {datasetDetail.status === DatasetStatusEnum.active && ( - - )} - {datasetDetail.status === DatasetStatusEnum.syncing && ( - - - - {t('core.dataset.status.syncing')} - - - )} - - ) : ( - - )} - - )} - + - - - - - - - - - - - - - {formatCollections.map((collection, index) => ( - + } - bg={dragTargetId === collection._id ? 'primary.100' : ''} - userSelect={'none'} - onDragStart={(e) => { - setDragStartId(collection._id); + onChange={(e) => { + setSearchText(e.target.value); + debounceRefetch(); }} - onDragOver={(e) => { - e.preventDefault(); - const targetId = e.currentTarget.getAttribute('data-drag-id'); - if (!targetId) return; - DatasetCollectionTypeEnum.folder && setDragTargetId(targetId); - }} - onDragLeave={(e) => { - e.preventDefault(); - setDragTargetId(undefined); - }} - onDrop={async (e) => { - e.preventDefault(); - if (!dragTargetId || !dragStartId || dragTargetId === dragStartId) return; - // update parentId - try { - await putDatasetCollectionById({ - id: dragStartId, - parentId: dragTargetId - }); - getData(pageNum); - } catch (error) {} - setDragTargetId(undefined); + onBlur={() => { + if (searchText === lastSearch.current) return; + getData(1); }} - title={ - collection.type === DatasetCollectionTypeEnum.folder - ? t('dataset.collections.Click to view folder') - : t('dataset.collections.Click to view file') - } - onClick={() => { - if (collection.type === DatasetCollectionTypeEnum.folder) { - router.replace({ - query: { - ...router.query, - parentId: collection._id - } - }); - } else { - router.replace({ - query: { - ...router.query, - collectionId: collection._id, - currentTab: TabEnum.dataCard - } - }); + onKeyDown={(e) => { + if (searchText === lastSearch.current) return; + if (e.key === 'Enter') { + getData(1); } }} - > - - - - - - - - ))} - -
#{t('common.Name')}{t('dataset.collections.Data Amount')}{t('core.dataset.Sync Time')}{t('common.Status')} -
{index + 1} - - {''} - - - {collection.name} - - - - {collection.dataAmount || '-'}{dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')} - - {t(collection.statusText)} - - e.stopPropagation()}> - {collection.canWrite && userInfo?.team?.role !== TeamMemberRoleEnum.visitor && ( - - - + /> + + )} + {datasetDetail?.type === DatasetTypeEnum.dataset && ( + <> + {userInfo?.team?.role !== TeamMemberRoleEnum.visitor && ( + + + + {t('dataset.collections.Create And Import')} + + + } + menuList={[ + { + child: ( + + {''} + {t('Folder')} + + ), + onClick: () => setEditFolderData({}) + }, + { + child: ( + + {''} + {t('core.dataset.Manual collection')} + + ), + onClick: () => { + onOpenCreateVirtualFileModal({ + defaultVal: '', + onSuccess: (name) => { + onCreateCollection({ name, type: DatasetCollectionTypeEnum.virtual }); + } + }); } - menuList={[ - ...(collection.type === DatasetCollectionTypeEnum.link - ? [ - { - child: ( - - - {t('core.dataset.collection.Sync')} - - ), - onClick: () => - openSyncConfirm(() => { - onclickStartSync(collection._id); - })() - } - ] - : []), - { - child: ( - - - {t('Move')} - - ), - onClick: () => setMoveCollectionData({ collectionId: collection._id }) - }, - { - child: ( - - - {t('Rename')} - - ), - onClick: () => - onOpenEditTitleModal({ - defaultVal: collection.name, - onSuccess: (newName) => { - onUpdateCollectionName({ - collectionId: collection._id, - name: newName - }); - } - }) - }, - { - child: ( - - - {t('common.Delete')} - - ), - onClick: () => - openDeleteConfirm( - () => { - onDelCollection(collection._id); - }, - undefined, - collection.type === DatasetCollectionTypeEnum.folder - ? t('dataset.collections.Confirm to delete the folder') - : t('dataset.Confirm to delete the file') - )() - } - ]} - /> + }, + { + child: ( + + {''} + {t('core.dataset.File collection')} + + ), + onClick: onOpenFileImportModal + } + ]} + /> + )} + + )} + {datasetDetail?.type === DatasetTypeEnum.websiteDataset && ( + <> + {datasetDetail?.websiteConfig?.url ? ( + + {datasetDetail.status === DatasetStatusEnum.active && ( + )} -
- {total > pageSize && ( - - - - )} - {total === 0 && ( - {datasetDetail.status === DatasetStatusEnum.syncing && ( - <>{t('core.dataset.status.syncing')} - )} - {datasetDetail.status === DatasetStatusEnum.active && ( - <> - {!datasetDetail?.websiteConfig?.url ? ( - <> - {t('core.dataset.collection.Website Empty Tip')} - {', '} - - {t('core.dataset.collection.Click top config website')} - - - ) : ( - <>{t('core.dataset.website.UnValid Website Tip')} - )} - + + + + {t('core.dataset.status.syncing')} + + )} - ) - } + ) : ( + + )} + + )} + + + + + + + + + + + + + + + {formatCollections.map((collection, index) => ( + { + setDragStartId(collection._id); + }} + onDragOver={(e) => { + e.preventDefault(); + const targetId = e.currentTarget.getAttribute('data-drag-id'); + if (!targetId) return; + DatasetCollectionTypeEnum.folder && setDragTargetId(targetId); + }} + onDragLeave={(e) => { + e.preventDefault(); + setDragTargetId(undefined); + }} + onDrop={async (e) => { + e.preventDefault(); + if (!dragTargetId || !dragStartId || dragTargetId === dragStartId) return; + // update parentId + try { + await putDatasetCollectionById({ + id: dragStartId, + parentId: dragTargetId + }); + getData(pageNum); + } catch (error) {} + setDragTargetId(undefined); + }} + title={ + collection.type === DatasetCollectionTypeEnum.folder + ? t('dataset.collections.Click to view folder') + : t('dataset.collections.Click to view file') + } + onClick={() => { + if (collection.type === DatasetCollectionTypeEnum.folder) { + router.replace({ + query: { + ...router.query, + parentId: collection._id + } + }); + } else { + router.replace({ + query: { + ...router.query, + collectionId: collection._id, + currentTab: TabEnum.dataCard + } + }); + } + }} + > + + + + + + + + ))} + +
+ # + + {t('common.Name')} + + {t('dataset.collections.Data Amount')} + + {t('core.dataset.Sync Time')} + + {t('common.Status')} + +
{index + 1} + + {''} + + + {collection.name} + + + + {collection.dataAmount || '-'}{dayjs(collection.updateTime).format('YYYY/MM/DD HH:mm')} + + {t(collection.statusText)} + + e.stopPropagation()}> + {collection.canWrite && userInfo?.team?.role !== TeamMemberRoleEnum.visitor && ( + + + + } + menuList={[ + ...(collection.type === DatasetCollectionTypeEnum.link + ? [ + { + child: ( + + + {t('core.dataset.collection.Sync')} + + ), + onClick: () => + openSyncConfirm(() => { + onclickStartSync(collection._id); + })() + } + ] + : []), + { + child: ( + + + {t('Move')} + + ), + onClick: () => setMoveCollectionData({ collectionId: collection._id }) + }, + { + child: ( + + + {t('Rename')} + + ), + onClick: () => + onOpenEditTitleModal({ + defaultVal: collection.name, + onSuccess: (newName) => { + onUpdateCollectionName({ + collectionId: collection._id, + name: newName + }); + } + }) + }, + { + child: ( + + + {t('common.Delete')} + + ), + onClick: () => + openDeleteConfirm( + () => { + onDelCollection(collection._id); + }, + undefined, + collection.type === DatasetCollectionTypeEnum.folder + ? t('dataset.collections.Confirm to delete the folder') + : t('dataset.Confirm to delete the file') + )() + } + ]} + /> + )} +
+ {total > pageSize && ( + + + + )} + {total === 0 && ( + + {datasetDetail.status === DatasetStatusEnum.syncing && ( + <>{t('core.dataset.status.syncing')} + )} + {datasetDetail.status === DatasetStatusEnum.active && ( + <> + {!datasetDetail?.websiteConfig?.url ? ( + <> + {t('core.dataset.collection.Website Empty Tip')} + {', '} + + {t('core.dataset.collection.Click top config website')} + + + ) : ( + <>{t('core.dataset.website.UnValid Website Tip')} + )} + + )} + + ) + } + /> + )} + + + + + + + {isOpenFileImportModal && ( + { + getData(1); + onCloseFileImportModal(); + }} + onClose={onCloseFileImportModal} /> )} - - - - - - - - {isOpenFileImportModal && ( - { - getData(1); - onCloseFileImportModal(); - }} - onClose={onCloseFileImportModal} - /> - )} - {!!editFolderData && ( - setEditFolderData(undefined)} - editCallback={async (name) => { - try { - if (editFolderData.id) { - await putDatasetCollectionById({ - id: editFolderData.id, - name - }); - getData(pageNum); - } else { - onCreateCollection({ - name, - type: DatasetCollectionTypeEnum.folder - }); + {!!editFolderData && ( + setEditFolderData(undefined)} + editCallback={async (name) => { + try { + if (editFolderData.id) { + await putDatasetCollectionById({ + id: editFolderData.id, + name + }); + getData(pageNum); + } else { + onCreateCollection({ + name, + type: DatasetCollectionTypeEnum.folder + }); + } + } catch (error) { + return Promise.reject(error); } - } catch (error) { - return Promise.reject(error); - } - }} - isEdit={!!editFolderData.id} - name={editFolderData.name} - /> - )} - {!!moveCollectionData && ( - setMoveCollectionData(undefined)} - onSuccess={async ({ parentId }) => { - await putDatasetCollectionById({ - id: moveCollectionData.collectionId, - parentId - }); - getData(pageNum); - setMoveCollectionData(undefined); - toast({ - status: 'success', - title: t('common.folder.Move Success') - }); - }} - /> - )} - {isOpenWebsiteModal && ( - - )} - + }} + isEdit={!!editFolderData.id} + name={editFolderData.name} + /> + )} + {!!moveCollectionData && ( + setMoveCollectionData(undefined)} + onSuccess={async ({ parentId }) => { + await putDatasetCollectionById({ + id: moveCollectionData.collectionId, + parentId + }); + getData(pageNum); + setMoveCollectionData(undefined); + toast({ + status: 'success', + title: t('common.folder.Move Success') + }); + }} + /> + )} + {isOpenWebsiteModal && ( + + )} + + ); }; diff --git a/projects/app/src/pages/dataset/detail/components/DataCard.tsx b/projects/app/src/pages/dataset/detail/components/DataCard.tsx index 285ffe3e0a7..45b6f215ae9 100644 --- a/projects/app/src/pages/dataset/detail/components/DataCard.tsx +++ b/projects/app/src/pages/dataset/detail/components/DataCard.tsx @@ -32,16 +32,17 @@ import { useRouter } from 'next/router'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyInput from '@/components/MyInput'; import { useLoading } from '@/web/common/hooks/useLoading'; -import InputDataModal, { RawSourceText, type InputDataType } from '../components/InputDataModal'; +import InputDataModal from '../components/InputDataModal'; +import RawSourceBox from '@/components/core/dataset/RawSourceBox'; import type { DatasetDataListItemType } from '@/global/core/dataset/type.d'; import { TabEnum } from '..'; import { useUserStore } from '@/web/support/user/useUserStore'; import { TeamMemberRoleEnum } from '@fastgpt/global/support/user/team/constant'; -import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils'; import { useSystemStore } from '@/web/common/system/useSystemStore'; import { DatasetCollectionTypeMap, - DatasetCollectionTrainingTypeMap + TrainingModeEnum, + TrainingTypeMap } from '@fastgpt/global/core/dataset/constant'; import { formatTime2YMDHM } from '@fastgpt/global/common/string/time'; import { formatFileSize } from '@fastgpt/global/common/file/tools'; @@ -90,7 +91,7 @@ const DataCard = () => { } }); - const [editInputData, setEditInputData] = useState(); + const [editDataId, setEditDataId] = useState(); // get first page data const getFirstData = useCallback( @@ -154,7 +155,7 @@ const DataCard = () => { }, { label: t('core.dataset.collection.metadata.Training Type'), - value: t(DatasetCollectionTrainingTypeMap[collection.trainingType]?.label) + value: t(TrainingTypeMap[collection.trainingType]?.label) }, { label: t('core.dataset.collection.metadata.Chunk Size'), @@ -193,7 +194,7 @@ const DataCard = () => { /> - { size={['sm', 'md']} onClick={() => { if (!collection) return; - setEditInputData({ - q: '', - indexes: [getDefaultIndex({ dataId: `${Date.now()}` })] - }); + setEditDataId(''); }} > {t('dataset.Insert Data')} @@ -297,12 +295,7 @@ const DataCard = () => { }} onClick={() => { if (!collection) return; - setEditInputData({ - id: item._id, - q: item.q, - a: item.a, - indexes: item.indexes - }); + setEditDataId(item._id); }} > @@ -424,11 +417,11 @@ const DataCard = () => { )} - {editInputData !== undefined && collection && ( + {editDataId !== undefined && collection && ( setEditInputData(undefined)} + dataId={editDataId} + onClose={() => setEditDataId(undefined)} onSuccess={() => getData(pageNum)} onDelete={() => getData(pageNum)} /> diff --git a/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx b/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx index f5b5588d10f..834c54cff31 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/FileSelect.tsx @@ -4,14 +4,8 @@ import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; import { useToast } from '@/web/common/hooks/useToast'; import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { simpleText } from '@fastgpt/global/common/string/tools'; -import { - fileDownload, - readCsvContent, - readPdfContent, - readDocContent -} from '@/web/common/file/utils'; -import { readFileRawText, readMdFile, readHtmlFile } from '@fastgpt/web/common/file/read'; -import { getUploadMdImgController, uploadFiles } from '@/web/common/file/controller'; +import { fileDownload, readCsvContent } from '@/web/common/file/utils'; +import { getUploadBase64ImgController, uploadFiles } from '@/web/common/file/controller'; import { Box, Flex, useDisclosure, type BoxProps } from '@chakra-ui/react'; import React, { DragEvent, useCallback, useState } from 'react'; import { useTranslation } from 'next-i18next'; @@ -25,6 +19,8 @@ import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; import { DatasetCollectionTypeEnum } from '@fastgpt/global/core/dataset/constant'; import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d'; import { UrlFetchResponse } from '@fastgpt/global/common/file/api.d'; +import { readFileRawContent } from '@fastgpt/web/common/file/read/index'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; const UrlFetchModal = dynamic(() => import('./UrlFetchModal')); const CreateFileModal = dynamic(() => import('./CreateFileModal')); @@ -168,36 +164,22 @@ const FileSelect = ({ } // parse and upload files - let text = await (async () => { - switch (extension) { - case 'txt': - return readFileRawText(file); - case 'md': - return readMdFile({ - file, - uploadImgController: (base64Img) => - getUploadMdImgController({ base64Img, metadata: { fileId } }) - }); - case 'html': - return readHtmlFile({ - file, - uploadImgController: (base64Img) => - getUploadMdImgController({ base64Img, metadata: { fileId } }) - }); - case 'pdf': - return readPdfContent(file); - case 'docx': - return readDocContent(file, { + let { rawText } = await readFileRawContent({ + file, + uploadBase64Controller: (base64Img) => + getUploadBase64ImgController({ + base64Img, + type: MongoImageTypeEnum.docImage, + metadata: { fileId - }); - } - return ''; - })(); + } + }) + }); - if (text) { - text = simpleText(text); + if (rawText) { + rawText = simpleText(rawText); const { chunks, tokens } = splitText2Chunks({ - text, + text: rawText, chunkLen, overlapRatio, customReg: customSplitChar ? [customSplitChar] : [] @@ -207,7 +189,7 @@ const FileSelect = ({ id: nanoid(), filename: file.name, icon, - rawText: text, + rawText, tokens, type: DatasetCollectionTypeEnum.file, fileId, diff --git a/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx b/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx index 1848e947b9a..5b9a0853565 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/ImportModal.tsx @@ -10,10 +10,7 @@ const CsvImport = dynamic(() => import('./Csv'), {}); import MyModal from '@/components/MyModal'; import Provider from './Provider'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; -import { - DatasetCollectionTrainingModeEnum, - TrainingModeEnum -} from '@fastgpt/global/core/dataset/constant'; +import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; export enum ImportTypeEnum { chunk = 'chunk', @@ -46,24 +43,21 @@ const ImportData = ({ chunkOverlapRatio: 0.2, inputPrice: vectorModel?.inputPrice || 0, outputPrice: 0, - mode: TrainingModeEnum.chunk, - collectionTrainingType: DatasetCollectionTrainingModeEnum.chunk + collectionTrainingType: TrainingModeEnum.chunk }, [ImportTypeEnum.qa]: { defaultChunkLen: agentModel?.maxContext * 0.55 || 8000, chunkOverlapRatio: 0, inputPrice: agentModel?.inputPrice || 0, outputPrice: agentModel?.outputPrice || 0, - mode: TrainingModeEnum.qa, - collectionTrainingType: DatasetCollectionTrainingModeEnum.qa + collectionTrainingType: TrainingModeEnum.qa }, [ImportTypeEnum.csv]: { defaultChunkLen: 0, chunkOverlapRatio: 0, inputPrice: vectorModel?.inputPrice || 0, outputPrice: 0, - mode: TrainingModeEnum.chunk, - collectionTrainingType: DatasetCollectionTrainingModeEnum.manual + collectionTrainingType: TrainingModeEnum.chunk } }; return map[importType]; diff --git a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx index 246d74539fc..def0a6ada25 100644 --- a/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx +++ b/projects/app/src/pages/dataset/detail/components/Import/Provider.tsx @@ -16,10 +16,7 @@ import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { hashStr } from '@fastgpt/global/common/string/tools'; import { useToast } from '@/web/common/hooks/useToast'; import { getErrText } from '@fastgpt/global/common/error/utils'; -import { - DatasetCollectionTrainingModeEnum, - TrainingModeEnum -} from '@fastgpt/global/core/dataset/constant'; +import { TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; import { Box, Flex, Image, useTheme } from '@chakra-ui/react'; import { CloseIcon } from '@chakra-ui/icons'; import DeleteIcon, { hoverDeleteStyles } from '@fastgpt/web/components/common/Icon/delete'; @@ -104,7 +101,6 @@ const Provider = ({ parentId, inputPrice, outputPrice, - mode, collectionTrainingType, vectorModel, agentModel, @@ -118,8 +114,7 @@ const Provider = ({ parentId: string; inputPrice: number; outputPrice: number; - mode: `${TrainingModeEnum}`; - collectionTrainingType: `${DatasetCollectionTrainingModeEnum}`; + collectionTrainingType: `${TrainingModeEnum}`; vectorModel: string; agentModel: string; defaultChunkLen: number; @@ -147,14 +142,14 @@ const Provider = ({ const totalTokens = useMemo(() => files.reduce((sum, file) => sum + file.tokens, 0), [files]); const price = useMemo(() => { - if (mode === TrainingModeEnum.qa) { + if (collectionTrainingType === TrainingModeEnum.qa) { const inputTotal = totalTokens * inputPrice; const outputTotal = totalTokens * 0.5 * outputPrice; return formatModelPrice2Read(inputTotal + outputTotal); } return formatModelPrice2Read(totalTokens * inputPrice); - }, [inputPrice, mode, outputPrice, totalTokens]); + }, [collectionTrainingType, inputPrice, outputPrice, totalTokens]); /* start upload data @@ -169,7 +164,7 @@ const Provider = ({ for await (const file of files) { // create training bill const billId = await postCreateTrainingBill({ - name: t('dataset.collections.Create Training Data', { filename: file.filename }), + name: file.filename, vectorModel, agentModel }); @@ -180,11 +175,15 @@ const Provider = ({ parentId, name: file.filename, type: file.type, + + trainingType: collectionTrainingType, + chunkSize: chunkLen, + chunkSplitter: customSplitChar, + qaPrompt: collectionTrainingType === TrainingModeEnum.qa ? prompt : '', + fileId: file.fileId, rawLink: file.rawLink, - chunkSize: chunkLen, - trainingType: collectionTrainingType, - qaPrompt: mode === TrainingModeEnum.qa ? prompt : '', + rawTextLength: file.rawText.length, hashRawText: hashStr(file.rawText), metadata: file.metadata @@ -195,8 +194,8 @@ const Provider = ({ const { insertLen } = await chunksUpload({ collectionId, billId, + trainingMode: collectionTrainingType, chunks, - mode, onUploading: (insertLen) => { setSuccessChunks((state) => state + insertLen); }, diff --git a/projects/app/src/pages/dataset/detail/components/Info.tsx b/projects/app/src/pages/dataset/detail/components/Info.tsx index d0cc8582349..787f2c772ff 100644 --- a/projects/app/src/pages/dataset/detail/components/Info.tsx +++ b/projects/app/src/pages/dataset/detail/components/Info.tsx @@ -1,10 +1,9 @@ -import React, { useCallback, useState, useMemo } from 'react'; +import React, { useState, useMemo } from 'react'; import { useRouter } from 'next/router'; import { Box, Flex, Button, IconButton, Input, Textarea } from '@chakra-ui/react'; import { DeleteIcon } from '@chakra-ui/icons'; import { delDatasetById } from '@/web/core/dataset/api'; import { useSelectFile } from '@/web/common/file/hooks/useSelectFile'; -import { useToast } from '@/web/common/hooks/useToast'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; import { useConfirm } from '@/web/common/hooks/useConfirm'; import { useForm } from 'react-hook-form'; @@ -17,6 +16,7 @@ import PermissionRadio from '@/components/support/permission/Radio'; import MySelect from '@/components/Select'; import { qaModelList } from '@/web/common/system/staticData'; import { useRequest } from '@/web/common/hooks/useRequest'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; const Info = ({ datasetId }: { datasetId: string }) => { const { t } = useTranslation(); @@ -70,6 +70,7 @@ const Info = ({ datasetId }: { datasetId: string }) => { const file = e[0]; if (!file) return Promise.resolve(null); return compressImgFileAndUpload({ + type: MongoImageTypeEnum.datasetAvatar, file, maxW: 300, maxH: 300 diff --git a/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx b/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx index 53856041450..a2cef7f4f42 100644 --- a/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx +++ b/projects/app/src/pages/dataset/detail/components/InputDataModal.tsx @@ -1,44 +1,38 @@ import React, { useMemo, useState } from 'react'; -import { Box, Flex, Button, Textarea, BoxProps, Image, useTheme, Grid } from '@chakra-ui/react'; +import { Box, Flex, Button, Textarea, useTheme, Grid } from '@chakra-ui/react'; import { useFieldArray, useForm } from 'react-hook-form'; import { postInsertData2Dataset, putDatasetDataById, delOneDatasetDataById, - getDatasetCollectionById + getDatasetCollectionById, + getDatasetDataItemById } from '@/web/core/dataset/api'; import { useToast } from '@/web/common/hooks/useToast'; -import { getErrText } from '@fastgpt/global/common/error/utils'; import MyIcon from '@fastgpt/web/components/common/Icon'; import MyModal from '@/components/MyModal'; import MyTooltip from '@/components/MyTooltip'; import { QuestionOutlineIcon } from '@chakra-ui/icons'; import { useQuery } from '@tanstack/react-query'; import { useTranslation } from 'next-i18next'; -import { getFileAndOpen } from '@/web/core/dataset/utils'; -import { useSystemStore } from '@/web/common/system/useSystemStore'; import { useRequest } from '@/web/common/hooks/useRequest'; import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; import { useConfirm } from '@/web/common/hooks/useConfirm'; -import { getDefaultIndex, getSourceNameIcon } from '@fastgpt/global/core/dataset/utils'; -import { feConfigs, vectorModelList } from '@/web/common/system/staticData'; +import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils'; +import { vectorModelList } from '@/web/common/system/staticData'; import { DatasetDataIndexTypeEnum } from '@fastgpt/global/core/dataset/constant'; import { DatasetDataIndexItemType } from '@fastgpt/global/core/dataset/type'; import SideTabs from '@/components/SideTabs'; -import { useLoading } from '@/web/common/hooks/useLoading'; import DeleteIcon from '@fastgpt/web/components/common/Icon/delete'; import { defaultCollectionDetail } from '@/constants/dataset'; import { getDocPath } from '@/web/common/system/doc'; +import RawSourceBox from '@/components/core/dataset/RawSourceBox'; +import MyBox from '@/components/common/MyBox'; +import { getErrText } from '@fastgpt/global/common/error/utils'; -export type RawSourceTextProps = BoxProps & { - sourceName?: string; - sourceId?: string; - canView?: boolean; -}; export type InputDataType = { - id?: string; q: string; - a?: string; + a: string; indexes: (Omit & { dataId?: string; // pg data id })[]; @@ -53,26 +47,25 @@ enum TabEnum { const InputDataModal = ({ collectionId, + dataId, defaultValue, onClose, onSuccess, onDelete }: { collectionId: string; - defaultValue: InputDataType; + dataId?: string; + defaultValue?: { q: string; a?: string }; onClose: () => void; - onSuccess: (data: InputDataType) => void; + onSuccess: (data: InputDataType & { dataId: string }) => void; onDelete?: () => void; }) => { const { t } = useTranslation(); const theme = useTheme(); const { toast } = useToast(); - const { Loading } = useLoading(); const [currentTab, setCurrentTab] = useState(TabEnum.content); - const { register, handleSubmit, reset, control } = useForm({ - defaultValues: defaultValue - }); + const { register, handleSubmit, reset, control } = useForm(); const { fields: indexes, append: appendIndexes, @@ -89,14 +82,15 @@ const InputDataModal = ({ id: TabEnum.index, icon: 'kbTest' }, - ...(defaultValue.id + ...(dataId ? [{ label: t('dataset.data.edit.Delete'), id: TabEnum.delete, icon: 'delete' }] : []), { label: t('dataset.data.edit.Course'), id: TabEnum.doc, icon: 'common/courseLight' } ]; const { ConfirmModal, openConfirm } = useConfirm({ - content: t('dataset.data.Delete Tip') + content: t('dataset.data.Delete Tip'), + type: 'delete' }); const { data: collection = defaultCollectionDetail } = useQuery( @@ -105,6 +99,37 @@ const InputDataModal = ({ return getDatasetCollectionById(collectionId); } ); + const { isFetching: isFetchingData } = useQuery( + ['getDatasetDataItemById', dataId], + () => { + if (dataId) return getDatasetDataItemById(dataId); + return null; + }, + { + onSuccess(res) { + if (res) { + reset({ + q: res.q, + a: res.a, + indexes: res.indexes + }); + } else if (defaultValue) { + reset({ + q: defaultValue.q, + a: defaultValue.a, + indexes: [getDefaultIndex({ dataId: `${Date.now()}` })] + }); + } + }, + onError(err) { + toast({ + status: 'error', + title: getErrText(err) + }); + onClose(); + } + } + ); const maxToken = useMemo(() => { const vectorModel = @@ -130,7 +155,7 @@ const InputDataModal = ({ const data = { ...e }; - data.id = await postInsertData2Dataset({ + const dataId = await postInsertData2Dataset({ collectionId: collection._id, q: e.q, a: e.a, @@ -140,7 +165,10 @@ const InputDataModal = ({ ) }); - return data; + return { + ...data, + dataId + }; }, successToast: t('dataset.data.Input Success Tip'), onSuccess(e) { @@ -158,17 +186,18 @@ const InputDataModal = ({ // update const { mutate: onUpdateData, isLoading: isUpdating } = useRequest({ mutationFn: async (e: InputDataType) => { - if (!e.id) return e; + if (!dataId) return e; // not exactly same await putDatasetDataById({ - id: e.id, - q: e.q, - a: e.a, - indexes: e.indexes + id: dataId, + ...e }); - return e; + return { + dataId, + ...e + }; }, successToast: t('dataset.data.Update Success Tip'), errorToast: t('common.error.unKnow'), @@ -180,8 +209,8 @@ const InputDataModal = ({ // delete const { mutate: onDeleteData, isLoading: isDeleting } = useRequest({ mutationFn: () => { - if (!onDelete || !defaultValue.id) return Promise.resolve(null); - return delOneDatasetDataById(defaultValue.id); + if (!onDelete || !dataId) return Promise.resolve(null); + return delOneDatasetDataById(dataId); }, onSuccess() { if (!onDelete) return; @@ -192,13 +221,16 @@ const InputDataModal = ({ errorToast: t('common.error.unKnow') }); - const loading = useMemo(() => isImporting || isUpdating, [isImporting, isUpdating]); + const isLoading = useMemo( + () => isImporting || isUpdating || isFetchingData || isDeleting, + [isImporting, isUpdating, isFetchingData, isDeleting] + ); return ( - + - {currentTab === TabEnum.content && ( - <>{defaultValue.id ? t('dataset.data.Update Data') : t('dataset.data.Input Data')} + <>{dataId ? t('dataset.data.Update Data') : t('dataset.data.Input Data')} )} {currentTab === TabEnum.index && <> {t('dataset.data.Index Edit')}} @@ -351,82 +383,24 @@ const InputDataModal = ({ )} - - + - ); }; -export default InputDataModal; - -export function RawSourceText({ - sourceId, - sourceName = '', - canView = true, - ...props -}: RawSourceTextProps) { - const { t } = useTranslation(); - const { toast } = useToast(); - const { setLoading } = useSystemStore(); - - const canPreview = useMemo(() => !!sourceId && canView, [canView, sourceId]); - - const icon = useMemo(() => getSourceNameIcon({ sourceId, sourceName }), [sourceId, sourceName]); - - return ( - - { - setLoading(true); - try { - await getFileAndOpen(sourceId as string); - } catch (error) { - toast({ - title: t(getErrText(error, 'error.fileNotFound')), - status: 'error' - }); - } - setLoading(false); - } - } - : {})} - {...props} - > - - - {sourceName || t('common.UnKnow Source')} - - - - ); -} +export default React.memo(InputDataModal); diff --git a/projects/app/src/pages/dataset/detail/components/Test.tsx b/projects/app/src/pages/dataset/detail/components/Test.tsx index bb97919e234..1040659f4e5 100644 --- a/projects/app/src/pages/dataset/detail/components/Test.tsx +++ b/projects/app/src/pages/dataset/detail/components/Test.tsx @@ -233,7 +233,7 @@ const Test = ({ datasetId }: { datasetId: string }) => { h={'100%'} resize={'none'} variant={'unstyled'} - maxLength={datasetDetail.vectorModel.maxToken} + maxLength={datasetDetail.vectorModel?.maxToken} placeholder={t('core.dataset.test.Test Text Placeholder')} onFocus={() => setIsFocus(true)} {...register('inputText', { @@ -314,7 +314,7 @@ const Test = ({ datasetId }: { datasetId: string }) => { {/* result show */} - + @@ -384,6 +384,9 @@ const TestHistories = React.memo(function TestHistories({ }} cursor={'pointer'} fontSize={'sm'} + {...(item.id === datasetTestItem?.id && { + bg: 'primary.50' + })} onClick={() => setDatasetTestItem(item)} > diff --git a/projects/app/src/pages/dataset/detail/index.tsx b/projects/app/src/pages/dataset/detail/index.tsx index ad7f7ca0254..16d03783806 100644 --- a/projects/app/src/pages/dataset/detail/index.tsx +++ b/projects/app/src/pages/dataset/detail/index.tsx @@ -16,8 +16,6 @@ import { serviceSideProps } from '@/web/common/utils/i18n'; import { useTranslation } from 'next-i18next'; import { getTrainingQueueLen } from '@/web/core/dataset/api'; import MyTooltip from '@/components/MyTooltip'; -import { QuestionOutlineIcon } from '@chakra-ui/icons'; -import { feConfigs } from '@/web/common/system/staticData'; import Script from 'next/script'; import CollectionCard from './components/CollectionCard'; import { useDatasetStore } from '@/web/core/dataset/store/dataset'; @@ -29,6 +27,7 @@ import { } from '@fastgpt/global/core/dataset/constant'; import { useConfirm } from '@/web/common/hooks/useConfirm'; import { useRequest } from '@/web/common/hooks/useRequest'; +import DatasetTypeTag from '@/components/core/dataset/DatasetTypeTag'; const DataCard = dynamic(() => import('./components/DataCard'), { ssr: false @@ -150,50 +149,47 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T {isPc ? ( - - - - {datasetDetail.name} - - - {DatasetTypeMap[datasetDetail.type] && ( - - - {t(DatasetTypeMap[datasetDetail.type]?.label)} - {datasetDetail.type === DatasetTypeEnum.websiteDataset && - datasetDetail.status === DatasetStatusEnum.active && ( - - - openConfirmSync( - onUpdateDatasetWebsiteConfig, - undefined, - t('core.dataset.website.Confirm Create Tips') - )() - } - /> - - )} + + + + + {datasetDetail.name} + - )} + {DatasetTypeMap[datasetDetail.type] && ( + + + {datasetDetail.type === DatasetTypeEnum.websiteDataset && + datasetDetail.status === DatasetStatusEnum.active && ( + + + openConfirmSync( + onUpdateDatasetWebsiteConfig, + undefined, + t('core.dataset.website.Confirm Create Tips') + )() + } + /> + + )} + + )} + - + {t('core.dataset.training.Agent queue')}({agentTrainingMap.tip}) @@ -229,6 +225,7 @@ const Detail = ({ datasetId, currentTab }: { datasetId: string; currentTab: `${T /> + void; parentId?: string }) => { const { t } = useTranslation(); @@ -49,6 +50,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st if (!file) return; try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.datasetAvatar, file, maxW: 300, maxH: 300 @@ -62,7 +64,7 @@ const CreateModal = ({ onClose, parentId }: { onClose: () => void; parentId?: st }); } }, - [setValue, toast] + [setValue, t, toast] ); /* create a new kb and router to it */ diff --git a/projects/app/src/pages/dataset/list/index.tsx b/projects/app/src/pages/dataset/list/index.tsx index 522f7fe7bf5..c5832433e77 100644 --- a/projects/app/src/pages/dataset/list/index.tsx +++ b/projects/app/src/pages/dataset/list/index.tsx @@ -22,7 +22,7 @@ import { putDatasetById, postCreateDataset } from '@/web/core/dataset/api'; -import { checkTeamExportDatasetLimit } from '@/web/support/user/api'; +import { checkTeamExportDatasetLimit } from '@/web/support/user/team/api'; import { useTranslation } from 'next-i18next'; import Avatar from '@/components/Avatar'; import MyIcon from '@fastgpt/web/components/common/Icon'; @@ -44,6 +44,7 @@ import PermissionIconText from '@/components/support/permission/IconText'; import { PermissionTypeEnum } from '@fastgpt/global/support/permission/constant'; import { DatasetItemType } from '@fastgpt/global/core/dataset/type'; import ParentPaths from '@/components/common/ParentPaths'; +import DatasetTypeTag from '@/components/core/dataset/DatasetTypeTag'; const CreateModal = dynamic(() => import('./component/CreateModal'), { ssr: false }); const MoveModal = dynamic(() => import('./component/MoveModal'), { ssr: false }); @@ -409,8 +410,9 @@ const Kb = () => { - - {t(dataset.label)} + {dataset.type !== DatasetTypeEnum.folder && ( + + )} ))} diff --git a/projects/app/src/pages/plugin/list/component/EditModal.tsx b/projects/app/src/pages/plugin/list/component/EditModal.tsx index 60fc6af5396..b9d152e0cbb 100644 --- a/projects/app/src/pages/plugin/list/component/EditModal.tsx +++ b/projects/app/src/pages/plugin/list/component/EditModal.tsx @@ -17,6 +17,7 @@ import { useConfirm } from '@/web/common/hooks/useConfirm'; import MyIcon from '@fastgpt/web/components/common/Icon'; import { CreateOnePluginParams } from '@fastgpt/global/core/plugin/controller'; import { customAlphabet } from 'nanoid'; +import { MongoImageTypeEnum } from '@fastgpt/global/common/file/image/constants'; const nanoid = customAlphabet('abcdefghijklmnopqrstuvwxyz1234567890', 12); export type FormType = CreateOnePluginParams & { @@ -92,6 +93,7 @@ const CreateModal = ({ if (!file) return; try { const src = await compressImgFileAndUpload({ + type: MongoImageTypeEnum.pluginAvatar, file, maxW: 300, maxH: 300 diff --git a/projects/app/src/service/common/system/cron.ts b/projects/app/src/service/common/system/cron.ts new file mode 100644 index 00000000000..33434c50576 --- /dev/null +++ b/projects/app/src/service/common/system/cron.ts @@ -0,0 +1,18 @@ +import { initSystemConfig } from '@/pages/api/common/system/getInitData'; +import { generateQA } from '@/service/events/generateQA'; +import { generateVector } from '@/service/events/generateVector'; +import { setCron } from '@fastgpt/service/common/system/cron'; + +export const setUpdateSystemConfigCron = () => { + setCron('*/5 * * * *', () => { + initSystemConfig(); + console.log('refresh system config'); + }); +}; + +export const setTrainingQueueCron = () => { + setCron('*/3 * * * *', () => { + generateVector(); + generateQA(); + }); +}; diff --git a/projects/app/src/service/core/ai/rerank.ts b/projects/app/src/service/core/ai/rerank.ts index 4b1cf781a3d..5ced9f11015 100644 --- a/projects/app/src/service/core/ai/rerank.ts +++ b/projects/app/src/service/core/ai/rerank.ts @@ -27,7 +27,7 @@ export function reRankRecall({ query, inputs }: PostReRankProps) { return data; }) .catch((err) => { - console.log(err); + console.log('rerank error:', err); return []; }); diff --git a/projects/app/src/service/core/dataset/data/controller.ts b/projects/app/src/service/core/dataset/data/controller.ts index 839519946dd..eacd5fda3ee 100644 --- a/projects/app/src/service/core/dataset/data/controller.ts +++ b/projects/app/src/service/core/dataset/data/controller.ts @@ -14,7 +14,8 @@ import { DatasetDataIndexTypeEnum, DatasetSearchModeEnum, DatasetSearchModeMap, - SearchScoreTypeEnum + SearchScoreTypeEnum, + TrainingModeEnum } from '@fastgpt/global/core/dataset/constant'; import { getDefaultIndex } from '@fastgpt/global/core/dataset/utils'; import { jiebaSplit } from '@/service/common/string/jieba'; @@ -27,7 +28,173 @@ import { } from '@fastgpt/global/core/dataset/type'; import { reRankRecall } from '../../ai/rerank'; import { countPromptTokens } from '@fastgpt/global/common/string/tiktoken'; -import { hashStr } from '@fastgpt/global/common/string/tools'; +import { hashStr, simpleText } from '@fastgpt/global/common/string/tools'; +import type { PushDatasetDataProps } from '@/global/core/dataset/api.d'; +import type { PushDataResponse } from '@/global/core/api/datasetRes'; +import { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api'; +import { MongoDatasetTraining } from '@fastgpt/service/core/dataset/training/schema'; +import { startQueue } from '@/service/utils/tools'; +import { getCollectionWithDataset } from '@fastgpt/service/core/dataset/controller'; +import { getQAModel, getVectorModel } from '../../ai/model'; +import { delay } from '@fastgpt/global/common/system/utils'; + +export async function pushDataToDatasetCollection({ + teamId, + tmbId, + collectionId, + data, + prompt, + billId, + trainingMode +}: { + teamId: string; + tmbId: string; +} & PushDatasetDataProps): Promise { + const checkModelValid = async ({ collectionId }: { collectionId: string }) => { + const { + datasetId: { _id: datasetId, vectorModel, agentModel } + } = await getCollectionWithDataset(collectionId); + + if (trainingMode === TrainingModeEnum.chunk) { + if (!collectionId) return Promise.reject(`CollectionId is empty`); + const vectorModelData = getVectorModel(vectorModel); + if (!vectorModelData) { + return Promise.reject(`Model ${vectorModel} is inValid`); + } + + return { + datasetId, + maxToken: vectorModelData.maxToken * 1.5, + model: vectorModelData.model, + weight: vectorModelData.weight + }; + } + + if (trainingMode === TrainingModeEnum.qa) { + const qaModelData = getQAModel(agentModel); + if (!qaModelData) { + return Promise.reject(`Model ${agentModel} is inValid`); + } + return { + datasetId, + maxToken: qaModelData.maxContext * 0.8, + model: qaModelData.model, + weight: 0 + }; + } + return Promise.reject(`Mode ${trainingMode} is inValid`); + }; + + const { datasetId, model, maxToken, weight } = await checkModelValid({ + collectionId + }); + + // format q and a, remove empty char + data.forEach((item) => { + item.q = simpleText(item.q); + item.a = simpleText(item.a); + + item.indexes = item.indexes + ?.map((index) => { + return { + ...index, + text: simpleText(index.text) + }; + }) + .filter(Boolean); + }); + + // filter repeat or equal content + const set = new Set(); + const filterResult: Record = { + success: [], + overToken: [], + repeat: [], + error: [] + }; + + data.forEach((item) => { + if (!item.q) { + filterResult.error.push(item); + return; + } + + const text = item.q + item.a; + + // count q token + const token = countPromptTokens(item.q); + + if (token > maxToken) { + filterResult.overToken.push(item); + return; + } + + if (set.has(text)) { + console.log('repeat', item); + filterResult.repeat.push(item); + } else { + filterResult.success.push(item); + set.add(text); + } + }); + + // 插入记录 + const insertData = async (dataList: PushDatasetDataChunkProps[], retry = 3): Promise => { + try { + const results = await MongoDatasetTraining.insertMany( + dataList.map((item, i) => ({ + teamId, + tmbId, + datasetId, + collectionId, + billId, + mode: trainingMode, + prompt, + model, + q: item.q, + a: item.a, + chunkIndex: item.chunkIndex ?? i, + weight: weight ?? 0, + indexes: item.indexes + })) + ); + await delay(500); + return results.length; + } catch (error) { + if (retry > 0) { + await delay(1000); + return insertData(dataList, retry - 1); + } + return Promise.reject(error); + } + }; + + let insertLen = 0; + const chunkSize = 50; + const chunkList = filterResult.success.reduce( + (acc, cur) => { + const lastChunk = acc[acc.length - 1]; + if (lastChunk.length < chunkSize) { + lastChunk.push(cur); + } else { + acc.push([cur]); + } + return acc; + }, + [[]] as PushDatasetDataChunkProps[][] + ); + for await (const chunks of chunkList) { + insertLen += await insertData(chunks); + } + + startQueue(); + delete filterResult.success; + + return { + insertLen, + ...filterResult + }; +} /* insert data. * 1. create data id @@ -439,7 +606,9 @@ export async function searchDatasetData(props: { })) }); - if (!Array.isArray(results)) return []; + if (!Array.isArray(results)) { + return []; + } // add new score to data const mergeResult = results @@ -457,7 +626,6 @@ export async function searchDatasetData(props: { return mergeResult; } catch (error) { - usingReRank = false; return []; } }; diff --git a/projects/app/src/service/events/generateQA.ts b/projects/app/src/service/events/generateQA.ts index d4c58aaa846..cb3684a9f77 100644 --- a/projects/app/src/service/events/generateQA.ts +++ b/projects/app/src/service/events/generateQA.ts @@ -8,20 +8,15 @@ import { addLog } from '@fastgpt/service/common/system/log'; import { splitText2Chunks } from '@fastgpt/global/common/string/textSplitter'; import { replaceVariable } from '@fastgpt/global/common/string/tools'; import { Prompt_AgentQA } from '@/global/core/prompt/agent'; -import { pushDataToDatasetCollection } from '@/pages/api/core/dataset/data/pushData'; import { getErrText } from '@fastgpt/global/common/error/utils'; import { authTeamBalance } from '../support/permission/auth/bill'; import type { PushDatasetDataChunkProps } from '@fastgpt/global/core/dataset/api.d'; import { UserErrEnum } from '@fastgpt/global/common/error/code/user'; import { lockTrainingDataByTeamId } from '@fastgpt/service/core/dataset/training/controller'; +import { pushDataToDatasetCollection } from '@/service/core/dataset/data/controller'; -const reduceQueue = (retry = false) => { +const reduceQueue = () => { global.qaQueueLen = global.qaQueueLen > 0 ? global.qaQueueLen - 1 : 0; - if (global.qaQueueLen === 0 && retry) { - setTimeout(() => { - generateQA(); - }, 60000); - } return global.vectorQueueLen === 0; }; @@ -144,11 +139,11 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`; teamId: data.teamId, tmbId: data.tmbId, collectionId: data.collectionId, + trainingMode: TrainingModeEnum.chunk, data: qaArr.map((item) => ({ ...item, chunkIndex: data.chunkIndex })), - mode: TrainingModeEnum.chunk, billId: data.billId }); @@ -178,7 +173,7 @@ ${replaceVariable(Prompt_AgentQA.fixedText, { text })}`; reduceQueue(); generateQA(); } catch (err: any) { - reduceQueue(true); + reduceQueue(); // log if (err?.response) { addLog.info('openai error: 生成QA错误', { diff --git a/projects/app/src/service/events/generateVector.ts b/projects/app/src/service/events/generateVector.ts index 217f76d85cf..831aad97105 100644 --- a/projects/app/src/service/events/generateVector.ts +++ b/projects/app/src/service/events/generateVector.ts @@ -9,15 +9,9 @@ import { pushGenerateVectorBill } from '@/service/support/wallet/bill/push'; import { UserErrEnum } from '@fastgpt/global/common/error/code/user'; import { lockTrainingDataByTeamId } from '@fastgpt/service/core/dataset/training/controller'; -const reduceQueue = (retry = false) => { +const reduceQueue = () => { global.vectorQueueLen = global.vectorQueueLen > 0 ? global.vectorQueueLen - 1 : 0; - if (global.vectorQueueLen === 0 && retry) { - setTimeout(() => { - generateVector(); - }, 60000); - } - return global.vectorQueueLen === 0; }; @@ -159,7 +153,7 @@ export async function generateVector(): Promise { console.log(`embedding finished, time: ${Date.now() - start}ms`); } catch (err: any) { - reduceQueue(true); + reduceQueue(); // log if (err?.response) { addLog.info('openai error: 生成向量错误', { diff --git a/projects/app/src/service/moduleDispatch/chat/oneapi.ts b/projects/app/src/service/moduleDispatch/chat/oneapi.ts index 6f862326396..36ffeca5bf4 100644 --- a/projects/app/src/service/moduleDispatch/chat/oneapi.ts +++ b/projects/app/src/service/moduleDispatch/chat/oneapi.ts @@ -214,7 +214,7 @@ export const dispatchChatCompletion = async (props: ChatProps): Promise { return connectMongo({ beforeHook: () => {}, - afterHook: () => { + afterHook: async () => { initVectorStore(); // start queue startQueue(); - return initRootUser(); + // init system config + getInitConfig(); + + // cron + setUpdateSystemConfigCron(); + setTrainingQueueCron(); + + initRootUser(); } }); } diff --git a/projects/app/src/service/utils/chat/saveChat.ts b/projects/app/src/service/utils/chat/saveChat.ts index 78ef20da28b..3b6d25048f0 100644 --- a/projects/app/src/service/utils/chat/saveChat.ts +++ b/projects/app/src/service/utils/chat/saveChat.ts @@ -60,7 +60,6 @@ export async function saveChat({ })) ) ]; - console.log(metadataUpdate); const title = chatContentReplaceBlock(content[0].value).slice(0, 20) || diff --git a/projects/app/src/service/utils/tools.ts b/projects/app/src/service/utils/tools.ts index b5111841923..207dc4274fb 100644 --- a/projects/app/src/service/utils/tools.ts +++ b/projects/app/src/service/utils/tools.ts @@ -2,20 +2,9 @@ import { generateQA } from '../events/generateQA'; import { generateVector } from '../events/generateVector'; /* start task */ -export const startQueue = (limit?: number) => { +export const startQueue = () => { if (!global.systemEnv) return; - if (limit) { - for (let i = 0; i < limit; i++) { - generateVector(); - generateQA(); - } - return; - } - for (let i = 0; i < global.systemEnv.qaMaxProcess; i++) { - generateQA(); - } - for (let i = 0; i < global.systemEnv.vectorMaxProcess; i++) { - generateVector(); - } + generateQA(); + generateVector(); }; diff --git a/projects/app/src/web/common/file/controller.ts b/projects/app/src/web/common/file/controller.ts index 89773e7770d..c45161aa1b3 100644 --- a/projects/app/src/web/common/file/controller.ts +++ b/projects/app/src/web/common/file/controller.ts @@ -1,10 +1,8 @@ import { postUploadImg, postUploadFiles } from '@/web/common/file/api'; import { UploadImgProps } from '@fastgpt/global/common/file/api'; import { BucketNameEnum } from '@fastgpt/global/common/file/constants'; -import { - compressBase64ImgAndUpload as compressBase64ImgAndUploadControl, - type CompressImgProps -} from '@fastgpt/web/common/file/img'; +import { preUploadImgProps } from '@fastgpt/global/common/file/api'; +import { compressBase64Img, type CompressImgProps } from '@fastgpt/web/common/file/img'; /** * upload file to mongo gridfs @@ -34,57 +32,45 @@ export const uploadFiles = ({ }); }; -export const getUploadMdImgController = ({ - base64Img, - metadata -}: { - base64Img: string; - metadata: Record; -}) => - compressBase64ImgAndUpload({ - base64Img, +export const getUploadBase64ImgController = (props: CompressImgProps & UploadImgProps) => + compressBase64Img({ maxW: 4000, maxH: 4000, maxSize: 1024 * 1024 * 5, - metadata + ...props }); /** * compress image. response base64 * @param maxSize The max size of the compressed image */ -export const compressBase64ImgAndUpload = ({ - expiredTime, - metadata, - shareId, +export const compressBase64ImgAndUpload = async ({ + base64Img, + maxW, + maxH, + maxSize, ...props }: UploadImgProps & CompressImgProps) => { - return compressBase64ImgAndUploadControl({ + const compressUrl = await compressBase64Img({ + base64Img, + maxW, + maxH, + maxSize + }); + + return postUploadImg({ ...props, - uploadController: (base64Img) => - postUploadImg({ - shareId, - base64Img, - expiredTime, - metadata - }) + base64Img: compressUrl }); }; + export const compressImgFileAndUpload = async ({ file, - maxW, - maxH, - maxSize, - expiredTime, - shareId -}: { - file: File; - maxW?: number; - maxH?: number; - maxSize?: number; - expiredTime?: Date; - shareId?: string; -}) => { + ...props +}: preUploadImgProps & + CompressImgProps & { + file: File; + }) => { const reader = new FileReader(); reader.readAsDataURL(file); @@ -94,16 +80,12 @@ export const compressImgFileAndUpload = async ({ }; reader.onerror = (err) => { console.log(err); - reject('压缩图片异常'); + reject('Load image error'); }; }); return compressBase64ImgAndUpload({ base64Img, - maxW, - maxH, - maxSize, - expiredTime, - shareId + ...props }); }; diff --git a/projects/app/src/web/common/file/utils.ts b/projects/app/src/web/common/file/utils.ts index d71617eb7be..23dcafc8fc1 100644 --- a/projects/app/src/web/common/file/utils.ts +++ b/projects/app/src/web/common/file/utils.ts @@ -1,80 +1,5 @@ -import mammoth from 'mammoth'; import Papa from 'papaparse'; -import { compressBase64ImgAndUpload } from './controller'; -import { simpleMarkdownText } from '@fastgpt/global/common/string/markdown'; -import { htmlStr2Md } from '@fastgpt/web/common/string/markdown'; -import { readPdfFile } from '@fastgpt/global/common/file/read/index'; -import { readFileRawText } from '@fastgpt/web/common/file/read'; - -/** - * read pdf to raw text - */ -export const readPdfContent = (file: File) => - new Promise((resolve, reject) => { - try { - let reader = new FileReader(); - reader.readAsArrayBuffer(file); - reader.onload = async (event) => { - if (!event?.target?.result) return reject('解析 PDF 失败'); - try { - const content = await readPdfFile({ pdf: event.target.result }); - - resolve(content); - } catch (err) { - console.log(err, 'pdf load error'); - reject('解析 PDF 失败'); - } - }; - reader.onerror = (err) => { - console.log(err, 'pdf load error'); - reject('解析 PDF 失败'); - }; - } catch (error) { - reject('浏览器不支持文件内容读取'); - } - }); - -/** - * read docx to markdown - */ -export const readDocContent = (file: File, metadata: Record) => - new Promise((resolve, reject) => { - try { - const reader = new FileReader(); - reader.readAsArrayBuffer(file); - reader.onload = async ({ target }) => { - if (!target?.result) return reject('读取 doc 文件失败'); - try { - const buffer = target.result as ArrayBuffer; - const { value: html } = await mammoth.convertToHtml({ - arrayBuffer: buffer - }); - const md = htmlStr2Md(html); - - const rawText = await uploadMarkdownBase64(md, metadata); - - resolve(rawText); - } catch (error) { - window.umami?.track('wordReadError', { - err: error?.toString() - }); - console.log('error doc read:', error); - - reject('读取 doc 文件失败, 请转换成 PDF'); - } - }; - reader.onerror = (err) => { - window.umami?.track('wordReadError', { - err: err?.toString() - }); - console.log('error doc read:', err); - - reject('读取 doc 文件失败'); - }; - } catch (error) { - reject('浏览器不支持文件内容读取'); - } - }); +import { readFileRawText } from '@fastgpt/web/common/file/read/rawText'; /** * read csv to json @@ -85,7 +10,7 @@ export const readDocContent = (file: File, metadata: Record) => */ export const readCsvContent = async (file: File) => { try { - const textArr = await readFileRawText(file); + const { rawText: textArr } = await readFileRawText(file); const csvArr = Papa.parse(textArr).data as string[][]; if (csvArr.length === 0) { throw new Error('csv 解析失败'); @@ -99,44 +24,6 @@ export const readCsvContent = async (file: File) => { } }; -/** - * format markdown - * 1. upload base64 - * 2. replace \ - */ -export const uploadMarkdownBase64 = async (rawText: string = '', metadata: Record) => { - // match base64, upload and replace it - const base64Regex = /data:image\/.*;base64,([^\)]+)/g; - const base64Arr = rawText.match(base64Regex) || []; - // upload base64 and replace it - await Promise.all( - base64Arr.map(async (base64Img) => { - try { - const str = await compressBase64ImgAndUpload({ - base64Img, - maxW: 4329, - maxH: 4329, - maxSize: 1024 * 1024 * 5, - metadata - }); - - rawText = rawText.replace(base64Img, str); - } catch (error) { - rawText = rawText.replace(base64Img, ''); - rawText = rawText.replace(/!\[.*\]\(\)/g, ''); - } - }) - ); - - // Remove white space on both sides of the picture - const trimReg = /(!\[.*\]\(.*\))\s*/g; - if (trimReg.test(rawText)) { - rawText = rawText.replace(trimReg, '$1'); - } - - return simpleMarkdownText(rawText); -}; - /** * file download by text */ diff --git a/projects/app/src/web/common/hooks/useConfirm.tsx b/projects/app/src/web/common/hooks/useConfirm.tsx index f856616ea78..ec01215bf8a 100644 --- a/projects/app/src/web/common/hooks/useConfirm.tsx +++ b/projects/app/src/web/common/hooks/useConfirm.tsx @@ -105,7 +105,7 @@ export const useConfirm = (props?: { )}