From 8b71ce23bbb6329bd5e1f516b716b1880c01e6b0 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Wed, 17 Jul 2024 22:01:32 +0200 Subject: [PATCH 001/212] refac: styling --- src/lib/components/chat/Messages/UserMessage.svelte | 7 ++++++- src/lib/components/common/FileItem.svelte | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib/components/chat/Messages/UserMessage.svelte b/src/lib/components/chat/Messages/UserMessage.svelte index ac36dc7fdb..748fcd61b3 100644 --- a/src/lib/components/chat/Messages/UserMessage.svelte +++ b/src/lib/components/chat/Messages/UserMessage.svelte @@ -100,7 +100,12 @@ {#if file.type === 'image'} input {:else} - + {/if} {/each} diff --git a/src/lib/components/common/FileItem.svelte b/src/lib/components/common/FileItem.svelte index b20c5a80be..6ae011caea 100644 --- a/src/lib/components/common/FileItem.svelte +++ b/src/lib/components/common/FileItem.svelte @@ -5,6 +5,7 @@ const dispatch = createEventDispatcher(); export let className = 'w-72'; + export let colorClassName = 'bg-white dark:bg-gray-800'; export let url: string | null = null; export let clickHandler: Function | null = null; @@ -18,7 +19,7 @@
From 7e03624408098bda2113f7b4662a866c1f91fbb9 Mon Sep 17 00:00:00 2001 From: Jonathan Rohde Date: Thu, 18 Jul 2024 12:36:48 +0200 Subject: [PATCH 005/212] fix: send tags to upload doc handler in modal dialog --- src/lib/components/documents/AddDocModal.svelte | 4 ++-- src/lib/components/workspace/Documents.svelte | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/lib/components/documents/AddDocModal.svelte b/src/lib/components/documents/AddDocModal.svelte index 147d3d91b6..10164be97d 100644 --- a/src/lib/components/documents/AddDocModal.svelte +++ b/src/lib/components/documents/AddDocModal.svelte @@ -35,12 +35,12 @@ SUPPORTED_FILE_TYPE.includes(file['type']) || SUPPORTED_FILE_EXTENSIONS.includes(file.name.split('.').at(-1)) ) { - uploadDoc(file); + uploadDoc(file, tags); } else { toast.error( `Unknown File Type '${file['type']}', but accepting and treating as plain text` ); - uploadDoc(file); + uploadDoc(file, tags); } } diff --git a/src/lib/components/workspace/Documents.svelte b/src/lib/components/workspace/Documents.svelte index 635753d1a6..80f528af15 100644 --- a/src/lib/components/workspace/Documents.svelte +++ b/src/lib/components/workspace/Documents.svelte @@ -51,7 +51,7 @@ await documents.set(await getDocs(localStorage.token)); }; - const uploadDoc = async (file) => { + const uploadDoc = async (file, tags?: object) => { console.log(file); // Check if the file is an audio file and transcribe/convert it to text file if (['audio/mpeg', 'audio/wav'].includes(file['type'])) { @@ -84,7 +84,12 @@ res.collection_name, res.filename, transformFileName(res.filename), - res.filename + res.filename, + tags?.length > 0 + ? { + tags: tags + } + : null ).catch((error) => { toast.error(error); return null; From c6eba8c0a1bd47a77cb225482383bd079cdaa66a Mon Sep 17 00:00:00 2001 From: Jonathan Rohde Date: Thu, 18 Jul 2024 14:47:04 +0200 Subject: [PATCH 006/212] enh: add e2e tests for document list --- cypress/e2e/documents.cy.ts | 46 +++++++++++++++++ cypress/support/e2e.ts | 49 +++++++++++++++++++ cypress/support/index.d.ts | 2 + .../components/common/Tags/TagInput.svelte | 3 +- src/lib/components/workspace/Documents.svelte | 3 ++ 5 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 cypress/e2e/documents.cy.ts diff --git a/cypress/e2e/documents.cy.ts b/cypress/e2e/documents.cy.ts new file mode 100644 index 0000000000..6ca14980d2 --- /dev/null +++ b/cypress/e2e/documents.cy.ts @@ -0,0 +1,46 @@ +// eslint-disable-next-line @typescript-eslint/triple-slash-reference +/// + +describe('Documents', () => { + const timestamp = Date.now(); + + before(() => { + cy.uploadTestDocument(timestamp); + }); + + after(() => { + cy.deleteTestDocument(timestamp); + }); + + context('Admin', () => { + beforeEach(() => { + // Login as the admin user + cy.loginAdmin(); + // Visit the home page + cy.visit('/workspace/documents'); + cy.get('button').contains('#cypress-test').click(); + }); + + it('can see documents', () => { + cy.get('div').contains(`document-test-initial-${timestamp}.txt`).should('have.length', 1); + }); + + it('can see edit button', () => { + cy.get('div') + .contains(`document-test-initial-${timestamp}.txt`) + .get("button[aria-label='Edit Doc']") + .should('exist'); + }); + + it('can see delete button', () => { + cy.get('div') + .contains(`document-test-initial-${timestamp}.txt`) + .get("button[aria-label='Delete Doc']") + .should('exist'); + }); + + it('can see upload button', () => { + cy.get("button[aria-label='Add Docs']").should('exist'); + }); + }); +}); diff --git a/cypress/support/e2e.ts b/cypress/support/e2e.ts index 1eedc98dfe..9847887333 100644 --- a/cypress/support/e2e.ts +++ b/cypress/support/e2e.ts @@ -1,4 +1,6 @@ /// +// eslint-disable-next-line @typescript-eslint/triple-slash-reference +/// export const adminUser = { name: 'Admin User', @@ -10,6 +12,9 @@ const login = (email: string, password: string) => { return cy.session( email, () => { + // Make sure to test against us english to have stable tests, + // regardless on local language preferences + localStorage.setItem('locale', 'en-US'); // Visit auth page cy.visit('/auth'); // Fill out the form @@ -68,6 +73,50 @@ Cypress.Commands.add('register', (name, email, password) => register(name, email Cypress.Commands.add('registerAdmin', () => registerAdmin()); Cypress.Commands.add('loginAdmin', () => loginAdmin()); +Cypress.Commands.add('uploadTestDocument', (suffix: any) => { + // Login as admin + cy.loginAdmin(); + // upload example document + cy.visit('/workspace/documents'); + // Create a document + cy.get("button[aria-label='Add Docs']").click(); + cy.readFile('cypress/data/example-doc.txt').then((text) => { + // select file + cy.get('#upload-doc-input').selectFile( + { + contents: Cypress.Buffer.from(text + Date.now()), + fileName: `document-test-initial-${suffix}.txt`, + mimeType: 'text/plain', + lastModified: Date.now() + }, + { + force: true + } + ); + // open tag input + cy.get("button[aria-label='Add Tag']").click(); + cy.get("input[placeholder='Add a tag']").type('cypress-test'); + cy.get("button[aria-label='Save Tag']").click(); + + // submit to upload + cy.get("button[type='submit']").click(); + + // wait for upload to finish + cy.get('button').contains('#cypress-test').should('exist'); + cy.get('div').contains(`document-test-initial-${suffix}.txt`).should('exist'); + }); +}); + +Cypress.Commands.add('deleteTestDocument', (suffix: any) => { + cy.loginAdmin(); + cy.visit('/workspace/documents'); + // clean up uploaded documents + cy.get('div') + .contains(`document-test-initial-${suffix}.txt`) + .find("button[aria-label='Delete Doc']") + .click(); +}); + before(() => { cy.registerAdmin(); }); diff --git a/cypress/support/index.d.ts b/cypress/support/index.d.ts index e6c69121a9..647db92115 100644 --- a/cypress/support/index.d.ts +++ b/cypress/support/index.d.ts @@ -7,5 +7,7 @@ declare namespace Cypress { register(name: string, email: string, password: string): Chainable; registerAdmin(): Chainable; loginAdmin(): Chainable; + uploadTestDocument(suffix: any): Chainable; + deleteTestDocument(suffix: any): Chainable; } } diff --git a/src/lib/components/common/Tags/TagInput.svelte b/src/lib/components/common/Tags/TagInput.svelte index 549a46cae9..dbda5a175c 100644 --- a/src/lib/components/common/Tags/TagInput.svelte +++ b/src/lib/components/common/Tags/TagInput.svelte @@ -42,7 +42,7 @@ {/each} - + {#if $config?.features.enable_username_password_login} +
+ - {#if $config?.features.enable_signup} -
- {mode === 'signin' - ? $i18n.t("Don't have an account?") - : $i18n.t('Already have an account?')} + {#if $config?.features.enable_signup} +
+ {mode === 'signin' + ? $i18n.t("Don't have an account?") + : $i18n.t('Already have an account?')} - -
- {/if} -
+ +
+ {/if} + + {/if} {#if Object.keys($config?.oauth?.providers ?? {}).length > 0}
- {$i18n.t('or')} + {#if $config?.features.enable_username_password_login} + {$i18n.t('or')} + {/if}
{#if $config?.oauth?.providers?.google} From a8d2072e9fd2fb3319992a1c857e304e88d84709 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Wed, 24 Jul 2024 11:19:42 +0100 Subject: [PATCH 051/212] refac --- src/lib/components/chat/ModelSelector/Selector.svelte | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/components/chat/ModelSelector/Selector.svelte b/src/lib/components/chat/ModelSelector/Selector.svelte index 0601a64b69..d517db6fdb 100644 --- a/src/lib/components/chat/ModelSelector/Selector.svelte +++ b/src/lib/components/chat/ModelSelector/Selector.svelte @@ -282,8 +282,8 @@
Model ImageURl {item.label}
From edff071cd2c8a7a7aeb698e3a985de776bb4a9e1 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Wed, 24 Jul 2024 11:25:07 +0100 Subject: [PATCH 052/212] refac --- backend/apps/webui/models/chats.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/backend/apps/webui/models/chats.py b/backend/apps/webui/models/chats.py index 6419ac8ee5..abde4f2b31 100644 --- a/backend/apps/webui/models/chats.py +++ b/backend/apps/webui/models/chats.py @@ -265,19 +265,17 @@ def get_chat_title_id_list_by_user_id( ).all() ) # result has to be destrctured from sqlalchemy `row` and mapped to a dict since the `ChatModel`is not the returned dataclass. - return list( - map( - lambda row: ChatTitleIdResponse.model_validate( - { - "id": row[0], - "title": row[1], - "updated_at": row[2], - "created_at": row[3], - } - ), - all_chats, + return [ + ChatTitleIdResponse.model_validate( + { + "id": chat[0], + "title": chat[1], + "updated_at": chat[2], + "created_at": chat[3], + } ) - ) + for chat in all_chats + ] def get_chat_list_by_chat_ids( self, chat_ids: List[str], skip: int = 0, limit: int = 50 From 23e69bcdb4de7cae2695b10ce2685bfc0da15138 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Wed, 24 Jul 2024 11:29:57 +0100 Subject: [PATCH 053/212] enh: AsyncGenerator support --- backend/apps/webui/main.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/backend/apps/webui/main.py b/backend/apps/webui/main.py index 570cad9f19..997a05974b 100644 --- a/backend/apps/webui/main.py +++ b/backend/apps/webui/main.py @@ -54,7 +54,7 @@ import time import json -from typing import Iterator, Generator, Optional +from typing import Iterator, Generator, AsyncGenerator, Optional from pydantic import BaseModel app = FastAPI() @@ -411,6 +411,25 @@ async def stream_content(): yield f"data: {json.dumps(finish_message)}\n\n" yield f"data: [DONE]" + if isinstance(res, AsyncGenerator): + async for line in res: + if isinstance(line, BaseModel): + line = line.model_dump_json() + line = f"data: {line}" + if isinstance(line, dict): + line = f"data: {json.dumps(line)}" + + try: + line = line.decode("utf-8") + except: + pass + + if line.startswith("data:"): + yield f"{line}\n\n" + else: + line = stream_message_template(form_data["model"], line) + yield f"data: {json.dumps(line)}\n\n" + return StreamingResponse(stream_content(), media_type="text/event-stream") else: @@ -434,9 +453,12 @@ async def stream_content(): message = "" if isinstance(res, str): message = res - if isinstance(res, Generator): + elif isinstance(res, Generator): for stream in res: message = f"{message}{stream}" + elif isinstance(res, AsyncGenerator): + async for stream in res: + message = f"{message}{stream}" return { "id": f"{form_data['model']}-{str(uuid.uuid4())}", From 413a9031eb49d523ecbde5d93892cf59d50bcc22 Mon Sep 17 00:00:00 2001 From: "Timothy J. Baek" Date: Wed, 24 Jul 2024 11:36:41 +0100 Subject: [PATCH 054/212] chore: format --- src/lib/i18n/locales/th-TH/translation.json | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/lib/i18n/locales/th-TH/translation.json b/src/lib/i18n/locales/th-TH/translation.json index 701abca38a..572ede11ec 100644 --- a/src/lib/i18n/locales/th-TH/translation.json +++ b/src/lib/i18n/locales/th-TH/translation.json @@ -27,6 +27,7 @@ "Add Memory": "เพิ่มความจำ", "Add message": "เพิ่มข้อความ", "Add Model": "เพิ่มโมเดล", + "Add Tag": "", "Add Tags": "เพิ่มแท็ก", "Add User": "เพิ่มผู้ใช้", "Adjusting these settings will apply changes universally to all users.": "การปรับการตั้งค่าเหล่านี้จะนำไปใช้กับผู้ใช้ทั้งหมด", @@ -168,6 +169,7 @@ "Delete chat": "ลบแชท", "Delete Chat": "ลบแชท", "Delete chat?": "ลบแชท?", + "Delete Doc": "", "Delete function?": "ลบฟังก์ชัน?", "Delete prompt?": "ลบพรอมต์?", "delete this link": "ลบลิงก์นี้", @@ -211,6 +213,7 @@ "Edit Doc": "แก้ไขเอกสาร", "Edit Memory": "แก้ไขความจำ", "Edit User": "แก้ไขผู้ใช้", + "ElevenLabs": "", "Email": "อีเมล", "Embedding Batch Size": "ขนาดชุดการฝัง", "Embedding Model": "โมเดลการฝัง", @@ -355,7 +358,7 @@ "Local Models": "โมเดลท้องถิ่น", "LTR": "LTR", "Made by OpenWebUI Community": "สร้างโดยชุมชน OpenWebUI", - "Make sure you have the correct URL and try again.": "ตรวจสอบให้แน่ใจว่าคุณมี URL ที่ถูกต้องและลองอีกครั้ง", + "Make sure to enclose them with": "", "Manage": "จัดการ", "Manage Models": "จัดการโมเดล", "Manage Ollama Models": "จัดการโมเดล Ollama", @@ -385,7 +388,7 @@ "Model {{modelName}} is not vision capable": "โมเดล {{modelName}} ไม่มีคุณสมบัติวิสชั่น", "Model {{name}} is now {{status}}": "โมเดล {{name}} ขณะนี้ {{status}}", "Model created successfully!": "สร้างโมเดลสำเร็จ!", - "Model filesystem path detected. Model shortname is required for update, cannot continue.": "ตรวจพบเส้นทางระบบไฟล์ของโมเดล ต้องการชื่อย่อของโมเดลสำหรับการอัปเดต ไม่สามารถดำเนินการต่อได้", + "Model filesystem path detected. Model shortname is required for update, cannot continue.": "ตรวจพบเส้นทางระบบไฟล์ของโมเดล ต้องการชื่อย่อของโมเดลสำหรับการอัปเดต ไม่สามารถดำเนินการต่อได้", "Model ID": "รหัสโมเดล", "Model not selected": "ยังไม่ได้เลือกโมเดล", "Model Params": "พารามิเตอร์ของโมเดล", @@ -465,9 +468,9 @@ "Prompt": "พรอมต์", "Prompt (e.g. Tell me a fun fact about the Roman Empire)": "พรอมต์ (เช่น บอกข้อเท็จจริงที่น่าสนุกเกี่ยวกับจักรวรรดิโรมัน)", "Prompt Content": "เนื้อหาพรอมต์", - "Prompt Suggestions": "คำแนะนำพรอมต์", + "Prompt suggestions": "", "Prompts": "พรอมต์", - "Prompts imported successfully": "นำเข้าพรอมต์สำเร็จ", + "Pull \"{{searchValue}}\" from Ollama.com": "", "Pull a model from Ollama.com": "", "Query Params": "พารามิเตอร์การค้นหา", "RAG Template": "แม่แบบ RAG", @@ -500,6 +503,7 @@ "Save": "บันทึก", "Save & Create": "บันทึกและสร้าง", "Save & Update": "บันทึกและอัปเดต", + "Save Tag": "", "Saving chat logs directly to your browser's storage is no longer supported. Please take a moment to download and delete your chat logs by clicking the button below. Don't worry, you can easily re-import your chat logs to the backend through": "การบันทึกบันทึกการสนทนาโดยตรงไปยังที่จัดเก็บในเบราว์เซอร์ของคุณไม่ได้รับการสนับสนุนอีกต่อไป โปรดสละเวลาสักครู่เพื่อดาวน์โหลดและลบบันทึกการสนทนาของคุณโดยคลิกที่ปุ่มด้านล่าง ไม่ต้องกังวล คุณสามารถนำเข้าบันทึกการสนทนาของคุณกลับไปยังส่วนแบ็กเอนด์ได้อย่างง่ายดายผ่าน", "Scan": "สแกน", "Scan complete!": "การสแกนเสร็จสมบูรณ์!", From dbf88a2ecacd8a207a41a13284c9df7d9ac4f25c Mon Sep 17 00:00:00 2001 From: Aryan Kothari <87589047+thearyadev@users.noreply.github.com> Date: Wed, 24 Jul 2024 09:01:01 -0400 Subject: [PATCH 055/212] refactor: rename `pseudoSelectedIndex` to `selectedModelIdx` --- .../chat/ModelSelector/Selector.svelte | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/lib/components/chat/ModelSelector/Selector.svelte b/src/lib/components/chat/ModelSelector/Selector.svelte index b055cd4eb2..868a514046 100644 --- a/src/lib/components/chat/ModelSelector/Selector.svelte +++ b/src/lib/components/chat/ModelSelector/Selector.svelte @@ -44,7 +44,7 @@ let searchValue = ''; let ollamaVersion = null; - let pseudoSelectedIndex = 0; + let selectedModelIdx = 0; $: filteredItems = items.filter( (item) => @@ -205,7 +205,7 @@ bind:open={show} onOpenChange={async () => { searchValue = ''; - pseudoSelectedIndex = 0; // when the dropdown is closed, reset the selected index + selectedModelIdx = 0; window.setTimeout(() => document.getElementById('model-search-input')?.focus(), 0); }} closeFocus={false} @@ -244,19 +244,19 @@ autocomplete="off" on:keydown={(e) => { if (e.code === 'Enter') { - value = filteredItems[pseudoSelectedIndex].value; + value = filteredItems[selectedModelIdx].value; show = false; return; // dont need to scroll on selection } else if (e.code === 'ArrowDown') { - pseudoSelectedIndex = Math.min(pseudoSelectedIndex + 1, filteredItems.length - 1); + selectedModelIdx = Math.min(selectedModelIdx + 1, filteredItems.length - 1); } else if (e.code === 'ArrowUp') { - pseudoSelectedIndex = Math.max(pseudoSelectedIndex - 1, 0); + selectedModelIdx = Math.max(selectedModelIdx - 1, 0); } else { // if the user types something, reset to the top selection. - pseudoSelectedIndex = 0; + selectedModelIdx = 0; } - const item = document.querySelector(`[data-pseudo-selected="true"]`); + const item = document.querySelector(`[data-arrow-selected="true"]`); item?.scrollIntoView({ block: 'center', inline: 'nearest', behavior: 'instant' }); }} /> @@ -270,13 +270,13 @@
- {#if $config?.features.enable_username_password_login} + {#if $config?.features.enable_login_form}
{#if mode === 'signup'}
@@ -218,7 +218,7 @@
{/if} - {#if $config?.features.enable_username_password_login} + {#if $config?.features.enable_login_form}
-
+
{#if chatFiles.length > 0} -
-
{$i18n.t('Files')}
- -
+ +
{#each chatFiles as file, fileIdx} {/each}
-
+
{/if} @@ -67,27 +66,25 @@
{/if} -
-
{$i18n.t('System Prompt')}
- -
+ +