diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml new file mode 100644 index 000000000..dd7494f98 --- /dev/null +++ b/.github/workflows/playwright.yml @@ -0,0 +1,32 @@ +name: Playwright Tests +on: + push: + branches: [main, master] + pull_request: + branches: [main, master] +jobs: + test: + timeout-minutes: 60 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: lts/* + - name: Install dependencies + run: npm install -g pnpm && pnpm install + - name: Install Playwright Browsers + run: pnpm exec playwright install --with-deps + - name: Run Playwright tests + env: + AUTH_SECRET: ${{ secrets.AUTH_SECRET }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + POSTGRES_URL: ${{ secrets.POSTGRES_URL }} + TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }} + run: pnpm exec playwright test + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: playwright-report + path: playwright-report/ + retention-days: 30 diff --git a/.gitignore b/.gitignore index dd019e403..c28770353 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,7 @@ yarn-error.log* .vercel .vscode .env*.local +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ diff --git a/components/block-messages.tsx b/components/block-messages.tsx index 1e8687928..099acd58a 100644 --- a/components/block-messages.tsx +++ b/components/block-messages.tsx @@ -52,6 +52,7 @@ function PureBlockMessages({ setMessages={setMessages} reload={reload} isReadonly={isReadonly} + index={index} /> ))} diff --git a/components/message-editor.tsx b/components/message-editor.tsx index 35fbcf9b4..9a53af066 100644 --- a/components/message-editor.tsx +++ b/components/message-editor.tsx @@ -56,6 +56,7 @@ export function MessageEditor({ className="bg-transparent outline-none overflow-hidden resize-none !text-base rounded-xl w-full" value={draftContent} onChange={handleInput} + data-testid="message-editor" />
@@ -104,6 +105,7 @@ export function MessageEditor({ setMode('view'); reload(); }} + data-testid="message-editor-send-button" > {isSubmitting ? 'Sending...' : 'Send'} diff --git a/components/message.tsx b/components/message.tsx index daea57fdc..9fdee970d 100644 --- a/components/message.tsx +++ b/components/message.tsx @@ -28,6 +28,7 @@ const PurePreviewMessage = ({ setMessages, reload, isReadonly, + index, }: { chatId: string; message: Message; @@ -40,6 +41,7 @@ const PurePreviewMessage = ({ chatRequestOptions?: ChatRequestOptions, ) => Promise; isReadonly: boolean; + index: number; }) => { const [mode, setMode] = useState<'view' | 'edit'>('view'); @@ -50,6 +52,7 @@ const PurePreviewMessage = ({ initial={{ y: 5, opacity: 0 }} animate={{ y: 0, opacity: 1 }} data-role={message.role} + data-testid={`message-${message.role}-${index}`} >
{message.experimental_attachments && ( -
+
{message.experimental_attachments.map((attachment) => ( diff --git a/components/messages.tsx b/components/messages.tsx index 4ba90b253..cddc0e194 100644 --- a/components/messages.tsx +++ b/components/messages.tsx @@ -36,7 +36,7 @@ function PureMessages({ return (
{/* {messages.length === 0 && } */} @@ -54,6 +54,7 @@ function PureMessages({ setMessages={setMessages} reload={reload} isReadonly={isReadonly} + index={index} /> ))} diff --git a/components/multimodal-input.tsx b/components/multimodal-input.tsx index 191470534..c47217a59 100644 --- a/components/multimodal-input.tsx +++ b/components/multimodal-input.tsx @@ -208,10 +208,14 @@ function PureMultimodalInput({ multiple onChange={handleFileChange} tabIndex={-1} + data-testid="attachments-input" /> {(attachments.length > 0 || uploadQueue.length > 0) && ( -
+
{attachments.map((attachment) => ( ))} @@ -252,6 +256,7 @@ function PureMultimodalInput({ } } }} + data-testid="multimodal-input" />
@@ -300,6 +305,7 @@ function PureAttachmentsButton({ }} disabled={isLoading} variant="ghost" + data-testid="attachments-button" > @@ -323,6 +329,7 @@ function PureStopButton({ stop(); setMessages((messages) => sanitizeUIMessages(messages)); }} + data-testid="stop-button" > @@ -348,6 +355,7 @@ function PureSendButton({ submitForm(); }} disabled={input.length === 0 || uploadQueue.length > 0} + data-testid="send-button" > diff --git a/components/preview-attachment.tsx b/components/preview-attachment.tsx index 327edc39f..2b2e7ef2a 100644 --- a/components/preview-attachment.tsx +++ b/components/preview-attachment.tsx @@ -12,7 +12,7 @@ export const PreviewAttachment = ({ const { name, url, contentType } = attachment; return ( -
+
{contentType ? ( contentType.startsWith('image') ? ( @@ -32,7 +32,10 @@ export const PreviewAttachment = ({ )} {isUploading && ( -
+
)} diff --git a/components/suggested-actions.tsx b/components/suggested-actions.tsx index b2d14d63d..a8921c8bb 100644 --- a/components/suggested-actions.tsx +++ b/components/suggested-actions.tsx @@ -38,7 +38,10 @@ function PureSuggestedActions({ chatId, append }: SuggestedActionsProps) { ]; return ( -
+
{suggestedActions.map((suggestedAction, index) => ( =14'} + '@playwright/test@1.49.1': + resolution: {integrity: sha512-Ky+BVzPz8pL6PQxHqNRW1k3mIyv933LML7HktS8uik0bUXNCdPhoS/kLihiO1tMf/egaJb4IutXd7UywvXEW+g==} + engines: {node: '>=18'} + hasBin: true + '@radix-ui/number@1.1.0': resolution: {integrity: sha512-V3gRzhVNU1ldS5XhAPTom1fOIo4ccrjjJgmE+LI2h/WaFpHmx0MQApT+KZHnx8abG6Avtfcz4WoEciMnpFT3HQ==} @@ -2309,6 +2317,11 @@ packages: fs.realpath@1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -3063,6 +3076,16 @@ packages: resolution: {integrity: sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==} engines: {node: '>= 6'} + playwright-core@1.49.1: + resolution: {integrity: sha512-BzmpVcs4kE2CH15rWfzpjzVGhWERJfmnXmniSyKeRZUs9Ws65m+RGIi7mjJK/euCegfn3i7jvqWeWyHe9y3Vgg==} + engines: {node: '>=18'} + hasBin: true + + playwright@1.49.1: + resolution: {integrity: sha512-VYL8zLoNTBxVOrJBbDuRgDWa3i+mfQgDTrL8Ah9QXZ7ax4Dsj0MSq5bYgytRnDVVe+njoKnfsYkH3HzqVj5UZA==} + engines: {node: '>=18'} + hasBin: true + possible-typed-array-names@1.0.0: resolution: {integrity: sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==} engines: {node: '>= 0.4'} @@ -4352,6 +4375,10 @@ snapshots: '@pkgjs/parseargs@0.11.0': optional: true + '@playwright/test@1.49.1': + dependencies: + playwright: 1.49.1 + '@radix-ui/number@1.1.0': {} '@radix-ui/primitive@1.1.0': {} @@ -4842,11 +4869,11 @@ snapshots: '@ungap/structured-clone@1.2.0': {} - '@vercel/analytics@1.3.2(next@15.0.3-canary.2(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021)': + '@vercel/analytics@1.3.2(next@15.0.3-canary.2(@opentelemetry/api@1.9.0)(@playwright/test@1.49.1)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021)': dependencies: server-only: 0.0.1 optionalDependencies: - next: 15.0.3-canary.2(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021) + next: 15.0.3-canary.2(@opentelemetry/api@1.9.0)(@playwright/test@1.49.1)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021) react: 19.0.0-rc-45804af1-20241021 '@vercel/blob@0.24.1': @@ -5699,6 +5726,9 @@ snapshots: fs.realpath@1.0.0: {} + fsevents@2.3.2: + optional: true + fsevents@2.3.3: optional: true @@ -5713,9 +5743,9 @@ snapshots: functions-have-names@1.2.3: {} - geist@1.3.1(next@15.0.3-canary.2(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021)): + geist@1.3.1(next@15.0.3-canary.2(@opentelemetry/api@1.9.0)(@playwright/test@1.49.1)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021)): dependencies: - next: 15.0.3-canary.2(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021) + next: 15.0.3-canary.2(@opentelemetry/api@1.9.0)(@playwright/test@1.49.1)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021) get-intrinsic@1.2.4: dependencies: @@ -6494,10 +6524,10 @@ snapshots: natural-compare@1.4.0: {} - next-auth@5.0.0-beta.25(next@15.0.3-canary.2(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021): + next-auth@5.0.0-beta.25(next@15.0.3-canary.2(@opentelemetry/api@1.9.0)(@playwright/test@1.49.1)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021): dependencies: '@auth/core': 0.37.2 - next: 15.0.3-canary.2(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021) + next: 15.0.3-canary.2(@opentelemetry/api@1.9.0)(@playwright/test@1.49.1)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021) react: 19.0.0-rc-45804af1-20241021 next-themes@0.3.0(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021): @@ -6505,7 +6535,7 @@ snapshots: react: 19.0.0-rc-45804af1-20241021 react-dom: 19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021) - next@15.0.3-canary.2(@opentelemetry/api@1.9.0)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021): + next@15.0.3-canary.2(@opentelemetry/api@1.9.0)(@playwright/test@1.49.1)(react-dom@19.0.0-rc-45804af1-20241021(react@19.0.0-rc-45804af1-20241021))(react@19.0.0-rc-45804af1-20241021): dependencies: '@next/env': 15.0.3-canary.2 '@swc/counter': 0.1.3 @@ -6526,6 +6556,7 @@ snapshots: '@next/swc-win32-arm64-msvc': 15.0.3-canary.2 '@next/swc-win32-x64-msvc': 15.0.3-canary.2 '@opentelemetry/api': 1.9.0 + '@playwright/test': 1.49.1 sharp: 0.33.5 transitivePeerDependencies: - '@babel/core' @@ -6658,6 +6689,14 @@ snapshots: pirates@4.0.6: {} + playwright-core@1.49.1: {} + + playwright@1.49.1: + dependencies: + playwright-core: 1.49.1 + optionalDependencies: + fsevents: 2.3.2 + possible-typed-array-names@1.0.0: {} postcss-import@15.1.0(postcss@8.4.47): diff --git a/public/images/mouth of the seine, monet.jpg b/public/images/mouth of the seine, monet.jpg new file mode 100644 index 000000000..62515e565 Binary files /dev/null and b/public/images/mouth of the seine, monet.jpg differ diff --git a/tests/authentication.test.ts b/tests/authentication.test.ts new file mode 100644 index 000000000..7e19dab97 --- /dev/null +++ b/tests/authentication.test.ts @@ -0,0 +1,50 @@ +import { test, expect } from '@playwright/test'; +import { getUnixTime } from 'date-fns'; + +const testEmail = `auth-${getUnixTime(new Date())}@playwright.com`; +const testPassword = process.env.TEST_USER_PASSWORD ?? ''; + +test.describe('authentication', () => { + test('redirect to login page when unauthenticated', async ({ page }) => { + await page.goto('/'); + await expect(page.getByRole('heading')).toContainText('Sign In'); + }); + + test('register a test account', async ({ page }) => { + await page.goto('/register'); + await expect(page.getByRole('heading')).toContainText('Sign Up'); + await page.getByPlaceholder('user@acme.com').click(); + await page.getByPlaceholder('user@acme.com').fill(testEmail); + await page.getByLabel('Password').click(); + await page.getByLabel('Password').fill(testPassword); + await page.getByRole('button', { name: 'Sign Up' }).click(); + await expect(page).toHaveURL('/'); + await expect(page.getByRole('status')).toContainText( + 'Account created successfully', + ); + }); + + // test("register test account with existing email", async ({ page }) => { + // await page.goto("/register"); + // await expect(page.getByRole("heading")).toContainText("Sign Up"); + // await page.getByPlaceholder("user@acme.com").click(); + // await page.getByPlaceholder("user@acme.com").fill(testEmail); + // await page.getByLabel("Password").click(); + // await page.getByLabel("Password").fill(testPassword); + // await page.getByRole("button", { name: "Sign Up" }).click(); + // await expect(page.locator("li")).toContainText("Account already exists"); + // }); + + // test("log into account", async ({ page }) => { + // await page.goto("/login"); + // await expect(page.getByRole("heading")).toContainText("Sign In"); + // await page.getByPlaceholder("user@acme.com").click(); + // await page.getByPlaceholder("user@acme.com").fill(testEmail); + // await page.getByLabel("Password").click(); + // await page.getByLabel("Password").fill(testPassword); + // await page.getByRole("button", { name: "Sign in" }).click(); + // await page.waitForURL("/"); + // await expect(page).toHaveURL("/"); + // await expect(page.getByPlaceholder("Send a message...")).toBeVisible(); + // }); +}); diff --git a/tests/chat.test.ts b/tests/chat.test.ts new file mode 100644 index 000000000..2fb8591cc --- /dev/null +++ b/tests/chat.test.ts @@ -0,0 +1,200 @@ +import path from 'path'; +import fs from 'fs'; +import { test as baseTest, expect, Page } from '@playwright/test'; + +type Fixtures = { + authenticatedPage: Page; +}; + +const testEmail = `chat@playwright.com`; +const testPassword = process.env.TEST_USER_PASSWORD ?? ''; + +const test = baseTest.extend({ + authenticatedPage: async ({ page }, use) => { + await page.goto('/login'); + await expect(page.getByRole('heading')).toContainText('Sign In'); + await page.getByPlaceholder('user@acme.com').click(); + await page.getByPlaceholder('user@acme.com').fill(testEmail); + await page.getByLabel('Password').click(); + await page.getByLabel('Password').fill(testPassword); + await page.getByRole('button', { name: 'Sign In' }).click(); + await expect(page.getByPlaceholder('Send a message...')).toBeVisible(); + await use(page); + }, +}); + +test.describe('chat', () => { + test('submit a user message and receive response', async ({ + authenticatedPage, + }) => { + await authenticatedPage.getByPlaceholder('Send a message...').click(); + await authenticatedPage + .getByPlaceholder('Send a message...') + .fill('this is a test message, respond with "test"'); + await authenticatedPage.keyboard.press('Enter'); + await expect(authenticatedPage.getByRole('main')).toContainText('test'); + }); + + test('redirect to /chat/:id after submitting message', async ({ + authenticatedPage, + }) => { + await authenticatedPage.getByPlaceholder('Send a message...').click(); + await authenticatedPage + .getByPlaceholder('Send a message...') + .fill('this is a test message, respond with "test"'); + await authenticatedPage.keyboard.press('Enter'); + await expect(authenticatedPage.getByRole('main')).toContainText('test'); + await expect(authenticatedPage).toHaveURL( + /^http:\/\/localhost:3000\/chat\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/, + ); + }); + + test('submit message through suggested actions', async ({ + authenticatedPage, + }) => { + await authenticatedPage + .getByRole('button', { name: 'What are the advantages of' }) + .click(); + await expect( + authenticatedPage.getByText('What are the advantages of'), + ).toBeVisible(); + }); + + test('toggle between send/stop button based on activity', async ({ + authenticatedPage, + }) => { + await expect(authenticatedPage.getByTestId('send-button')).toBeDisabled(); + await authenticatedPage.getByPlaceholder('Send a message...').click(); + await authenticatedPage + .getByPlaceholder('Send a message...') + .fill('this is a test message, respond with "test"'); + await expect( + authenticatedPage.getByTestId('send-button'), + ).not.toBeDisabled(); + await authenticatedPage.keyboard.press('Enter'); + + await expect(authenticatedPage.getByTestId('stop-button')).toBeVisible(); + await expect(authenticatedPage.getByRole('main')).toContainText('test'); + await expect(authenticatedPage.getByTestId('send-button')).toBeVisible(); + }); + + test('stop generation after submission', async ({ authenticatedPage }) => { + await authenticatedPage.getByPlaceholder('Send a message...').click(); + await authenticatedPage + .getByPlaceholder('Send a message...') + .fill('this is a test message, respond with "test"'); + await authenticatedPage.keyboard.press('Enter'); + await expect(authenticatedPage.getByRole('main')).toContainText('test'); + await authenticatedPage.getByTestId('stop-button').click(); + await expect(authenticatedPage.getByTestId('send-button')).toBeVisible(); + }); + + test('edit user message and resubmit', async ({ authenticatedPage }) => { + await authenticatedPage.getByTestId('multimodal-input').click(); + await authenticatedPage + .getByTestId('multimodal-input') + .fill('this is a test message, respond with "test"'); + await authenticatedPage.keyboard.press('Enter'); + + await expect(authenticatedPage.getByTestId('message-user-0')).toContainText( + 'this is a test message, respond with "test"', + ); + + await expect( + authenticatedPage.getByTestId('message-assistant-1'), + ).toContainText('test'); + + await authenticatedPage.getByTestId('edit-user-0').click(); + await authenticatedPage + .getByTestId('message-editor') + .fill('this is a test message, respond with "edited test"'); + + await authenticatedPage.getByTestId('message-editor-send-button').click(); + + await expect(authenticatedPage.getByTestId('message-user-0')).toContainText( + 'this is a test message, respond with "edited test"', + ); + + await expect( + authenticatedPage.getByTestId('message-assistant-1'), + ).toContainText('edited test'); + }); + + test('hide suggested actions after sending message', async ({ + authenticatedPage, + }) => { + await expect( + authenticatedPage.getByTestId('suggested-actions'), + ).toBeVisible(); + + await authenticatedPage + .getByRole('button', { name: 'What are the advantages of' }) + .click(); + await expect( + authenticatedPage.getByText('What are the advantages of'), + ).toBeVisible(); + + await authenticatedPage.getByPlaceholder('Send a message...').click(); + await authenticatedPage + .getByPlaceholder('Send a message...') + .fill('this is a test message, respond with "test"'); + await authenticatedPage.keyboard.press('Enter'); + + await expect( + authenticatedPage.getByTestId('suggested-actions'), + ).not.toBeVisible(); + }); + + test('show file selector on clicking attachments button', async ({ + authenticatedPage, + }) => { + const fileChooserPromise = authenticatedPage.waitForEvent('filechooser'); + await authenticatedPage.getByTestId('attachments-button').click(); + await fileChooserPromise; + }); + + test('handle file upload and send image attachment with message', async ({ + authenticatedPage, + }) => { + authenticatedPage.on('filechooser', async (fileChooser) => { + const filePath = path.join( + process.cwd(), + 'public', + 'images', + 'mouth of the seine, monet.jpg', + ); + const imageBuffer = fs.readFileSync(filePath); + + await fileChooser.setFiles({ + name: 'mouth of the seine, monet.jpg', + mimeType: 'image/jpeg', + buffer: imageBuffer, + }); + }); + + await authenticatedPage.getByTestId('attachments-button').click(); + + await expect( + authenticatedPage.getByTestId('attachments-preview'), + ).toBeVisible(); + + await expect( + authenticatedPage.getByTestId('input-attachment-loader'), + ).toBeVisible(); + + await authenticatedPage.getByTestId('multimodal-input').click(); + await authenticatedPage + .getByTestId('multimodal-input') + .fill('this is a test message, respond with "test"'); + + await expect(authenticatedPage.getByTestId('send-button')).toBeVisible(); + await authenticatedPage.getByTestId('send-button').click(); + + await expect( + authenticatedPage.getByTestId('message-attachments-0'), + ).toBeVisible(); + await expect( + authenticatedPage.getByTestId('message-assistant-1'), + ).toBeVisible(); + }); +});