diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index 4249429a364c8..11c1e3236b708 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -32,6 +32,10 @@ type Annotation = { type ErrorDetails = { message: string; location?: Location; + actual?: any; + expected?: any; + log?: string[]; + shortMessage?: string; }; type TestSummary = { @@ -471,9 +475,11 @@ export function formatError(error: TestError, highlightCode: boolean): ErrorDeta if (parsedStack && !location) location = parsedStack.location; + const { actual, expected, log, shortMessage } = error; return { location, message: tokens.join('\n'), + actual, expected, log, shortMessage }; } diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 6ab3f7d77f16f..e927df52b6291 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -287,6 +287,10 @@ export interface JSONReportTest { export interface JSONReportError { message: string; location?: Location; + actual?: any; + expected?: any; + log?: Array; + shortMessage?: string; } export interface JSONReportTestResult { diff --git a/tests/playwright-test/reporter-errors.spec.ts b/tests/playwright-test/reporter-errors.spec.ts new file mode 100644 index 0000000000000..6169f1a2d5b06 --- /dev/null +++ b/tests/playwright-test/reporter-errors.spec.ts @@ -0,0 +1,143 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; + +test('should report matcherResults for generic matchers', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect as baseExpect } from '@playwright/test'; + const expect = baseExpect.soft; + test('fail', ({}) => { + expect(1).toBe(2); + expect(1).toBeCloseTo(2); + expect(undefined).toBeDefined(); + expect(1).toBeFalsy(); + expect(1).toBeGreaterThan(2); + expect(1).toBeGreaterThanOrEqual(2); + expect('a').toBeInstanceOf(Number); + expect(2).toBeLessThan(1); + expect(2).toBeLessThanOrEqual(1); + expect(1).toBeNaN(); + expect(1).toBeNull(); + expect(0).toBeTruthy(); + expect(1).toBeUndefined(); + expect([1]).toContain(2); + expect([1]).toContainEqual(2); + expect([1]).toEqual([2]); + expect([1]).toHaveLength(2); + expect({ a: 1 }).toHaveProperty('b'); + expect('a').toMatch(/b/); + expect({ a: 1 }).toMatchObject({ b: 2 }); + expect({ a: 1 }).toStrictEqual({ b: 2 }); + expect(() => {}).toThrow(); + expect(() => {}).toThrowError('a'); + }); + ` + }, { }); + expect(result.exitCode).toBe(1); + + const { errors } = result.report.suites[0].specs[0].tests[0].results[0]; + expect( errors).toEqual([ + { expected: 2, actual: 1 }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { }, + { expected: [2], actual: [1] }, + { }, + { }, + { }, + { }, + { expected: { b: 2 }, actual: { a: 1 } }, + { }, + { }, + ].map(e => expect.objectContaining(e))); +}); + +test('should report matcherResults for web matchers', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + import { test, expect as baseExpect } from '@playwright/test'; + + const expect = baseExpect.configure({ soft: true, timeout: 1 }); + test('fail', async ({ page }) => { + await page.setContent('Hello
World
'); + await expect(page.locator('input')).toBeChecked(); + await expect(page.locator('input')).toBeDisabled(); + await expect(page.locator('textarea')).not.toBeEditable(); + await expect(page.locator('span')).toBeEmpty(); + await expect(page.locator('button')).not.toBeEnabled(); + await expect(page.locator('button')).toBeFocused(); + await expect(page.locator('span')).toBeHidden(); + await expect(page.locator('div')).not.toBeInViewport(); + await expect(page.locator('div')).not.toBeVisible(); + await expect(page.locator('span')).toContainText('World'); + await expect(page.locator('span')).toHaveAccessibleDescription('World'); + await expect(page.locator('span')).toHaveAccessibleName('World'); + await expect(page.locator('span')).toHaveAttribute('name', 'value'); + await expect(page.locator('span')).toHaveAttribute('name'); + await expect(page.locator('span')).toHaveClass('name'); + await expect(page.locator('span')).toHaveCount(2); + await expect(page.locator('span')).toHaveCSS('width', '10'); + await expect(page.locator('span')).toHaveId('id'); + await expect(page.locator('span')).toHaveJSProperty('name', 'value'); + await expect(page.locator('span')).toHaveRole('role'); + await expect(page.locator('span')).toHaveText('World'); + await expect(page.locator('textarea')).toHaveValue('value'); + await expect(page.locator('select')).toHaveValues(['value']); + }); + ` + }, { }); + expect(result.exitCode).toBe(1); + + const { errors } = result.report.suites[0].specs[0].tests[0].results[0]; + expect(errors).toEqual([ + { expected: 'checked', actual: 'unchecked' }, + { expected: 'disabled', actual: 'enabled' }, + { expected: 'editable', actual: 'editable' }, + { expected: 'empty', actual: 'notEmpty' }, + { expected: 'enabled', actual: 'enabled' }, + { expected: 'focused', actual: 'inactive' }, + { expected: 'hidden', actual: 'visible' }, + { expected: 'in viewport', actual: 'in viewport' }, + { expected: 'visible', actual: 'visible' }, + { expected: 'World', actual: 'Hello' }, + { expected: 'World', actual: '' }, + { expected: 'World', actual: '' }, + { expected: 'value', actual: null }, + { expected: 'have attribute', actual: 'not have attribute' }, + { expected: 'name', actual: '' }, + { expected: 2, actual: 1 }, + { expected: '10', actual: 'auto' }, + { expected: 'id', actual: '' }, + { expected: 'value' }, + { expected: 'role', actual: '' }, + { expected: 'World', actual: 'Hello' }, + { expected: 'value', actual: '' }, + { expected: ['value'], actual: [] }, + ].map(e => expect.objectContaining(e))); +}); diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index b2e8f0e9e66ad..3fb7ec1acae1a 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -105,6 +105,11 @@ export interface JSONReportTest { export interface JSONReportError { message: string; location?: Location; + + actual?: any; + expected?: any; + log?: Array; + shortMessage?: string; } export interface JSONReportTestResult {