From 16536beed61528366be37adee43225983c2b65f3 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 1 Oct 2024 16:40:12 +0200 Subject: [PATCH 01/17] start --- docs/src/test-api/class-testconfig.md | 8 ++++---- packages/playwright/src/common/config.ts | 10 ++++++++-- packages/playwright/src/runner/tasks.ts | 20 ++++++++++++++------ packages/playwright/types/test.d.ts | 10 ++++++---- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index d013f5e4ea10c..de11e985751e2 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -110,9 +110,9 @@ export default defineConfig({ ## property: TestConfig.globalSetup * since: v1.10 -- type: ?<[string]> +- type: ?<[string]|[Array]<[string]>> -Path to the global setup file. This file will be required and run before all the tests. It must export a single function that takes a [FullConfig] argument. +Path to the global setup file. This file will be required and run before all the tests. It must export a single function that takes a [FullConfig] argument. Pass an array for multiple global setup files. Learn more about [global setup and teardown](../test-global-setup-teardown.md). @@ -128,9 +128,9 @@ export default defineConfig({ ## property: TestConfig.globalTeardown * since: v1.10 -- type: ?<[string]> +- type: ?<[string]|[Array]<[string]>> -Path to the global teardown file. This file will be required and run after all the tests. It must export a single function. See also [`property: TestConfig.globalSetup`]. +Path to the global teardown file. This file will be required and run after all the tests. It must export a single function. See also [`property: TestConfig.globalSetup`]. Pass an array for multiple global teardown files. Learn more about [global setup and teardown](../test-global-setup-teardown.md). diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index d7fb49964581b..759e23f1c3168 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -57,6 +57,9 @@ export class FullConfigInternal { testIdMatcher?: Matcher; defineConfigWasUsed = false; + globalSetups: string[] = []; + globalTeardowns: string[] = []; + constructor(location: ConfigLocation, userConfig: Config, configCLIOverrides: ConfigCLIOverrides) { if (configCLIOverrides.projects && userConfig.projects) throw new Error(`Cannot use --browser option when configuration file defines projects. Specify browserName in the projects instead.`); @@ -70,13 +73,16 @@ export class FullConfigInternal { const privateConfiguration = (userConfig as any)['@playwright/test']; this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p })); + this.globalSetups = (Array.isArray(userConfig.globalSetup) ? userConfig.globalSetup : [userConfig.globalSetup]).map(s => resolveScript(s, configDir)).filter((script): script is string => !!script); + this.globalTeardowns = (Array.isArray(userConfig.globalTeardown) ? userConfig.globalTeardown : [userConfig.globalTeardown]).map(s => resolveScript(s, configDir)).filter((script): script is string => !!script); + this.config = { configFile: resolvedConfigFile, rootDir: pathResolve(configDir, userConfig.testDir) || configDir, forbidOnly: takeFirst(configCLIOverrides.forbidOnly, userConfig.forbidOnly, false), fullyParallel: takeFirst(configCLIOverrides.fullyParallel, userConfig.fullyParallel, false), - globalSetup: takeFirst(resolveScript(userConfig.globalSetup, configDir), null), - globalTeardown: takeFirst(resolveScript(userConfig.globalTeardown, configDir), null), + globalSetup: takeFirst(...this.globalSetups, null), + globalTeardown: takeFirst(...this.globalTeardowns, null), globalTimeout: takeFirst(configCLIOverrides.globalTimeout, userConfig.globalTimeout, 0), grep: takeFirst(userConfig.grep, defaultGrep), grepInvert: takeFirst(userConfig.grepInvert, null), diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index 77d84419f4dec..25e33a7f2b3b5 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -98,8 +98,11 @@ export function createGlobalSetupTasks(config: FullConfigInternal) { if (!config.configCLIOverrides.preserveOutputDir && !process.env.PW_TEST_NO_REMOVE_OUTPUT_DIRS) tasks.push(createRemoveOutputDirsTask()); tasks.push(...createPluginSetupTasks(config)); - if (config.config.globalSetup || config.config.globalTeardown) - tasks.push(createGlobalSetupTask()); + if (config.globalSetups.length || config.globalTeardowns.length) { + const length = Math.max(config.globalSetups.length, config.globalTeardowns.length); + for (let i = 0; i < length; i++) + tasks.push(createGlobalSetupTask(i, length)); + } return tasks; } @@ -161,15 +164,20 @@ function createPluginBeginTask(plugin: TestRunnerPluginRegistration): Task { +function createGlobalSetupTask(index: number, length: number): Task { let globalSetupResult: any; let globalSetupFinished = false; let teardownHook: any; + + let title = 'global setup'; + if (length > 1) + title += ` (${index + 1}/${length})`; + return { - title: 'global setup', + title, setup: async ({ config }) => { - const setupHook = config.config.globalSetup ? await loadGlobalHook(config, config.config.globalSetup) : undefined; - teardownHook = config.config.globalTeardown ? await loadGlobalHook(config, config.config.globalTeardown) : undefined; + const setupHook = config.globalSetups[index] ? await loadGlobalHook(config, config.globalSetups[index]) : undefined; + teardownHook = config.globalTeardowns[index] ? await loadGlobalHook(config, config.globalTeardowns[index]) : undefined; globalSetupResult = setupHook ? await setupHook(config.config) : undefined; globalSetupFinished = true; }, diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 400d7cbcf3eef..e480a90936d6a 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1077,7 +1077,8 @@ interface TestConfig { /** * Path to the global setup file. This file will be required and run before all the tests. It must export a single - * function that takes a [FullConfig](https://playwright.dev/docs/api/class-fullconfig) argument. + * function that takes a [FullConfig](https://playwright.dev/docs/api/class-fullconfig) argument. Pass an array for + * multiple global setup files. * * Learn more about [global setup and teardown](https://playwright.dev/docs/test-global-setup-teardown). * @@ -1093,12 +1094,13 @@ interface TestConfig { * ``` * */ - globalSetup?: string; + globalSetup?: string|Array; /** * Path to the global teardown file. This file will be required and run after all the tests. It must export a single * function. See also - * [testConfig.globalSetup](https://playwright.dev/docs/api/class-testconfig#test-config-global-setup). + * [testConfig.globalSetup](https://playwright.dev/docs/api/class-testconfig#test-config-global-setup). Pass an array + * for multiple global teardown files. * * Learn more about [global setup and teardown](https://playwright.dev/docs/test-global-setup-teardown). * @@ -1114,7 +1116,7 @@ interface TestConfig { * ``` * */ - globalTeardown?: string; + globalTeardown?: string|Array; /** * Maximum time in milliseconds the whole test suite can run. Zero timeout (default) disables this behavior. Useful on From d7b52fbed7dc796dc9bedabe8bbccbe453b9ee15 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 4 Oct 2024 13:01:48 +0200 Subject: [PATCH 02/17] some more --- .../playwright/src/common/configLoader.ts | 16 +++++++- packages/playwright/src/runner/tasks.ts | 16 +++++--- tests/playwright-test/global-setup.spec.ts | 41 +++++++++++++++++++ 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/packages/playwright/src/common/configLoader.ts b/packages/playwright/src/common/configLoader.ts index 5ed1c68ea7c21..1675219b6d0cf 100644 --- a/packages/playwright/src/common/configLoader.ts +++ b/packages/playwright/src/common/configLoader.ts @@ -137,13 +137,25 @@ function validateConfig(file: string, config: Config) { } if ('globalSetup' in config && config.globalSetup !== undefined) { - if (typeof config.globalSetup !== 'string') + if (Array.isArray(config.globalSetup)) { + config.globalSetup.forEach((item, index) => { + if (typeof item !== 'string') + throw errorWithFile(file, `config.globalSetup[${index}] must be a string`); + }); + } else if (typeof config.globalSetup !== 'string') { throw errorWithFile(file, `config.globalSetup must be a string`); + } } if ('globalTeardown' in config && config.globalTeardown !== undefined) { - if (typeof config.globalTeardown !== 'string') + if (Array.isArray(config.globalTeardown)) { + config.globalTeardown.forEach((item, index) => { + if (typeof item !== 'string') + throw errorWithFile(file, `config.globalTeardown[${index}] must be a string`); + }); + } else if (typeof config.globalTeardown !== 'string') { throw errorWithFile(file, `config.globalTeardown must be a string`); + } } if ('globalTimeout' in config && config.globalTimeout !== undefined) { diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index 25e33a7f2b3b5..b0220a6984f83 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -99,9 +99,8 @@ export function createGlobalSetupTasks(config: FullConfigInternal) { tasks.push(createRemoveOutputDirsTask()); tasks.push(...createPluginSetupTasks(config)); if (config.globalSetups.length || config.globalTeardowns.length) { - const length = Math.max(config.globalSetups.length, config.globalTeardowns.length); - for (let i = 0; i < length; i++) - tasks.push(createGlobalSetupTask(i, length)); + for (let i = 0; i < Math.max(config.globalSetups.length, config.globalTeardowns.length); i++) + tasks.push(createGlobalSetupTask(i, config)); } return tasks; } @@ -164,14 +163,19 @@ function createPluginBeginTask(plugin: TestRunnerPluginRegistration): Task { +function createGlobalSetupTask(index: number, config: FullConfigInternal): Task { let globalSetupResult: any; let globalSetupFinished = false; let teardownHook: any; let title = 'global setup'; - if (length > 1) - title += ` (${index + 1}/${length})`; + if (config.globalSetups.length > 1 || config.globalSetups.length > 1) { + const files = [ + config.globalSetups[index], + config.globalTeardowns[index], + ].filter(Boolean).join(', '); + title += ` (${files})`; + } return { title, diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 3d28be82cddc0..001299d857ecf 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -386,3 +386,44 @@ test('teardown after error', async ({ runInlineTest }) => { 'teardown 1', ]); }); + +function expectInOrder(output: string, tokens: string[]) { + for (const token of tokens) + expect(output).toContain(token); + + const positions = tokens.map(token => `${output.indexOf(token).toString().padStart(3, '0')} ${token}`); + expect(positions, 'order is correct').toEqual(positions.toSorted()); +} + +test('globalSetup should support multiple', async ({ runInlineTest }) => { + const { passed, output } = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + globalSetup: ['./globalSetup1.ts','./globalSetup2.ts','./globalSetup3.ts'], + globalTeardown: ['./globalTeardown1.ts', './globalTeardown2.ts'], + }; + `, + 'globalSetup1.ts': `module.exports = () => console.log('globalSetup1');`, + 'globalSetup2.ts': `module.exports = () => console.log('globalSetup2');`, + 'globalSetup3.ts': `module.exports = () => console.log('globalSetup3');`, + 'globalTeardown1.ts': `module.exports = () => console.log('globalTeardown1');`, + 'globalTeardown2.ts': `module.exports = () => console.log('globalTeardown2');`, + + 'a.test.js': ` + import { test } from '@playwright/test'; + test('a', () => console.log('test a')); + test('b', () => console.log('test b')); + `, + }, { reporter: 'line' }); + + expect(passed).toBe(2); + expectInOrder(output, [ + 'globalSetup1', + 'globalSetup2', + 'globalSetup3', + 'test a', + 'test b', + 'globalTeardown2', + 'globalTeardown1', + ]); +}); From b24f625578f4318957f8da7f6c2713fc22cf1141 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 4 Oct 2024 13:12:01 +0200 Subject: [PATCH 03/17] assert on title --- packages/playwright/src/runner/tasks.ts | 16 ++++++--------- tests/playwright-test/global-setup.spec.ts | 23 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index b0220a6984f83..24a6c7048b9d7 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -99,8 +99,9 @@ export function createGlobalSetupTasks(config: FullConfigInternal) { tasks.push(createRemoveOutputDirsTask()); tasks.push(...createPluginSetupTasks(config)); if (config.globalSetups.length || config.globalTeardowns.length) { - for (let i = 0; i < Math.max(config.globalSetups.length, config.globalTeardowns.length); i++) - tasks.push(createGlobalSetupTask(i, config)); + const length = Math.max(config.globalSetups.length, config.globalTeardowns.length); + for (let i = 0; i < length; i++) + tasks.push(createGlobalSetupTask(i, length)); } return tasks; } @@ -163,19 +164,14 @@ function createPluginBeginTask(plugin: TestRunnerPluginRegistration): Task { +function createGlobalSetupTask(index: number, length: number): Task { let globalSetupResult: any; let globalSetupFinished = false; let teardownHook: any; let title = 'global setup'; - if (config.globalSetups.length > 1 || config.globalSetups.length > 1) { - const files = [ - config.globalSetups[index], - config.globalTeardowns[index], - ].filter(Boolean).join(', '); - title += ` (${files})`; - } + if (length) + title += ` #${index}`; return { title, diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 001299d857ecf..12bdff21dda78 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -427,3 +427,26 @@ test('globalSetup should support multiple', async ({ runInlineTest }) => { 'globalTeardown1', ]); }); + +test('debug logs show index of setup/teardown', async ({ runInlineTest }) => { + const { output } = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + globalSetup: ['./globalSetup1.ts','./globalSetup2.ts'], + globalTeardown: ['./globalTeardown1.ts', './globalTeardown2.ts'], + }; + `, + 'globalSetup1.ts': `module.exports = () => console.log('globalSetup1');`, + 'globalSetup2.ts': `module.exports = () => console.log('globalSetup2');`, + 'globalTeardown1.ts': `module.exports = () => console.log('globalTeardown1');`, + 'globalTeardown2.ts': `module.exports = () => console.log('globalTeardown2');`, + + 'a.test.js': ` + import { test } from '@playwright/test'; + test('a', () => console.log('test a')); + `, + }, { reporter: 'line' }, { DEBUG: 'pw:test:task' }); + + expect(output).toContain('pw:test:task "global setup #0" started'); + expect(output).toContain('pw:test:task "teardown for global setup #0" started'); +}); From 4316a0178dcf0f70afc700ec1fe8a4ea620f2e73 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Mon, 7 Oct 2024 07:19:59 +0200 Subject: [PATCH 04/17] remove test --- tests/playwright-test/global-setup.spec.ts | 23 ---------------------- 1 file changed, 23 deletions(-) diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 12bdff21dda78..001299d857ecf 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -427,26 +427,3 @@ test('globalSetup should support multiple', async ({ runInlineTest }) => { 'globalTeardown1', ]); }); - -test('debug logs show index of setup/teardown', async ({ runInlineTest }) => { - const { output } = await runInlineTest({ - 'playwright.config.ts': ` - module.exports = { - globalSetup: ['./globalSetup1.ts','./globalSetup2.ts'], - globalTeardown: ['./globalTeardown1.ts', './globalTeardown2.ts'], - }; - `, - 'globalSetup1.ts': `module.exports = () => console.log('globalSetup1');`, - 'globalSetup2.ts': `module.exports = () => console.log('globalSetup2');`, - 'globalTeardown1.ts': `module.exports = () => console.log('globalTeardown1');`, - 'globalTeardown2.ts': `module.exports = () => console.log('globalTeardown2');`, - - 'a.test.js': ` - import { test } from '@playwright/test'; - test('a', () => console.log('test a')); - `, - }, { reporter: 'line' }, { DEBUG: 'pw:test:task' }); - - expect(output).toContain('pw:test:task "global setup #0" started'); - expect(output).toContain('pw:test:task "teardown for global setup #0" started'); -}); From 38b7dee4423f24645a1e43e88a4aac3f7217834e Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Mon, 7 Oct 2024 07:22:55 +0200 Subject: [PATCH 05/17] use %% notation --- tests/playwright-test/global-setup.spec.ts | 28 ++++++++-------------- 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 001299d857ecf..0ec6c05840977 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -387,37 +387,29 @@ test('teardown after error', async ({ runInlineTest }) => { ]); }); -function expectInOrder(output: string, tokens: string[]) { - for (const token of tokens) - expect(output).toContain(token); - - const positions = tokens.map(token => `${output.indexOf(token).toString().padStart(3, '0')} ${token}`); - expect(positions, 'order is correct').toEqual(positions.toSorted()); -} - test('globalSetup should support multiple', async ({ runInlineTest }) => { - const { passed, output } = await runInlineTest({ + const result = await runInlineTest({ 'playwright.config.ts': ` module.exports = { globalSetup: ['./globalSetup1.ts','./globalSetup2.ts','./globalSetup3.ts'], globalTeardown: ['./globalTeardown1.ts', './globalTeardown2.ts'], }; `, - 'globalSetup1.ts': `module.exports = () => console.log('globalSetup1');`, - 'globalSetup2.ts': `module.exports = () => console.log('globalSetup2');`, - 'globalSetup3.ts': `module.exports = () => console.log('globalSetup3');`, - 'globalTeardown1.ts': `module.exports = () => console.log('globalTeardown1');`, - 'globalTeardown2.ts': `module.exports = () => console.log('globalTeardown2');`, + 'globalSetup1.ts': `module.exports = () => console.log('%%globalSetup1');`, + 'globalSetup2.ts': `module.exports = () => console.log('%%globalSetup2');`, + 'globalSetup3.ts': `module.exports = () => console.log('%%globalSetup3');`, + 'globalTeardown1.ts': `module.exports = () => console.log('%%globalTeardown1');`, + 'globalTeardown2.ts': `module.exports = () => console.log('%%globalTeardown2');`, 'a.test.js': ` import { test } from '@playwright/test'; - test('a', () => console.log('test a')); - test('b', () => console.log('test b')); + test('a', () => console.log('%%test a')); + test('b', () => console.log('%%test b')); `, }, { reporter: 'line' }); - expect(passed).toBe(2); - expectInOrder(output, [ + expect(result.passed).toBe(2); + expect(result.outputLines).toEqual([ 'globalSetup1', 'globalSetup2', 'globalSetup3', From 3ef8178cca9b70d4b9df39f4374e663750d8ae91 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Mon, 7 Oct 2024 07:40:58 +0200 Subject: [PATCH 06/17] extend tests --- packages/playwright/src/runner/tasks.ts | 4 ++-- tests/playwright-test/global-setup.spec.ts | 23 ++++++++++++++-------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index 24a6c7048b9d7..dfb081d36be24 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -182,10 +182,10 @@ function createGlobalSetupTask(index: number, length: number): Task { globalSetupFinished = true; }, teardown: async ({ config }) => { - if (typeof globalSetupResult === 'function') - await globalSetupResult(); if (globalSetupFinished) await teardownHook?.(config.config); + if (typeof globalSetupResult === 'function') + await globalSetupResult(); }, }; } diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 0ec6c05840977..64472add420d1 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -391,15 +391,16 @@ test('globalSetup should support multiple', async ({ runInlineTest }) => { const result = await runInlineTest({ 'playwright.config.ts': ` module.exports = { - globalSetup: ['./globalSetup1.ts','./globalSetup2.ts','./globalSetup3.ts'], - globalTeardown: ['./globalTeardown1.ts', './globalTeardown2.ts'], + globalSetup: ['./globalSetup1.ts','./globalSetup2.ts','./globalSetup3.ts','./globalSetup4.ts'], + globalTeardown: ['./globalTeardown2.ts', './globalTeardown3.ts'], }; `, - 'globalSetup1.ts': `module.exports = () => console.log('%%globalSetup1');`, + 'globalSetup1.ts': `module.exports = () => { console.log('%%globalSetup1'); return () => console.log('%%globalSetup1Function'); };`, 'globalSetup2.ts': `module.exports = () => console.log('%%globalSetup2');`, - 'globalSetup3.ts': `module.exports = () => console.log('%%globalSetup3');`, - 'globalTeardown1.ts': `module.exports = () => console.log('%%globalTeardown1');`, - 'globalTeardown2.ts': `module.exports = () => console.log('%%globalTeardown2');`, + 'globalSetup3.ts': `module.exports = () => { console.log('%%globalSetup3'); return () => console.log('%%globalSetup3Function'); }`, + 'globalSetup4.ts': `module.exports = () => console.log('%%globalSetup4');`, + 'globalTeardown2.ts': `module.exports = () => { console.log('%%globalTeardown2'); throw new Error('kaboom'); }`, + 'globalTeardown3.ts': `module.exports = () => console.log('%%globalTeardown3');`, 'a.test.js': ` import { test } from '@playwright/test'; @@ -407,15 +408,21 @@ test('globalSetup should support multiple', async ({ runInlineTest }) => { test('b', () => console.log('%%test b')); `, }, { reporter: 'line' }); - expect(result.passed).toBe(2); + + // behaviour: setups in order, teardowns in reverse order. + // setup-returned functions inherit their position, and take precedence over `globalTeardown` scripts. expect(result.outputLines).toEqual([ 'globalSetup1', 'globalSetup2', 'globalSetup3', + 'globalSetup4', 'test a', 'test b', + 'globalSetup3Function', + 'globalTeardown3', 'globalTeardown2', - 'globalTeardown1', + // 'globalSetup1Function' is missing, because globalTeardown2 errored out. ]); + expect(result.output).toContain('Error: kaboom'); }); From d1273ad1b614d831c572b4120a833af700c2c7c2 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 8 Oct 2024 16:55:22 +0200 Subject: [PATCH 07/17] revert ordering --- packages/playwright/src/runner/tasks.ts | 4 ++-- tests/playwright-test/global-setup.spec.ts | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index dfb081d36be24..24a6c7048b9d7 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -182,10 +182,10 @@ function createGlobalSetupTask(index: number, length: number): Task { globalSetupFinished = true; }, teardown: async ({ config }) => { - if (globalSetupFinished) - await teardownHook?.(config.config); if (typeof globalSetupResult === 'function') await globalSetupResult(); + if (globalSetupFinished) + await teardownHook?.(config.config); }, }; } diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 64472add420d1..9528a42953288 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -392,15 +392,15 @@ test('globalSetup should support multiple', async ({ runInlineTest }) => { 'playwright.config.ts': ` module.exports = { globalSetup: ['./globalSetup1.ts','./globalSetup2.ts','./globalSetup3.ts','./globalSetup4.ts'], - globalTeardown: ['./globalTeardown2.ts', './globalTeardown3.ts'], + globalTeardown: ['./globalTeardown1.ts', './globalTeardown2.ts'], }; `, - 'globalSetup1.ts': `module.exports = () => { console.log('%%globalSetup1'); return () => console.log('%%globalSetup1Function'); };`, + 'globalSetup1.ts': `module.exports = () => { console.log('%%globalSetup1'); return () => { console.log('%%globalSetup1Function'); throw new Error('kaboom'); } };`, 'globalSetup2.ts': `module.exports = () => console.log('%%globalSetup2');`, 'globalSetup3.ts': `module.exports = () => { console.log('%%globalSetup3'); return () => console.log('%%globalSetup3Function'); }`, 'globalSetup4.ts': `module.exports = () => console.log('%%globalSetup4');`, - 'globalTeardown2.ts': `module.exports = () => { console.log('%%globalTeardown2'); throw new Error('kaboom'); }`, - 'globalTeardown3.ts': `module.exports = () => console.log('%%globalTeardown3');`, + 'globalTeardown1.ts': `module.exports = () => console.log('%%globalTeardown1')`, + 'globalTeardown2.ts': `module.exports = () => console.log('%%globalTeardown2');`, 'a.test.js': ` import { test } from '@playwright/test'; @@ -420,9 +420,9 @@ test('globalSetup should support multiple', async ({ runInlineTest }) => { 'test a', 'test b', 'globalSetup3Function', - 'globalTeardown3', 'globalTeardown2', - // 'globalSetup1Function' is missing, because globalTeardown2 errored out. + 'globalSetup1Function', + // 'globalTeardown1' is missing, because globalTeardown1 errored out. ]); expect(result.output).toContain('Error: kaboom'); }); From 87407618d91f9b6031354ea28a32141ac0c2d1fd Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 8 Oct 2024 17:54:26 +0200 Subject: [PATCH 08/17] only add index when length > 1 --- packages/playwright/src/runner/tasks.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index 24a6c7048b9d7..528cac47cd1d4 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -170,7 +170,7 @@ function createGlobalSetupTask(index: number, length: number): Task { let teardownHook: any; let title = 'global setup'; - if (length) + if (length > 1) title += ` #${index}`; return { From c8c483cacbfe313bbe2b88283e65c95a065fcbb7 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Wed, 16 Oct 2024 08:53:47 +0200 Subject: [PATCH 09/17] docs change --- docs/src/test-api/class-testconfig.md | 4 ++-- packages/playwright/types/test.d.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/src/test-api/class-testconfig.md b/docs/src/test-api/class-testconfig.md index de11e985751e2..003d04843a500 100644 --- a/docs/src/test-api/class-testconfig.md +++ b/docs/src/test-api/class-testconfig.md @@ -112,7 +112,7 @@ export default defineConfig({ * since: v1.10 - type: ?<[string]|[Array]<[string]>> -Path to the global setup file. This file will be required and run before all the tests. It must export a single function that takes a [FullConfig] argument. Pass an array for multiple global setup files. +Path to the global setup file. This file will be required and run before all the tests. It must export a single function that takes a [FullConfig] argument. Pass an array of paths to specify multiple global setup files. Learn more about [global setup and teardown](../test-global-setup-teardown.md). @@ -130,7 +130,7 @@ export default defineConfig({ * since: v1.10 - type: ?<[string]|[Array]<[string]>> -Path to the global teardown file. This file will be required and run after all the tests. It must export a single function. See also [`property: TestConfig.globalSetup`]. Pass an array for multiple global teardown files. +Path to the global teardown file. This file will be required and run after all the tests. It must export a single function. See also [`property: TestConfig.globalSetup`]. Pass an array of paths to specify multiple global teardown files. Learn more about [global setup and teardown](../test-global-setup-teardown.md). diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index e480a90936d6a..e27b3f3f4ed48 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1077,8 +1077,8 @@ interface TestConfig { /** * Path to the global setup file. This file will be required and run before all the tests. It must export a single - * function that takes a [FullConfig](https://playwright.dev/docs/api/class-fullconfig) argument. Pass an array for - * multiple global setup files. + * function that takes a [FullConfig](https://playwright.dev/docs/api/class-fullconfig) argument. Pass an array of + * paths to specify multiple global setup files. * * Learn more about [global setup and teardown](https://playwright.dev/docs/test-global-setup-teardown). * @@ -1100,7 +1100,7 @@ interface TestConfig { * Path to the global teardown file. This file will be required and run after all the tests. It must export a single * function. See also * [testConfig.globalSetup](https://playwright.dev/docs/api/class-testconfig#test-config-global-setup). Pass an array - * for multiple global teardown files. + * of paths to specify multiple global teardown files. * * Learn more about [global setup and teardown](https://playwright.dev/docs/test-global-setup-teardown). * From 5eb3734bb28d23bd5ff17117be8151204506122c Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Wed, 16 Oct 2024 08:54:46 +0200 Subject: [PATCH 10/17] apply nit --- packages/playwright/src/common/config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 759e23f1c3168..cf126efa29883 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -81,8 +81,8 @@ export class FullConfigInternal { rootDir: pathResolve(configDir, userConfig.testDir) || configDir, forbidOnly: takeFirst(configCLIOverrides.forbidOnly, userConfig.forbidOnly, false), fullyParallel: takeFirst(configCLIOverrides.fullyParallel, userConfig.fullyParallel, false), - globalSetup: takeFirst(...this.globalSetups, null), - globalTeardown: takeFirst(...this.globalTeardowns, null), + globalSetup: this.globalSetups[0] ?? null, + globalTeardown: this.globalTeardowns[0] ?? null, globalTimeout: takeFirst(configCLIOverrides.globalTimeout, userConfig.globalTimeout, 0), grep: takeFirst(userConfig.grep, defaultGrep), grepInvert: takeFirst(userConfig.grepInvert, null), From 02d723699e51888af2eadab25ebc9a3f8c72b08a Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 17 Oct 2024 14:29:14 +0200 Subject: [PATCH 11/17] Update tests/playwright-test/global-setup.spec.ts Co-authored-by: Dmitry Gozman Signed-off-by: Simon Knott --- tests/playwright-test/global-setup.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 9528a42953288..1145679281bfb 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -422,7 +422,7 @@ test('globalSetup should support multiple', async ({ runInlineTest }) => { 'globalSetup3Function', 'globalTeardown2', 'globalSetup1Function', - // 'globalTeardown1' is missing, because globalTeardown1 errored out. + // 'globalTeardown1' is missing, because globalSetup1Function errored out. ]); expect(result.output).toContain('Error: kaboom'); }); From d544059250a1e2c01ed72d05d9b30a63c8d64563 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Thu, 17 Oct 2024 14:50:04 +0200 Subject: [PATCH 12/17] use typescript 5.5 magic --- packages/playwright/src/common/config.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index cf126efa29883..affa72f7727cb 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -73,8 +73,8 @@ export class FullConfigInternal { const privateConfiguration = (userConfig as any)['@playwright/test']; this.plugins = (privateConfiguration?.plugins || []).map((p: any) => ({ factory: p })); - this.globalSetups = (Array.isArray(userConfig.globalSetup) ? userConfig.globalSetup : [userConfig.globalSetup]).map(s => resolveScript(s, configDir)).filter((script): script is string => !!script); - this.globalTeardowns = (Array.isArray(userConfig.globalTeardown) ? userConfig.globalTeardown : [userConfig.globalTeardown]).map(s => resolveScript(s, configDir)).filter((script): script is string => !!script); + this.globalSetups = (Array.isArray(userConfig.globalSetup) ? userConfig.globalSetup : [userConfig.globalSetup]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined); + this.globalTeardowns = (Array.isArray(userConfig.globalTeardown) ? userConfig.globalTeardown : [userConfig.globalTeardown]).map(s => resolveScript(s, configDir)).filter(script => script !== undefined); this.config = { configFile: resolvedConfigFile, From a72ec6e8245a1725995400429a4ed26481c7cc9b Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 18 Oct 2024 08:48:31 +0200 Subject: [PATCH 13/17] ensure that teardown throwing isn't a showstopper --- tests/playwright-test/global-setup.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 1145679281bfb..f1bd7b7458348 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -400,7 +400,7 @@ test('globalSetup should support multiple', async ({ runInlineTest }) => { 'globalSetup3.ts': `module.exports = () => { console.log('%%globalSetup3'); return () => console.log('%%globalSetup3Function'); }`, 'globalSetup4.ts': `module.exports = () => console.log('%%globalSetup4');`, 'globalTeardown1.ts': `module.exports = () => console.log('%%globalTeardown1')`, - 'globalTeardown2.ts': `module.exports = () => console.log('%%globalTeardown2');`, + 'globalTeardown2.ts': `module.exports = () => { console.log('%%globalTeardown2'); throw new Error('kaboom'); }`, 'a.test.js': ` import { test } from '@playwright/test'; From 9b785662f6e04f13f183985cf1f3b341e3480db3 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 18 Oct 2024 08:53:39 +0200 Subject: [PATCH 14/17] make behaviour congruent for globalTeardown --- packages/playwright/src/runner/tasks.ts | 7 ++++++- tests/playwright-test/global-setup.spec.ts | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index 528cac47cd1d4..aeaceadc8b01a 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -182,10 +182,15 @@ function createGlobalSetupTask(index: number, length: number): Task { globalSetupFinished = true; }, teardown: async ({ config }) => { + let firstError: any; if (typeof globalSetupResult === 'function') - await globalSetupResult(); + try { await globalSetupResult(); } catch (error) { firstError = error; } + if (globalSetupFinished) await teardownHook?.(config.config); + + if (firstError) + throw firstError; }, }; } diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index f1bd7b7458348..177d29a91d2d6 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -422,7 +422,7 @@ test('globalSetup should support multiple', async ({ runInlineTest }) => { 'globalSetup3Function', 'globalTeardown2', 'globalSetup1Function', - // 'globalTeardown1' is missing, because globalSetup1Function errored out. + 'globalTeardown1', ]); expect(result.output).toContain('Error: kaboom'); }); From 13a4d1976457a3c3fcb9c3d7f0d078a72b0fc8a1 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 18 Oct 2024 08:58:22 +0200 Subject: [PATCH 15/17] don't swallow the first error if the second throws --- packages/playwright/src/runner/tasks.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index aeaceadc8b01a..aeeafe7b41596 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -182,15 +182,18 @@ function createGlobalSetupTask(index: number, length: number): Task { globalSetupFinished = true; }, teardown: async ({ config }) => { - let firstError: any; + const errors = []; if (typeof globalSetupResult === 'function') - try { await globalSetupResult(); } catch (error) { firstError = error; } + try { await globalSetupResult(); } catch (error) { errors.push(error); } if (globalSetupFinished) - await teardownHook?.(config.config); + try { await teardownHook?.(config.config); } catch (error) { errors.push(error); } - if (firstError) - throw firstError; + if (errors.length === 1) + throw errors[0]; + + if (errors.length > 1) + throw new AggregateError(errors); }, }; } From d8d044e5edf36f85951c51ae86b462a7ba748105 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 18 Oct 2024 11:00:33 +0200 Subject: [PATCH 16/17] Revert "don't swallow the first error if the second throws" This reverts commit 13a4d1976457a3c3fcb9c3d7f0d078a72b0fc8a1. --- packages/playwright/src/runner/tasks.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index aeeafe7b41596..aeaceadc8b01a 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -182,18 +182,15 @@ function createGlobalSetupTask(index: number, length: number): Task { globalSetupFinished = true; }, teardown: async ({ config }) => { - const errors = []; + let firstError: any; if (typeof globalSetupResult === 'function') - try { await globalSetupResult(); } catch (error) { errors.push(error); } + try { await globalSetupResult(); } catch (error) { firstError = error; } if (globalSetupFinished) - try { await teardownHook?.(config.config); } catch (error) { errors.push(error); } + await teardownHook?.(config.config); - if (errors.length === 1) - throw errors[0]; - - if (errors.length > 1) - throw new AggregateError(errors); + if (firstError) + throw firstError; }, }; } From d841c675ccceae9e0d044d98ab0753917539f2ce Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Fri, 18 Oct 2024 11:00:36 +0200 Subject: [PATCH 17/17] Revert "make behaviour congruent for globalTeardown" This reverts commit 9b785662f6e04f13f183985cf1f3b341e3480db3. --- packages/playwright/src/runner/tasks.ts | 7 +------ tests/playwright-test/global-setup.spec.ts | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index aeaceadc8b01a..528cac47cd1d4 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -182,15 +182,10 @@ function createGlobalSetupTask(index: number, length: number): Task { globalSetupFinished = true; }, teardown: async ({ config }) => { - let firstError: any; if (typeof globalSetupResult === 'function') - try { await globalSetupResult(); } catch (error) { firstError = error; } - + await globalSetupResult(); if (globalSetupFinished) await teardownHook?.(config.config); - - if (firstError) - throw firstError; }, }; } diff --git a/tests/playwright-test/global-setup.spec.ts b/tests/playwright-test/global-setup.spec.ts index 177d29a91d2d6..f1bd7b7458348 100644 --- a/tests/playwright-test/global-setup.spec.ts +++ b/tests/playwright-test/global-setup.spec.ts @@ -422,7 +422,7 @@ test('globalSetup should support multiple', async ({ runInlineTest }) => { 'globalSetup3Function', 'globalTeardown2', 'globalSetup1Function', - 'globalTeardown1', + // 'globalTeardown1' is missing, because globalSetup1Function errored out. ]); expect(result.output).toContain('Error: kaboom'); });