diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index 26f8d619e22ca..67619c914e1f7 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -317,6 +317,6 @@ Resolved configuration. The root suite that contains the projects, files and test cases that will run. -The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. It contains only the top-level projects being run — setup and dependency projects are not included and cannot be excluded from here. +The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here. The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocessSuite`] returns, unless the returned `implementsSharding` is `true`. diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 991ac8304ef0d..598a041d8a81e 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -58,7 +58,7 @@ export class Suite extends Base { _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; _fullProject: FullProjectInternal | undefined; _fileId: string | undefined; - _preprocessing = false; + _preprocessMode: 'editable' | 'readonly' | undefined = undefined; readonly _type: 'root' | 'project' | 'file' | 'describe'; skip: (reason?: string) => void; @@ -270,22 +270,28 @@ export class Suite extends Base { } private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void { - if (!this._rootSuite()._preprocessing) + const mode = this._resolvePreprocessMode(); + if (!mode) throw new Error(`Suite.${type}() can only be called from Reporter.preprocessSuite().`); + if (mode === 'readonly') + throw new Error(`Suite.${type}() cannot be called on a setup or teardown project; these always run in full.`); for (const test of this.allTests()) test._applyPlanAnnotation({ type, description: reason, location }); } exclude(): void { - if (!this._rootSuite()._preprocessing) + const mode = this._resolvePreprocessMode(); + if (!mode) throw new Error(`Suite.exclude() can only be called from Reporter.preprocessSuite().`); if (!this.parent) throw new Error(`Suite.exclude() cannot be called on the root suite.`); + if (mode === 'readonly') + throw new Error(`Suite.exclude() cannot be called on a setup or teardown project; these always run in full.`); this.parent._detach(this); } - _rootSuite(): Suite { - return this.parent?._rootSuite() ?? this; + _resolvePreprocessMode(): 'editable' | 'readonly' | undefined { + return this._preprocessMode ?? this.parent?._resolvePreprocessMode(); } } @@ -353,8 +359,11 @@ export class TestCase extends Base implements reporterTypes.TestCase { } private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void { - if (!this._rootSuite()._preprocessing) + const mode = this.parent._resolvePreprocessMode(); + if (!mode) throw new Error(`TestCase.${type}() can only be called from Reporter.preprocessSuite().`); + if (mode === 'readonly') + throw new Error(`TestCase.${type}() cannot be called on a setup or teardown project test; these always run in full.`); this._applyPlanAnnotation({ type, description: reason, location }); } @@ -368,15 +377,14 @@ export class TestCase extends Base implements reporterTypes.TestCase { } exclude(): void { - if (!this._rootSuite()._preprocessing) + const mode = this.parent._resolvePreprocessMode(); + if (!mode) throw new Error(`TestCase.exclude() can only be called from Reporter.preprocessSuite().`); + if (mode === 'readonly') + throw new Error(`TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.`); this.parent._detach(this); } - _rootSuite(): Suite { - return this.parent._rootSuite(); - } - _serialize(): any { return { kind: 'test', diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index 7766ec11f6f8b..f4f092cd5d9ea 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -55,12 +55,7 @@ export class InternalReporter implements ReporterV2 { } async preprocessSuite(config: FullConfig, suite: testNs.Suite) { - suite._preprocessing = true; - try { - return await this._reporter.preprocessSuite?.(config, suite); - } finally { - suite._preprocessing = false; - } + return await this._reporter.preprocessSuite?.(config, suite); } onBegin(suite: testNs.Suite) { diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 9485a923ba5a1..9878b14702c8b 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -165,7 +165,30 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho } } - const preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite); + // Temporarily prepend unfiltered dependency projects for preprocessing. + const dependencySuites = new Map(); + for (const [project, type] of projectClosure) { + if (type !== 'dependency') + continue; + const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!); + dependencySuite._preprocessMode = 'readonly'; + dependencySuites.set(project, dependencySuite); + rootSuite._prependSuite(dependencySuite); + } + + rootSuite._preprocessMode = 'editable'; + let preprocessResult: Awaited>; + try { + preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite); + } finally { + // Continue the existing sharding and filtering pipeline with top-level projects only. + rootSuite._preprocessMode = undefined; + for (const dependencySuite of dependencySuites.values()) { + dependencySuite._preprocessMode = undefined; + rootSuite._detach(dependencySuite); + } + } + // Shard only the top-level projects. if (config.config.shard && !preprocessResult?.implementsSharding) { // Create test groups for top-level projects. @@ -193,16 +216,13 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho suiteUtils.filterTestsRemoveEmptySuites(rootSuite, test => testRun.postShardTestFilters.every(filter => filter(test))); const topLevelProjects = []; - // Now prepend dependency projects without filtration. { // Filtering 'only' and sharding might have reduced the number of top-level projects. // Build the project closure to only include dependencies that are still needed. - const projectClosure = new Map(buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!))); - - // Clone file suites for dependency projects. - for (const [project, level] of projectClosure.entries()) { - if (level === 'dependency') - rootSuite._prependSuite(buildProjectSuite(project, projectSuites.get(project)!)); + const finalProjectClosure = buildProjectsClosure(rootSuite.suites.map(suite => suite._fullProject!)); + for (const [project, type] of finalProjectClosure) { + if (type === 'dependency') + rootSuite._prependSuite(dependencySuites.get(project)!); else topLevelProjects.push(project); } diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 47e406bcd1d82..f7809c29c65fe 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -153,8 +153,7 @@ export interface Reporter { * @param suite The root suite that contains the projects, files and test cases that will run. * * The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that - * match the current invocation. It contains only the top-level projects being run — setup and dependency projects are - * not included and cannot be excluded from here. + * match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here. * * The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its * built-in sharding after diff --git a/tests/playwright-test/reporter-preprocess-suite.spec.ts b/tests/playwright-test/reporter-preprocess-suite.spec.ts index 7e2f16342851e..d9139a34c518d 100644 --- a/tests/playwright-test/reporter-preprocess-suite.spec.ts +++ b/tests/playwright-test/reporter-preprocess-suite.spec.ts @@ -392,15 +392,40 @@ test('multiple reporters declaring implementsSharding throws', async ({ runInlin expect(result.outputLines.join('\n')).toContain(`Multiple reporters declare 'implementsSharding'`); }); -test('plan.suite contains only top-level projects, not dependency/setup projects', async ({ runInlineTest }) => { +test('plan.suite exposes setup/teardown dependency projects but they are read-only', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` class Reporter { async preprocessSuite(config, suite) { - // The suite only exposes top-level projects, so a reporter has no handle on - // setup/dependency project tests and therefore cannot exclude them. console.log('%% plan projects: ' + suite.suites.map(s => s.title).join(',')); console.log('%% plan tests: ' + suite.allTests().map(t => t.title).join(',')); + this.preprocessedTests = new Set(suite.allTests()); + const setupTest = suite.allTests().find(t => t.title === 'setup-test'); + for (const method of ['skip', 'fixme', 'fail', 'exclude']) { + try { + setupTest[method](); + console.log('%% dep-' + method + ': no-throw'); + } catch (e) { + console.log('%% dep-' + method + ': ' + e.message); + } + } + const setupProject = suite.suites.find(s => s.title === 'setup'); + try { + setupProject.exclude(); + console.log('%% dep-suite-exclude: no-throw'); + } catch (e) { + console.log('%% dep-suite-exclude: ' + e.message); + } + } + onBegin(config, suite) { + console.log('%% same test objects: ' + suite.allTests().every(test => this.preprocessedTests.has(test))); + const setupTest = suite.allTests().find(t => t.title === 'setup-test'); + try { + setupTest.skip(); + console.log('%% dep-after-preprocess: no-throw'); + } catch (e) { + console.log('%% dep-after-preprocess: ' + e.message); + } } onTestEnd(test, result) { console.log('%% ran ' + test.parent.project().name + '/' + test.title); @@ -412,7 +437,8 @@ test('plan.suite contains only top-level projects, not dependency/setup projects module.exports = { reporter: './reporter.ts', projects: [ - { name: 'setup', testMatch: /a\\.setup\\.ts/ }, + { name: 'setup', testMatch: /a\\.setup\\.ts/, teardown: 'teardown' }, + { name: 'teardown', testMatch: /a\\.teardown\\.ts/ }, { name: 'main', testMatch: /a\\.test\\.ts/, dependencies: ['setup'] }, ], }; @@ -421,6 +447,10 @@ test('plan.suite contains only top-level projects, not dependency/setup projects import { test } from '@playwright/test'; test('setup-test', async () => {}); `, + 'a.teardown.ts': ` + import { test } from '@playwright/test'; + test('teardown-test', async () => {}); + `, 'a.test.ts': ` import { test } from '@playwright/test'; test('main-test', async () => {}); @@ -428,11 +458,65 @@ test('plan.suite contains only top-level projects, not dependency/setup projects }, { reporter: '', workers: 1 }, undefined, { additionalArgs: ['--project=main'] }); expect(result.exitCode).toBe(0); - // plan only sees the top-level 'main' project; the 'setup' dependency is prepended afterwards. - expect(result.outputLines).toContain('plan projects: main'); - // 'setup-test' is absent from the plan suite, proving setup/dependency tests are not exposed. - expect(result.outputLines).toContain('plan tests: main-test'); - // Both the dependency and the main project still run. - expect(result.outputLines).toContain('ran setup/setup-test'); - expect(result.outputLines).toContain('ran main/main-test'); + expect(result.outputLines).toEqual([ + 'plan projects: teardown,setup,main', + 'plan tests: teardown-test,setup-test,main-test', + 'dep-skip: TestCase.skip() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-fixme: TestCase.fixme() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-fail: TestCase.fail() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-exclude: TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-suite-exclude: Suite.exclude() cannot be called on a setup or teardown project; these always run in full.', + 'same test objects: true', + 'dep-after-preprocess: TestCase.skip() can only be called from Reporter.preprocessSuite().', + 'ran setup/setup-test', + 'ran main/main-test', + 'ran teardown/teardown-test', + ]); +}); + +test('plan.suite temporarily exposes dependencies without changing final project selection', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': ` + class Reporter { + async preprocessSuite(config, suite) { + console.log('%% plan projects: ' + suite.suites.map(suite => suite.title).join(',')); + suite.suites.find(suite => suite.title === 'main').exclude(); + } + onTestEnd(test, result) { + console.log('%% ran ' + test.parent.project().name + '/' + test.title); + } + } + module.exports = Reporter; + `, + 'playwright.config.ts': ` + module.exports = { + reporter: './reporter.ts', + projects: [ + { name: 'setup', testMatch: /setup\\.spec\\.ts/ }, + { name: 'main', testMatch: /main\\.spec\\.ts/, dependencies: ['setup'] }, + { name: 'keep', testMatch: /keep\\.spec\\.ts/ }, + ], + }; + `, + 'setup.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('setup-test', async () => { + expect(1).toBe(2); + }); + `, + 'main.spec.ts': ` + import { test } from '@playwright/test'; + test('main-test', async () => {}); + `, + 'keep.spec.ts': ` + import { test } from '@playwright/test'; + test('keep-test', async () => {}); + `, + }, { reporter: '', workers: 1 }, undefined, { additionalArgs: ['setup.spec.ts', 'main.spec.ts', 'keep.spec.ts'] }); + + expect(result.exitCode).toBe(0); + expect(result.outputLines).toEqual([ + 'plan projects: setup,main,keep', + 'ran keep/keep-test', + ]); });