From 4b62ee7385788d15da2fff5ffe572e956e58f5d0 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Wed, 15 Jul 2026 11:55:01 +0200 Subject: [PATCH 1/2] feat(test runner): expose TestRun to reporters Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cc00d7ab-1f1b-4571-9feb-3301d11ef143 --- docs/src/test-reporter-api/class-reporter.md | 26 ++-- docs/src/test-reporter-api/class-suite.md | 38 ------ docs/src/test-reporter-api/class-testcase.md | 38 ------ docs/src/test-reporter-api/class-testrun.md | 67 ++++++++++ docs/src/test-reporter-api/class-teststep.md | 2 +- packages/playwright/src/common/test.ts | 61 +++------ .../src/reporters/internalReporter.ts | 40 +++++- .../playwright/src/reporters/multiplexer.ts | 10 +- .../playwright/src/reporters/reporterV2.ts | 14 +- packages/playwright/src/runner/loadUtils.ts | 25 ++-- packages/playwright/types/testReporter.d.ts | 120 +++++++----------- tests/bidi/expectationReporter.ts | 4 +- .../reporter-preprocess-suite.spec.ts | 118 ++++++++--------- .../overrides-testReporter.d.ts | 2 +- 14 files changed, 264 insertions(+), 301 deletions(-) create mode 100644 docs/src/test-reporter-api/class-testrun.md diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index 67619c914e1f7..a0c481f6ba79e 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -298,25 +298,21 @@ Result of the test run. Whether this reporter uses stdio for reporting. When it does not, Playwright Test could add some output to enhance user experience. If your reporter does not print to the terminal, it is strongly recommended to return `false`. -## optional async method: Reporter.preprocessSuite -* since: v1.61 +## optional async method: Reporter.preprocess +* since: v1.62 - `result` ?<[Object]> - - `implementsSharding` ?<[boolean]> When `true`, Playwright skips its built-in shard filter for this run, leaving sharding to the reporter (typically implemented by calling [`method: TestCase.exclude`] on out-of-shard tests). + - `implementsSharding` ?<[boolean]> When `true`, Playwright skips its built-in shard filter for this run, leaving sharding to the reporter (typically implemented by calling [`method: TestRun.exclude`] on out-of-shard tests). Called after the configuration has been resolved and before [`method: Reporter.onBegin`]. Allows a reporter to mark individual tests as skipped, excluded, fixed or failing. -### param: Reporter.preprocessSuite.config -* since: v1.61 -- `config` <[FullConfig]> - -Resolved configuration. - -### param: Reporter.preprocessSuite.suite -* since: v1.61 -- `suite` <[Suite]> +### param: Reporter.preprocess.params +* since: v1.62 +- `params` <[Object]> + - `config` <[FullConfig]> Resolved configuration. + - `suite` <[Suite]> The root suite that contains the projects, files and test cases that will run. + - `testRun` <[TestRun]> Control which tests will run and their expected status. -The root suite that contains the projects, files and test cases that will run. -The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here. +The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. Setup and dependency projects are readonly and cannot be changed through [TestRun]. -The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocessSuite`] returns, unless the returned `implementsSharding` is `true`. +The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocess`] returns, unless the returned `implementsSharding` is `true`. diff --git a/docs/src/test-reporter-api/class-suite.md b/docs/src/test-reporter-api/class-suite.md index 9f93dcabd6f4c..1d458c842b3f0 100644 --- a/docs/src/test-reporter-api/class-suite.md +++ b/docs/src/test-reporter-api/class-suite.md @@ -85,41 +85,3 @@ Returns a list of titles from the root down to this suite. Returns the type of the suite. The Suites form the following hierarchy: `root` -> `project` -> `file` -> `describe` -> ...`describe` -> `test`. - -## method: Suite.skip -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`]. Mark every [TestCase] of this suite as skipped, see [`method: TestCase.skip`]. - -### param: Suite.skip.reason -* since: v1.61 -- `reason` ?<[string]> - -Optional explanation surfaced as the annotation description. - -## method: Suite.fixme -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`]. Mark every [TestCase] of this suite as fixme, see [`method: TestCase.fixme`]. - -### param: Suite.fixme.reason -* since: v1.61 -- `reason` ?<[string]> - -Optional explanation surfaced as the annotation description. - -## method: Suite.fail -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`]. Mark every [TestCase] of this suite as expected-to-fail, see [`method: TestCase.fail`]. - -### param: Suite.fail.reason -* since: v1.61 -- `reason` ?<[string]> - -Optional explanation surfaced as the annotation description. - -## method: Suite.exclude -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`], exclude this suite from the run. Excluded tests do not appear in the report and their body is not executed. diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index 95e6052e32ae7..22a8588fb0934 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -107,41 +107,3 @@ Returns a list of titles from the root down to this test. - returns: <[TestCaseType]<"test">> Returns "test". Useful for detecting test cases in [`method: Suite.entries`]. - -## method: TestCase.skip -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`], skip this test. The test body is not executed and the test is reported as skipped. - -### param: TestCase.skip.reason -* since: v1.61 -- `reason` ?<[string]> - -Optional explanation surfaced as the annotation description. - -## method: TestCase.fixme -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`], mark this test as fixme. The test body is not executed and the test is reported as skipped, with the intention to fix it. - -### param: TestCase.fixme.reason -* since: v1.61 -- `reason` ?<[string]> - -Optional explanation surfaced as the annotation description. - -## method: TestCase.fail -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`], mark this test as "should fail". Playwright runs the test and ensures it is actually failing, useful for documenting broken functionality until it is fixed. - -### param: TestCase.fail.reason -* since: v1.61 -- `reason` ?<[string]> - -Optional explanation surfaced as the annotation description. - -## method: TestCase.exclude -* since: v1.61 - -Must be called from inside [`method: Reporter.preprocessSuite`], exclude this test from the run. Excluded tests do not appear in the report and their body is not executed. diff --git a/docs/src/test-reporter-api/class-testrun.md b/docs/src/test-reporter-api/class-testrun.md new file mode 100644 index 0000000000000..fa6e01066901f --- /dev/null +++ b/docs/src/test-reporter-api/class-testrun.md @@ -0,0 +1,67 @@ +# class: TestRun +* since: v1.62 +* langs: js + +Controls which tests will run and their expected status. A [TestRun] is available during [`method: Reporter.preprocess`]. + +## method: TestRun.exclude +* since: v1.62 + +Excludes a test or suite from the run. Excluded tests do not appear in the report and their bodies are not executed. + +### param: TestRun.exclude.test +* since: v1.62 +- `test` <[TestCase]|[Suite]> + +Test or suite to exclude. The root suite and setup or teardown projects cannot be excluded. + +## method: TestRun.fail +* since: v1.62 + +Marks a test or every test in a suite as "should fail". Playwright runs the tests and ensures they are actually failing, useful for documenting broken functionality until it is fixed. + +### param: TestRun.fail.test +* since: v1.62 +- `test` <[TestCase]|[Suite]> + +Test or suite to mark as expected-to-fail. Setup and teardown projects cannot be changed. + +### param: TestRun.fail.reason +* since: v1.62 +- `reason` ?<[string]> + +Optional explanation surfaced as the annotation description. + +## method: TestRun.fixme +* since: v1.62 + +Marks a test or every test in a suite as fixme. The test bodies are not executed and the tests are reported as skipped, with the intention to fix them. + +### param: TestRun.fixme.test +* since: v1.62 +- `test` <[TestCase]|[Suite]> + +Test or suite to mark as fixme. Setup and teardown projects cannot be changed. + +### param: TestRun.fixme.reason +* since: v1.62 +- `reason` ?<[string]> + +Optional explanation surfaced as the annotation description. + +## method: TestRun.skip +* since: v1.62 + +Skips a test or every test in a suite. The test bodies are not executed and the tests are reported as skipped. + +### param: TestRun.skip.test +* since: v1.62 +- `test` <[TestCase]|[Suite]> + +Test or suite to skip. Setup and teardown projects cannot be changed. + +### param: TestRun.skip.reason +* since: v1.62 +- `reason` ?<[string]> + +Optional explanation surfaced as the annotation description. diff --git a/docs/src/test-reporter-api/class-teststep.md b/docs/src/test-reporter-api/class-teststep.md index 4c312ad2a75c5..0c18e6f768ddf 100644 --- a/docs/src/test-reporter-api/class-teststep.md +++ b/docs/src/test-reporter-api/class-teststep.md @@ -2,7 +2,7 @@ * since: v1.10 * langs: js -Represents a step in the [TestRun]. +Represents a step in a [TestResult]. ## property: TestStep.category * since: v1.10 diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 598a041d8a81e..8236ea529e9bc 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -16,7 +16,6 @@ import { rootTestType } from './testType'; import { computeTestCaseOutcome } from '../isomorphic/teleReceiver'; -import { wrapFunctionWithLocation } from '../transform/transform'; import type { FixturesWithLocation, FullProjectInternal } from './config'; import type { FixturePool } from './fixtures'; import type { TestTypeImpl } from './testType'; @@ -58,19 +57,12 @@ export class Suite extends Base { _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; _fullProject: FullProjectInternal | undefined; _fileId: string | undefined; - _preprocessMode: 'editable' | 'readonly' | undefined = undefined; + _preprocessReadonly = false; readonly _type: 'root' | 'project' | 'file' | 'describe'; - skip: (reason?: string) => void; - fixme: (reason?: string) => void; - fail: (reason?: string) => void; - constructor(title: string, type: 'root' | 'project' | 'file' | 'describe') { super(title); this._type = type; - this.skip = wrapFunctionWithLocation((location, reason?: string) => this._modifier('skip', location, reason)); - this.fixme = wrapFunctionWithLocation((location, reason?: string) => this._modifier('fixme', location, reason)); - this.fail = wrapFunctionWithLocation((location, reason?: string) => this._modifier('fail', location, reason)); } get type(): 'root' | 'project' | 'file' | 'describe' { @@ -269,29 +261,25 @@ export class Suite extends Base { return this._fullProject?.project || this.parent?.project(); } - private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void { - const mode = this._resolvePreprocessMode(); - if (!mode) - throw new Error(`Suite.${type}() can only be called from Reporter.preprocessSuite().`); - if (mode === 'readonly') - throw new Error(`Suite.${type}() cannot be called on a setup or teardown project; these always run in full.`); + _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined) { + if (!this.parent) + throw new Error(`TestRun.${type}() cannot be called on the root suite.`); + if (this._resolvePreprocessReadonly()) + throw new Error(`TestRun.${type}() cannot be called on a setup or teardown project; these always run in full.`); for (const test of this.allTests()) test._applyPlanAnnotation({ type, description: reason, location }); } - exclude(): void { - const mode = this._resolvePreprocessMode(); - if (!mode) - throw new Error(`Suite.exclude() can only be called from Reporter.preprocessSuite().`); + _exclude() { + if (this._resolvePreprocessReadonly()) + throw new Error(`TestRun.exclude() cannot be called on a setup or teardown project; these always run in full.`); if (!this.parent) - throw new Error(`Suite.exclude() cannot be called on the root suite.`); - if (mode === 'readonly') - throw new Error(`Suite.exclude() cannot be called on a setup or teardown project; these always run in full.`); + throw new Error(`TestRun.exclude() cannot be called on the root suite.`); this.parent._detach(this); } - _resolvePreprocessMode(): 'editable' | 'readonly' | undefined { - return this._preprocessMode ?? this.parent?._resolvePreprocessMode(); + _resolvePreprocessReadonly(): boolean { + return this._preprocessReadonly || !!this.parent?._resolvePreprocessReadonly(); } } @@ -318,18 +306,11 @@ export class TestCase extends Base implements reporterTypes.TestCase { _tags: string[] = []; _planAnnotations: TestAnnotation[] = []; - skip: (reason?: string) => void; - fixme: (reason?: string) => void; - fail: (reason?: string) => void; - constructor(title: string, fn: Function, testType: TestTypeImpl, location: Location) { super(title); this.fn = fn; this._testType = testType; this.location = location; - this.skip = wrapFunctionWithLocation((location, reason?: string) => this._modifier('skip', location, reason)); - this.fixme = wrapFunctionWithLocation((location, reason?: string) => this._modifier('fixme', location, reason)); - this.fail = wrapFunctionWithLocation((location, reason?: string) => this._modifier('fail', location, reason)); } titlePath(): string[] { @@ -358,12 +339,9 @@ export class TestCase extends Base implements reporterTypes.TestCase { ]; } - private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined): void { - const mode = this.parent._resolvePreprocessMode(); - if (!mode) - throw new Error(`TestCase.${type}() can only be called from Reporter.preprocessSuite().`); - if (mode === 'readonly') - throw new Error(`TestCase.${type}() cannot be called on a setup or teardown project test; these always run in full.`); + _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined) { + if (this.parent._resolvePreprocessReadonly()) + throw new Error(`TestRun.${type}() cannot be called on a setup or teardown project test; these always run in full.`); this._applyPlanAnnotation({ type, description: reason, location }); } @@ -376,12 +354,9 @@ export class TestCase extends Base implements reporterTypes.TestCase { this.expectedStatus = 'failed'; } - exclude(): void { - const mode = this.parent._resolvePreprocessMode(); - if (!mode) - throw new Error(`TestCase.exclude() can only be called from Reporter.preprocessSuite().`); - if (mode === 'readonly') - throw new Error(`TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.`); + _exclude() { + if (this.parent._resolvePreprocessReadonly()) + throw new Error(`TestRun.exclude() cannot be called on a setup or teardown project test; these always run in full.`); this.parent._detach(this); } diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index f4f092cd5d9ea..c14c035fc2efb 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -20,13 +20,38 @@ import { monotonicTime } from '@isomorphic/time'; import { internalScreen, prepareErrorStack, relativeFilePath } from './base'; import { Multiplexer } from './multiplexer'; -import { test as testNs } from '../common'; +import { test as testNs, transform } from '../common'; import * as babel from '../transform/babelBundle'; import { wrapReporterAsV2 } from './reporterV2'; -import type { AnyReporter, ReporterV2 } from './reporterV2'; -import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep, WorkerInfo } from '../../types/testReporter'; +import type { AnyReporter, ReporterPreprocessParams, ReporterV2 } from './reporterV2'; +import type { FullConfig, FullResult, Location, Suite as ReporterSuite, TestCase, TestError, TestResult, TestRun as ReporterTestRun, TestStep, WorkerInfo } from '../../types/testReporter'; +type ReporterTestRunTarget = ReporterSuite | TestCase; + +class ReporterTestRunImpl implements ReporterTestRun { + private _active = true; + + close() { + this._active = false; + } + + skip = transform.wrapFunctionWithLocation((location: Location, target: ReporterTestRunTarget, reason?: string) => this._modifier('skip', location, target, reason)); + fixme = transform.wrapFunctionWithLocation((location: Location, target: ReporterTestRunTarget, reason?: string) => this._modifier('fixme', location, target, reason)); + fail = transform.wrapFunctionWithLocation((location: Location, target: ReporterTestRunTarget, reason?: string) => this._modifier('fail', location, target, reason)); + + exclude(target: ReporterTestRunTarget) { + if (!this._active) + throw new Error(`TestRun.exclude() can only be called from Reporter.preprocess().`); + (target as testNs.Suite | testNs.TestCase)._exclude(); + } + + private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, target: ReporterTestRunTarget, reason: string | undefined) { + if (!this._active) + throw new Error(`TestRun.${type}() can only be called from Reporter.preprocess().`); + (target as testNs.Suite | testNs.TestCase)._modifier(type, location, reason); + } +} export class InternalReporter implements ReporterV2 { private _reporter: Multiplexer; @@ -54,8 +79,13 @@ export class InternalReporter implements ReporterV2 { this._reporter.onConfigure?.(config); } - async preprocessSuite(config: FullConfig, suite: testNs.Suite) { - return await this._reporter.preprocessSuite?.(config, suite); + async preprocess(params: Omit) { + const testRun = new ReporterTestRunImpl(); + try { + return await this._reporter.preprocess({ ...params, testRun }); + } finally { + testRun.close(); + } } onBegin(suite: testNs.Suite) { diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.ts index 9a57c13ed72ca..b5a051d379f33 100644 --- a/packages/playwright/src/reporters/multiplexer.ts +++ b/packages/playwright/src/reporters/multiplexer.ts @@ -16,7 +16,7 @@ import { serializeError } from '../util'; -import type { ReportConfigureParams, ReportEndParams, ReporterV2 } from './reporterV2'; +import type { ReportConfigureParams, ReportEndParams, ReporterPreprocessParams, ReporterV2 } from './reporterV2'; import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep, WorkerInfo } from '../../types/testReporter'; import type { test } from '../common'; @@ -41,14 +41,14 @@ export class Multiplexer implements ReporterV2 { this._wrap(() => reporter.onConfigure?.(config)); } - async preprocessSuite(config: FullConfig, suite: test.Suite) { - // Unlike other reporter callbacks, `preprocessSuite` errors are NOT swallowed — - // they propagate so the run aborts before onBegin. Reporters use preprocessSuite + async preprocess(params: ReporterPreprocessParams) { + // Unlike other reporter callbacks, `preprocess` errors are NOT swallowed — + // they propagate so the run aborts before onBegin. Reporters use preprocess // to mutate the corpus; silently dropping a planning error would let // an inconsistent (partial-mutation) state reach the workers. const shardingReporters: ReporterV2[] = []; for (const reporter of this._reporters) { - const result = await reporter.preprocessSuite?.(config, suite); + const result = await reporter.preprocess?.(params); if (result?.implementsSharding) shardingReporters.push(reporter); } diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.ts index 3d5a65a53d46a..a894c59bb1810 100644 --- a/packages/playwright/src/reporters/reporterV2.ts +++ b/packages/playwright/src/reporters/reporterV2.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError, TestResult, TestStep, WorkerInfo } from '../../types/testReporter'; +import type { FullConfig, FullResult, Reporter, Suite, TestCase, TestError, TestResult, TestRun, TestStep, WorkerInfo } from '../../types/testReporter'; export interface ReportConfigureParams { config: FullConfig; @@ -26,9 +26,15 @@ export interface ReportEndParams { result: FullResult; } +export type ReporterPreprocessParams = { + config: FullConfig; + suite: Suite; + testRun: TestRun; +}; + export interface ReporterV2 { onConfigure?(config: FullConfig): void; - preprocessSuite?(config: FullConfig, suite: Suite): { implementsSharding?: boolean } | Promise<{ implementsSharding?: boolean } | undefined | void> | void; + preprocess?(params: ReporterPreprocessParams): { implementsSharding?: boolean } | Promise<{ implementsSharding?: boolean } | undefined | void> | void; onBegin?(suite: Suite): void; onTestBegin?(test: TestCase, result: TestResult): void; onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; @@ -80,8 +86,8 @@ class ReporterV2Wrapper implements ReporterV2 { this._config = config; } - async preprocessSuite(config: FullConfig, suite: Suite) { - return await this._reporter.preprocessSuite?.(config, suite); + preprocess(params: ReporterPreprocessParams) { + return this._reporter.preprocess?.(params); } onBegin(suite: Suite) { diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 9878b14702c8b..8b1bfdd167f4f 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -23,7 +23,7 @@ import { toPosixPath } from '@utils/fileUtils'; import { InProcessLoaderHost, OutOfProcessLoaderHost } from './loaderHost'; import { createTitleMatcher, errorWithFile, parseLocationArg } from '../util'; import { buildProjectsClosure, collectFilesForProject } from './projectUtils'; -import { createTestGroups, filterForShard } from './testGroups'; +import { createTestGroups, filterForShard } from './testGroups'; import { cc, config as commonConfig, FullConfigInternal, suiteUtils, test as testNs, transform } from '../common'; import type { RawSourceMap } from 'source-map'; @@ -32,7 +32,6 @@ import type { TestGroup } from './testGroups'; import type { FullConfig, Reporter, TestError } from '../../types/testReporter'; import type { Matcher, TestCaseFilter } from '../util'; - export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTestsOutsideProjectFilter: boolean) { const fsCache = new Map(); const sourceMapCache = new Map(); @@ -171,23 +170,19 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho if (type !== 'dependency') continue; const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!); - dependencySuite._preprocessMode = 'readonly'; + dependencySuite._preprocessReadonly = true; dependencySuites.set(project, dependencySuite); rootSuite._prependSuite(dependencySuite); } - rootSuite._preprocessMode = 'editable'; - let preprocessResult: Awaited>; - try { - preprocessResult = await testRun.reporter.preprocessSuite(config.config, rootSuite); - } finally { - // Continue the existing sharding and filtering pipeline with top-level projects only. - rootSuite._preprocessMode = undefined; - for (const dependencySuite of dependencySuites.values()) { - dependencySuite._preprocessMode = undefined; - rootSuite._detach(dependencySuite); - } - } + const preprocessResult = await testRun.reporter.preprocess({ + config: config.config, + suite: rootSuite, + }); + + // Continue sharding and filtering pipeline with top-level projects only. + for (const dependencySuite of dependencySuites.values()) + rootSuite._detach(dependencySuite); // Shard only the top-level projects. if (config.config.shard && !preprocessResult?.implementsSharding) { diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index f7809c29c65fe..b031fa03f1155 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -149,18 +149,16 @@ export interface Reporter { * Called after the configuration has been resolved and before * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin). Allows a * reporter to mark individual tests as skipped, excluded, fixed or failing. - * @param config Resolved configuration. - * @param suite The root suite that contains the projects, files and test cases that will run. - * - * The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that - * match the current invocation. Setup and dependency projects are readonly and cannot be excluded from here. + * @param params The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that + * match the current invocation. Setup and dependency projects are readonly and cannot be changed through + * [TestRun](https://playwright.dev/docs/api/class-testrun). * * The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its * built-in sharding after - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite) - * returns, unless the returned `implementsSharding` is `true`. + * [reporter.preprocess(params)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess) returns, unless + * the returned `implementsSharding` is `true`. */ - preprocessSuite?(config: FullConfig, suite: Suite): Promise<{ implementsSharding?: boolean } | undefined | void> | { implementsSharding?: boolean } | void; + preprocess?(params: { config: FullConfig, suite: Suite, testRun: TestRun }): Promise<{ implementsSharding?: boolean } | undefined | void> | { implementsSharding?: boolean } | void; /** * Called after all tests have been run, or testing has been interrupted. Note that this method may return a [Promise] * and Playwright Test will await it. Reporter is allowed to override the status and hence affect the exit code of the @@ -384,45 +382,11 @@ export interface Suite { */ entries(): Array; - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), - * exclude this suite from the run. Excluded tests do not appear in the report and their body is not executed. - */ - exclude(): void; - - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite). - * Mark every [TestCase](https://playwright.dev/docs/api/class-testcase) of this suite as expected-to-fail, see - * [testCase.fail([reason])](https://playwright.dev/docs/api/class-testcase#test-case-fail). - * @param reason Optional explanation surfaced as the annotation description. - */ - fail(reason?: string): void; - - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite). - * Mark every [TestCase](https://playwright.dev/docs/api/class-testcase) of this suite as fixme, see - * [testCase.fixme([reason])](https://playwright.dev/docs/api/class-testcase#test-case-fixme). - * @param reason Optional explanation surfaced as the annotation description. - */ - fixme(reason?: string): void; - /** * Configuration of the project this suite belongs to, or [void] for the root suite. */ project(): FullProject|undefined; - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite). - * Mark every [TestCase](https://playwright.dev/docs/api/class-testcase) of this suite as skipped, see - * [testCase.skip([reason])](https://playwright.dev/docs/api/class-testcase#test-case-skip). - * @param reason Optional explanation surfaced as the annotation description. - */ - skip(reason?: string): void; - /** * Returns a list of titles from the root down to this suite. */ @@ -477,31 +441,6 @@ export interface Suite { * projects' suites. */ export interface TestCase { - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), - * exclude this test from the run. Excluded tests do not appear in the report and their body is not executed. - */ - exclude(): void; - - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), - * mark this test as "should fail". Playwright runs the test and ensures it is actually failing, useful for - * documenting broken functionality until it is fixed. - * @param reason Optional explanation surfaced as the annotation description. - */ - fail(reason?: string): void; - - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), - * mark this test as fixme. The test body is not executed and the test is reported as skipped, with the intention to - * fix it. - * @param reason Optional explanation surfaced as the annotation description. - */ - fixme(reason?: string): void; - /** * Whether the test is considered running fine. Non-ok tests fail the test run with non-zero exit code. */ @@ -515,14 +454,6 @@ export interface TestCase { */ outcome(): "skipped"|"expected"|"unexpected"|"flaky"; - /** - * Must be called from inside - * [reporter.preprocessSuite(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess-suite), - * skip this test. The test body is not executed and the test is reported as skipped. - * @param reason Optional explanation surfaced as the annotation description. - */ - skip(reason?: string): void; - /** * Returns a list of titles from the root down to this test. */ @@ -802,7 +733,44 @@ export interface TestResult { } /** - * Represents a step in the [TestRun]. + * Controls which tests will run and their expected status. A [TestRun](https://playwright.dev/docs/api/class-testrun) + * is available during + * [reporter.preprocess(params)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess). + */ +export interface TestRun { + /** + * Excludes a test or suite from the run. Excluded tests do not appear in the report and their bodies are not + * executed. + * @param test Test or suite to exclude. The root suite and setup or teardown projects cannot be excluded. + */ + exclude(test: TestCase|Suite): void; + + /** + * Marks a test or every test in a suite as "should fail". Playwright runs the tests and ensures they are actually + * failing, useful for documenting broken functionality until it is fixed. + * @param test Test or suite to mark as expected-to-fail. Setup and teardown projects cannot be changed. + * @param reason Optional explanation surfaced as the annotation description. + */ + fail(test: TestCase|Suite, reason?: string): void; + + /** + * Marks a test or every test in a suite as fixme. The test bodies are not executed and the tests are reported as + * skipped, with the intention to fix them. + * @param test Test or suite to mark as fixme. Setup and teardown projects cannot be changed. + * @param reason Optional explanation surfaced as the annotation description. + */ + fixme(test: TestCase|Suite, reason?: string): void; + + /** + * Skips a test or every test in a suite. The test bodies are not executed and the tests are reported as skipped. + * @param test Test or suite to skip. Setup and teardown projects cannot be changed. + * @param reason Optional explanation surfaced as the annotation description. + */ + skip(test: TestCase|Suite, reason?: string): void; +} + +/** + * Represents a step in a [TestResult](https://playwright.dev/docs/api/class-testresult). */ export interface TestStep { /** diff --git a/tests/bidi/expectationReporter.ts b/tests/bidi/expectationReporter.ts index d58c6be80d5df..734095aa9c34f 100644 --- a/tests/bidi/expectationReporter.ts +++ b/tests/bidi/expectationReporter.ts @@ -35,7 +35,7 @@ class ExpectationReporter implements Reporter { this._options = options; } - async preprocessSuite(config: FullConfig, suite: Suite) { + async preprocess({ suite, testRun }: Parameters>[0]) { if (!process.env.PWTEST_USE_BIDI_EXPECTATIONS) return; if (this._options.rebase) @@ -45,7 +45,7 @@ class ExpectationReporter implements Reporter { for (const test of project.allTests()) { const expectation = expectations.get(expectationKey(test)); if (expectation && ['flaky', 'fail', 'timeout'].includes(expectation)) - test.fixme(`marked as ${expectation} in bidi expectations`); + testRun.fixme(test, `marked as ${expectation} in bidi expectations`); } } } diff --git a/tests/playwright-test/reporter-preprocess-suite.spec.ts b/tests/playwright-test/reporter-preprocess-suite.spec.ts index d9139a34c518d..1a3d67dc09eef 100644 --- a/tests/playwright-test/reporter-preprocess-suite.spec.ts +++ b/tests/playwright-test/reporter-preprocess-suite.spec.ts @@ -16,15 +16,15 @@ import { test, expect } from './playwright-test-fixtures'; -test('preprocessSuite sees the filtered corpus, can skip tests, and records the caller location', async ({ runInlineTest }) => { - // preprocessSuite runs between project setup and onBegin and sees the .only-narrowed corpus. +test('preprocess sees the filtered corpus, can skip tests, and records the caller location', async ({ runInlineTest }) => { + // preprocess runs between project setup and onBegin and sees the .only-narrowed corpus. const only = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { console.log('%% plan: ' + suite.allTests().map(t => t.title).join(',')); for (const t of suite.allTests()) - if (t.title.includes('skip-me')) t.skip('planned skip'); + if (t.title.includes('skip-me')) testRun.skip(t, 'planned skip'); } onBegin(config, suite) { console.log('%% onBegin: ' + suite.allTests().map(t => t.title).join(',')); @@ -51,15 +51,15 @@ test('preprocessSuite sees the filtered corpus, can skip tests, and records the 'plan: run-me,skip-me', 'onBegin: run-me,skip-me', 'end run-me status=passed expected=passed ann= loc=none', - // The skip annotation location points at the reporter's `t.skip(...)` call (line 6 of reporter.ts). + // The skip annotation location points at the reporter's `testRun.skip(...)` call (line 6 of reporter.ts). 'end skip-me status=skipped expected=skipped ann=skip:planned skip loc=reporter.ts:6', ]); - // preprocessSuite respects --grep. + // preprocess respects --grep. const grep = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite }) { console.log('%% plan: ' + suite.allTests().map(t => t.title).join(',')); } } @@ -75,11 +75,11 @@ test('preprocessSuite sees the filtered corpus, can skip tests, and records the expect(grep.exitCode).toBe(0); expect(grep.outputLines).toEqual(['plan: foo-one']); - // preprocessSuite respects --project. + // preprocess respects --project. const project = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite }) { console.log('%% plan projects: ' + suite.suites.map(s => s.title).join(',')); } } @@ -104,11 +104,11 @@ test('TestCase.exclude and Suite.exclude remove entries from the run and report' const result = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { for (const t of suite.allTests()) - if (t.title === 'excluded-test') t.exclude(); + if (t.title === 'excluded-test') testRun.exclude(t); const visit = (s) => { - if (s.title === 'excluded-suite') s.exclude(); + if (s.title === 'excluded-suite') testRun.exclude(s); else for (const child of s.suites || []) visit(child); }; visit(suite); @@ -144,9 +144,9 @@ test('Suite.skip cascades to all descendants', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { const visit = (s) => { - if (s.title === 'doomed') s.skip('whole group'); + if (s.title === 'doomed') testRun.skip(s, 'whole group'); for (const child of s.suites || []) visit(child); }; visit(suite); @@ -176,14 +176,15 @@ test('Suite.skip cascades to all descendants', async ({ runInlineTest }) => { ]); }); -test('disposition methods throw when called outside preprocessSuite, and the root suite cannot be excluded', async ({ runInlineTest }) => { +test('TestRun methods throw outside preprocess, and the root suite cannot be excluded', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { - // Excluding the root suite is banned even during preprocessSuite. + async preprocess({ config, suite, testRun }) { + this.testRun = testRun; + // Excluding the root suite is banned even during preprocess. try { - suite.exclude(); + testRun.exclude(suite); console.log('%% root-exclude: no-throw'); } catch (e) { console.log('%% root-exclude: ' + e.message); @@ -195,7 +196,7 @@ test('disposition methods throw when called outside preprocessSuite, and the roo for (const [label, obj] of [['TestCase', testCase], ['Suite', fileSuite]]) { for (const method of ['skip', 'fixme', 'fail', 'exclude']) { try { - obj[method](); + this.testRun[method](obj); console.log('%% ' + label + '.' + method + ': no-throw'); } catch (e) { console.log('%% ' + label + '.' + method + ': ' + e.message); @@ -215,23 +216,23 @@ test('disposition methods throw when called outside preprocessSuite, and the roo expect(result.exitCode).toBe(0); expect(result.outputLines).toEqual([ - 'root-exclude: Suite.exclude() cannot be called on the root suite.', - 'TestCase.skip: TestCase.skip() can only be called from Reporter.preprocessSuite().', - 'TestCase.fixme: TestCase.fixme() can only be called from Reporter.preprocessSuite().', - 'TestCase.fail: TestCase.fail() can only be called from Reporter.preprocessSuite().', - 'TestCase.exclude: TestCase.exclude() can only be called from Reporter.preprocessSuite().', - 'Suite.skip: Suite.skip() can only be called from Reporter.preprocessSuite().', - 'Suite.fixme: Suite.fixme() can only be called from Reporter.preprocessSuite().', - 'Suite.fail: Suite.fail() can only be called from Reporter.preprocessSuite().', - 'Suite.exclude: Suite.exclude() can only be called from Reporter.preprocessSuite().', + 'root-exclude: TestRun.exclude() cannot be called on the root suite.', + 'TestCase.skip: TestRun.skip() can only be called from Reporter.preprocess().', + 'TestCase.fixme: TestRun.fixme() can only be called from Reporter.preprocess().', + 'TestCase.fail: TestRun.fail() can only be called from Reporter.preprocess().', + 'TestCase.exclude: TestRun.exclude() can only be called from Reporter.preprocess().', + 'Suite.skip: TestRun.skip() can only be called from Reporter.preprocess().', + 'Suite.fixme: TestRun.fixme() can only be called from Reporter.preprocess().', + 'Suite.fail: TestRun.fail() can only be called from Reporter.preprocess().', + 'Suite.exclude: TestRun.exclude() can only be called from Reporter.preprocess().', ]); }); -test('preprocessSuite throwing aborts the run before onBegin', async ({ runInlineTest }) => { +test('preprocess throwing aborts the run before onBegin', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite }) { throw new Error('plan-aborted'); } onBegin(config, suite) { @@ -256,15 +257,15 @@ test('preprocessSuite throwing aborts the run before onBegin', async ({ runInlin expect(result.outputLines).not.toContain('onBegin: 1'); }); -test('multiple reporters: preprocessSuite called in order, annotations accumulate, exclude prunes for next reporter', async ({ runInlineTest }) => { +test('multiple reporters: preprocess called in order, annotations accumulate, exclude prunes for next reporter', async ({ runInlineTest }) => { const result = await runInlineTest({ 'first.ts': ` class R { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { console.log('%% first plan sees: ' + suite.allTests().map(t => t.title).join(',')); for (const t of suite.allTests()) { - if (t.title === 'gone') t.exclude(); - else t.fail('first reason'); + if (t.title === 'gone') testRun.exclude(t); + else testRun.fail(t, 'first reason'); } } onTestEnd(test, result) { @@ -275,9 +276,9 @@ test('multiple reporters: preprocessSuite called in order, annotations accumulat `, 'second.ts': ` class R { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { console.log('%% second plan sees: ' + suite.allTests().map(t => t.title).join(',')); - suite.allTests()[0].skip('second reason'); + testRun.skip(suite.allTests()[0], 'second reason'); } } module.exports = R; @@ -303,15 +304,15 @@ test('multiple reporters: a later reporter observes an earlier reporter Suite.sk const result = await runInlineTest({ 'first.ts': ` class R { - async preprocessSuite(config, suite) { - suite.allTests()[0].parent.skip('first reason'); + async preprocess({ config, suite, testRun }) { + testRun.skip(suite.allTests()[0].parent, 'first reason'); } } module.exports = R; `, 'second.ts': ` class R { - async preprocessSuite(config, suite) { + async preprocess({ config, suite }) { const skipped = suite.allTests().filter(t => t.expectedStatus === 'skipped').map(t => t.title); console.log('%% second sees skipped: ' + skipped.join(',')); } @@ -333,16 +334,16 @@ test('multiple reporters: a later reporter observes an earlier reporter Suite.sk expect(result.outputLines).toContain('second sees skipped: one,two'); }); -test('implementsSharding disables the built-in shard filter; preprocessSuite sees the full corpus', async ({ runInlineTest }) => { +test('implementsSharding disables the built-in shard filter; preprocess sees the full corpus', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` class R { - async preprocessSuite(config, suite) { - // preprocessSuite observes the full, un-sharded corpus regardless of --shard. + async preprocess({ config, suite, testRun }) { + // preprocess observes the full, un-sharded corpus regardless of --shard. console.log('%% plan: ' + suite.allTests().map(t => t.title).join(',')); let i = 0; for (const t of suite.allTests()) { - if (i++ % 2 === 1) t.exclude(); + if (i++ % 2 === 1) testRun.exclude(t); } return { implementsSharding: true }; } @@ -361,7 +362,7 @@ test('implementsSharding disables the built-in shard filter; preprocessSuite see }, { reporter: '', workers: 1 }); expect(result.exitCode).toBe(0); - // preprocessSuite sees all four tests even though --shard=1/2 was configured. + // preprocess sees all four tests even though --shard=1/2 was configured. expect(result.outputLines).toContain('plan: t0,t1,t2,t3'); // The reporter's own exclusions define the shard; the built-in shard filter did NOT run // (it would have produced a different split), so t0,t2 remain. @@ -372,13 +373,13 @@ test('multiple reporters declaring implementsSharding throws', async ({ runInlin const result = await runInlineTest({ 'reporter-a.ts': ` class A { - preprocessSuite() { return { implementsSharding: true }; } + preprocess() { return { implementsSharding: true }; } onError(err) { console.log('%% error: ' + err.message); } } module.exports = A; `, 'reporter-b.ts': ` - class B { preprocessSuite() { return { implementsSharding: true }; } } + class B { preprocess() { return { implementsSharding: true }; } } module.exports = B; `, 'playwright.config.ts': `module.exports = { reporter: [['./reporter-a.ts'], ['./reporter-b.ts']] };`, @@ -396,14 +397,15 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on const result = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { + this.testRun = testRun; console.log('%% plan projects: ' + suite.suites.map(s => s.title).join(',')); console.log('%% plan tests: ' + suite.allTests().map(t => t.title).join(',')); this.preprocessedTests = new Set(suite.allTests()); const setupTest = suite.allTests().find(t => t.title === 'setup-test'); for (const method of ['skip', 'fixme', 'fail', 'exclude']) { try { - setupTest[method](); + testRun[method](setupTest); console.log('%% dep-' + method + ': no-throw'); } catch (e) { console.log('%% dep-' + method + ': ' + e.message); @@ -411,7 +413,7 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on } const setupProject = suite.suites.find(s => s.title === 'setup'); try { - setupProject.exclude(); + testRun.exclude(setupProject); console.log('%% dep-suite-exclude: no-throw'); } catch (e) { console.log('%% dep-suite-exclude: ' + e.message); @@ -421,7 +423,7 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on console.log('%% same test objects: ' + suite.allTests().every(test => this.preprocessedTests.has(test))); const setupTest = suite.allTests().find(t => t.title === 'setup-test'); try { - setupTest.skip(); + this.testRun.skip(setupTest); console.log('%% dep-after-preprocess: no-throw'); } catch (e) { console.log('%% dep-after-preprocess: ' + e.message); @@ -461,13 +463,13 @@ test('plan.suite exposes setup/teardown dependency projects but they are read-on expect(result.outputLines).toEqual([ 'plan projects: teardown,setup,main', 'plan tests: teardown-test,setup-test,main-test', - 'dep-skip: TestCase.skip() cannot be called on a setup or teardown project test; these always run in full.', - 'dep-fixme: TestCase.fixme() cannot be called on a setup or teardown project test; these always run in full.', - 'dep-fail: TestCase.fail() cannot be called on a setup or teardown project test; these always run in full.', - 'dep-exclude: TestCase.exclude() cannot be called on a setup or teardown project test; these always run in full.', - 'dep-suite-exclude: Suite.exclude() cannot be called on a setup or teardown project; these always run in full.', + 'dep-skip: TestRun.skip() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-fixme: TestRun.fixme() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-fail: TestRun.fail() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-exclude: TestRun.exclude() cannot be called on a setup or teardown project test; these always run in full.', + 'dep-suite-exclude: TestRun.exclude() cannot be called on a setup or teardown project; these always run in full.', 'same test objects: true', - 'dep-after-preprocess: TestCase.skip() can only be called from Reporter.preprocessSuite().', + 'dep-after-preprocess: TestRun.skip() can only be called from Reporter.preprocess().', 'ran setup/setup-test', 'ran main/main-test', 'ran teardown/teardown-test', @@ -478,9 +480,9 @@ test('plan.suite temporarily exposes dependencies without changing final project const result = await runInlineTest({ 'reporter.ts': ` class Reporter { - async preprocessSuite(config, suite) { + async preprocess({ config, suite, testRun }) { console.log('%% plan projects: ' + suite.suites.map(suite => suite.title).join(',')); - suite.suites.find(suite => suite.title === 'main').exclude(); + testRun.exclude(suite.suites.find(suite => suite.title === 'main')); } onTestEnd(test, result) { console.log('%% ran ' + test.parent.project().name + '/' + test.title); diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 26633625a568f..d08bf9c8665e3 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -42,7 +42,7 @@ export interface FullResult { } export interface Reporter { - preprocessSuite?(config: FullConfig, suite: Suite): Promise<{ implementsSharding?: boolean } | undefined | void> | { implementsSharding?: boolean } | void; + preprocess?(params: { config: FullConfig, suite: Suite, testRun: TestRun }): Promise<{ implementsSharding?: boolean } | undefined | void> | { implementsSharding?: boolean } | void; onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void; } From 7626493a6e307a9f5b852ae758177429393e8721 Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Wed, 15 Jul 2026 14:35:58 +0200 Subject: [PATCH 2/2] feat(test runner): refine reporter preprocessing API Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: cc00d7ab-1f1b-4571-9feb-3301d11ef143 --- docs/src/test-reporter-api/class-reporter.md | 4 +- docs/src/test-reporter-api/class-testrun.md | 15 ++-- packages/playwright/src/common/test.ts | 33 -------- .../src/reporters/internalReporter.ts | 39 +-------- .../playwright/src/reporters/multiplexer.ts | 11 +-- .../playwright/src/reporters/reporterV2.ts | 2 +- packages/playwright/src/runner/loadUtils.ts | 9 +- .../playwright/src/runner/reporterTestRun.ts | 83 +++++++++++++++++++ packages/playwright/types/testReporter.d.ts | 62 +++++++++----- ...te.spec.ts => reporter-preprocess.spec.ts} | 12 +-- .../overrides-testReporter.d.ts | 1 - 11 files changed, 156 insertions(+), 115 deletions(-) create mode 100644 packages/playwright/src/runner/reporterTestRun.ts rename tests/playwright-test/{reporter-preprocess-suite.spec.ts => reporter-preprocess.spec.ts} (97%) diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index a0c481f6ba79e..68a2a21dab9b5 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -300,8 +300,6 @@ Whether this reporter uses stdio for reporting. When it does not, Playwright Tes ## optional async method: Reporter.preprocess * since: v1.62 -- `result` ?<[Object]> - - `implementsSharding` ?<[boolean]> When `true`, Playwright skips its built-in shard filter for this run, leaving sharding to the reporter (typically implemented by calling [`method: TestRun.exclude`] on out-of-shard tests). Called after the configuration has been resolved and before [`method: Reporter.onBegin`]. Allows a reporter to mark individual tests as skipped, excluded, fixed or failing. @@ -315,4 +313,4 @@ Called after the configuration has been resolved and before [`method: Reporter.o The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that match the current invocation. Setup and dependency projects are readonly and cannot be changed through [TestRun]. -The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocess`] returns, unless the returned `implementsSharding` is `true`. +The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its built-in sharding after [`method: Reporter.preprocess`] returns, unless the reporter calls [`method: TestRun.skipSharding`]. diff --git a/docs/src/test-reporter-api/class-testrun.md b/docs/src/test-reporter-api/class-testrun.md index fa6e01066901f..563accf59f293 100644 --- a/docs/src/test-reporter-api/class-testrun.md +++ b/docs/src/test-reporter-api/class-testrun.md @@ -2,7 +2,7 @@ * since: v1.62 * langs: js -Controls which tests will run and their expected status. A [TestRun] is available during [`method: Reporter.preprocess`]. +Controls which tests will run and their expected status. A [TestRun] is available during [`method: Reporter.preprocess`]. Setup and teardown projects cannot be changed. ## method: TestRun.exclude * since: v1.62 @@ -13,7 +13,7 @@ Excludes a test or suite from the run. Excluded tests do not appear in the repor * since: v1.62 - `test` <[TestCase]|[Suite]> -Test or suite to exclude. The root suite and setup or teardown projects cannot be excluded. +Test or suite to exclude. The root suite cannot be excluded. ## method: TestRun.fail * since: v1.62 @@ -24,7 +24,7 @@ Marks a test or every test in a suite as "should fail". Playwright runs the test * since: v1.62 - `test` <[TestCase]|[Suite]> -Test or suite to mark as expected-to-fail. Setup and teardown projects cannot be changed. +Test or suite to mark as expected-to-fail. ### param: TestRun.fail.reason * since: v1.62 @@ -41,7 +41,7 @@ Marks a test or every test in a suite as fixme. The test bodies are not executed * since: v1.62 - `test` <[TestCase]|[Suite]> -Test or suite to mark as fixme. Setup and teardown projects cannot be changed. +Test or suite to mark as fixme. ### param: TestRun.fixme.reason * since: v1.62 @@ -58,10 +58,15 @@ Skips a test or every test in a suite. The test bodies are not executed and the * since: v1.62 - `test` <[TestCase]|[Suite]> -Test or suite to skip. Setup and teardown projects cannot be changed. +Test or suite to skip. ### param: TestRun.skip.reason * since: v1.62 - `reason` ?<[string]> Optional explanation surfaced as the annotation description. + +## method: TestRun.skipSharding +* since: v1.62 + +Disables Playwright's built-in shard filter for this run, leaving sharding to the reporter. Reporters typically implement their own sharding by calling [`method: TestRun.exclude`] on out-of-shard tests. diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 8236ea529e9bc..60d1fc2e2156f 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -57,7 +57,6 @@ export class Suite extends Base { _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; _fullProject: FullProjectInternal | undefined; _fileId: string | undefined; - _preprocessReadonly = false; readonly _type: 'root' | 'project' | 'file' | 'describe'; constructor(title: string, type: 'root' | 'project' | 'file' | 'describe') { @@ -261,26 +260,6 @@ export class Suite extends Base { return this._fullProject?.project || this.parent?.project(); } - _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined) { - if (!this.parent) - throw new Error(`TestRun.${type}() cannot be called on the root suite.`); - if (this._resolvePreprocessReadonly()) - throw new Error(`TestRun.${type}() cannot be called on a setup or teardown project; these always run in full.`); - for (const test of this.allTests()) - test._applyPlanAnnotation({ type, description: reason, location }); - } - - _exclude() { - if (this._resolvePreprocessReadonly()) - throw new Error(`TestRun.exclude() cannot be called on a setup or teardown project; these always run in full.`); - if (!this.parent) - throw new Error(`TestRun.exclude() cannot be called on the root suite.`); - this.parent._detach(this); - } - - _resolvePreprocessReadonly(): boolean { - return this._preprocessReadonly || !!this.parent?._resolvePreprocessReadonly(); - } } export class TestCase extends Base implements reporterTypes.TestCase { @@ -339,12 +318,6 @@ export class TestCase extends Base implements reporterTypes.TestCase { ]; } - _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, reason: string | undefined) { - if (this.parent._resolvePreprocessReadonly()) - throw new Error(`TestRun.${type}() cannot be called on a setup or teardown project test; these always run in full.`); - this._applyPlanAnnotation({ type, description: reason, location }); - } - _applyPlanAnnotation(annotation: TestAnnotation): void { this.annotations.push(annotation); this._planAnnotations.push(annotation); @@ -354,12 +327,6 @@ export class TestCase extends Base implements reporterTypes.TestCase { this.expectedStatus = 'failed'; } - _exclude() { - if (this.parent._resolvePreprocessReadonly()) - throw new Error(`TestRun.exclude() cannot be called on a setup or teardown project test; these always run in full.`); - this.parent._detach(this); - } - _serialize(): any { return { kind: 'test', diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index c14c035fc2efb..26e9ba42dcb34 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -20,38 +20,12 @@ import { monotonicTime } from '@isomorphic/time'; import { internalScreen, prepareErrorStack, relativeFilePath } from './base'; import { Multiplexer } from './multiplexer'; -import { test as testNs, transform } from '../common'; +import { test as testNs } from '../common'; import * as babel from '../transform/babelBundle'; import { wrapReporterAsV2 } from './reporterV2'; import type { AnyReporter, ReporterPreprocessParams, ReporterV2 } from './reporterV2'; -import type { FullConfig, FullResult, Location, Suite as ReporterSuite, TestCase, TestError, TestResult, TestRun as ReporterTestRun, TestStep, WorkerInfo } from '../../types/testReporter'; - -type ReporterTestRunTarget = ReporterSuite | TestCase; - -class ReporterTestRunImpl implements ReporterTestRun { - private _active = true; - - close() { - this._active = false; - } - - skip = transform.wrapFunctionWithLocation((location: Location, target: ReporterTestRunTarget, reason?: string) => this._modifier('skip', location, target, reason)); - fixme = transform.wrapFunctionWithLocation((location: Location, target: ReporterTestRunTarget, reason?: string) => this._modifier('fixme', location, target, reason)); - fail = transform.wrapFunctionWithLocation((location: Location, target: ReporterTestRunTarget, reason?: string) => this._modifier('fail', location, target, reason)); - - exclude(target: ReporterTestRunTarget) { - if (!this._active) - throw new Error(`TestRun.exclude() can only be called from Reporter.preprocess().`); - (target as testNs.Suite | testNs.TestCase)._exclude(); - } - - private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, target: ReporterTestRunTarget, reason: string | undefined) { - if (!this._active) - throw new Error(`TestRun.${type}() can only be called from Reporter.preprocess().`); - (target as testNs.Suite | testNs.TestCase)._modifier(type, location, reason); - } -} +import type { FullConfig, FullResult, TestCase, TestError, TestResult, TestStep, WorkerInfo } from '../../types/testReporter'; export class InternalReporter implements ReporterV2 { private _reporter: Multiplexer; @@ -79,13 +53,8 @@ export class InternalReporter implements ReporterV2 { this._reporter.onConfigure?.(config); } - async preprocess(params: Omit) { - const testRun = new ReporterTestRunImpl(); - try { - return await this._reporter.preprocess({ ...params, testRun }); - } finally { - testRun.close(); - } + async preprocess(params: ReporterPreprocessParams) { + await this._reporter.preprocess(params); } onBegin(suite: testNs.Suite) { diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.ts index b5a051d379f33..2a370e8fd8d1c 100644 --- a/packages/playwright/src/reporters/multiplexer.ts +++ b/packages/playwright/src/reporters/multiplexer.ts @@ -46,15 +46,8 @@ export class Multiplexer implements ReporterV2 { // they propagate so the run aborts before onBegin. Reporters use preprocess // to mutate the corpus; silently dropping a planning error would let // an inconsistent (partial-mutation) state reach the workers. - const shardingReporters: ReporterV2[] = []; - for (const reporter of this._reporters) { - const result = await reporter.preprocess?.(params); - if (result?.implementsSharding) - shardingReporters.push(reporter); - } - if (shardingReporters.length > 1) - throw new Error(`Multiple reporters declare 'implementsSharding': ${shardingReporters.map(r => r.constructor?.name ?? 'reporter').join(', ')}. Only one reporter may handle sharding.`); - return { implementsSharding: shardingReporters.length > 0 }; + for (const reporter of this._reporters) + await reporter.preprocess?.(params); } onBegin(suite: test.Suite) { diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.ts index a894c59bb1810..131e2475a5438 100644 --- a/packages/playwright/src/reporters/reporterV2.ts +++ b/packages/playwright/src/reporters/reporterV2.ts @@ -34,7 +34,7 @@ export type ReporterPreprocessParams = { export interface ReporterV2 { onConfigure?(config: FullConfig): void; - preprocess?(params: ReporterPreprocessParams): { implementsSharding?: boolean } | Promise<{ implementsSharding?: boolean } | undefined | void> | void; + preprocess?(params: ReporterPreprocessParams): Promise | void; onBegin?(suite: Suite): void; onTestBegin?(test: TestCase, result: TestResult): void; onStdOut?(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index 8b1bfdd167f4f..e409d110210e1 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -23,6 +23,7 @@ import { toPosixPath } from '@utils/fileUtils'; import { InProcessLoaderHost, OutOfProcessLoaderHost } from './loaderHost'; import { createTitleMatcher, errorWithFile, parseLocationArg } from '../util'; import { buildProjectsClosure, collectFilesForProject } from './projectUtils'; +import { ReporterTestRunImpl } from './reporterTestRun'; import { createTestGroups, filterForShard } from './testGroups'; import { cc, config as commonConfig, FullConfigInternal, suiteUtils, test as testNs, transform } from '../common'; @@ -170,22 +171,24 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho if (type !== 'dependency') continue; const dependencySuite = buildProjectSuite(project, projectSuites.get(project)!); - dependencySuite._preprocessReadonly = true; dependencySuites.set(project, dependencySuite); rootSuite._prependSuite(dependencySuite); } - const preprocessResult = await testRun.reporter.preprocess({ + const reporterTestRun = new ReporterTestRunImpl(rootSuite, new Set(dependencySuites.values())); + await testRun.reporter.preprocess({ config: config.config, suite: rootSuite, + testRun: reporterTestRun, }); + reporterTestRun.close(); // Continue sharding and filtering pipeline with top-level projects only. for (const dependencySuite of dependencySuites.values()) rootSuite._detach(dependencySuite); // Shard only the top-level projects. - if (config.config.shard && !preprocessResult?.implementsSharding) { + if (config.config.shard && !reporterTestRun.shouldSkipSharding()) { // Create test groups for top-level projects. const testGroups: TestGroup[] = []; for (const projectSuite of rootSuite.suites) { diff --git a/packages/playwright/src/runner/reporterTestRun.ts b/packages/playwright/src/runner/reporterTestRun.ts new file mode 100644 index 0000000000000..c0e0c11ecaebf --- /dev/null +++ b/packages/playwright/src/runner/reporterTestRun.ts @@ -0,0 +1,83 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test as testNs, transform } from '../common'; + +import type { Location, Suite as ReporterSuite, TestCase as ReporterTestCase, TestRun as ReporterTestRun } from '../../types/testReporter'; + +type ReporterTestRunTarget = ReporterSuite | ReporterTestCase; + +export class ReporterTestRunImpl implements ReporterTestRun { + private _active = true; + private _skipSharding = false; + + constructor(private _rootSuite: testNs.Suite, private _readonlyProjectSuites: Set) {} + + close() { + this._active = false; + } + + shouldSkipSharding() { + return this._skipSharding; + } + + skipSharding() { + if (!this._active) + throw new Error(`TestRun.skipSharding() can only be called from Reporter.preprocess().`); + if (this._skipSharding) + throw new Error(`Multiple reporters called 'skipSharding'. Only one reporter may handle sharding.`); + this._skipSharding = true; + } + + skip = transform.wrapFunctionWithLocation((location: Location, target: ReporterTestRunTarget, reason?: string) => this._modifier('skip', location, target, reason)); + fixme = transform.wrapFunctionWithLocation((location: Location, target: ReporterTestRunTarget, reason?: string) => this._modifier('fixme', location, target, reason)); + fail = transform.wrapFunctionWithLocation((location: Location, target: ReporterTestRunTarget, reason?: string) => this._modifier('fail', location, target, reason)); + + exclude(target: ReporterTestRunTarget) { + if (!this._active) + throw new Error(`TestRun.exclude() can only be called from Reporter.preprocess().`); + const internalTarget = target as testNs.Suite | testNs.TestCase; + this._checkTarget('exclude', internalTarget); + internalTarget.parent!._detach(internalTarget); + } + + private _modifier(type: 'skip' | 'fixme' | 'fail', location: Location, _target: ReporterTestRunTarget, reason: string | undefined) { + if (!this._active) + throw new Error(`TestRun.${type}() can only be called from Reporter.preprocess().`); + const target = _target as testNs.Suite | testNs.TestCase; + this._checkTarget(type, target); + const annotation = { type, description: reason, location }; + if (target instanceof testNs.Suite) { + for (const test of target.allTests()) + test._applyPlanAnnotation(annotation); + } else { + target._applyPlanAnnotation(annotation); + } + } + + private _checkTarget(method: string, target: testNs.Suite | testNs.TestCase) { + if (target === this._rootSuite) + throw new Error(`TestRun.${method}() cannot be called on the root suite.`); + let suite: testNs.Suite | undefined = target instanceof testNs.Suite ? target : target.parent; + while (suite) { + if (this._readonlyProjectSuites.has(suite)) { + const targetType = target instanceof testNs.TestCase ? ' test' : ''; + throw new Error(`TestRun.${method}() cannot be called on a setup or teardown project${targetType}; these always run in full.`); + } + suite = suite.parent; + } + } +} diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index b031fa03f1155..550b27c86090a 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -145,20 +145,6 @@ export interface FullResult { * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin). */ export interface Reporter { - /** - * Called after the configuration has been resolved and before - * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin). Allows a - * reporter to mark individual tests as skipped, excluded, fixed or failing. - * @param params The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that - * match the current invocation. Setup and dependency projects are readonly and cannot be changed through - * [TestRun](https://playwright.dev/docs/api/class-testrun). - * - * The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its - * built-in sharding after - * [reporter.preprocess(params)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess) returns, unless - * the returned `implementsSharding` is `true`. - */ - preprocess?(params: { config: FullConfig, suite: Suite, testRun: TestRun }): Promise<{ implementsSharding?: boolean } | undefined | void> | { implementsSharding?: boolean } | void; /** * Called after all tests have been run, or testing has been interrupted. Note that this method may return a [Promise] * and Playwright Test will await it. Reporter is allowed to override the status and hence affect the exit code of the @@ -241,6 +227,36 @@ export interface Reporter { */ onTestEnd?(test: TestCase, result: TestResult): void; + /** + * Called after the configuration has been resolved and before + * [reporter.onBegin(config, suite)](https://playwright.dev/docs/api/class-reporter#reporter-on-begin). Allows a + * reporter to mark individual tests as skipped, excluded, fixed or failing. + * @param params The suite reflects `--project`, `--grep`/`--grep-invert` and `.only` filtering, so it only contains tests that + * match the current invocation. Setup and dependency projects are readonly and cannot be changed through + * [TestRun](https://playwright.dev/docs/api/class-testrun). + * + * The suite ignores the `--shard` argument: it always contains the full, un-sharded corpus. Playwright applies its + * built-in sharding after + * [reporter.preprocess(params)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess) returns, unless + * the reporter calls [testRun.skipSharding()](https://playwright.dev/docs/api/class-testrun#test-run-skip-sharding). + */ + preprocess?(params: { + /** + * Resolved configuration. + */ + config: FullConfig; + + /** + * The root suite that contains the projects, files and test cases that will run. + */ + suite: Suite; + + /** + * Control which tests will run and their expected status. + */ + testRun: TestRun; + }): Promise; + /** * Whether this reporter uses stdio for reporting. When it does not, Playwright Test could add some output to enhance * user experience. If your reporter does not print to the terminal, it is strongly recommended to return `false`. @@ -735,20 +751,21 @@ export interface TestResult { /** * Controls which tests will run and their expected status. A [TestRun](https://playwright.dev/docs/api/class-testrun) * is available during - * [reporter.preprocess(params)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess). + * [reporter.preprocess(params)](https://playwright.dev/docs/api/class-reporter#reporter-preprocess). Setup and + * teardown projects cannot be changed. */ export interface TestRun { /** * Excludes a test or suite from the run. Excluded tests do not appear in the report and their bodies are not * executed. - * @param test Test or suite to exclude. The root suite and setup or teardown projects cannot be excluded. + * @param test Test or suite to exclude. The root suite cannot be excluded. */ exclude(test: TestCase|Suite): void; /** * Marks a test or every test in a suite as "should fail". Playwright runs the tests and ensures they are actually * failing, useful for documenting broken functionality until it is fixed. - * @param test Test or suite to mark as expected-to-fail. Setup and teardown projects cannot be changed. + * @param test Test or suite to mark as expected-to-fail. * @param reason Optional explanation surfaced as the annotation description. */ fail(test: TestCase|Suite, reason?: string): void; @@ -756,17 +773,24 @@ export interface TestRun { /** * Marks a test or every test in a suite as fixme. The test bodies are not executed and the tests are reported as * skipped, with the intention to fix them. - * @param test Test or suite to mark as fixme. Setup and teardown projects cannot be changed. + * @param test Test or suite to mark as fixme. * @param reason Optional explanation surfaced as the annotation description. */ fixme(test: TestCase|Suite, reason?: string): void; /** * Skips a test or every test in a suite. The test bodies are not executed and the tests are reported as skipped. - * @param test Test or suite to skip. Setup and teardown projects cannot be changed. + * @param test Test or suite to skip. * @param reason Optional explanation surfaced as the annotation description. */ skip(test: TestCase|Suite, reason?: string): void; + + /** + * Disables Playwright's built-in shard filter for this run, leaving sharding to the reporter. Reporters typically + * implement their own sharding by calling + * [testRun.exclude(test)](https://playwright.dev/docs/api/class-testrun#test-run-exclude) on out-of-shard tests. + */ + skipSharding(): void; } /** diff --git a/tests/playwright-test/reporter-preprocess-suite.spec.ts b/tests/playwright-test/reporter-preprocess.spec.ts similarity index 97% rename from tests/playwright-test/reporter-preprocess-suite.spec.ts rename to tests/playwright-test/reporter-preprocess.spec.ts index 1a3d67dc09eef..12930c32f57d9 100644 --- a/tests/playwright-test/reporter-preprocess-suite.spec.ts +++ b/tests/playwright-test/reporter-preprocess.spec.ts @@ -334,18 +334,18 @@ test('multiple reporters: a later reporter observes an earlier reporter Suite.sk expect(result.outputLines).toContain('second sees skipped: one,two'); }); -test('implementsSharding disables the built-in shard filter; preprocess sees the full corpus', async ({ runInlineTest }) => { +test('skipSharding disables the built-in shard filter; preprocess sees the full corpus', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': ` class R { async preprocess({ config, suite, testRun }) { // preprocess observes the full, un-sharded corpus regardless of --shard. console.log('%% plan: ' + suite.allTests().map(t => t.title).join(',')); + testRun.skipSharding(); let i = 0; for (const t of suite.allTests()) { if (i++ % 2 === 1) testRun.exclude(t); } - return { implementsSharding: true }; } onBegin(config, suite) { console.log('%% begin: ' + suite.allTests().map(t => t.title).join(',')); @@ -369,17 +369,17 @@ test('implementsSharding disables the built-in shard filter; preprocess sees the expect(result.outputLines).toContain('begin: t0,t2'); }); -test('multiple reporters declaring implementsSharding throws', async ({ runInlineTest }) => { +test('multiple reporters declaring custom sharding throws', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter-a.ts': ` class A { - preprocess() { return { implementsSharding: true }; } + preprocess({ testRun }) { testRun.skipSharding(); } onError(err) { console.log('%% error: ' + err.message); } } module.exports = A; `, 'reporter-b.ts': ` - class B { preprocess() { return { implementsSharding: true }; } } + class B { preprocess({ testRun }) { testRun.skipSharding(); } } module.exports = B; `, 'playwright.config.ts': `module.exports = { reporter: [['./reporter-a.ts'], ['./reporter-b.ts']] };`, @@ -390,7 +390,7 @@ test('multiple reporters declaring implementsSharding throws', async ({ runInlin }, { reporter: '', workers: 1 }); expect(result.exitCode).not.toBe(0); - expect(result.outputLines.join('\n')).toContain(`Multiple reporters declare 'implementsSharding'`); + expect(result.outputLines.join('\n')).toContain(`Multiple reporters called 'skipSharding'`); }); test('plan.suite exposes setup/teardown dependency projects but they are read-only', async ({ runInlineTest }) => { diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index d08bf9c8665e3..6ca13fdcf2157 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -42,7 +42,6 @@ export interface FullResult { } export interface Reporter { - preprocess?(params: { config: FullConfig, suite: Suite, testRun: TestRun }): Promise<{ implementsSharding?: boolean } | undefined | void> | { implementsSharding?: boolean } | void; onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void; }