diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 0e8babce10cfd..eece9a746f76f 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -94,6 +94,7 @@ export class FullConfigInternal { reporter: takeFirst(configCLIOverrides.reporter, resolveReporters(userConfig.reporter, configDir), [[defaultReporter]]), reportSlowTests: takeFirst(userConfig.reportSlowTests, { max: 5, threshold: 15000 }), quiet: takeFirst(configCLIOverrides.quiet, userConfig.quiet, false), + failDependentTests: takeFirst(configCLIOverrides.failDependentTests, userConfig.failDependentTests, false), projects: [], shard: takeFirst(configCLIOverrides.shard, userConfig.shard, null), updateSnapshots: takeFirst(configCLIOverrides.updateSnapshots, userConfig.updateSnapshots, 'missing'), diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts index 909df3dc8fb13..9fdf8581f62d0 100644 --- a/packages/playwright/src/common/ipc.ts +++ b/packages/playwright/src/common/ipc.ts @@ -30,6 +30,7 @@ export type ConfigCLIOverrides = { outputDir?: string; preserveOutputDir?: boolean; quiet?: boolean; + failDependentTests?: boolean; repeatEach?: number; retries?: number; reporter?: ReporterDescription[]; diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 3e7fb30a1ae81..47979ec85b255 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -354,6 +354,24 @@ export class TestCase extends Base implements reporterTypes.TestCase { return result; } + _appendFailedTestResult(): reporterTypes.TestResult { + const result: reporterTypes.TestResult = { + retry: this.results.length, + parallelIndex: -1, + workerIndex: -1, + duration: 0, + startTime: new Date(), + stdout: [], + stderr: [], + attachments: [], + status: 'failed', + steps: [], + errors: [{ message: 'dependency test failed' }] + }; + this.results.push(result); + return result; + } + _grepTitle() { const path: string[] = []; this.parent._collectGrepTitlePath(path); diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index f96547d427a1f..13436ce01be22 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -592,6 +592,7 @@ export const baseFullConfig: reporterTypes.FullConfig = { configFile: '', rootDir: '', quiet: false, + failDependentTests: false, shard: null, updateSnapshots: 'missing', updateSourceMethod: 'patch', diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index ea0a48fe6a446..c9a97205d64ff 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -295,6 +295,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid maxFailures: options.x ? 1 : (options.maxFailures ? parseInt(options.maxFailures, 10) : undefined), outputDir: options.output ? path.resolve(process.cwd(), options.output) : undefined, quiet: options.quiet ? options.quiet : undefined, + failDependentTests: options.failDependentTests ? options.failDependentTests : undefined, repeatEach: options.repeatEach ? parseInt(options.repeatEach, 10) : undefined, retries: options.retries ? parseInt(options.retries, 10) : undefined, reporter: resolveReporterOption(options.reporter), @@ -375,6 +376,7 @@ const testOptions: [string, string][] = [ ['--pass-with-no-tests', `Makes test run succeed even if no tests were found`], ['--project ', `Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects)`], ['--quiet', `Suppress stdio`], + ['--failDependentTests', `Automatically fail all project tests if project dependencies fail`], ['--repeat-each ', `Run each test N times (default: 1)`], ['--reporter ', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`], ['--retries ', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`], diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index 84cffac573f60..9efaf83737273 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -355,7 +355,7 @@ function createRunTestsTask(): Task { // We don't want to run the test groups belonging to the projects // that depend on the projects that failed previously. const phaseTestGroups: TestGroup[] = []; - for (const { project, testGroups } of projects) { + for (const { project, testGroups, projectSuite } of projects) { // Inherit extra environment variables from dependencies. let extraEnv: Record = {}; for (const dep of project.deps) @@ -367,8 +367,9 @@ function createRunTestsTask(): Task { const hasFailedDeps = project.deps.some(p => !successfulProjects.has(p)); if (!hasFailedDeps) phaseTestGroups.push(...testGroups); + else if (project.fullConfig.config.failDependentTests) + projectSuite.allTests().forEach(test => test._appendFailedTestResult()); } - if (phaseTestGroups.length) { await dispatcher!.run(phaseTestGroups, extraEnvByProjectId); await dispatcher.stop(); diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index caed95b8d5673..fed27975a8d78 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -1332,6 +1332,23 @@ interface TestConfig { */ quiet?: boolean; + /** + * Whether to automatically fail all project tests if one of its dependencies fail + * + * **Usage** + * + * ```js + * // playwright.config.ts + * import { defineConfig } from '@playwright/test'; + * + * export default defineConfig({ + * failDependentTests: true + * }); + * ``` + * + */ + failDependentTests?: boolean; + /** * The number of times to repeat each test, useful for debugging flaky tests. * @@ -1805,6 +1822,8 @@ export interface FullConfig { */ quiet: boolean; + failDependentTests: boolean; + /** * See [testConfig.reportSlowTests](https://playwright.dev/docs/api/class-testconfig#test-config-report-slow-tests). */ diff --git a/tests/playwright-test/playwright.spec.ts b/tests/playwright-test/playwright.spec.ts index 8cd83c40b14a7..da56df7a154aa 100644 --- a/tests/playwright-test/playwright.spec.ts +++ b/tests/playwright-test/playwright.spec.ts @@ -894,3 +894,30 @@ test('page.pause() should disable test timeout', async ({ runInlineTest }) => { expect(result.passed).toBe(1); expect(result.output).toContain('success!'); }); + +test('should automatically fail tests when their dependency tests fail', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = { + failDependentTests: true, + projects: [ { testMatch: /dependent\.test\.ts/, dependencies: ['a'] }, { name: 'a', testMatch: /setup\.test\.ts/ } ] + } + `, + 'setup.test.ts': ` + import { test, expect } from '@playwright/test'; + test('fail', async ({ page }) => { + test.expect(1).toBeFalsy(); + }); + `, + 'dependent.test.ts': ` + import { test, expect } from '@playwright/test'; + test('pass', async ({ page }) => { + test.expect(0).toBeFalsy(); + }); + ` + }, { workers: 1 }); + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(0); + expect(result.failed).toBe(2); + expect(result.skipped).toBe(0); +}); \ No newline at end of file