diff --git a/src/generate/formatter.ts b/src/generate/formatter.ts index 1c5bab56..2f080dd3 100644 --- a/src/generate/formatter.ts +++ b/src/generate/formatter.ts @@ -74,13 +74,14 @@ export class Formatter { } workerHooksCall(type: WorkerHookType, fixturesNames: Set, bddDataVar: string) { - const runWorkerHooksFixture = '$runWorkerHooks'; + const runWorkerHooksFixture = + type === 'beforeAll' ? '$runBeforeAllHooks' : '$registerAfterAllHooks'; const fixturesStr = [...fixturesNames].join(', '); const allFixturesStr = [runWorkerHooksFixture, ...fixturesNames].join(', '); const title = type === 'beforeAll' ? 'BeforeAll Hooks' : 'AfterAll Hooks'; return [ // eslint-disable-next-line max-len - `test.${type}(${this.quoted(title)}, ({ ${allFixturesStr} }) => ${runWorkerHooksFixture}(test, ${this.quoted(type)}, { ${fixturesStr} }, ${bddDataVar}));`, + `test.${type}(${this.quoted(title)}, ({ ${allFixturesStr} }) => ${runWorkerHooksFixture}(test, { ${fixturesStr} }, ${bddDataVar}));`, ]; } diff --git a/src/generate/hooks.ts b/src/generate/hooks.ts index a902a53d..e9cdf2fd 100644 --- a/src/generate/hooks.ts +++ b/src/generate/hooks.ts @@ -27,7 +27,8 @@ import { WorkerHook, WorkerHookType, } from '../hooks/worker'; -import { toBoolean } from '../utils'; +import { areSetsEqual, toBoolean } from '../utils'; +import { exit } from '../utils/exit'; import { Formatter } from './formatter'; import { TestGen } from './test'; @@ -41,10 +42,11 @@ export class TestFileHooks { fillFromTests(tests: TestGen[]) { tests.forEach((test) => { - this.beforeAll.registerHooksForTest(test.tags); - this.afterAll.registerHooksForTest(test.tags); - this.before.registerHooksForTest(test.tags); - this.after.registerHooksForTest(test.tags); + // todo: filter skipped tests + this.beforeAll.registerHooksForTest(test); + this.afterAll.registerHooksForTest(test); + this.before.registerHooksForTest(test); + this.after.registerHooksForTest(test); }); } @@ -77,8 +79,8 @@ class TestFileScenarioHooks { private formatter: Formatter, ) {} - registerHooksForTest(testTags: string[]) { - getScenarioHooksToRun(this.type, testTags).forEach((hook) => this.hooks.add(hook)); + registerHooksForTest(test: TestGen) { + getScenarioHooksToRun(this.type, test.tags).forEach((hook) => this.hooks.add(hook)); } getCustomTestInstances() { @@ -97,14 +99,27 @@ class TestFileScenarioHooks { class TestFileWorkerHooks { private hooks = new Set(); + private tests: TestGen[] = []; constructor( private type: T, private formatter: Formatter, ) {} - registerHooksForTest(testTags: string[]) { - getWorkerHooksToRun(this.type, testTags).forEach((hook) => this.hooks.add(hook)); + registerHooksForTest(test: TestGen) { + /** + * For worker hooks (beforeAll, afterAll) we require + * that each test match exactly the same set of hooks. + * Otherwise, in fully-parallel mode, we will run all worker hooks + * in each worker for each test, that actually makes test-level tags useless. + */ + const hooksForTest = new Set(getWorkerHooksToRun(this.type, test.tags)); + if (this.tests.length === 0) { + this.hooks = hooksForTest; + } else { + this.ensureHooksEqual(test, hooksForTest); + } + this.tests.push(test); } getCustomTestInstances() { @@ -120,4 +135,17 @@ class TestFileWorkerHooks { BddDataRenderer.varName, ); } + + private ensureHooksEqual(test: TestGen, hooksForTest: Set) { + if (areSetsEqual(this.hooks, hooksForTest)) return; + const prevTest = this.tests.at(-1)!; + exit( + [ + `Tagged ${this.type} hooks should be the same for all scenarios in a feature.`, + `Feature: ${test.featureUri}`, + ` - ${this.hooks.size} hook(s): ${prevTest.testTitle} ${prevTest.tags.join(' ')}`, + ` - ${hooksForTest.size} hook(s): ${test.testTitle} ${test.tags.join(' ')}`, + ].join('\n'), + ); + } } diff --git a/src/generate/test/index.ts b/src/generate/test/index.ts index d78ac076..16b06bd2 100644 --- a/src/generate/test/index.ts +++ b/src/generate/test/index.ts @@ -49,13 +49,13 @@ export class TestGen { // eslint-disable-next-line max-params constructor( private config: BDDConfig, - private featureUri: string, + public featureUri: string, private i18nKeywordsMap: KeywordsMap | undefined, private stepFinder: StepFinder, private formatter: Formatter, private backgrounds: BackgroundGen[], public pickle: PickleWithLocation, - private testTitle: string, + public testTitle: string, private scenarioSteps: readonly Step[], ownTestTags: string[], ) { diff --git a/src/hooks/scenario.ts b/src/hooks/scenario.ts index 346a3d95..78b9a5fb 100644 --- a/src/hooks/scenario.ts +++ b/src/hooks/scenario.ts @@ -79,15 +79,16 @@ export function createBeforeAfter< } // eslint-disable-next-line visual/complexity -export async function runScenarioHooks(type: ScenarioHookType, fixtures: ScenarioHookFixtures) { - const hooksToRun = getScenarioHooksToRun(type, fixtures.$bddContext.tags); - +export async function runScenarioHooks( + hooks: GeneralScenarioHook[], + fixtures: ScenarioHookFixtures, +) { let error; - for (const hook of hooksToRun) { + for (const hook of hooks) { try { await runScenarioHook(hook, fixtures); } catch (e) { - if (type === 'before') throw e; + if (hook.type === 'before') throw e; if (!error) error = e; } } @@ -155,12 +156,7 @@ function setTagsExpression(hook: GeneralScenarioHook) { function addHook(hook: GeneralScenarioHook) { setTagsExpression(hook); - if (hook.type === 'before') { - scenarioHooks.push(hook); - } else { - // 'after' hooks run in reverse order - scenarioHooks.unshift(hook); - } + scenarioHooks.push(hook); } function getTimeoutMessage(hook: GeneralScenarioHook) { diff --git a/src/hooks/worker.ts b/src/hooks/worker.ts index fa75ce1d..0537130b 100644 --- a/src/hooks/worker.ts +++ b/src/hooks/worker.ts @@ -11,7 +11,6 @@ import { KeyValue, PlaywrightLocation, TestTypeCommon } from '../playwright/type import { callWithTimeout } from '../utils'; import { getLocationByOffset } from '../playwright/getLocationInFile'; import { runStepWithLocation } from '../playwright/runStepWithLocation'; -import { BddFileData } from '../bddData/types'; export type WorkerHookType = 'beforeAll' | 'afterAll'; @@ -40,6 +39,12 @@ export type WorkerHook = { executed?: boolean; }; +export type WorkerHookRunInfo = { + test: TestTypeCommon; + hook: WorkerHook; + fixtures: WorkerHookFixtures; +}; + /** * When calling BeforeAll() / AfterAll() you can pass: * 1. hook fn @@ -73,37 +78,32 @@ export function createBeforeAllAfterAll( }; } -// eslint-disable-next-line visual/complexity, max-params -export async function runWorkerHooks( - test: TestTypeCommon, - type: WorkerHookType, - fixtures: WorkerHookFixtures, - bddFileData: BddFileData, -) { - // collect worker hooks for all tests in the file - const hooksToRun = new Set(); - bddFileData.forEach((bddTestData) => { - getWorkerHooksToRun(type, bddTestData.tags).forEach((hook) => hooksToRun.add(hook)); - }); - +// eslint-disable-next-line visual/complexity +export async function runWorkerHooks(hooksRunInfo: Map) { let error; - for (const hook of hooksToRun) { + for (const runInfo of hooksRunInfo.values()) { try { - await runWorkerHook(test, hook, fixtures); + await runWorkerHook(runInfo); } catch (e) { - if (type === 'beforeAll') throw e; + if (runInfo.hook.type === 'beforeAll') throw e; if (!error) error = e; } } if (error) throw error; } -async function runWorkerHook(test: TestTypeCommon, hook: WorkerHook, fixtures: WorkerHookFixtures) { - if (!hook.executed) { - hook.executed = true; - const hookFn = wrapHookFnWithTimeout(hook, fixtures); - const stepTitle = getHookStepTitle(hook); +async function runWorkerHook({ test, hook, fixtures }: WorkerHookRunInfo) { + if (hook.executed) return; + hook.executed = true; + const hookFn = wrapHookFnWithTimeout(hook, fixtures); + const stepTitle = getHookStepTitle(hook); + // test.step() is not available for afterAll hooks. + // See: https://github.com/microsoft/playwright/issues/33750 + // So all afterAll hooks are called under AfterAll step (with type = 'hook' in reporter) + if (hook.type === 'beforeAll') { await runStepWithLocation(test, stepTitle, hook.location, hookFn); + } else { + await hookFn(); } } @@ -150,12 +150,7 @@ function getFnFromArgs(args: unknown[]) { function addHook(hook: WorkerHook) { setTagsExpression(hook); - if (hook.type === 'beforeAll') { - workerHooks.push(hook); - } else { - // 'afterAll' hooks run in reverse order - workerHooks.unshift(hook); - } + workerHooks.push(hook); } function getTimeoutMessage(hook: WorkerHook) { diff --git a/src/run/bddFixtures/test.ts b/src/run/bddFixtures/test.ts index 162b2d8e..c4caa0c6 100644 --- a/src/run/bddFixtures/test.ts +++ b/src/run/bddFixtures/test.ts @@ -6,7 +6,7 @@ import { test as base } from './worker'; import { BDDConfig } from '../../config/types'; -import { runScenarioHooks } from '../../hooks/scenario'; +import { getScenarioHooksToRun, runScenarioHooks } from '../../hooks/scenario'; import { createStepInvoker, StepKeywordFixture } from '../StepInvoker'; import { TestTypeCommon } from '../../playwright/types'; import { TestInfo } from '@playwright/test'; @@ -145,17 +145,20 @@ export const test = base.extend({ // unused dependency '$afterEach' is important to run afterEach // in case of error in beforeEach. $beforeEach: [ - async ({ $bddContext, $beforeEachFixtures, $afterEach }, use) => { - await runScenarioHooks('before', { $bddContext, ...$beforeEachFixtures }); + async ({ $bddContext, $beforeEachFixtures, $tags, $afterEach }, use) => { + const hooksToRun = getScenarioHooksToRun('before', $tags); + await runScenarioHooks(hooksToRun, { $bddContext, ...$beforeEachFixtures }); await use(); }, fixtureOptions, ], // runs afterEach hooks $afterEach: [ - async ({ $bddContext, $afterEachFixtures }, use) => { + async ({ $bddContext, $afterEachFixtures, $tags }, use) => { await use(); - await runScenarioHooks('after', { $bddContext, ...$afterEachFixtures }); + const hooksToRun = getScenarioHooksToRun('after', $tags); + hooksToRun.reverse(); + await runScenarioHooks(hooksToRun, { $bddContext, ...$afterEachFixtures }); }, fixtureOptions, ], diff --git a/src/run/bddFixtures/worker.ts b/src/run/bddFixtures/worker.ts index c57f0094..78fe0457 100644 --- a/src/run/bddFixtures/worker.ts +++ b/src/run/bddFixtures/worker.ts @@ -5,8 +5,15 @@ import { BDDConfig } from '../../config/types'; import { test as base, WorkerInfo } from '@playwright/test'; import { getConfigFromEnv } from '../../config/env'; -import { runWorkerHooks } from '../../hooks/worker'; +import { + getWorkerHooksToRun, + runWorkerHooks, + WorkerHook, + WorkerHookRunInfo, +} from '../../hooks/worker'; import { loadSteps, resolveStepFiles } from '../../steps/loader'; +import { BddFileData } from '../../bddData/types'; +import { TestTypeCommon } from '../../playwright/types'; // BDD fixtures prefixed with '$' to avoid collision with user's fixtures. @@ -15,10 +22,17 @@ import { loadSteps, resolveStepFiles } from '../../steps/loader'; // make type coercion to satisfy TS in early PW versions const fixtureOptions = { scope: 'worker', box: true } as { scope: 'worker' }; +type WorkerHooksFixture = ( + test: TestTypeCommon, + fixtures: Record, + bddFileData: BddFileData, +) => unknown; + export type BddFixturesWorker = { $workerInfo: WorkerInfo; $bddConfig: BDDConfig; - $runWorkerHooks: typeof runWorkerHooks; + $runBeforeAllHooks: WorkerHooksFixture; + $registerAfterAllHooks: WorkerHooksFixture; }; export const test = base.extend, BddFixturesWorker>({ @@ -32,12 +46,57 @@ export const test = base.extend, BddFixturesWorker>({ }, fixtureOptions, ], - $runWorkerHooks: [ + $runBeforeAllHooks: [ + // Important unused dependency: + // - $bddConfig: to load bdd config before hooks + // eslint-disable-next-line @typescript-eslint/no-unused-vars + async ({ $bddConfig }, use, $workerInfo) => { + await use(async (test, customFixtures, bddFileData) => { + // Collect worker hooks for all tests in the file + // In fact, we could use only first test from bddFileData to get hooks, + // b/c we require that all tests in the file have the same beforeAll hooks. + // todo: filter out skipped tests + const hooksToRun = new Map(); + const fixtures = { $workerInfo, ...customFixtures }; + bddFileData.forEach((bddTestData) => { + // eslint-disable-next-line max-nested-callbacks + getWorkerHooksToRun('beforeAll', bddTestData.tags).forEach((hook) => { + hooksToRun.set(hook, { test, hook, fixtures }); + }); + }); + await runWorkerHooks(hooksToRun); + }); + }, + fixtureOptions, + ], + // Execution of AfterAll hooks is different from BeforeAll hooks. + // The main challenge is with tagged AfterAll hooks: + // We can't detect when the last test for tagged AfterAll hook is executed, + // especially when there are several test files in a worker or in fullyParallel mode. + // The solution: collect and run all AfterAll hooks in worker teardown phase. + // A list of afterAll hooks is populated from test.afterAll() call in each test file. + + $registerAfterAllHooks: [ // Important unused dependency: // - $bddConfig: to load bdd config before hooks // eslint-disable-next-line @typescript-eslint/no-unused-vars - async ({ $bddConfig }, use) => { - await use(runWorkerHooks); + async ({ $bddConfig }, use, $workerInfo) => { + const hooksToRun = new Map(); + + await use((test, customFixtures, bddFileData) => { + const fixtures = { $workerInfo, ...customFixtures }; + bddFileData.forEach((bddTestData) => { + getWorkerHooksToRun('afterAll', bddTestData.tags) + // eslint-disable-next-line max-nested-callbacks + .filter((hook) => !hooksToRun.has(hook)) + // eslint-disable-next-line max-nested-callbacks + .forEach((hook) => hooksToRun.set(hook, { test, hook, fixtures })); + }); + }); + + // run AfterAll hooks in FILO order + const reversedHooksToRun = new Map([...hooksToRun].reverse()); + await runWorkerHooks(reversedHooksToRun); }, fixtureOptions, ], diff --git a/src/utils/index.ts b/src/utils/index.ts index 3c12bf3a..15ce4543 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -107,3 +107,7 @@ export function saveFileSync(filePath: string, content: string) { export function toBoolean(value: T): value is NonNullable { return Boolean(value); } + +export function areSetsEqual(set1: Set, set2: Set) { + return set1.size === set2.size && [...set1].every((val) => set2.has(val)); +} diff --git a/test/_helpers/index.mjs b/test/_helpers/index.mjs index 201ed3d2..eb1a2449 100644 --- a/test/_helpers/index.mjs +++ b/test/_helpers/index.mjs @@ -9,6 +9,9 @@ import { expect } from '@playwright/test'; export * from './runPlaywright.mjs'; export * from './TestDir.mjs'; +// At some point we will be able to run test.only() +// See: https://github.com/nodejs/node/issues/47945 + export { test, expect, normalize }; export const playwrightVersion = getPackageVersion('@playwright/test'); diff --git a/test/_helpers/track.ts b/test/_helpers/track.ts deleted file mode 100644 index e27f653a..00000000 --- a/test/_helpers/track.ts +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Fixture to track calls in a worker. - */ -// don't import from 'playwright-bdd' as it will use another instance -import { test as base } from '../node_modules/playwright-bdd'; - -const logger = console; - -export const test = base.extend unknown }>({ - track: [ - async ({}, use, workerInfo) => { - const calls: string[] = ['']; - const fn = (title: string) => { - calls.push(`worker ${workerInfo.workerIndex}: ${title}`); - }; - await use(fn); - logger.log(calls.join('\n')); - }, - { scope: 'worker', auto: true }, - ], -}); diff --git a/test/_helpers/withLog.ts b/test/_helpers/withLog.ts new file mode 100644 index 00000000..50b86f06 --- /dev/null +++ b/test/_helpers/withLog.ts @@ -0,0 +1,29 @@ +/** + * Fixture to track calls in a worker. + * Important to use this fixture to log events for later checking, + * instead of using console.log directly in the test. + * Because Playwright's default reporter may overwrite the console.log output on CI. + */ +import { mergeTests, test as base } from '@playwright/test'; + +const logger = console; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +export function withLog(test: any) { + return mergeTests( + test, + base.extend unknown }>({ + log: [ + async ({}, use, workerInfo) => { + const calls: string[] = ['']; + const fn = (title: string) => { + calls.push(`worker ${workerInfo.workerIndex}: ${title}`); + }; + await use(fn); + logger.log(calls.join('\n')); + }, + { scope: 'worker', auto: true }, + ], + }), + ); +} diff --git a/test/hooks-error/features/fixtures.ts b/test/hooks-error/features/fixtures.ts deleted file mode 100644 index 28a067dd..00000000 --- a/test/hooks-error/features/fixtures.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { mergeTests } from '@playwright/test'; -import { test as base, createBdd } from 'playwright-bdd'; -import { test as testWithTrack } from '../../_helpers/track'; - -export const test = mergeTests(base, testWithTrack); - -export const { Given, Before, BeforeAll, After, AfterAll } = createBdd(test); diff --git a/test/hooks-error/features/steps.ts b/test/hooks-error/features/steps.ts deleted file mode 100644 index dd93a9bb..00000000 --- a/test/hooks-error/features/steps.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { Given, Before, BeforeAll, After, AfterAll } from './fixtures'; - -Before(async function ({ $testInfo, track }) { - track(`Before 1 ${$testInfo.title}`); - canThrow('Before 1'); -}); - -Before(async ({ $testInfo, track }) => { - track(`Before 2 ${$testInfo.title}`); - canThrow('Before 2'); -}); - -After(async function ({ $testInfo, track }) { - track(`After 1 ${$testInfo.title}`); - canThrow('After 1'); -}); - -After(async ({ $testInfo, track }) => { - track(`After 2 ${$testInfo.title}`); - canThrow('After 2'); -}); - -BeforeAll(async function ({ track }) { - track(`BeforeAll 1`); - canThrow('BeforeAll 1'); -}); - -BeforeAll(async ({ track }) => { - track(`BeforeAll 2`); - canThrow('BeforeAll 2'); -}); - -AfterAll(async function ({ track }) { - track(`AfterAll 1`); - canThrow('AfterAll 1'); -}); - -AfterAll(async ({ track }) => { - track(`AfterAll 2`); - canThrow('AfterAll 2'); -}); - -Given('Step {int}', async ({ track }, n: number) => { - track(`Step ${n}`); -}); - -Given('Step bg', async ({ track }) => { - track(`Step bg`); -}); - -function canThrow(hookTitle: string) { - const shouldThrow = process.env.ERROR && hookTitle.startsWith(process.env.ERROR); - if (shouldThrow) { - throw new Error(hookTitle); - } -} diff --git a/test/hooks-errors/not-same-worker-hooks/sample.feature b/test/hooks-errors/not-same-worker-hooks/sample.feature new file mode 100644 index 00000000..34b00086 --- /dev/null +++ b/test/hooks-errors/not-same-worker-hooks/sample.feature @@ -0,0 +1,8 @@ +Feature: feature 1 + + @scenario1 + Scenario: scenario 1 + Given a step + + Scenario: scenario 2 + Given a step diff --git a/test/hooks-errors/not-same-worker-hooks/steps.ts b/test/hooks-errors/not-same-worker-hooks/steps.ts new file mode 100644 index 00000000..1a9b7b57 --- /dev/null +++ b/test/hooks-errors/not-same-worker-hooks/steps.ts @@ -0,0 +1,7 @@ +import { createBdd } from 'playwright-bdd'; + +const { BeforeAll, Given } = createBdd(); + +BeforeAll({ tags: '@scenario1' }, () => {}); + +Given('a step', () => {}); diff --git a/test/hooks-error/package.json b/test/hooks-errors/package.json similarity index 100% rename from test/hooks-error/package.json rename to test/hooks-errors/package.json diff --git a/test/hooks-tags/playwright.config.ts b/test/hooks-errors/playwright.config.ts similarity index 64% rename from test/hooks-tags/playwright.config.ts rename to test/hooks-errors/playwright.config.ts index ed874a7e..ded25bd9 100644 --- a/test/hooks-tags/playwright.config.ts +++ b/test/hooks-errors/playwright.config.ts @@ -2,10 +2,10 @@ import { defineConfig } from '@playwright/test'; import { defineBddConfig } from 'playwright-bdd'; const testDir = defineBddConfig({ - featuresRoot: 'features', + outputDir: `.features-gen/${process.env.FEATURES_ROOT}`, + featuresRoot: process.env.FEATURES_ROOT!, }); export default defineConfig({ testDir, - workers: 1, }); diff --git a/test/hooks-errors/test.mjs b/test/hooks-errors/test.mjs new file mode 100644 index 00000000..3702d676 --- /dev/null +++ b/test/hooks-errors/test.mjs @@ -0,0 +1,18 @@ +import { test, TestDir, normalize, execPlaywrightTestWithError } from '../_helpers/index.mjs'; + +const testDir = new TestDir(import.meta); + +test(`${testDir.name} (not-same-worker-hooks)`, () => { + execPlaywrightTestWithError( + testDir.name, + [ + `Error: Tagged beforeAll hooks should be the same for all scenarios in a feature`, + `Feature: ${normalize('not-same-worker-hooks/sample.feature')}`, + `- 1 hook(s): scenario 1 @scenario1`, + `- 0 hook(s): scenario 2`, + ], + { + env: { FEATURES_ROOT: 'not-same-worker-hooks' }, + }, + ); +}); diff --git a/test/hooks-tags-worker/features/feature1.feature b/test/hooks-order-many-features/features/feature1.feature similarity index 100% rename from test/hooks-tags-worker/features/feature1.feature rename to test/hooks-order-many-features/features/feature1.feature diff --git a/test/hooks-tags-worker/features/feature2.feature b/test/hooks-order-many-features/features/feature2.feature similarity index 85% rename from test/hooks-tags-worker/features/feature2.feature rename to test/hooks-order-many-features/features/feature2.feature index b4a6adf7..9dcd4820 100644 --- a/test/hooks-tags-worker/features/feature2.feature +++ b/test/hooks-order-many-features/features/feature2.feature @@ -1,4 +1,4 @@ -@feature2 +# no tags Feature: feature 2 Scenario: scenario 2 diff --git a/test/hooks-order-many-features/features/fixtures.ts b/test/hooks-order-many-features/features/fixtures.ts new file mode 100644 index 00000000..b92a1cbf --- /dev/null +++ b/test/hooks-order-many-features/features/fixtures.ts @@ -0,0 +1,34 @@ +import { test as base, createBdd } from 'playwright-bdd'; +import { withLog } from '../../_helpers/withLog'; + +type WorkerFixtures = { + workerFixtureCommon: void; + workerFixture1: void; + workerFixture2: void; +}; + +export const test = withLog(base).extend({ + workerFixtureCommon: [ + async ({ log }, use) => { + log(`workerFixtureCommon setup`); + await use(); + }, + { scope: 'worker' }, + ], + workerFixture1: [ + async ({ log }, use) => { + log(`workerFixture1 setup`); + await use(); + }, + { scope: 'worker' }, + ], + workerFixture2: [ + async ({ log }, use) => { + log(`workerFixture2 setup`); + await use(); + }, + { scope: 'worker' }, + ], +}); + +export const { Given, BeforeAll, AfterAll } = createBdd(test); diff --git a/test/hooks-order-many-features/features/steps.ts b/test/hooks-order-many-features/features/steps.ts new file mode 100644 index 00000000..b84674f3 --- /dev/null +++ b/test/hooks-order-many-features/features/steps.ts @@ -0,0 +1,5 @@ +import { Given } from './fixtures'; + +Given('a step', ({ log, $testInfo }) => { + log(`a step of ${$testInfo.title}`); +}); diff --git a/test/hooks-order-many-features/features/workerHooks.ts b/test/hooks-order-many-features/features/workerHooks.ts new file mode 100644 index 00000000..c32519d3 --- /dev/null +++ b/test/hooks-order-many-features/features/workerHooks.ts @@ -0,0 +1,44 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { BeforeAll, AfterAll } from './fixtures'; + +BeforeAll(({ log, workerFixtureCommon }) => { + log(`BeforeAll 1`); +}); + +BeforeAll(({ log }) => { + log(`BeforeAll 2`); +}); + +BeforeAll({ tags: '@feature1' }, ({ log, workerFixture1 }) => { + log(`BeforeAll 3 (@feature1)`); +}); + +BeforeAll({ tags: 'not @feature1' }, ({ log, workerFixture2 }) => { + log(`BeforeAll 4 (not @feature1)`); +}); + +BeforeAll({ tags: '@unused' }, ({ log }) => { + log(`BeforeAll 5 (@unused)`); +}); + +// AfterAll + +AfterAll(({ log, workerFixtureCommon }) => { + log(`AfterAll 1`); +}); + +AfterAll(({ log }) => { + log(`AfterAll 2`); +}); + +AfterAll({ tags: '@feature1' }, ({ log, workerFixture1 }) => { + log(`AfterAll 3 (@feature1)`); +}); + +AfterAll({ tags: 'not @feature1' }, ({ log, workerFixture2 }) => { + log(`AfterAll 4 (not @feature1)`); +}); + +AfterAll({ tags: '@unused' }, ({ log }) => { + log(`AfterAll 5 (@unused)`); +}); diff --git a/test/hooks-order/package.json b/test/hooks-order-many-features/package.json similarity index 100% rename from test/hooks-order/package.json rename to test/hooks-order-many-features/package.json diff --git a/test/hooks-timeout/playwright.config.ts b/test/hooks-order-many-features/playwright.config.ts similarity index 80% rename from test/hooks-timeout/playwright.config.ts rename to test/hooks-order-many-features/playwright.config.ts index ed874a7e..b320a60e 100644 --- a/test/hooks-timeout/playwright.config.ts +++ b/test/hooks-order-many-features/playwright.config.ts @@ -7,5 +7,5 @@ const testDir = defineBddConfig({ export default defineConfig({ testDir, - workers: 1, + workers: Number(process.env.WORKERS) || undefined, }); diff --git a/test/hooks-order-many-features/test.mjs b/test/hooks-order-many-features/test.mjs new file mode 100644 index 00000000..ae1c1a00 --- /dev/null +++ b/test/hooks-order-many-features/test.mjs @@ -0,0 +1,56 @@ +import { test, expectCalls, TestDir, execPlaywrightTest } from '../_helpers/index.mjs'; + +const testDir = new TestDir(import.meta); + +// These tests mainly focus on worker hooks order across several workers. +// Checking of scenario hooks order is in tests/hooks-order-one-feature. + +test(`${testDir.name} (1 worker)`, () => { + const stdout = execPlaywrightTest(testDir.name, { env: { WORKERS: 1 } }); + + expectCalls('worker 0: ', stdout, [ + 'workerFixtureCommon setup', + 'workerFixture1 setup', + 'BeforeAll 1', + 'BeforeAll 2', + 'BeforeAll 3 (@feature1)', + 'a step of scenario 1', + 'workerFixture2 setup', + 'BeforeAll 4 (not @feature1)', + 'a step of scenario 2', + // all afterAll hooks run in worker teardown, + // not instantly after the corresponding test file + 'AfterAll 4 (not @feature1)', + 'AfterAll 3 (@feature1)', + 'AfterAll 2', + 'AfterAll 1', + ]); +}); + +test(`${testDir.name} (2 workers)`, () => { + const stdout = execPlaywrightTest(testDir.name, { env: { WORKERS: 2 } }); + + expectCalls('worker 0: ', stdout, [ + 'workerFixtureCommon setup', + 'workerFixture1 setup', + 'BeforeAll 1', + 'BeforeAll 2', + 'BeforeAll 3 (@feature1)', + 'a step of scenario 1', + 'AfterAll 3 (@feature1)', + 'AfterAll 2', + 'AfterAll 1', + ]); + + expectCalls('worker 1: ', stdout, [ + 'workerFixtureCommon setup', + 'workerFixture2 setup', + 'BeforeAll 1', + 'BeforeAll 2', + 'BeforeAll 4 (not @feature1)', + 'a step of scenario 2', + 'AfterAll 4 (not @feature1)', + 'AfterAll 2', + 'AfterAll 1', + ]); +}); diff --git a/test/hooks-order-one-feature/features/fixtures.ts b/test/hooks-order-one-feature/features/fixtures.ts new file mode 100644 index 00000000..26585c27 --- /dev/null +++ b/test/hooks-order-one-feature/features/fixtures.ts @@ -0,0 +1,24 @@ +import timers from 'node:timers/promises'; +import { test as base, createBdd } from 'playwright-bdd'; +import { withLog } from '../../_helpers/withLog'; + +type TestFixtures = { + testFixtureCommon: void; + testFixtureScenario1: void; +}; + +export const test = withLog(base).extend({ + testFixtureCommon: async ({ log }, use) => { + log(`testFixtureCommon setup`); + await use(); + }, + + testFixtureScenario1: async ({ log }, use) => { + // tiny delay to have always foo after bar + await timers.setTimeout(50); + log(`testFixtureScenario1 setup`); + await use(); + }, +}); + +export const { Given, Before, BeforeAll, After, AfterAll } = createBdd(test); diff --git a/test/hooks-order/features/sample.feature b/test/hooks-order-one-feature/features/sample.feature similarity index 55% rename from test/hooks-order/features/sample.feature rename to test/hooks-order-one-feature/features/sample.feature index ccb4c7f5..69a47d9c 100644 --- a/test/hooks-order/features/sample.feature +++ b/test/hooks-order-one-feature/features/sample.feature @@ -1,10 +1,11 @@ Feature: sample Background: bg - Given Step bg + Given bg step + @scenario1 Scenario: scenario 1 - Given Step 1 + Given a step Scenario: scenario 2 - Given Step 2 + Given a step diff --git a/test/hooks-order-one-feature/features/scenarioHooks.ts b/test/hooks-order-one-feature/features/scenarioHooks.ts new file mode 100644 index 00000000..957bbdfd --- /dev/null +++ b/test/hooks-order-one-feature/features/scenarioHooks.ts @@ -0,0 +1,26 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ +import { Before, After } from './fixtures'; + +Before(async ({ log, testFixtureCommon }) => { + log(`Before 1`); +}); + +Before(async ({ log }) => { + log(`Before 2`); +}); + +Before({ tags: '@scenario1' }, async ({ log, testFixtureScenario1 }) => { + log(`Before 3 (@scenario1)`); +}); + +After(async ({ log, testFixtureCommon }) => { + log(`After 1`); +}); + +After(async ({ log }) => { + log(`After 2`); +}); + +After({ tags: '@scenario1' }, async ({ log, testFixtureScenario1 }) => { + log(`After 3 (@scenario1)`); +}); diff --git a/test/hooks-order-one-feature/features/steps.ts b/test/hooks-order-one-feature/features/steps.ts new file mode 100644 index 00000000..176fc242 --- /dev/null +++ b/test/hooks-order-one-feature/features/steps.ts @@ -0,0 +1,9 @@ +import { Given } from './fixtures'; + +Given('bg step', async ({ log, $testInfo }) => { + log(`bg step of ${$testInfo.title}`); +}); + +Given('a step', async ({ log, $testInfo }) => { + log(`a step of ${$testInfo.title}`); +}); diff --git a/test/hooks-order-one-feature/features/workerHooks.ts b/test/hooks-order-one-feature/features/workerHooks.ts new file mode 100644 index 00000000..dfed5ed3 --- /dev/null +++ b/test/hooks-order-one-feature/features/workerHooks.ts @@ -0,0 +1,9 @@ +import { BeforeAll, AfterAll } from './fixtures'; + +BeforeAll(async ({ log }) => { + log(`BeforeAll 1`); +}); + +AfterAll(async ({ log }) => { + log(`AfterAll 1`); +}); diff --git a/test/hooks-tags/package.json b/test/hooks-order-one-feature/package.json similarity index 77% rename from test/hooks-tags/package.json rename to test/hooks-order-one-feature/package.json index de610253..61692124 100644 --- a/test/hooks-tags/package.json +++ b/test/hooks-order-one-feature/package.json @@ -1,3 +1,4 @@ { - "description": "This file is required for Playwright to consider this dir as a . It ensures to load 'playwright-bdd' from './test/node_modules/playwright-bdd' and output './test-results' here to avoid conflicts." + "description": "This file is required for Playwright to consider this dir as a . It ensures to load 'playwright-bdd' from './test/node_modules/playwright-bdd' and output './test-results' here to avoid conflicts.", + "smoke": true } diff --git a/test/hooks-tags-worker/playwright.config.ts b/test/hooks-order-one-feature/playwright.config.ts similarity index 82% rename from test/hooks-tags-worker/playwright.config.ts rename to test/hooks-order-one-feature/playwright.config.ts index d65725b6..dc9b4f8d 100644 --- a/test/hooks-tags-worker/playwright.config.ts +++ b/test/hooks-order-one-feature/playwright.config.ts @@ -8,4 +8,5 @@ const testDir = defineBddConfig({ export default defineConfig({ testDir, workers: Number(process.env.WORKERS), + fullyParallel: Boolean(process.env.FULLY_PARALLEL), }); diff --git a/test/hooks-order-one-feature/test.mjs b/test/hooks-order-one-feature/test.mjs new file mode 100644 index 00000000..141e977f --- /dev/null +++ b/test/hooks-order-one-feature/test.mjs @@ -0,0 +1,68 @@ +import { test, expectCalls, TestDir, execPlaywrightTest } from '../_helpers/index.mjs'; + +const testDir = new TestDir(import.meta); + +test(`${testDir.name} (1 worker)`, () => { + const stdout = execPlaywrightTest(testDir.name, { env: { WORKERS: 1 } }); + expectCalls('worker 0: ', stdout, [ + // scenario 1 + 'BeforeAll 1', + 'testFixtureCommon setup', + 'testFixtureScenario1 setup', + 'Before 1', + 'Before 2', + 'Before 3 (@scenario1)', + 'bg step of scenario 1', + 'a step of scenario 1', + 'After 3 (@scenario1)', + 'After 2', + 'After 1', + // scenario 2 + 'testFixtureCommon setup', + // Currently testFixtureScenario1 is executed for scenario 2 as well, + // b/c we have all scenario hooks fixtures in a single object ($beforeEachFixtures, ) + // todo: setup only needed fixtures for each test + 'testFixtureScenario1 setup', + 'Before 1', + 'Before 2', + 'bg step of scenario 2', + 'a step of scenario 2', + 'After 2', + 'After 1', + 'AfterAll 1', + ]); +}); + +test(`${testDir.name} (2 workers, fully-parallel)`, () => { + const stdout = execPlaywrightTest(testDir.name, { env: { WORKERS: 2, FULLY_PARALLEL: 1 } }); + + // worker 0 + expectCalls('worker 0: ', stdout, [ + 'BeforeAll 1', + 'testFixtureCommon setup', + 'testFixtureScenario1 setup', + 'Before 1', + 'Before 2', + 'Before 3 (@scenario1)', + 'bg step of scenario 1', + 'a step of scenario 1', + 'After 3 (@scenario1)', + 'After 2', + 'After 1', + 'AfterAll 1', + ]); + + // worker 1 + expectCalls('worker 1: ', stdout, [ + 'BeforeAll 1', + 'testFixtureCommon setup', + 'testFixtureScenario1 setup', + 'Before 1', + 'Before 2', + 'bg step of scenario 2', + 'a step of scenario 2', + 'After 2', + 'After 1', + 'AfterAll 1', + ]); +}); diff --git a/test/hooks-order/features/fixtures.ts b/test/hooks-order/features/fixtures.ts deleted file mode 100644 index 28a067dd..00000000 --- a/test/hooks-order/features/fixtures.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { mergeTests } from '@playwright/test'; -import { test as base, createBdd } from 'playwright-bdd'; -import { test as testWithTrack } from '../../_helpers/track'; - -export const test = mergeTests(base, testWithTrack); - -export const { Given, Before, BeforeAll, After, AfterAll } = createBdd(test); diff --git a/test/hooks-order/features/steps.ts b/test/hooks-order/features/steps.ts deleted file mode 100644 index 34223f56..00000000 --- a/test/hooks-order/features/steps.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Given, Before, BeforeAll, After, AfterAll } from './fixtures'; - -Before(async function ({ $testInfo, track }) { - track(`Before 1 ${$testInfo.title}`); -}); - -Before(async ({ $testInfo, track }) => { - track(`Before 2 ${$testInfo.title}`); -}); - -After(async function ({ $testInfo, track }) { - track(`After 1 ${$testInfo.title}`); -}); - -After(async ({ $testInfo, track }) => { - track(`After 2 ${$testInfo.title}`); -}); - -BeforeAll(async function ({ track }) { - track(`BeforeAll 1`); -}); - -BeforeAll(async ({ track }) => { - track(`BeforeAll 2`); -}); - -AfterAll(async function ({ track }) { - track(`AfterAll 1`); -}); - -AfterAll(async ({ track }) => { - track(`AfterAll 2`); -}); - -Given('Step {int}', async ({ track }, n: number) => { - track(`Step ${n}`); -}); - -Given('Step bg', async ({ track }) => { - track(`Step bg`); -}); diff --git a/test/hooks-order/test.mjs b/test/hooks-order/test.mjs deleted file mode 100644 index 7782e2ae..00000000 --- a/test/hooks-order/test.mjs +++ /dev/null @@ -1,57 +0,0 @@ -import { test, expectCalls, TestDir, execPlaywrightTest, DEFAULT_CMD } from '../_helpers/index.mjs'; - -const testDir = new TestDir(import.meta); - -test(`${testDir.name} (1 worker)`, () => { - const stdout = execPlaywrightTest(testDir.name); - expectCalls('worker 0: ', stdout, [ - 'BeforeAll 1', - 'BeforeAll 2', - 'Before 1 scenario 1', - 'Before 2 scenario 1', - 'Step bg', - 'Step 1', - 'After 2 scenario 1', - 'After 1 scenario 1', - 'Before 1 scenario 2', - 'Before 2 scenario 2', - 'Step bg', - 'Step 2', - 'After 2 scenario 2', - 'After 1 scenario 2', - 'AfterAll 2', - 'AfterAll 1', - ]); -}); - -test(`${testDir.name} (2 workers)`, () => { - const stdout = execPlaywrightTest(testDir.name, `${DEFAULT_CMD} --workers=2 --fully-parallel`); - - // worker 0 - expectCalls('worker 0: ', stdout, [ - 'BeforeAll 1', - 'BeforeAll 2', - 'Before 1 scenario 1', - 'Before 2 scenario 1', - 'Step bg', - 'Step 1', - 'After 2 scenario 1', - 'After 1 scenario 1', - 'AfterAll 2', - 'AfterAll 1', - ]); - - // worker 1 - expectCalls('worker 1: ', stdout, [ - 'BeforeAll 1', - 'BeforeAll 2', - 'Before 1 scenario 2', - 'Before 2 scenario 2', - 'Step bg', - 'Step 2', - 'After 2 scenario 2', - 'After 1 scenario 2', - 'AfterAll 2', - 'AfterAll 1', - ]); -}); diff --git a/test/hooks-tags-worker/features/fixtures.ts b/test/hooks-tags-worker/features/fixtures.ts deleted file mode 100644 index 4fc0fd0e..00000000 --- a/test/hooks-tags-worker/features/fixtures.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { test as base, createBdd } from 'playwright-bdd'; - -export const test = base.extend({ - myWorkerFixture: [ - async ({}, use) => { - await use('myWorkerFixture'); - }, - { scope: 'worker' }, - ], -}); - -export const { Given, BeforeAll, AfterAll } = createBdd(test); diff --git a/test/hooks-tags-worker/features/steps.ts b/test/hooks-tags-worker/features/steps.ts deleted file mode 100644 index 8cb2f9bb..00000000 --- a/test/hooks-tags-worker/features/steps.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Given, BeforeAll, AfterAll } from './fixtures'; - -const logger = console; - -// eslint-disable-next-line @typescript-eslint/no-unused-vars -BeforeAll(({ $workerInfo, myWorkerFixture }) => { - logger.log(`worker ${$workerInfo.workerIndex}: BeforeAll 1`); -}); - -BeforeAll(({ $workerInfo }) => { - logger.log(`worker ${$workerInfo.workerIndex}: BeforeAll 2`); -}); - -BeforeAll({ tags: '@feature1' }, ({ $workerInfo }) => { - logger.log(`worker ${$workerInfo.workerIndex}: BeforeAll for feature 1`); -}); - -BeforeAll({ tags: '@feature2' }, ({ $workerInfo }) => { - logger.log(`worker ${$workerInfo.workerIndex}: BeforeAll for feature 2`); -}); - -BeforeAll({ tags: '@feature1 or @feature2' }, ({ $workerInfo }) => { - logger.log(`worker ${$workerInfo.workerIndex}: BeforeAll for feature (1 or 2)`); -}); - -AfterAll(({ $workerInfo }) => { - logger.log(`worker ${$workerInfo.workerIndex}: AfterAll 1`); -}); - -AfterAll(({ $workerInfo }) => { - logger.log(`worker ${$workerInfo.workerIndex}: AfterAll 2`); -}); - -Given('a step', ({ $testInfo }) => { - logger.log(`worker ${$testInfo.workerIndex}: a step of ${$testInfo.title}`); -}); diff --git a/test/hooks-tags-worker/test.mjs b/test/hooks-tags-worker/test.mjs deleted file mode 100644 index 40e87e82..00000000 --- a/test/hooks-tags-worker/test.mjs +++ /dev/null @@ -1,21 +0,0 @@ -import { test, expect, TestDir, execPlaywrightTest, countOfSubstring } from '../_helpers/index.mjs'; - -const testDir = new TestDir(import.meta); - -test(`${testDir.name} (1 worker)`, () => { - const stdout = execPlaywrightTest(testDir.name, { env: { WORKERS: 1 } }); - - // feature 1 - expect(countOfSubstring(stdout, 'worker 0: BeforeAll 1')).toEqual(1); - expect(countOfSubstring(stdout, 'worker 0: BeforeAll 2')).toEqual(1); - expect(countOfSubstring(stdout, 'worker 0: BeforeAll for feature 1')).toEqual(1); - expect(countOfSubstring(stdout, 'worker 0: BeforeAll for feature (1 or 2)')).toEqual(1); - expect(countOfSubstring(stdout, 'worker 0: a step of scenario 1')).toEqual(1); - // after all should be called after feature 2 !!! - expect(countOfSubstring(stdout, 'worker 0: AfterAll 2')).toEqual(1); - expect(countOfSubstring(stdout, 'worker 0: AfterAll 1')).toEqual(1); - - // feature 2 - expect(countOfSubstring(stdout, 'worker 0: BeforeAll for feature 2')).toEqual(1); - expect(countOfSubstring(stdout, 'worker 0: a step of scenario 2')).toEqual(1); -}); diff --git a/test/hooks-tags/features/fixtures.ts b/test/hooks-tags/features/fixtures.ts deleted file mode 100644 index e1eb1ab4..00000000 --- a/test/hooks-tags/features/fixtures.ts +++ /dev/null @@ -1,23 +0,0 @@ -import timers from 'node:timers/promises'; -import { mergeTests } from '@playwright/test'; -import { test as base, createBdd } from 'playwright-bdd'; -import { test as testWithTrack } from '../../_helpers/track'; - -export const test = mergeTests(base, testWithTrack).extend<{ - fixtureForFoo: void; - fixtureForBar: void; -}>({ - fixtureForFoo: async ({ track }, use) => { - // tiny delay to have always foo after bar - await timers.setTimeout(50); - track(`setup fixture for foo`); - await use(); - }, - - fixtureForBar: async ({ track }, use) => { - track(`setup fixture for bar`); - await use(); - }, -}); - -export const { Given, Before, After, AfterAll } = createBdd(test); diff --git a/test/hooks-tags/features/sample.feature b/test/hooks-tags/features/sample.feature deleted file mode 100644 index c3d6a136..00000000 --- a/test/hooks-tags/features/sample.feature +++ /dev/null @@ -1,9 +0,0 @@ -@foo -Feature: hooks - - @bar - Scenario: scenario 1 - Given Step 1 - - Scenario: scenario 2 - Given Step 2 diff --git a/test/hooks-tags/features/sample2.feature b/test/hooks-tags/features/sample2.feature deleted file mode 100644 index a3439747..00000000 --- a/test/hooks-tags/features/sample2.feature +++ /dev/null @@ -1,4 +0,0 @@ -Feature: sample 2 - - Scenario: scenario 3 - Given Step 3 diff --git a/test/hooks-tags/features/steps.ts b/test/hooks-tags/features/steps.ts deleted file mode 100644 index 19e14863..00000000 --- a/test/hooks-tags/features/steps.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { expect } from '@playwright/test'; -import { Given, Before, After } from './fixtures'; - -// eslint-disable-next-line @typescript-eslint/no-unused-vars -Before('@bar', async function ({ $testInfo, $tags, track, fixtureForBar }) { - track(`Before @bar ${$testInfo.title}`); - expect($tags).toEqual(['@foo', '@bar']); -}); - -// eslint-disable-next-line @typescript-eslint/no-unused-vars -Before({ tags: '@foo and not @bar' }, async ({ $testInfo, track, fixtureForFoo }) => { - track(`Before @foo and not @bar ${$testInfo.title}`); -}); - -After('@bar', async function ({ $testInfo, $tags, track }) { - track(`After @bar ${$testInfo.title}`); - expect($tags).toEqual(['@foo', '@bar']); -}); - -After({ tags: '@foo and not @bar' }, async ({ $testInfo, track }) => { - track(`After @foo and not @bar ${$testInfo.title}`); -}); - -Given('Step {int}', async ({ $step, track }) => { - track($step.title); -}); diff --git a/test/hooks-tags/test.mjs b/test/hooks-tags/test.mjs deleted file mode 100644 index 4d6d4720..00000000 --- a/test/hooks-tags/test.mjs +++ /dev/null @@ -1,116 +0,0 @@ -import { - test, - expectCalls, - TestDir, - execPlaywrightTest, - playwrightVersion, - BDDGEN_CMD, - PLAYWRIGHT_CMD, -} from '../_helpers/index.mjs'; - -const testDir = new TestDir(import.meta); -const pwSupportsTags = playwrightVersion >= '1.42.0'; - -test(`${testDir.name} (run all tests)`, () => { - const stdout = execPlaywrightTest(testDir.name); - - expectCalls('worker 0: ', stdout, [ - 'setup fixture for bar', - 'setup fixture for foo', - 'Before @bar scenario 1', - 'Step 1', - 'After @bar scenario 1', - 'setup fixture for bar', - 'setup fixture for foo', - 'Before @foo and not @bar scenario 2', - 'Step 2', - 'After @foo and not @bar scenario 2', - 'Step 3', - ]); -}); - -test(`${testDir.name} (gen only bar)`, () => { - const stdout = execPlaywrightTest( - testDir.name, - `${BDDGEN_CMD} --tags "@bar" && ${PLAYWRIGHT_CMD}`, - ); - - expectCalls('worker 0: ', stdout, [ - 'setup fixture for bar', // prettier-ignore - 'Before @bar scenario 1', - 'Step 1', - 'After @bar scenario 1', - ]); -}); - -test(`${testDir.name} (gen not bar)`, () => { - const stdout = execPlaywrightTest( - testDir.name, - `${BDDGEN_CMD} --tags "not @bar" && ${PLAYWRIGHT_CMD}`, - ); - - expectCalls('worker 0: ', stdout, [ - 'setup fixture for foo', // prettier-ignore - 'Before @foo and not @bar scenario 2', - 'Step 2', - 'After @foo and not @bar scenario 2', - 'Step 3', - ]); -}); - -test(`${testDir.name} (run bar)`, { skip: !pwSupportsTags }, () => { - const stdout = execPlaywrightTest( - testDir.name, - `${BDDGEN_CMD} && ${PLAYWRIGHT_CMD} --grep "@bar"`, - ); - - expectCalls('worker 0: ', stdout, [ - // initialization of fixtureForFoo is expected, - // b/c both scenarios are in the same file. - // We can't know beforehand, which scenarios will be filtered with PW tags in runtime. - // Looks like a minor issue. - 'setup fixture for bar', // prettier-ignore - 'setup fixture for foo', - 'Before @bar scenario 1', - 'Step 1', - 'After @bar scenario 1', - ]); -}); - -test(`${testDir.name} (run not bar)`, { skip: !pwSupportsTags }, () => { - const stdout = execPlaywrightTest( - testDir.name, - `${BDDGEN_CMD} && ${PLAYWRIGHT_CMD} --grep-invert "@bar"`, - ); - - expectCalls('worker 0: ', stdout, [ - // initialization of fixtureForBar is expected, - // b/c both scenarios are in the same file. - // We can't know beforehand, which will it be filtered with PW tags in runtime. - // Looks like a minor issue. - 'setup fixture for bar', // prettier-ignore - 'setup fixture for foo', - 'Before @foo and not @bar scenario 2', - 'Step 2', - 'After @foo and not @bar scenario 2', - 'Step 3', - ]); -}); - -test(`${testDir.name} (gen not foo)`, () => { - const stdout = execPlaywrightTest( - testDir.name, - `${BDDGEN_CMD} --tags "not @foo" && ${PLAYWRIGHT_CMD}`, - ); - - expectCalls('worker 0: ', stdout, ['Step 3']); -}); - -test(`${testDir.name} (run not foo)`, { skip: !pwSupportsTags }, () => { - const stdout = execPlaywrightTest( - testDir.name, - `${BDDGEN_CMD} && ${PLAYWRIGHT_CMD} --grep-invert "@foo"`, - ); - - expectCalls('worker 0: ', stdout, ['Step 3']); -}); diff --git a/test/hooks-timeout/features/fixtures.ts b/test/hooks-timeout/features/fixtures.ts deleted file mode 100644 index 28a067dd..00000000 --- a/test/hooks-timeout/features/fixtures.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { mergeTests } from '@playwright/test'; -import { test as base, createBdd } from 'playwright-bdd'; -import { test as testWithTrack } from '../../_helpers/track'; - -export const test = mergeTests(base, testWithTrack); - -export const { Given, Before, BeforeAll, After, AfterAll } = createBdd(test); diff --git a/test/hooks-timeout/features/steps.ts b/test/hooks-timeout/features/steps.ts deleted file mode 100644 index 948c3b1f..00000000 --- a/test/hooks-timeout/features/steps.ts +++ /dev/null @@ -1,59 +0,0 @@ -import timers from 'node:timers/promises'; -import { Given, Before, BeforeAll, After, AfterAll } from './fixtures'; - -// worker hooks - -BeforeAll({ timeout: 5 }, async function ({ track }) { - track(`BeforeAll 1`); - await canTimeout(`BeforeAll 1`); -}); - -BeforeAll(async ({ track }) => { - track(`BeforeAll 2`); - await canTimeout(`BeforeAll 2`); -}); - -AfterAll(async function ({ track }) { - track(`AfterAll 1`); - await canTimeout(`AfterAll 1`); -}); - -AfterAll({ timeout: 5 }, async ({ track }) => { - track(`AfterAll 2`); - await canTimeout(`AfterAll 2`); -}); - -// scenario hooks - -Before({ timeout: 5 }, async function ({ $testInfo, track }) { - track(`Before 1 ${$testInfo.title}`); - await canTimeout(`Before 1`); -}); - -Before(async ({ $testInfo, track }) => { - track(`Before 2 ${$testInfo.title}`); - await canTimeout(`Before 2`); -}); - -After(async function ({ $testInfo, track }) { - track(`After 1 ${$testInfo.title}`); - await canTimeout(`After 1`); -}); - -After({ timeout: 5 }, async ({ $testInfo, track }) => { - track(`After 2 ${$testInfo.title}`); - await canTimeout(`After 2`); -}); - -// step - -Given('State {int}', async ({ $testInfo, track }) => { - track(`Step ${$testInfo.title}`); -}); - -async function canTimeout(hookTitle: string) { - const shouldTimeout = process.env.TIMEOUT && hookTitle.startsWith(process.env.TIMEOUT); - if (shouldTimeout) { - await timers.setTimeout(100); - } -} diff --git a/test/hooks-with-error/features/fixtures.ts b/test/hooks-with-error/features/fixtures.ts new file mode 100644 index 00000000..788d6da6 --- /dev/null +++ b/test/hooks-with-error/features/fixtures.ts @@ -0,0 +1,6 @@ +import { test as base, createBdd } from 'playwright-bdd'; +import { withLog } from '../../_helpers/withLog'; + +export const test = withLog(base); + +export const { Given, Before, BeforeAll, After, AfterAll } = createBdd(test); diff --git a/test/hooks-error/features/sample.feature b/test/hooks-with-error/features/sample.feature similarity index 100% rename from test/hooks-error/features/sample.feature rename to test/hooks-with-error/features/sample.feature diff --git a/test/hooks-with-error/features/steps.ts b/test/hooks-with-error/features/steps.ts new file mode 100644 index 00000000..073bd886 --- /dev/null +++ b/test/hooks-with-error/features/steps.ts @@ -0,0 +1,56 @@ +import { Given, Before, BeforeAll, After, AfterAll } from './fixtures'; + +Before(async function ({ $testInfo, log }) { + log(`Before 1 ${$testInfo.title}`); + canThrow('Before 1'); +}); + +Before(async ({ $testInfo, log }) => { + log(`Before 2 ${$testInfo.title}`); + canThrow('Before 2'); +}); + +After(async function ({ $testInfo, log }) { + log(`After 1 ${$testInfo.title}`); + canThrow('After 1'); +}); + +After(async ({ $testInfo, log }) => { + log(`After 2 ${$testInfo.title}`); + canThrow('After 2'); +}); + +BeforeAll(async function ({ log }) { + log(`BeforeAll 1`); + canThrow('BeforeAll 1'); +}); + +BeforeAll(async ({ log }) => { + log(`BeforeAll 2`); + canThrow('BeforeAll 2'); +}); + +AfterAll(async function ({ log }) { + log(`AfterAll 1`); + canThrow('AfterAll 1'); +}); + +AfterAll(async ({ log }) => { + log(`AfterAll 2`); + canThrow('AfterAll 2'); +}); + +Given('Step {int}', async ({ log }, n: number) => { + log(`Step ${n}`); +}); + +Given('Step bg', async ({ log }) => { + log(`Step bg`); +}); + +function canThrow(hookTitle: string) { + const shouldThrow = process.env.ERROR && hookTitle.startsWith(process.env.ERROR); + if (shouldThrow) { + throw new Error(hookTitle); + } +} diff --git a/test/hooks-timeout/package.json b/test/hooks-with-error/package.json similarity index 77% rename from test/hooks-timeout/package.json rename to test/hooks-with-error/package.json index de610253..61692124 100644 --- a/test/hooks-timeout/package.json +++ b/test/hooks-with-error/package.json @@ -1,3 +1,4 @@ { - "description": "This file is required for Playwright to consider this dir as a . It ensures to load 'playwright-bdd' from './test/node_modules/playwright-bdd' and output './test-results' here to avoid conflicts." + "description": "This file is required for Playwright to consider this dir as a . It ensures to load 'playwright-bdd' from './test/node_modules/playwright-bdd' and output './test-results' here to avoid conflicts.", + "smoke": true } diff --git a/test/hooks-error/playwright.config.ts b/test/hooks-with-error/playwright.config.ts similarity index 100% rename from test/hooks-error/playwright.config.ts rename to test/hooks-with-error/playwright.config.ts diff --git a/test/hooks-error/test.mjs b/test/hooks-with-error/test.mjs similarity index 100% rename from test/hooks-error/test.mjs rename to test/hooks-with-error/test.mjs diff --git a/test/hooks-with-timeout/features/fixtures.ts b/test/hooks-with-timeout/features/fixtures.ts new file mode 100644 index 00000000..788d6da6 --- /dev/null +++ b/test/hooks-with-timeout/features/fixtures.ts @@ -0,0 +1,6 @@ +import { test as base, createBdd } from 'playwright-bdd'; +import { withLog } from '../../_helpers/withLog'; + +export const test = withLog(base); + +export const { Given, Before, BeforeAll, After, AfterAll } = createBdd(test); diff --git a/test/hooks-timeout/features/sample.feature b/test/hooks-with-timeout/features/sample.feature similarity index 100% rename from test/hooks-timeout/features/sample.feature rename to test/hooks-with-timeout/features/sample.feature diff --git a/test/hooks-with-timeout/features/steps.ts b/test/hooks-with-timeout/features/steps.ts new file mode 100644 index 00000000..732d79e4 --- /dev/null +++ b/test/hooks-with-timeout/features/steps.ts @@ -0,0 +1,59 @@ +import timers from 'node:timers/promises'; +import { Given, Before, BeforeAll, After, AfterAll } from './fixtures'; + +// worker hooks + +BeforeAll({ timeout: 5 }, async function ({ log }) { + log(`BeforeAll 1`); + await canTimeout(`BeforeAll 1`); +}); + +BeforeAll(async ({ log }) => { + log(`BeforeAll 2`); + await canTimeout(`BeforeAll 2`); +}); + +AfterAll(async function ({ log }) { + log(`AfterAll 1`); + await canTimeout(`AfterAll 1`); +}); + +AfterAll({ timeout: 5 }, async ({ log }) => { + log(`AfterAll 2`); + await canTimeout(`AfterAll 2`); +}); + +// scenario hooks + +Before({ timeout: 5 }, async function ({ $testInfo, log }) { + log(`Before 1 ${$testInfo.title}`); + await canTimeout(`Before 1`); +}); + +Before(async ({ $testInfo, log }) => { + log(`Before 2 ${$testInfo.title}`); + await canTimeout(`Before 2`); +}); + +After(async function ({ $testInfo, log }) { + log(`After 1 ${$testInfo.title}`); + await canTimeout(`After 1`); +}); + +After({ timeout: 5 }, async ({ $testInfo, log }) => { + log(`After 2 ${$testInfo.title}`); + await canTimeout(`After 2`); +}); + +// step + +Given('State {int}', async ({ $testInfo, log }) => { + log(`Step ${$testInfo.title}`); +}); + +async function canTimeout(hookTitle: string) { + const shouldTimeout = process.env.TIMEOUT && hookTitle.startsWith(process.env.TIMEOUT); + if (shouldTimeout) { + await timers.setTimeout(100); + } +} diff --git a/test/hooks-tags-worker/package.json b/test/hooks-with-timeout/package.json similarity index 100% rename from test/hooks-tags-worker/package.json rename to test/hooks-with-timeout/package.json diff --git a/test/hooks-order/playwright.config.ts b/test/hooks-with-timeout/playwright.config.ts similarity index 100% rename from test/hooks-order/playwright.config.ts rename to test/hooks-with-timeout/playwright.config.ts diff --git a/test/hooks-timeout/test.mjs b/test/hooks-with-timeout/test.mjs similarity index 100% rename from test/hooks-timeout/test.mjs rename to test/hooks-with-timeout/test.mjs diff --git a/test/reporter-data/features/steps.ts b/test/reporter-data/features/steps.ts index 0a22d84f..54376a53 100644 --- a/test/reporter-data/features/steps.ts +++ b/test/reporter-data/features/steps.ts @@ -13,4 +13,5 @@ Before({ name: 'hook 1' }, async () => {}); Before(async () => {}); BeforeAll(async () => {}); BeforeAll({ name: 'named BeforeAll hook' }, async () => {}); +AfterAll(async () => {}); AfterAll({ name: 'named AfterAll hook' }, async () => {}); diff --git a/test/reporter-data/test.mjs b/test/reporter-data/test.mjs index 377f7fed..923b4f85 100644 --- a/test/reporter-data/test.mjs +++ b/test/reporter-data/test.mjs @@ -25,7 +25,6 @@ function checkStepLocations(output) { expect(output).toContain(`BeforeAll Hooks`); expect(output).toContain(`BeforeAll hook ${normalize('features/steps.ts')}:14:1`); expect(output).toContain(`named BeforeAll hook ${normalize('features/steps.ts')}:15:1`); - expect(output).toContain(`named AfterAll hook ${normalize('features/steps.ts')}:16:1`); expect(output).toContain(`BeforeEach Hooks`); expect(output).toContain(`hook 1 ${normalize('features/steps.ts')}:12:1`); @@ -34,6 +33,13 @@ function checkStepLocations(output) { expect(output).toContain( `Background ${normalize('.features-gen/features/sample.feature.spec.js')}:6:8`, ); + + // after hooks + expect(output).toContain(`AfterAll Hooks`); + // AfterAll hooks are not executed via test.step and don't have location. + // See: https://github.com/microsoft/playwright/issues/33750 + // todo: fix this + // expect(output).toContain(`named AfterAll hook ${normalize('features/steps.ts')}:17:1`); } function checkStepTitles(output) {