diff --git a/test/reporter-cucumber-html/.features-gen/features/sample.feature.spec.js-snapshots/rich-feature-failing-match-snapshot-1-darwin.txt b/test/reporter-cucumber-html/.features-gen/error-in-step/sample.feature.spec.js-snapshots/error-in-step-failing-match-snapshot-1-darwin.txt similarity index 100% rename from test/reporter-cucumber-html/.features-gen/features/sample.feature.spec.js-snapshots/rich-feature-failing-match-snapshot-1-darwin.txt rename to test/reporter-cucumber-html/.features-gen/error-in-step/sample.feature.spec.js-snapshots/error-in-step-failing-match-snapshot-1-darwin.txt diff --git a/test/reporter-cucumber-html/.features-gen/features/sample.feature.spec.js-snapshots/rich-feature-failing-match-snapshot-1-linux.txt b/test/reporter-cucumber-html/.features-gen/error-in-step/sample.feature.spec.js-snapshots/error-in-step-failing-match-snapshot-1-linux.txt similarity index 100% rename from test/reporter-cucumber-html/.features-gen/features/sample.feature.spec.js-snapshots/rich-feature-failing-match-snapshot-1-linux.txt rename to test/reporter-cucumber-html/.features-gen/error-in-step/sample.feature.spec.js-snapshots/error-in-step-failing-match-snapshot-1-linux.txt diff --git a/test/reporter-cucumber-html/.features-gen/features/sample.feature.spec.js-snapshots/rich-feature-failing-match-snapshot-1-win32.txt b/test/reporter-cucumber-html/.features-gen/error-in-step/sample.feature.spec.js-snapshots/error-in-step-failing-match-snapshot-1-win32.txt similarity index 100% rename from test/reporter-cucumber-html/.features-gen/features/sample.feature.spec.js-snapshots/rich-feature-failing-match-snapshot-1-win32.txt rename to test/reporter-cucumber-html/.features-gen/error-in-step/sample.feature.spec.js-snapshots/error-in-step-failing-match-snapshot-1-win32.txt diff --git a/test/reporter-cucumber-html/check-report/fixtures.ts b/test/reporter-cucumber-html/check-report/fixtures.ts new file mode 100644 index 00000000..264e3a9b --- /dev/null +++ b/test/reporter-cucumber-html/check-report/fixtures.ts @@ -0,0 +1,28 @@ +import { pathToFileURL } from 'node:url'; +import { test as base } from '@playwright/test'; +import { HtmlReport } from './poms/HtmlReport'; +import { Feature } from './poms/Feature'; +import { Scenario } from './poms/Scenario'; + +type Fixtures = { + htmlReport: HtmlReport; + featureUri: string; + feature: Feature; + scenario: Scenario; +}; + +export const test = base.extend({ + page: async ({ page }, use) => { + await page.goto(pathToFileURL('actual-reports/report.html').href); + await use(page); + }, + htmlReport: async ({ page }, use) => use(new HtmlReport(page)), + featureUri: ['', { option: true }], // will be overwritten in test files + feature: async ({ htmlReport, featureUri }, use) => { + if (!featureUri) throw new Error('Missing featureUri'); + await use(htmlReport.getFeature(featureUri)); + }, + scenario: async ({ feature }, use, testInfo) => { + await use(feature.getScenario(testInfo.title)); + }, +}); diff --git a/test/reporter-cucumber-html/check-report/helpers.ts b/test/reporter-cucumber-html/check-report/helpers.ts deleted file mode 100644 index 25a42818..00000000 --- a/test/reporter-cucumber-html/check-report/helpers.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { pathToFileURL } from 'node:url'; -import { Locator, Page } from '@playwright/test'; - -export async function openReport(page: Page) { - await page.goto(pathToFileURL('actual-reports/report.html').href); -} - -export function getFeature(page: Page) { - return new HtmlReport(page).getFeature('rich feature'); -} - -export function getScenario(page: Page, title: string) { - return getFeature(page).getScenario(title); -} - -export class HtmlReport { - constructor(public page: Page) {} - - getFeature(title: string) { - return new Feature(this.page, title); - } -} - -export class Feature { - root: Locator; - header: Locator; - constructor( - public page: Page, - public title: string, - ) { - this.header = page.getByRole('heading', { name: title, level: 1 }); - this.root = page.locator('section').filter({ has: this.header }); - } - - getBackground() { - return new Scenario(this, `Background:`); - } - - getScenario(title: string) { - return new Scenario(this, `Scenario:${title}`); - } - - getScenarioOutline(title: string) { - return new Scenario(this, `Scenario Outline:${title}`); - } - - getTags() { - return this.root.locator('> [aria-label="Tags"]').getByRole('listitem'); - } -} - -export class Scenario { - root: Locator; - header: Locator; - - constructor( - public feature: Feature, - public title: string, - ) { - this.header = this.page.getByRole('heading', { name: title, exact: true }); - this.root = feature.root.locator('section').filter({ has: this.header }); - } - - get page() { - return this.feature.page; - } - - getSteps(status: 'all' | 'failed' | 'passed' | 'skipped' = 'all') { - const steps = this.root.locator('[aria-label="Steps"]').getByRole('listitem'); - return status === 'all' - ? steps - : steps.filter({ - has: this.page.locator(`[data-status="${status.toUpperCase()}"]`), - }); - } - - getAttachments() { - return this.root.locator('details'); - } - - getLogs() { - return this.root.locator('div > pre').filter({ - hasNot: this.getErrors(), - }); - } - - getErrors() { - return this.getSteps() - .locator('div > pre') - .filter({ - hasText: /error|timeout|expected|browser .*?closed/i, - }); - } - - getTags() { - return this.root.locator('[aria-label="Tags"]').getByRole('listitem'); - } - - getDataTable() { - return this.root.getByRole('table'); - } - - getDocString() { - // todo: distinguish from error - return this.root.locator('div > pre'); - } - - getExamples() { - return this.root.locator('section'); - } -} diff --git a/test/reporter-cucumber-html/check-report/playwright.config.ts b/test/reporter-cucumber-html/check-report/playwright.config.ts index 4d5d5266..21e534d6 100644 --- a/test/reporter-cucumber-html/check-report/playwright.config.ts +++ b/test/reporter-cucumber-html/check-report/playwright.config.ts @@ -4,10 +4,12 @@ import { defineConfig } from '@playwright/test'; export default defineConfig({ - expect: { timeout: 500 }, + testDir: './specs', outputDir: './test-results', reporter: 'dot', use: { screenshot: 'only-on-failure', + viewport: { width: 800, height: 720 }, }, + expect: { timeout: 500 }, }); diff --git a/test/reporter-cucumber-html/check-report/poms/Feature.ts b/test/reporter-cucumber-html/check-report/poms/Feature.ts new file mode 100644 index 00000000..32d457d2 --- /dev/null +++ b/test/reporter-cucumber-html/check-report/poms/Feature.ts @@ -0,0 +1,43 @@ +import { Locator, Page } from '@playwright/test'; +import { Scenario } from './Scenario'; +import path from 'node:path'; + +export class Feature { + root: Locator; + + constructor( + public page: Page, + public featureUri: string, + ) { + const container = page + .locator('[data-accordion-component="AccordionItem"]') + .filter({ has: this.getFileHeader() }); + this.root = container.locator('[data-accordion-component="AccordionItemPanel"]'); + } + + getFileHeader() { + return this.page.locator('[data-accordion-component="AccordionItemHeading"]', { + hasText: path.normalize(this.featureUri), + }); + } + + getFeatureHeader() { + return this.root.getByRole('heading', { level: 1 }); + } + + getBackground() { + return new Scenario(this, `Background:`); + } + + getScenario(title: string) { + return new Scenario(this, `Scenario:${title}`); + } + + getScenarioOutline(title: string) { + return new Scenario(this, `Scenario Outline:${title}`); + } + + getTags() { + return this.root.locator('> section > [aria-label="Tags"]').getByRole('listitem'); + } +} diff --git a/test/reporter-cucumber-html/check-report/poms/HtmlReport.ts b/test/reporter-cucumber-html/check-report/poms/HtmlReport.ts new file mode 100644 index 00000000..c47ff198 --- /dev/null +++ b/test/reporter-cucumber-html/check-report/poms/HtmlReport.ts @@ -0,0 +1,10 @@ +import { Page } from '@playwright/test'; +import { Feature } from './Feature'; + +export class HtmlReport { + constructor(public page: Page) {} + + getFeature(featureUri: string) { + return new Feature(this.page, featureUri); + } +} diff --git a/test/reporter-cucumber-html/check-report/poms/Scenario.ts b/test/reporter-cucumber-html/check-report/poms/Scenario.ts new file mode 100644 index 00000000..1ec1de0c --- /dev/null +++ b/test/reporter-cucumber-html/check-report/poms/Scenario.ts @@ -0,0 +1,71 @@ +import { Locator } from '@playwright/test'; +import { Feature } from './Feature'; + +/** + * POM for scenario, scenario outline, and background + */ +export class Scenario { + root: Locator; + header: Locator; + + constructor( + public feature: Feature, + public titleWithKeyword: string, + ) { + // important to use this.page here, not feature.root + this.header = this.page.getByRole('heading', { name: titleWithKeyword, level: 2 }); + this.root = feature.root + .locator('section') + // filter out top section + .filter({ hasNot: this.page.getByRole('heading', { level: 1 }) }) + .filter({ has: this.header }); + } + + get page() { + return this.feature.page; + } + + getSteps(status: 'all' | 'failed' | 'passed' | 'skipped' = 'all') { + const steps = this.root.locator('[aria-label="Steps"]').getByRole('listitem'); + return status === 'all' + ? steps + : steps.filter({ + has: this.page.locator(`[data-status="${status.toUpperCase()}"]`), + }); + } + + getAttachments() { + return this.root.locator('details'); + } + + getLogs() { + return this.root.locator('div > pre').filter({ + hasNot: this.getErrors(), + }); + } + + getErrors() { + return this.getSteps() + .locator('div > pre') + .filter({ + hasText: /error|timeout|expected|browser .*?closed/i, + }); + } + + getTags() { + return this.root.locator('[aria-label="Tags"]').getByRole('listitem'); + } + + getDataTable() { + return this.root.getByRole('table'); + } + + getDocString() { + // todo: distinguish from error + return this.root.locator('div > pre'); + } + + getExamples() { + return this.root.locator('section'); + } +} diff --git a/test/reporter-cucumber-html/check-report/specs/error-in-after.test.ts b/test/reporter-cucumber-html/check-report/specs/error-in-after.test.ts new file mode 100644 index 00000000..e974aeee --- /dev/null +++ b/test/reporter-cucumber-html/check-report/specs/error-in-after.test.ts @@ -0,0 +1,83 @@ +import { normalize } from 'node:path'; +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; +import { getPackageVersion } from '../../../../src/utils'; + +// Automatic screenshot for failing fixtures teardown depends on pw version. +// see: https://github.com/microsoft/playwright/issues/29325 +const pwVersion = getPackageVersion('@playwright/test'); +const hasAutoScreenshotFixtureTeardown = pwVersion >= '1.42.0' && pwVersion < '1.45.0'; + +test.use({ featureUri: 'error-in-after/sample.feature' }); + +test('Failing by failingAfterFixtureNoStep', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + 'my attachment|before use', + 'Givenstep that uses failingAfterFixtureNoStep', + 'WhenAction 3', + `Hook "fixture: failingAfterFixtureNoStep" failed: ${normalize('features/error-in-after/fixtures.ts')}:`, + ]); + if (hasAutoScreenshotFixtureTeardown) { + await expect(scenario.getSteps()).toContainText(['screenshot']); + } + await expect(scenario.getAttachments()).toContainText([ + 'my attachment|before use', // prettier-ignore + 'my attachment|after use', + ]); + await expect(scenario.getSteps('passed')).toHaveCount(2); + await expect(scenario.getSteps('failed')).toHaveCount(1); + await expect(scenario.getSteps('skipped')).toHaveCount(0); + await expect(scenario.getErrors()).toContainText(['error in failingAfterFixtureNoStep']); +}); + +test('Failing by failingAfterFixtureWithStep', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + 'my attachment|outside step (before use)', + 'Givenstep that uses failingAfterFixtureWithStep', + 'WhenAction 4', + `Hook "step in failingAfterFixtureWithStep" failed: ${normalize('features/error-in-after/fixtures.ts')}:`, + 'my attachment|outside step (after use)', + ]); + if (hasAutoScreenshotFixtureTeardown) { + await expect(scenario.getSteps()).toContainText(['screenshot']); + } + await expect(scenario.getAttachments()).toContainText([ + 'my attachment|outside step (before use)', + 'my attachment|in step', + 'my attachment|outside step (after use)', + ]); + await expect(scenario.getSteps('passed')).toHaveCount(2); + await expect(scenario.getSteps('failed')).toHaveCount(1); + await expect(scenario.getSteps('skipped')).toHaveCount(0); + await expect(scenario.getErrors()).toContainText(['error in failingAfterFixtureWithStep']); +}); + +test('timeout in after fixture', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + 'GivenAction 0', + 'Givenstep that uses timeouted after fixture', + 'WhenAction 1', + /Hook "(After Hooks|fixture: timeoutedAfterFixture)" failed/, + ]); + // don't check screenshot as it's not reliable in timeouts + await expect(scenario.getSteps('passed')).toHaveCount(3); + await expect(scenario.getSteps('failed')).toHaveCount(1); + await expect(scenario.getErrors()).toContainText([ + /but tearing down "timeoutedAfterFixture" ran out of time|Tearing down "timeoutedAfterFixture" exceeded the test timeout/, + ]); +}); + +test('timeout in step and in after fixture', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + 'GivenAction 0', + 'Giventimeouted step', + 'WhenAction 1', + 'Givenstep that uses timeouted after fixture', + /Hook "(After Hooks|fixture: timeoutedAfterFixture)" failed/, + ]); + await expect(scenario.getSteps('passed')).toHaveCount(4); + await expect(scenario.getSteps('failed')).toHaveCount(1); + await expect(scenario.getErrors()).toContainText([ + /Test timeout of \d+ms exceeded|Tearing down "timeoutedAfterFixture" exceeded the test timeout/, + ]); +}); diff --git a/test/reporter-cucumber-html/check-report/specs/error-in-before.test.ts b/test/reporter-cucumber-html/check-report/specs/error-in-before.test.ts new file mode 100644 index 00000000..07276e81 --- /dev/null +++ b/test/reporter-cucumber-html/check-report/specs/error-in-before.test.ts @@ -0,0 +1,87 @@ +import { normalize } from 'node:path'; +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +test.use({ featureUri: 'error-in-before/sample.feature' }); + +test('Failing by anonymous before hook', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + `Hook "BeforeEach hook" failed: ${normalize('features/error-in-before/steps.ts')}:`, // prettier-ignore + 'GivenAction 1', + 'screenshotDownload trace', + ]); + await expect(scenario.getSteps('failed')).toHaveCount(1); + await expect(scenario.getSteps('skipped')).toHaveCount(1); + await expect(scenario.getTags()).toContainText(['@failing-anonymous-hook']); + await expect(scenario.getErrors()).toContainText(['Timed out']); + await expect(scenario.getErrors()).toContainText([`Before({ tags: '@failing-anonymous-hook' }`]); +}); + +test('Failing by named before hook', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + `Hook "failing named before hook" failed: ${normalize('features/error-in-before/steps.ts')}:`, + 'GivenAction 1', + 'screenshotDownload trace', + ]); + await expect(scenario.getSteps('failed')).toHaveCount(1); + await expect(scenario.getSteps('skipped')).toHaveCount(1); + await expect(scenario.getTags()).toContainText(['@failing-named-hook']); + await expect(scenario.getErrors()).toContainText(['Timed out']); + await expect(scenario.getErrors()).toContainText([`Before({ name: 'failing named before hook'`]); +}); + +test('Failing by failingBeforeFixtureNoStep', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + `Hook "fixture: failingBeforeFixtureNoStep" failed: ${normalize('features/error-in-before/fixtures.ts')}:`, + 'Givenstep that uses failingBeforeFixtureNoStep', + 'WhenAction 1', + 'screenshot', + ]); + await expect(scenario.getAttachments()).toHaveText([ + 'my attachment|outside step', + 'my attachment|in step', + 'screenshot', + ]); + await expect(scenario.getSteps('failed')).toHaveCount(1); + await expect(scenario.getSteps('skipped')).toHaveCount(2); + await expect(scenario.getErrors()).toContainText(['error in failingBeforeFixtureNoStep']); +}); + +test('Failing by failingBeforeFixtureWithStep', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + `Hook "my step" failed: ${normalize('features/error-in-before/fixtures.ts')}:`, + 'Givenstep that uses failingBeforeFixtureWithStep', + 'WhenAction 2', + 'screenshot', + ]); + await expect(scenario.getAttachments()).toHaveText([ + 'my attachment|in step', + 'my attachment|outside step', + 'screenshot', + ]); + await expect(scenario.getSteps('failed')).toHaveCount(1); + await expect(scenario.getSteps('skipped')).toHaveCount(2); + await expect(scenario.getErrors()).toContainText(['error in failingBeforeFixtureWithStep']); +}); + +test('timeout in before fixture', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + 'GivenAction 0', + 'Givenstep that uses timeouted before fixture', + 'WhenAction 1', + ]); + // 1. position of error message sometimes appears in After Hooks, so check it separately + // 2. here can be different error messages + await expect(scenario.getSteps()).toContainText([/Hook "(.+)" failed/]); + // screenshot position changes between PW versions, so check it separately + await expect(scenario.getSteps()).toContainText(['screenshot']); + // sometimes error is the following: + // "browser.newContext: Target page, context or browser has been closed" + // in that case there are two errors in test report. + expect(await scenario.getSteps('failed').count()).toBeGreaterThan(0); + await expect(scenario.getSteps('skipped')).toHaveCount(3); + await expect(scenario.getErrors()).toContainText([ + // here can be different error messages + /(Test timeout of \d+ms exceeded while setting up "timeoutedBeforeFixture")|(browser has been closed)|(Browser closed)/, + ]); +}); diff --git a/test/reporter-cucumber-html/check-report/specs/error-in-bg.test.ts b/test/reporter-cucumber-html/check-report/specs/error-in-bg.test.ts new file mode 100644 index 00000000..6888f15f --- /dev/null +++ b/test/reporter-cucumber-html/check-report/specs/error-in-bg.test.ts @@ -0,0 +1,31 @@ +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +test.use({ featureUri: 'error-in-bg/sample.feature' }); + +test('background is failing', async ({ feature }) => { + const background = feature.getBackground(); + await expect(background.getSteps()).toContainText([ + 'failing step', // prettier-ignore + ]); + await expect(background.getSteps('failed')).toHaveCount(1); + await expect(background.getErrors()).toContainText(['Timed out 1ms waiting for expect']); +}); + +test('scenario 1', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + 'GivenAction 1', // prettier-ignore + 'screenshot', + ]); + await expect(scenario.getSteps('skipped')).toHaveCount(1); + await expect(scenario.getErrors()).not.toBeVisible(); +}); + +test('scenario 2', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + 'GivenAction 2', // prettier-ignore + 'screenshot', + ]); + await expect(scenario.getSteps('skipped')).toHaveCount(1); + await expect(scenario.getErrors()).not.toBeVisible(); +}); diff --git a/test/reporter-cucumber-html/check-report/specs/error-in-step.test.ts b/test/reporter-cucumber-html/check-report/specs/error-in-step.test.ts new file mode 100644 index 00000000..687d4f9b --- /dev/null +++ b/test/reporter-cucumber-html/check-report/specs/error-in-step.test.ts @@ -0,0 +1,67 @@ +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; + +test.use({ featureUri: 'error-in-step/sample.feature' }); + +test('failing by step assertion', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + 'Givenfailing step', // prettier-ignore + 'screenshotDownload trace', + ]); + await expect(scenario.getErrors()).toContainText([ + 'Timed out 1ms waiting for expect', // prettier-ignore + ]); +}); + +test('failing by step timeout', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + 'GivenAction 0', + 'Giventimeouted step', + 'WhenAction 1', + // not 'screenshot', b/c no page + // 'screenshot', + 'Download trace', + ]); + // don't check passed/skipped steps counts b/c in different PW versions it's different + await expect(scenario.getErrors()).toContainText([/Test timeout of \d+ms exceeded/]); +}); + +test('failing match snapshot', async ({ scenario }) => { + await expect(scenario.getSteps()).toContainText([ + 'Whenopen page "https://example.com"', + 'Thenpage title snapshot matches the golden one', + ]); + await expect(scenario.getAttachments()).toHaveText([ + 'error-in-step-failing-match-snapshot-1-expected.txtbla-bla', + 'error-in-step-failing-match-snapshot-1-actual.txtExample Domain', + 'screenshot', + ]); + await expect(scenario.getSteps('passed')).toHaveCount(1); + await expect(scenario.getSteps('failed')).toHaveCount(1); + await expect(scenario.getSteps('skipped')).toHaveCount(0); + // since pw 1.49 error text for snapshots was changed. + // before: Snapshot comparison failed + // after: expect(string).toMatchSnapshot(expected) + await expect(scenario.getErrors()).toContainText( + /Snapshot comparison failed|expect\(string\)\.toMatchSnapshot\(expected\)/, + ); +}); + +test('soft assertions', async ({ scenario }) => { + await expect(scenario.getSteps()).toHaveText([ + /Givenfailing soft assertion "foo"/, + 'AndAction 1', + /Andfailing soft assertion "bar"/, + 'AndAction 2', + // not 'screenshot', b/c no page + // 'screenshot', + 'Download trace', + ]); + await expect(scenario.getSteps('passed')).toHaveCount(2); + await expect(scenario.getSteps('failed')).toHaveCount(2); + await expect(scenario.getSteps('skipped')).toHaveCount(0); + await expect(scenario.getErrors()).toContainText([ + 'Expected: "foo" Received: "xxx"', + 'Expected: "bar" Received: "xxx"', + ]); +}); diff --git a/test/reporter-cucumber-html/check-report/specs/failed.test.ts b/test/reporter-cucumber-html/check-report/specs/failed.test.ts deleted file mode 100644 index 7ffaec77..00000000 --- a/test/reporter-cucumber-html/check-report/specs/failed.test.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { normalize } from 'node:path'; -import { test, expect } from '@playwright/test'; -import { getFeature, getScenario, openReport } from '../helpers'; -import { getPackageVersion } from '../../../../src/utils'; - -const pwVersion = getPackageVersion('@playwright/test'); - -// Automatic screenshot for failing fixtures teardown depends on pw version. -// see: https://github.com/microsoft/playwright/issues/29325 -const hasAutoScreenshotFixtureTeardown = pwVersion >= '1.42.0' && pwVersion < '1.45.0'; - -test.beforeEach(async ({ page }) => { - await openReport(page); -}); - -test('Scenario: Failing by step', async ({ page }) => { - const scenario = getScenario(page, 'Failing by step'); - await expect(scenario.getSteps()).toContainText(['Givenfailing step', 'screenshot']); - await expect(scenario.getErrors()).toContainText(['Timed out']); -}); - -test('Scenario: Failing by background step', async ({ page }) => { - const scenario = getScenario(page, 'Failing by background step'); - await expect(scenario.getSteps()).toContainText(['GivenAction 1', 'screenshot']); - await expect(scenario.getSteps('skipped')).toHaveCount(1); - await expect(scenario.getErrors()).not.toBeVisible(); - - const background = getFeature(page).getBackground(); - await expect(background.getSteps()).toContainText([ - 'step failing for scenario "Failing by background step"', - ]); - await expect(background.getSteps('failed')).toHaveCount(1); - await expect(background.getErrors()).toContainText(['expect(true).toEqual(false)']); -}); - -test('Scenario: Failing by anonymous before hook', async ({ page }) => { - const scenario = getScenario(page, 'Failing by anonymous before hook'); - await expect(scenario.getSteps()).toContainText([ - `Hook "BeforeEach hook" failed: ${normalize('features/failed-steps.ts')}:`, // prettier-ignore - 'GivenAction 1', - 'screenshot', - ]); - await expect(scenario.getSteps('failed')).toHaveCount(1); - await expect(scenario.getSteps('skipped')).toHaveCount(1); - await expect(scenario.getTags()).toContainText(['@failing-anonymous-hook']); - await expect(scenario.getErrors()).toContainText(['Timed out']); - await expect(scenario.getErrors()).toContainText([`Before({ tags: '@failing-anonymous-hook' }`]); -}); - -test('Scenario: Failing by named before hook', async ({ page }) => { - const scenario = getScenario(page, 'Failing by named before hook'); - await expect(scenario.getSteps()).toContainText([ - `Hook "failing named before hook" failed: ${normalize('features/failed-steps.ts')}:`, - 'GivenAction 1', - 'screenshot', - ]); - await expect(scenario.getSteps('failed')).toHaveCount(1); - await expect(scenario.getSteps('skipped')).toHaveCount(1); - await expect(scenario.getTags()).toContainText(['@failing-named-hook']); - await expect(scenario.getErrors()).toContainText(['Timed out']); - await expect(scenario.getErrors()).toContainText([`Before({ name: 'failing named before hook'`]); -}); - -test('Scenario: Failing by failingBeforeFixtureNoStep', async ({ page }) => { - const scenario = getScenario(page, 'Failing by failingBeforeFixtureNoStep'); - await expect(scenario.getSteps()).toContainText([ - `Hook "fixture: failingBeforeFixtureNoStep" failed: ${normalize('features/fixtures.ts')}:`, - 'Givenstep that uses failingBeforeFixtureNoStep', - 'WhenAction 1', - 'screenshot', - ]); - await expect(scenario.getAttachments()).toHaveText([ - 'my attachment|outside step', - 'my attachment|in step', - 'screenshot', - ]); - await expect(scenario.getSteps('failed')).toHaveCount(1); - await expect(scenario.getSteps('skipped')).toHaveCount(2); - await expect(scenario.getErrors()).toContainText(['error in failingBeforeFixtureNoStep']); -}); - -test('Scenario: Failing by failingBeforeFixtureWithStep', async ({ page }) => { - const scenario = getScenario(page, 'Failing by failingBeforeFixtureWithStep'); - await expect(scenario.getSteps()).toContainText([ - `Hook "my step" failed: ${normalize('features/fixtures.ts')}:`, - 'Givenstep that uses failingBeforeFixtureWithStep', - 'WhenAction 2', - 'screenshot', - ]); - await expect(scenario.getAttachments()).toHaveText([ - 'my attachment|in step', - 'my attachment|outside step', - 'screenshot', - ]); - await expect(scenario.getSteps('failed')).toHaveCount(1); - await expect(scenario.getSteps('skipped')).toHaveCount(2); - await expect(scenario.getErrors()).toContainText(['error in failingBeforeFixtureWithStep']); -}); - -test('Scenario: Failing by failingAfterFixtureNoStep', async ({ page }) => { - const scenario = getScenario(page, 'Failing by failingAfterFixtureNoStep'); - await expect(scenario.getSteps()).toContainText([ - 'my attachment|before use', - 'Givenstep that uses failingAfterFixtureNoStep', - 'WhenAction 3', - `Hook "fixture: failingAfterFixtureNoStep" failed: ${normalize('features/fixtures.ts')}:`, - ]); - if (hasAutoScreenshotFixtureTeardown) { - await expect(scenario.getSteps()).toContainText(['screenshot']); - } - await expect(scenario.getAttachments()).toContainText([ - 'my attachment|before use', // prettier-ignore - 'my attachment|after use', - ]); - await expect(scenario.getSteps('passed')).toHaveCount(2); - await expect(scenario.getSteps('failed')).toHaveCount(1); - await expect(scenario.getSteps('skipped')).toHaveCount(0); - await expect(scenario.getErrors()).toContainText(['error in failingAfterFixtureNoStep']); -}); - -test('Scenario: Failing by failingAfterFixtureWithStep', async ({ page }) => { - const scenario = getScenario(page, 'Failing by failingAfterFixtureWithStep'); - await expect(scenario.getSteps()).toContainText([ - 'my attachment|outside step (before use)', - 'Givenstep that uses failingAfterFixtureWithStep', - 'WhenAction 4', - `Hook "step in failingAfterFixtureWithStep" failed: ${normalize('features/fixtures.ts')}:`, - 'my attachment|outside step (after use)', - ]); - if (hasAutoScreenshotFixtureTeardown) { - await expect(scenario.getSteps()).toContainText(['screenshot']); - } - await expect(scenario.getAttachments()).toContainText([ - 'my attachment|outside step (before use)', - 'my attachment|in step', - 'my attachment|outside step (after use)', - ]); - await expect(scenario.getSteps('passed')).toHaveCount(2); - await expect(scenario.getSteps('failed')).toHaveCount(1); - await expect(scenario.getSteps('skipped')).toHaveCount(0); - await expect(scenario.getErrors()).toContainText(['error in failingAfterFixtureWithStep']); -}); - -test('Scenario: failing match snapshot', async ({ page }) => { - const scenario = getScenario(page, 'failing match snapshot'); - await expect(scenario.getSteps()).toContainText([ - 'Whenopen page "https://example.com"', - 'Thenpage title snapshot matches the golden one', - ]); - await expect(scenario.getAttachments()).toHaveText([ - 'rich-feature-failing-match-snapshot-1-expected.txtbla-bla', - 'rich-feature-failing-match-snapshot-1-actual.txtExample Domain', - 'screenshot', - ]); - await expect(scenario.getSteps('passed')).toHaveCount(1); - await expect(scenario.getSteps('failed')).toHaveCount(1); - await expect(scenario.getSteps('skipped')).toHaveCount(0); - // since pw 1.49 error text for snapshots was changed. - // before: Snapshot comparison failed - // after: expect(string).toMatchSnapshot(expected) - await expect(scenario.getErrors()).toContainText( - /Snapshot comparison failed|expect\(string\)\.toMatchSnapshot\(expected\)/, - ); -}); - -test('Scenario: soft assertions', async ({ page }) => { - const scenario = getScenario(page, 'soft assertions'); - await expect(scenario.getSteps()).toHaveText([ - /Givenfailing soft assertion "foo"/, - 'AndAction 1', - /Andfailing soft assertion "bar"/, - 'AndAction 2', - 'screenshotDownload trace', - ]); - await expect(scenario.getSteps('passed')).toHaveCount(2); - await expect(scenario.getSteps('failed')).toHaveCount(2); - await expect(scenario.getSteps('skipped')).toHaveCount(0); - await expect(scenario.getErrors()).toContainText([ - 'Expected: "foo" Received: "xxx"', - 'Expected: "bar" Received: "xxx"', - ]); -}); diff --git a/test/reporter-cucumber-html/check-report/specs/header.test.ts b/test/reporter-cucumber-html/check-report/specs/header.test.ts deleted file mode 100644 index c7f16965..00000000 --- a/test/reporter-cucumber-html/check-report/specs/header.test.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { test, expect } from '@playwright/test'; -import { getFeature, openReport } from '../helpers'; - -// after changing this counts you should also update test/reporter-cucumber-junit -const failed = 14; -const passed = 8; -const skipped = 1; - -test.beforeEach(async ({ page }) => { - await openReport(page); -}); - -test('header info', async ({ page }) => { - await expect(page.getByText(/\d+ failed/)).toHaveText(`${failed} failed`); - await expect(page.getByText(/\d+ passed/)).toHaveText(`${passed} passed`); - await expect(page.getByText(/\d+ skipped/)).toHaveText(`${skipped} skipped`); - await expect(page.getByText(/\d+ executed/)).toHaveText(`${failed + passed + skipped} executed`); - - await expect(page.locator('dl').getByText('node.js')).toBeVisible(); - await expect(page.locator('dl').getByText('playwright-bdd')).toBeVisible(); - - await expect(getFeature(page).getTags()).toHaveText(['@feature-tag']); -}); diff --git a/test/reporter-cucumber-html/check-report/specs/skipped.test.ts b/test/reporter-cucumber-html/check-report/specs/skipped.test.ts deleted file mode 100644 index c2c05529..00000000 --- a/test/reporter-cucumber-html/check-report/specs/skipped.test.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { test, expect } from '@playwright/test'; -import { getScenario, openReport } from '../helpers'; - -test.beforeEach(async ({ page }) => { - await openReport(page); -}); - -test('Scenario: Skipped scenario', async ({ page }) => { - const scenario = getScenario(page, 'Skipped scenario'); - await expect(scenario.getSteps('skipped')).toHaveCount(2); -}); diff --git a/test/reporter-cucumber-html/check-report/specs/passed.test.ts b/test/reporter-cucumber-html/check-report/specs/success.test.ts similarity index 63% rename from test/reporter-cucumber-html/check-report/specs/passed.test.ts rename to test/reporter-cucumber-html/check-report/specs/success.test.ts index 570cb0d7..b4ecafd3 100644 --- a/test/reporter-cucumber-html/check-report/specs/passed.test.ts +++ b/test/reporter-cucumber-html/check-report/specs/success.test.ts @@ -1,14 +1,15 @@ -import { test, expect } from '@playwright/test'; -import { getFeature, getScenario, openReport } from '../helpers'; +import { expect } from '@playwright/test'; +import { test } from '../fixtures'; -test.beforeEach(async ({ page }) => { - await openReport(page); +test.use({ featureUri: 'success/sample.feature' }); + +test('Feature tags', async ({ feature }) => { + await expect(feature.getTags()).toHaveText(['@feature-tag']); }); // Cucumber html-formatter does not show failed retries // See: https://github.com/cucumber/react-components/issues/75 -test('Scenario: Scenario with retries', async ({ page }) => { - const scenario = getScenario(page, 'Scenario with retries'); +test('Scenario with retries', async ({ scenario }) => { await expect(scenario.getSteps()).toHaveText([ 'GivenAction 1', 'Andfails until retry 1', @@ -19,8 +20,7 @@ test('Scenario: Scenario with retries', async ({ page }) => { await expect(scenario.getSteps('passed')).toHaveCount(3); }); -test('Scenario: Scenario with data table', async ({ page }) => { - const scenario = getScenario(page, 'Scenario with data table'); +test('Scenario with data table', async ({ scenario }) => { await expect(scenario.getSteps()).toContainText(['Step with data table']); await expect(scenario.getDataTable().getByRole('cell')).toHaveText([ 'name', @@ -32,14 +32,12 @@ test('Scenario: Scenario with data table', async ({ page }) => { ]); }); -test('Scenario: Scenario with doc string', async ({ page }) => { - const scenario = getScenario(page, 'Scenario with doc string'); +test('Scenario with doc string', async ({ scenario }) => { await expect(scenario.getSteps()).toContainText(['Step with doc string']); await expect(scenario.getDocString()).toHaveText('some text'); }); -test('Scenario: Scenario with attachments', async ({ page }) => { - const scenario = getScenario(page, 'Scenario with attachments'); +test('Scenario with attachments', async ({ scenario }) => { await expect(scenario.getSteps()).toContainText([ 'attach text', 'attach image inline', @@ -53,8 +51,7 @@ test('Scenario: Scenario with attachments', async ({ page }) => { await expect(scenario.getLogs()).toHaveText(['123 some logs']); }); -test('Scenario: Scenario with all keywords and success hooks', async ({ page }) => { - const scenario = getScenario(page, 'Scenario with all keywords and success hooks'); +test('Scenario with all keywords and success hooks', async ({ scenario }) => { await expect(scenario.getSteps('passed')).toHaveCount(7); await expect(scenario.getSteps()).toHaveText([ '', // before hook @@ -69,9 +66,16 @@ test('Scenario: Scenario with all keywords and success hooks', async ({ page }) ]); }); -test('Scenario Outline: Check doubled', async ({ page }) => { - const scenario = getFeature(page).getScenarioOutline('Check doubled'); - await expect(scenario.getSteps()).toContainText(['GivenAction ', 'ThenAction ']); +test('Skipped scenario', async ({ scenario }) => { + await expect(scenario.getSteps('skipped')).toHaveCount(2); +}); + +test('Check doubled', async ({ feature }) => { + const scenario = feature.getScenarioOutline('Check doubled'); + await expect(scenario.getSteps()).toContainText([ + 'GivenAction ', // prettier-ignore + 'ThenAction ', + ]); await expect(scenario.getExamples().nth(0).getByRole('cell')).toContainText([ ' ', 'start', diff --git a/test/reporter-cucumber-html/check-report/specs/timeout.test.ts b/test/reporter-cucumber-html/check-report/specs/timeout.test.ts deleted file mode 100644 index 42cc804f..00000000 --- a/test/reporter-cucumber-html/check-report/specs/timeout.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { test, expect } from '@playwright/test'; -import { getScenario, openReport } from '../helpers'; - -test.beforeEach(async ({ page }) => { - await openReport(page); -}); - -test('Scenario: timeout in before fixture', async ({ page }) => { - const scenario = getScenario(page, 'timeout in before fixture'); - await expect(scenario.getSteps()).toContainText([ - 'GivenAction 0', - 'Givenstep that uses timeouted before fixture', - 'WhenAction 1', - ]); - // 1. position of error message sometimes appears in After Hooks, so check it separately - // 2. here can be different error messages - await expect(scenario.getSteps()).toContainText([/Hook "(.+)" failed/]); - // screenshot position changes between PW versions, so check it separately - await expect(scenario.getSteps()).toContainText(['screenshot']); - // sometimes error is the following: - // "browser.newContext: Target page, context or browser has been closed" - // in that case there are two errors in test report. - expect(await scenario.getSteps('failed').count()).toBeGreaterThan(0); - await expect(scenario.getSteps('skipped')).toHaveCount(3); - await expect(scenario.getErrors()).toContainText([ - // here can be different error messages - /(Test timeout of \d+ms exceeded while setting up "timeoutedBeforeFixture")|(browser has been closed)|(Browser closed)/, - ]); -}); - -test('Scenario: timeout in step', async ({ page }) => { - const scenario = getScenario(page, 'timeout in step'); - await expect(scenario.getSteps()).toContainText([ - 'GivenAction 0', - 'Giventimeouted step', - 'WhenAction 1', - 'screenshot', - ]); - // don't check passed/skipped steps counts b/c in different PW versions it's different - await expect(scenario.getErrors()).toContainText([/Test timeout of \d+ms exceeded/]); -}); - -test('Scenario: timeout in after fixture', async ({ page }) => { - const scenario = getScenario(page, 'timeout in after fixture'); - await expect(scenario.getSteps()).toContainText([ - 'GivenAction 0', - 'Givenstep that uses timeouted after fixture', - 'WhenAction 1', - /Hook "(After Hooks|fixture: timeoutedAfterFixture)" failed/, - ]); - // don't check screenshot as it's not reliable in timeouts - await expect(scenario.getSteps('passed')).toHaveCount(3); - await expect(scenario.getSteps('failed')).toHaveCount(1); - await expect(scenario.getErrors()).toContainText([ - /but tearing down "timeoutedAfterFixture" ran out of time|Tearing down "timeoutedAfterFixture" exceeded the test timeout/, - ]); -}); - -test('Scenario: timeout in step and in after fixture', async ({ page }) => { - const scenario = getScenario(page, 'timeout in step and in after fixture'); - await expect(scenario.getSteps()).toContainText([ - 'GivenAction 0', - 'Giventimeouted step', - 'WhenAction 1', - 'Givenstep that uses timeouted after fixture', - /Hook "(After Hooks|fixture: timeoutedAfterFixture)" failed/, - ]); - await expect(scenario.getSteps('passed')).toHaveCount(4); - await expect(scenario.getSteps('failed')).toHaveCount(1); - await expect(scenario.getErrors()).toContainText([ - /Test timeout of \d+ms exceeded|Tearing down "timeoutedAfterFixture" exceeded the test timeout/, - ]); -}); diff --git a/test/reporter-cucumber-html/features/error-in-after/fixtures.ts b/test/reporter-cucumber-html/features/error-in-after/fixtures.ts new file mode 100644 index 00000000..f2527f2b --- /dev/null +++ b/test/reporter-cucumber-html/features/error-in-after/fixtures.ts @@ -0,0 +1,27 @@ +import { test as base } from 'playwright-bdd'; + +export const test = base.extend<{ + failingAfterFixtureNoStep: void; + failingAfterFixtureWithStep: void; + timeoutedAfterFixture: void; +}>({ + failingAfterFixtureNoStep: async ({}, use, testInfo) => { + await testInfo.attach('my attachment', { body: '|before use' }); + await use(); + await testInfo.attach('my attachment', { body: '|after use' }); + throw new Error('error in failingAfterFixtureNoStep'); + }, + failingAfterFixtureWithStep: async ({}, use, testInfo) => { + await testInfo.attach('my attachment', { body: '|outside step (before use)' }); + await use(); + await testInfo.attach('my attachment', { body: '|outside step (after use)' }); + await test.step('step in failingAfterFixtureWithStep', async () => { + await testInfo.attach('my attachment', { body: '|in step' }); + throw new Error('error in failingAfterFixtureWithStep'); + }); + }, + timeoutedAfterFixture: async ({}, use, testInfo) => { + await use(); + await new Promise((r) => setTimeout(r, testInfo.timeout + 100)); + }, +}); diff --git a/test/reporter-cucumber-html/features/error-in-after/sample.feature b/test/reporter-cucumber-html/features/error-in-after/sample.feature new file mode 100644 index 00000000..11be0aec --- /dev/null +++ b/test/reporter-cucumber-html/features/error-in-after/sample.feature @@ -0,0 +1,21 @@ +Feature: error-in-after + + Scenario: Failing by failingAfterFixtureNoStep + Given step that uses failingAfterFixtureNoStep + When Action 3 + + Scenario: Failing by failingAfterFixtureWithStep + Given step that uses failingAfterFixtureWithStep + When Action 4 + + # see: https://github.com/microsoft/playwright/issues/30175 + Scenario: timeout in after fixture + Given Action 0 + Given step that uses timeouted after fixture + When Action 1 + + Scenario: timeout in step and in after fixture + Given Action 0 + Given timeouted step + When Action 1 + Given step that uses timeouted after fixture diff --git a/test/reporter-cucumber-html/features/error-in-after/steps.ts b/test/reporter-cucumber-html/features/error-in-after/steps.ts new file mode 100644 index 00000000..52bc3642 --- /dev/null +++ b/test/reporter-cucumber-html/features/error-in-after/steps.ts @@ -0,0 +1,18 @@ +import { createBdd } from 'playwright-bdd'; +import { test } from './fixtures'; + +const { Given } = createBdd(test); + +// todo: error in after hook + +Given('step that uses failingAfterFixtureNoStep', async ({ failingAfterFixtureNoStep }) => { + return failingAfterFixtureNoStep; +}); + +Given('step that uses failingAfterFixtureWithStep', async ({ failingAfterFixtureWithStep }) => { + return failingAfterFixtureWithStep; +}); + +Given('step that uses timeouted after fixture', async ({ timeoutedAfterFixture }) => { + return timeoutedAfterFixture; +}); diff --git a/test/reporter-cucumber-html/features/error-in-before-all/sample.feature b/test/reporter-cucumber-html/features/error-in-before-all/sample.feature new file mode 100644 index 00000000..89df1568 --- /dev/null +++ b/test/reporter-cucumber-html/features/error-in-before-all/sample.feature @@ -0,0 +1,3 @@ +Feature: error-in-before-all + + diff --git a/test/reporter-cucumber-html/features/fixtures.ts b/test/reporter-cucumber-html/features/error-in-before/fixtures.ts similarity index 54% rename from test/reporter-cucumber-html/features/fixtures.ts rename to test/reporter-cucumber-html/features/error-in-before/fixtures.ts index b3613a1e..a5bd36cd 100644 --- a/test/reporter-cucumber-html/features/fixtures.ts +++ b/test/reporter-cucumber-html/features/error-in-before/fixtures.ts @@ -3,9 +3,6 @@ import { test as base } from 'playwright-bdd'; export const test = base.extend<{ failingBeforeFixtureNoStep: void; failingBeforeFixtureWithStep: void; - failingAfterFixtureNoStep: void; - failingAfterFixtureWithStep: void; - setTestTimeout: void; timeoutedBeforeFixture: void; timeoutedAfterFixture: void; }>({ @@ -27,27 +24,8 @@ export const test = base.extend<{ await use(); await testInfo.attach('my attachment', { body: 'should not attach' }); }, - failingAfterFixtureNoStep: async ({}, use, testInfo) => { - await testInfo.attach('my attachment', { body: '|before use' }); - await use(); - await testInfo.attach('my attachment', { body: '|after use' }); - throw new Error('error in failingAfterFixtureNoStep'); - }, - failingAfterFixtureWithStep: async ({}, use, testInfo) => { - await testInfo.attach('my attachment', { body: '|outside step (before use)' }); - await use(); - await testInfo.attach('my attachment', { body: '|outside step (after use)' }); - await test.step('step in failingAfterFixtureWithStep', async () => { - await testInfo.attach('my attachment', { body: '|in step' }); - throw new Error('error in failingAfterFixtureWithStep'); - }); - }, timeoutedBeforeFixture: async ({}, use, testInfo) => { await new Promise((r) => setTimeout(r, testInfo.timeout + 100)); await use(); }, - timeoutedAfterFixture: async ({}, use, testInfo) => { - await use(); - await new Promise((r) => setTimeout(r, testInfo.timeout + 100)); - }, }); diff --git a/test/reporter-cucumber-html/features/error-in-before/sample.feature b/test/reporter-cucumber-html/features/error-in-before/sample.feature new file mode 100644 index 00000000..f09ef3fc --- /dev/null +++ b/test/reporter-cucumber-html/features/error-in-before/sample.feature @@ -0,0 +1,22 @@ +Feature: error-in-before + + @failing-anonymous-hook + Scenario: Failing by anonymous before hook + Given Action 1 + + @failing-named-hook + Scenario: Failing by named before hook + Given Action 1 + + Scenario: Failing by failingBeforeFixtureNoStep + Given step that uses failingBeforeFixtureNoStep + When Action 1 + + Scenario: Failing by failingBeforeFixtureWithStep + Given step that uses failingBeforeFixtureWithStep + When Action 2 + + Scenario: timeout in before fixture + Given Action 0 + Given step that uses timeouted before fixture + When Action 1 diff --git a/test/reporter-cucumber-html/features/error-in-before/steps.ts b/test/reporter-cucumber-html/features/error-in-before/steps.ts new file mode 100644 index 00000000..702b699b --- /dev/null +++ b/test/reporter-cucumber-html/features/error-in-before/steps.ts @@ -0,0 +1,25 @@ +import { expect } from '@playwright/test'; +import { createBdd } from 'playwright-bdd'; +import { test } from './fixtures'; + +const { Given, Before } = createBdd(test); + +Before({ name: 'failing named before hook', tags: '@failing-named-hook' }, async ({ page }) => { + await expect(page).toHaveTitle('Some title1'); +}); + +Before({ tags: '@failing-anonymous-hook' }, async ({ page }) => { + await expect(page).toHaveTitle('Some title2'); +}); + +Given('step that uses failingBeforeFixtureNoStep', async ({ failingBeforeFixtureNoStep }) => { + return failingBeforeFixtureNoStep; +}); + +Given('step that uses failingBeforeFixtureWithStep', async ({ failingBeforeFixtureWithStep }) => { + return failingBeforeFixtureWithStep; +}); + +Given('step that uses timeouted before fixture', async ({ timeoutedBeforeFixture }) => { + return timeoutedBeforeFixture; +}); diff --git a/test/reporter-cucumber-html/features/error-in-bg/sample.feature b/test/reporter-cucumber-html/features/error-in-bg/sample.feature new file mode 100644 index 00000000..17ea3cb5 --- /dev/null +++ b/test/reporter-cucumber-html/features/error-in-bg/sample.feature @@ -0,0 +1,10 @@ +Feature: error-in-bg + + Background: + Given failing step + + Scenario: scenario 1 + Given Action 1 + + Scenario: scenario 2 + Given Action 2 diff --git a/test/reporter-cucumber-html/features/error-in-step/sample.feature b/test/reporter-cucumber-html/features/error-in-step/sample.feature new file mode 100644 index 00000000..e0332f3b --- /dev/null +++ b/test/reporter-cucumber-html/features/error-in-step/sample.feature @@ -0,0 +1,24 @@ +Feature: error-in-step + + # - Increased timeout b/c this test opens page + @timeout:10_000 + Scenario: failing by step assertion + Given failing step + + Scenario: failing by step timeout + Given Action 0 + Given timeouted step + When Action 1 + + # - If this scenario name changed, snapshot file names should also change + # - Increased timeout b/c this test opens page + @timeout:10_000 + Scenario: failing match snapshot + When open page "https://example.com" + Then page title snapshot matches the golden one + + Scenario: soft assertions + Given failing soft assertion "foo" + And Action 1 + And failing soft assertion "bar" + And Action 2 diff --git a/test/reporter-cucumber-html/features/error-in-step/steps.ts b/test/reporter-cucumber-html/features/error-in-step/steps.ts new file mode 100644 index 00000000..a05c72ae --- /dev/null +++ b/test/reporter-cucumber-html/features/error-in-step/steps.ts @@ -0,0 +1,22 @@ +import { createBdd } from 'playwright-bdd'; +import { expect } from '@playwright/test'; + +const { Given, When, Then } = createBdd(); + +Given('timeouted step', async ({ $testInfo }) => { + await new Promise((r) => setTimeout(r, $testInfo.timeout + 100)); +}); + +When('failing soft assertion {string}', async ({}, msg: string) => { + expect.soft('xxx').toEqual(msg); +}); + +When('fails until retry {int}', async ({ $testInfo }, retry: number) => { + if ($testInfo.retry < retry) { + expect(1).toEqual(2); + } +}); + +Then('page title snapshot matches the golden one', async ({ page }) => { + expect(await page.title()).toMatchSnapshot(); +}); diff --git a/test/reporter-cucumber-html/features/failed-steps.ts b/test/reporter-cucumber-html/features/failed-steps.ts deleted file mode 100644 index c56bf857..00000000 --- a/test/reporter-cucumber-html/features/failed-steps.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { expect } from '@playwright/test'; -import { createBdd } from 'playwright-bdd'; -import { test } from './fixtures'; - -const { Given, Before, After } = createBdd(test); - -Before({ name: 'failing named before hook', tags: '@failing-named-hook' }, async ({ page }) => { - await expect(page).toHaveTitle('Some title1'); -}); - -Before({ tags: '@failing-anonymous-hook' }, async ({ page }) => { - await expect(page).toHaveTitle('Some title2'); -}); - -Given('step that uses failingBeforeFixtureNoStep', async ({ failingBeforeFixtureNoStep }) => { - return failingBeforeFixtureNoStep; -}); - -Given('step that uses failingBeforeFixtureWithStep', async ({ failingBeforeFixtureWithStep }) => { - return failingBeforeFixtureWithStep; -}); - -Given('step that uses failingAfterFixtureNoStep', async ({ failingAfterFixtureNoStep }) => { - return failingAfterFixtureNoStep; -}); - -Given('step that uses failingAfterFixtureWithStep', async ({ failingAfterFixtureWithStep }) => { - return failingAfterFixtureWithStep; -}); - -Given('step failing for scenario {string}', async ({ $testInfo }, title: string) => { - if ($testInfo.title === title) expect(true).toEqual(false); -}); - -Given('failing step', async ({ page }) => { - await page.goto('https://example.com'); - await expect(page.getByText('missing string')).toBeVisible(); -}); - -Given('timeouted step', async ({ $testInfo }) => { - await new Promise((r) => setTimeout(r, $testInfo.timeout + 100)); -}); - -Given('step that uses timeouted before fixture', async ({ timeoutedBeforeFixture }) => { - return timeoutedBeforeFixture; -}); - -Given('step that uses timeouted after fixture', async ({ timeoutedAfterFixture }) => { - return timeoutedAfterFixture; -}); - -Before({ name: 'success before hook', tags: '@success-before-hook' }, async ({}) => {}); - -After({ name: 'success after hook', tags: '@success-after-hook' }, async ({}) => {}); diff --git a/test/reporter-cucumber-html/features/sample.feature b/test/reporter-cucumber-html/features/sample.feature deleted file mode 100644 index 0b1cbaec..00000000 --- a/test/reporter-cucumber-html/features/sample.feature +++ /dev/null @@ -1,126 +0,0 @@ -@feature-tag -Feature: rich feature - - Background: - Given step failing for scenario "Failing by background step" - - # - Increased timeout b/c this test opens page - @timeout:10_000 - Scenario: Failing by step - Given failing step - - Scenario: Failing by background step - Given Action 1 - - @failing-anonymous-hook - Scenario: Failing by anonymous before hook - Given Action 1 - - @failing-named-hook - Scenario: Failing by named before hook - Given Action 1 - - Scenario: Failing by failingBeforeFixtureNoStep - Given step that uses failingBeforeFixtureNoStep - When Action 1 - - Scenario: Failing by failingBeforeFixtureWithStep - Given step that uses failingBeforeFixtureWithStep - When Action 2 - - Scenario: Failing by failingAfterFixtureNoStep - Given step that uses failingAfterFixtureNoStep - When Action 3 - - Scenario: Failing by failingAfterFixtureWithStep - Given step that uses failingAfterFixtureWithStep - When Action 4 - - # - If this scenario name changed, snapshot file names should also change - # - Increased timeout b/c this test opens page - @timeout:10_000 - Scenario: failing match snapshot - When open page "https://example.com" - Then page title snapshot matches the golden one - - Scenario: timeout in before fixture - Given Action 0 - Given step that uses timeouted before fixture - When Action 1 - - Scenario: timeout in step - Given Action 0 - Given timeouted step - When Action 1 - - # see: https://github.com/microsoft/playwright/issues/30175 - Scenario: timeout in after fixture - Given Action 0 - Given step that uses timeouted after fixture - When Action 1 - - Scenario: timeout in step and in after fixture - Given Action 0 - Given timeouted step - When Action 1 - Given step that uses timeouted after fixture - - Scenario: soft assertions - Given failing soft assertion "foo" - And Action 1 - And failing soft assertion "bar" - And Action 2 - - # ----- Success scenarios ------ - @retries:1 - Scenario: Scenario with retries - Given Action 1 - And fails until retry 1 - And Action 2 - - Scenario: Scenario with data table - When Step with data table - | name | value | - | foo | bar | - | x | 42 | - - Scenario: Scenario with doc string - Then Step with doc string - ``` - some text - ``` - - Scenario: Scenario with attachments - Given attach text - And attach image inline - And attach image as file - And attach stdout - - @success-before-hook - @success-after-hook - Scenario: Scenario with all keywords and success hooks - Given Action 1 - And Action 2 - When Action 3 - And Action 4 - Then Action 5 - But Action 6 - * Action 7 - - @skip - Scenario: Skipped scenario - Given Action 1 - And Action 2 - - Scenario Outline: Check doubled - Given Action - Then Action - - Examples: - | start | end | - | 2 | 4 | - | 3 | 6 | - - Examples: - | start | end | - | 10 | 20 | diff --git a/test/reporter-cucumber-html/features/shared.steps.ts b/test/reporter-cucumber-html/features/shared.steps.ts new file mode 100644 index 00000000..3d5d0007 --- /dev/null +++ b/test/reporter-cucumber-html/features/shared.steps.ts @@ -0,0 +1,16 @@ +import { createBdd } from 'playwright-bdd'; +import { expect } from '@playwright/test'; + +const { Given, When } = createBdd(); + +When('Action {int}', () => {}); + +Given('failing step', async ({ page }) => { + // using 'page' here to have a screenshot in report + await page.goto('https://example.com'); + await expect(page.getByText('missing string')).toBeVisible(); +}); + +When('open page {string}', async ({ page }, url: string) => { + await page.goto(url); +}); diff --git a/test/reporter-cucumber-html/features/cucumber.png b/test/reporter-cucumber-html/features/success/cucumber.png similarity index 100% rename from test/reporter-cucumber-html/features/cucumber.png rename to test/reporter-cucumber-html/features/success/cucumber.png diff --git a/test/reporter-cucumber-html/features/success/sample.feature b/test/reporter-cucumber-html/features/success/sample.feature new file mode 100644 index 00000000..16469d0d --- /dev/null +++ b/test/reporter-cucumber-html/features/success/sample.feature @@ -0,0 +1,55 @@ +@feature-tag +Feature: success + + @retries:1 + Scenario: Scenario with retries + Given Action 1 + And fails until retry 1 + And Action 2 + + Scenario: Scenario with data table + When Step with data table + | name | value | + | foo | bar | + | x | 42 | + + Scenario: Scenario with doc string + Then Step with doc string + ``` + some text + ``` + + Scenario: Scenario with attachments + Given attach text + And attach image inline + And attach image as file + And attach stdout + + @success-before-hook + @success-after-hook + Scenario: Scenario with all keywords and success hooks + Given Action 1 + And Action 2 + When Action 3 + And Action 4 + Then Action 5 + But Action 6 + * Action 7 + + @skip + Scenario: Skipped scenario + Given Action 1 + And Action 2 + + Scenario Outline: Check doubled + Given Action + Then Action + + Examples: + | start | end | + | 2 | 4 | + | 3 | 6 | + + Examples: + | start | end | + | 10 | 20 | diff --git a/test/reporter-cucumber-html/features/steps.ts b/test/reporter-cucumber-html/features/success/steps.ts similarity index 58% rename from test/reporter-cucumber-html/features/steps.ts rename to test/reporter-cucumber-html/features/success/steps.ts index 89287552..c93e3811 100644 --- a/test/reporter-cucumber-html/features/steps.ts +++ b/test/reporter-cucumber-html/features/success/steps.ts @@ -1,12 +1,9 @@ import fs from 'node:fs'; import path from 'node:path'; import { createBdd } from 'playwright-bdd'; -import { test } from './fixtures'; -import { expect } from '@playwright/test'; -const { When, Then } = createBdd(test); +const { When, Then, Before, After } = createBdd(); -When('Action {int}', () => {}); When('Step with data table', () => {}); When('Step with doc string', () => {}); @@ -33,20 +30,6 @@ Then('attach stdout', async () => { // don't test console.error b/c it poisons the output }); -When('open page {string}', async ({ page }, url: string) => { - await page.goto(url); -}); - -When('failing soft assertion {string}', async ({}, msg: string) => { - expect.soft('xxx').toEqual(msg); -}); +Before({ name: 'success before hook', tags: '@success-before-hook' }, async ({}) => {}); -When('fails until retry {int}', async ({ $testInfo }, retry: number) => { - if ($testInfo.retry < retry) { - expect(1).toEqual(2); - } -}); - -Then('page title snapshot matches the golden one', async ({ page }) => { - expect(await page.title()).toMatchSnapshot(); -}); +After({ name: 'success after hook', tags: '@success-after-hook' }, async ({}) => {}); diff --git a/test/reporter-cucumber-html/playwright.config.ts b/test/reporter-cucumber-html/playwright.config.ts index f022368e..fc69cfc9 100644 --- a/test/reporter-cucumber-html/playwright.config.ts +++ b/test/reporter-cucumber-html/playwright.config.ts @@ -3,8 +3,7 @@ import { defineBddConfig, cucumberReporter } from 'playwright-bdd'; import { testTimeout } from './timeout'; const testDir = defineBddConfig({ - paths: ['features/*.feature'], - require: ['features/*.ts'], + featuresRoot: './features', }); export default defineConfig({ diff --git a/test/reporter-cucumber-html/test.mjs b/test/reporter-cucumber-html/test.mjs index a7549db0..a43035a5 100644 --- a/test/reporter-cucumber-html/test.mjs +++ b/test/reporter-cucumber-html/test.mjs @@ -1,3 +1,6 @@ +/** + * Generate Cucumber HTML reporter and check the report via Playwright. + */ import { test, TestDir,