test: reporter-cucumber-html: support several feature files

test/reporter-cucumber-html/check-report/fixtures.ts

@@ -0,0 +1,28 @@
+import { pathToFileURL } from 'node:url';
+import { test as base } from '@playwright/test';
+import { HtmlReport } from './poms/HtmlReport';
+import { Feature } from './poms/Feature';
+import { Scenario } from './poms/Scenario';
+
+type Fixtures = {
+  htmlReport: HtmlReport;
+  featureUri: string;
+  feature: Feature;
+  scenario: Scenario;
+};
+
+export const test = base.extend<Fixtures>({
+  page: async ({ page }, use) => {
+    await page.goto(pathToFileURL('actual-reports/report.html').href);
+    await use(page);
+  },
+  htmlReport: async ({ page }, use) => use(new HtmlReport(page)),
+  featureUri: ['', { option: true }], // will be overwritten in test files
+  feature: async ({ htmlReport, featureUri }, use) => {
+    if (!featureUri) throw new Error('Missing featureUri');
+    await use(htmlReport.getFeature(featureUri));
+  },
+  scenario: async ({ feature }, use, testInfo) => {
+    await use(feature.getScenario(testInfo.title));
+  },
+});

test/reporter-cucumber-html/check-report/playwright.config.ts

@@ -4,10 +4,12 @@
 import { defineConfig } from '@playwright/test';
 
 export default defineConfig({
-  expect: { timeout: 500 },
+  testDir: './specs',
   outputDir: './test-results',
   reporter: 'dot',
   use: {
     screenshot: 'only-on-failure',
+    viewport: { width: 800, height: 720 },
   },
+  expect: { timeout: 500 },
 });

test/reporter-cucumber-html/check-report/poms/Feature.ts

@@ -0,0 +1,43 @@
+import { Locator, Page } from '@playwright/test';
+import { Scenario } from './Scenario';
+import path from 'node:path';
+
+export class Feature {
+  root: Locator;
+
+  constructor(
+    public page: Page,
+    public featureUri: string,
+  ) {
+    const container = page
+      .locator('[data-accordion-component="AccordionItem"]')
+      .filter({ has: this.getFileHeader() });
+    this.root = container.locator('[data-accordion-component="AccordionItemPanel"]');
+  }
+
+  getFileHeader() {
+    return this.page.locator('[data-accordion-component="AccordionItemHeading"]', {
+      hasText: path.normalize(this.featureUri),
+    });
+  }
+
+  getFeatureHeader() {
+    return this.root.getByRole('heading', { level: 1 });
+  }
+
+  getBackground() {
+    return new Scenario(this, `Background:`);
+  }
+
+  getScenario(title: string) {
+    return new Scenario(this, `Scenario:${title}`);
+  }
+
+  getScenarioOutline(title: string) {
+    return new Scenario(this, `Scenario Outline:${title}`);
+  }
+
+  getTags() {
+    return this.root.locator('> section > [aria-label="Tags"]').getByRole('listitem');
+  }
+}

test/reporter-cucumber-html/check-report/poms/HtmlReport.ts

@@ -0,0 +1,10 @@
+import { Page } from '@playwright/test';
+import { Feature } from './Feature';
+
+export class HtmlReport {
+  constructor(public page: Page) {}
+
+  getFeature(featureUri: string) {
+    return new Feature(this.page, featureUri);
+  }
+}

test/reporter-cucumber-html/check-report/poms/Scenario.ts

@@ -0,0 +1,71 @@
+import { Locator } from '@playwright/test';
+import { Feature } from './Feature';
+
+/**
+ * POM for scenario, scenario outline, and background
+ */
+export class Scenario {
+  root: Locator;
+  header: Locator;
+
+  constructor(
+    public feature: Feature,
+    public titleWithKeyword: string,
+  ) {
+    // important to use this.page here, not feature.root
+    this.header = this.page.getByRole('heading', { name: titleWithKeyword, level: 2 });
+    this.root = feature.root
+      .locator('section')
+      // filter out top section
+      .filter({ hasNot: this.page.getByRole('heading', { level: 1 }) })
+      .filter({ has: this.header });
+  }
+
+  get page() {
+    return this.feature.page;
+  }
+
+  getSteps(status: 'all' | 'failed' | 'passed' | 'skipped' = 'all') {
+    const steps = this.root.locator('[aria-label="Steps"]').getByRole('listitem');
+    return status === 'all'
+      ? steps
+      : steps.filter({
+          has: this.page.locator(`[data-status="${status.toUpperCase()}"]`),
+        });
+  }
+
+  getAttachments() {
+    return this.root.locator('details');
+  }
+
+  getLogs() {
+    return this.root.locator('div > pre').filter({
+      hasNot: this.getErrors(),
+    });
+  }
+
+  getErrors() {
+    return this.getSteps()
+      .locator('div > pre')
+      .filter({
+        hasText: /error|timeout|expected|browser .*?closed/i,
+      });
+  }
+
+  getTags() {
+    return this.root.locator('[aria-label="Tags"]').getByRole('listitem');
+  }
+
+  getDataTable() {
+    return this.root.getByRole('table');
+  }
+
+  getDocString() {
+    // todo: distinguish from error
+    return this.root.locator('div > pre');
+  }
+
+  getExamples() {
+    return this.root.locator('section');
+  }
+}

test/reporter-cucumber-html/check-report/specs/error-in-after.test.ts

@@ -0,0 +1,83 @@
+import { normalize } from 'node:path';
+import { expect } from '@playwright/test';
+import { test } from '../fixtures';
+import { getPackageVersion } from '../../../../src/utils';
+
+// Automatic screenshot for failing fixtures teardown depends on pw version.
+// see: https://github.com/microsoft/playwright/issues/29325
+const pwVersion = getPackageVersion('@playwright/test');
+const hasAutoScreenshotFixtureTeardown = pwVersion >= '1.42.0' && pwVersion < '1.45.0';
+
+test.use({ featureUri: 'error-in-after/sample.feature' });
+
+test('Failing by failingAfterFixtureNoStep', async ({ scenario }) => {
+  await expect(scenario.getSteps()).toContainText([
+    'my attachment|before use',
+    'Givenstep that uses failingAfterFixtureNoStep',
+    'WhenAction 3',
+    `Hook "fixture: failingAfterFixtureNoStep" failed: ${normalize('features/error-in-after/fixtures.ts')}:`,
+  ]);
+  if (hasAutoScreenshotFixtureTeardown) {
+    await expect(scenario.getSteps()).toContainText(['screenshot']);
+  }
+  await expect(scenario.getAttachments()).toContainText([
+    'my attachment|before use', // prettier-ignore
+    'my attachment|after use',
+  ]);
+  await expect(scenario.getSteps('passed')).toHaveCount(2);
+  await expect(scenario.getSteps('failed')).toHaveCount(1);
+  await expect(scenario.getSteps('skipped')).toHaveCount(0);
+  await expect(scenario.getErrors()).toContainText(['error in failingAfterFixtureNoStep']);
+});
+
+test('Failing by failingAfterFixtureWithStep', async ({ scenario }) => {
+  await expect(scenario.getSteps()).toContainText([
+    'my attachment|outside step (before use)',
+    'Givenstep that uses failingAfterFixtureWithStep',
+    'WhenAction 4',
+    `Hook "step in failingAfterFixtureWithStep" failed: ${normalize('features/error-in-after/fixtures.ts')}:`,
+    'my attachment|outside step (after use)',
+  ]);
+  if (hasAutoScreenshotFixtureTeardown) {
+    await expect(scenario.getSteps()).toContainText(['screenshot']);
+  }
+  await expect(scenario.getAttachments()).toContainText([
+    'my attachment|outside step (before use)',
+    'my attachment|in step',
+    'my attachment|outside step (after use)',
+  ]);
+  await expect(scenario.getSteps('passed')).toHaveCount(2);
+  await expect(scenario.getSteps('failed')).toHaveCount(1);
+  await expect(scenario.getSteps('skipped')).toHaveCount(0);
+  await expect(scenario.getErrors()).toContainText(['error in failingAfterFixtureWithStep']);
+});
+
+test('timeout in after fixture', async ({ scenario }) => {
+  await expect(scenario.getSteps()).toContainText([
+    'GivenAction 0',
+    'Givenstep that uses timeouted after fixture',
+    'WhenAction 1',
+    /Hook "(After Hooks|fixture: timeoutedAfterFixture)" failed/,
+  ]);
+  // don't check screenshot as it's not reliable in timeouts
+  await expect(scenario.getSteps('passed')).toHaveCount(3);
+  await expect(scenario.getSteps('failed')).toHaveCount(1);
+  await expect(scenario.getErrors()).toContainText([
+    /but tearing down "timeoutedAfterFixture" ran out of time|Tearing down "timeoutedAfterFixture" exceeded the test timeout/,
+  ]);
+});
+
+test('timeout in step and in after fixture', async ({ scenario }) => {
+  await expect(scenario.getSteps()).toContainText([
+    'GivenAction 0',
+    'Giventimeouted step',
+    'WhenAction 1',
+    'Givenstep that uses timeouted after fixture',
+    /Hook "(After Hooks|fixture: timeoutedAfterFixture)" failed/,
+  ]);
+  await expect(scenario.getSteps('passed')).toHaveCount(4);
+  await expect(scenario.getSteps('failed')).toHaveCount(1);
+  await expect(scenario.getErrors()).toContainText([
+    /Test timeout of \d+ms exceeded|Tearing down "timeoutedAfterFixture" exceeded the test timeout/,
+  ]);
+});