test: migrate from /reports -> /actual-reports + expected-reports

.gitignore

@@ -19,7 +19,6 @@ blob-report
 /todo.txt
 
 /test/**/report.*
-/test/**/reports
 /test/**/actual-reports
 
 /test/reporter-merge/features

test/reporter-cucumber-custom/playwright.config.ts

@@ -10,7 +10,7 @@ export default defineConfig({
   testDir,
   reporter: [
     cucumberReporter('./custom-reporter.ts', {
-      outputFile: 'reports/custom.ndjson',
+      outputFile: 'actual-reports/custom.ndjson',
       foo: 'bar',
     }),
     cucumberReporter('@cucumber/pretty-formatter', {

test/reporter-cucumber-custom/test.mjs

@@ -5,9 +5,9 @@ import { test, TestDir, execPlaywrightTest } from '../helpers.mjs';
 const testDir = new TestDir(import.meta);
 
 test(testDir.name, async () => {
-  testDir.clearDir('reports');
+  testDir.clearDir('actual-reports');
   const stdout = execPlaywrightTest(testDir.name);
-  checkCustomFormatter('reports/custom.ndjson');
+  checkCustomFormatter('actual-reports/custom.ndjson');
   checkPrettyFormatter(stdout);
 });
 

test/reporter-cucumber-html/check-report/helpers.ts

@@ -82,7 +82,7 @@ export class Scenario {
     return this.getSteps()
       .locator('div > pre')
       .filter({
-        hasText: /error|timeout/i,
+        hasText: /error|timeout|expected/i,
       });
   }
 

test/reporter-cucumber-msg/cucumber.config.js

@@ -18,8 +18,8 @@ module.exports = {
     requireModule: ['ts-node/register'],
     retry: featureDir === 'retry' ? 2 : 0,
     format: [
-      ['message', `features/${featureDir}/expected/messages.ndjson`],
-      ['json', `features/${featureDir}/expected/json-report.json`],
+      ['message', `features/${featureDir}/expected-reports/messages.ndjson`],
+      ['json', `features/${featureDir}/expected-reports/json-report.json`],
     ],
   },
 };

test/reporter-cucumber-msg/helpers/run-c.mjs

@@ -12,7 +12,7 @@ import { execSync } from 'node:child_process';
 const featureDir = process.argv[2];
 if (!featureDir) throw new Error(`Empty <featureDir>`);
 
-console.log(`Generating: features/${featureDir}/expected/messages.ndjson`);
+console.log(`Generating: features/${featureDir}/expected-reports/messages.ndjson`);
 
 try {
   execSync(`npx cucumber-js -c cucumber.config.js`, {

test/reporter-cucumber-msg/playwright.config.ts

@@ -15,18 +15,18 @@ export default defineConfig({
   retries: featureDir === 'retry' ? 2 : 0,
   reporter: [
     cucumberReporter('message', {
-      outputFile: `features/${featureDir}/reports/messages.ndjson`,
+      outputFile: `features/${featureDir}/actual-reports/messages.ndjson`,
     }),
     cucumberReporter('json', {
-      outputFile: `features/${featureDir}/reports/json-report.json`,
+      outputFile: `features/${featureDir}/actual-reports/json-report.json`,
     }),
     featureDir === 'attachments' ? jsonReporterNoAttachments() : null,
   ].filter((r): r is NonNullable<typeof r> => Boolean(r)),
 });
 
 function jsonReporterNoAttachments() {
   return cucumberReporter('json', {
-    outputFile: `features/${featureDir}/reports/json-report-no-attachments.json`,
+    outputFile: `features/${featureDir}/actual-reports/json-report-no-attachments.json`,
     skipAttachments: true,
   });
 }

test/reporter-cucumber-msg/test.mjs

@@ -55,20 +55,22 @@ async function checkFeature(featureDir) {
   assertJsonReport(absFeatureDir);
 
   if (featureDir === 'attachments') {
-    const actualJson = getJsonFromFile(`${absFeatureDir}/reports/json-report-no-attachments.json`);
+    const actualJson = getJsonFromFile(
+      `${absFeatureDir}/actual-reports/json-report-no-attachments.json`,
+    );
     expect(JSON.stringify(actualJson, null, 2)).not.toContain('embeddings');
   }
 }
 
 function assertMessagesReport(absFeatureDir) {
-  const actualMessages = getMessagesFromFile(`${absFeatureDir}/reports/messages.ndjson`);
-  const expectedMessages = getMessagesFromFile(`${absFeatureDir}/expected/messages.ndjson`);
+  const actualMessages = getMessagesFromFile(`${absFeatureDir}/actual-reports/messages.ndjson`);
+  const expectedMessages = getMessagesFromFile(`${absFeatureDir}/expected-reports/messages.ndjson`);
   assertShape(actualMessages, expectedMessages, messageReportFields);
 }
 
 function assertJsonReport(absFeatureDir) {
-  const actualJson = getJsonFromFile(`${absFeatureDir}/reports/json-report.json`);
-  const expectedJson = getJsonFromFile(`${absFeatureDir}/expected/json-report.json`);
+  const actualJson = getJsonFromFile(`${absFeatureDir}/actual-reports/json-report.json`);
+  const expectedJson = getJsonFromFile(`${absFeatureDir}/expected-reports/json-report.json`);
   assertShape(actualJson, expectedJson, jsonReportFields);
 }
 

test/reporter-cucumber-projects/playwright.config.ts

@@ -3,15 +3,15 @@ import { defineBddConfig, cucumberReporter } from 'playwright-bdd';
 
 export default defineConfig({
   reporter: [
-    cucumberReporter('message', { outputFile: 'reports/messages.ndjson' }),
-    cucumberReporter('html', { outputFile: 'reports/report.html' }),
+    cucumberReporter('message', { outputFile: 'actual-reports/messages.ndjson' }),
+    cucumberReporter('html', { outputFile: 'actual-reports/report.html' }),
     cucumberReporter('json', {
-      outputFile: 'reports/report.json',
+      outputFile: 'actual-reports/report.json',
       addProjectToFeatureName: true,
       addMetadata: 'list',
     }),
     cucumberReporter('junit', {
-      outputFile: 'reports/report.xml',
+      outputFile: 'actual-reports/report.xml',
       suiteName: 'my suite',
     }),
   ],

test/reporter-cucumber-projects/test.mjs

@@ -19,21 +19,21 @@ test(testDir.name, async () => {
 });
 
 function assertMessagesReport() {
-  const actualMessages = getMessagesFromFile(testDir.getAbsPath(`reports/messages.ndjson`));
+  const actualMessages = getMessagesFromFile(testDir.getAbsPath(`actual-reports/messages.ndjson`));
   const expectedMessages = getMessagesFromFile(
     testDir.getAbsPath(`expected-reports/messages.ndjson`),
   );
   assertShape(actualMessages, expectedMessages, messageReportFields);
 }
 
 function assertJsonReport() {
-  const actualJson = getJsonFromFile(testDir.getAbsPath(`reports/report.json`));
+  const actualJson = getJsonFromFile(testDir.getAbsPath(`actual-reports/report.json`));
   const expectedJson = getJsonFromFile(testDir.getAbsPath(`expected-reports/json-report.json`));
   assertShape(actualJson, expectedJson, jsonReportFields);
 }
 
 async function assertJunitReport() {
-  const actualJson = await getJsonFromXmlFile(testDir.getAbsPath('reports/report.xml'));
+  const actualJson = await getJsonFromXmlFile(testDir.getAbsPath('actual-reports/report.xml'));
   const expectedJson = await getJsonFromXmlFile(
     testDir.getAbsPath('expected-reports/junit-report.xml'),
   );

test/reporter-merge/test.mjs

@@ -20,7 +20,7 @@ const skip = pwVersion < '1.37';
 // skip until figure out how to map attachments in merge-reports mode
 // See: https://github.com/microsoft/playwright/issues/29323#issuecomment-1927311800
 test(testDir.name, { skip }, () => {
-  testDir.clearDir('reports');
+  testDir.clearDir('actual-reports');
   testDir.clearDir('blob-report');
 
   copyFeatures(); // uses features from reporter-cucumber-html