Improve E2E Test Failure Handling and Reporting

.github/scripts/handle-test-failures.js

@@ -0,0 +1,53 @@
+module.exports = async ({ github, context, browser }) => {
+    const fs = require('fs');
+    const path = require('path');
+
+    // Read the test results
+    const testResultsDir = './test-results';
+    const files = fs.readdirSync(testResultsDir);
+
+    for (const file of files) {
+      const content = fs.readFileSync(path.join(testResultsDir, file), 'utf8');
+      let result;
+      try {
+        result = JSON.parse(content);
+      } catch (error) {
+        console.error(`Error parsing JSON from ${file}: ${error}`);
+        continue;
+      }
+
+      if (result.status === 'failed') {
+        const testName = result.name;
+        const sauceLabsLink = `https://app.saucelabs.com/tests/${result.id}`;
+
+        // Search for an existing issue
+        const issues = await github.rest.issues.listForRepo({
+          owner: context.repo.owner,
+          repo: context.repo.repo,
+          state: 'open',
+          labels: ['test-failure', browser]
+        });
+
+        const existingIssue = issues.data.find(issue => issue.title.includes(testName));
+
+        if (existingIssue) {
+          // Update existing issue
+          await github.rest.issues.createComment({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            issue_number: existingIssue.number,
+            body: `Test failed again in the latest run.\nBrowser: ${browser}\nRun: ${context.runId}\n[View on SauceLabs](${sauceLabsLink})`
+          });
+        } else {
+          // Create new issue
+          await github.rest.issues.create({
+            owner: context.repo.owner,
+            repo: context.repo.repo,
+            title: `Test Failure: ${testName} (${browser})`,
+            body: `The test "${testName}" failed in the ${browser} browser.\n\nRun: ${context.runId}\n[View on SauceLabs](${sauceLabsLink})`,
+            labels: ['test-failure', browser]
+          });
+        }
+      }
+    }
+  };

.github/workflows/dev.yaml

@@ -9,6 +9,7 @@ env:
   SAUCE_CAPABILITIES_OVERRIDES_PATH: "sauceLabsCapabilities.json"
   EDGE_BASE_PATH: ee-pre-prd
   ALLOY_ENV: int
+  SAUCE_TUNNEL_ID: github-action-tunnel
 
 jobs:
   linting:

.github/workflows/prod.yml

@@ -143,22 +143,14 @@ jobs:
             echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV
           fi
 
-      - name: Notify Slack on Failure
+      - name: Handle Test Failures
         if: failure()
-        uses: 8398a7/action-slack@v3
+        uses: actions/github-script@v7
         with:
-          status: ${{ job.status }}
-          fields: repo,message,commit,author,action,eventName,ref,workflow
-          text: |
-            Alloy Prod Firefox Tests Failed
-            Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }}
-            Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }}
-            Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }}
-            Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }}
-            Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const handleTestFailures = require('./.github/scripts/handle-test-failures.js');
+            await handleTestFailures({ github, context, browser: 'safari' });
 
   alloy-prod-e2e-firefox:
     name: Firefox
@@ -244,22 +236,14 @@ jobs:
             echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV
           fi
 
-      - name: Notify Slack on Failure
+      - name: Handle Test Failures
         if: failure()
-        uses: 8398a7/action-slack@v3
+        uses: actions/github-script@v7
         with:
-          status: ${{ job.status }}
-          fields: repo,message,commit,author,action,eventName,ref,workflow
-          text: |
-            Alloy Prod Firefox Tests Failed
-            Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }}
-            Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }}
-            Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }}
-            Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }}
-            Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const handleTestFailures = require('./.github/scripts/handle-test-failures.js');
+            await handleTestFailures({ github, context, browser: 'firefox' });
 
   alloy-prod-e2e-chrome:
     name: Chrome
@@ -328,19 +312,11 @@ jobs:
             ALLOY_ENV=prod
           config-file: ./.sauce/prod-chrome.yml
 
-      - name: Notify Slack on Failure
+      - name: Handle Test Failures
         if: failure()
-        uses: 8398a7/action-slack@v3
+        uses: actions/github-script@v7
         with:
-          status: ${{ job.status }}
-          fields: repo,message,commit,author,action,eventName,ref,workflow
-          text: |
-            Alloy Prod Firefox Tests Failed
-            Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }}
-            Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }}
-            Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }}
-            Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }}
-            Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const handleTestFailures = require('./.github/scripts/handle-test-failures.js');
+            await handleTestFailures({ github, context, browser: 'chrome' });

.sauce/dev.yml

@@ -14,7 +14,7 @@ testcafe:
   version: 3.6.0
 nodeVersion: "22"
 speed: 1
-pageLoadTimeout: 3000
+pageLoadTimeout: 5000
 disablePageCaching: true
 timeZone: New_York
 npm:
@@ -26,22 +26,8 @@ npm:
     - "@adobe/reactor-cookie"
     - "@adobe/reactor-query-string"
 suites:
-  - name: "Edge"
-    browserName: "microsoftedge"
-    headless: true
-    browserVersion: "119"
-    src:
-      - "test/functional/specs/**/*.js"
-    platformName: "Windows 11"
-    excludedTestFiles:
-      [
-        "test/functional/specs/Personalization/C782718.js",
-        "test/functional/specs/Privacy/C5594870.js",
-        "test/functional/specs/Personalization/C8631577.js",
-      ]
   - name: "Safari"
     browserName: "safari"
-    headless: true
     browserVersion: "latest"
     src:
       - "test/functional/specs/**/*.js"

.sauce/prod-chrome.yml

@@ -2,19 +2,21 @@ apiVersion: v1alpha
 kind: testcafe
 sauce:
   region: us-west-1
+  retries: 3
 metadata:
   name: Alloy Prod
 rootDir: ./
 env:
   ALLOY_ENV: prod
   ALLOY_PROD_VERSION: $ALLOY_PROD_VERSION
   NPM_PACKAGE_VERSION: $NPM_PACKAGE_VERSION
+  PACKAGE_VERSION: $PACKAGE_VERSION
 testcafe:
   version: "package.json"
 nodeVersion: "$NODE_VERSION"
-speed: 1
-pageLoadTimeout: 3000
-disablePageCaching: true
+speed: 0.5
+pageLoadTimeout: 8000
+disablePageCaching: false
 timeZone: New_York
 npm:
   dependencies:
@@ -31,6 +33,8 @@ npm:
     - "@adobe/reactor-object-assign"
 suites:
   - name: "Chrome"
+    smartRetry:
+      failedOnly: true
     browserName: "chrome"
     browserVersion: "latest"
     src:

.sauce/prod-firefox.yml

@@ -2,21 +2,22 @@ apiVersion: v1alpha
 kind: testcafe
 sauce:
   region: us-west-1
+  retries: 3
 metadata:
   name: Alloy Prod
 rootDir: ./
 env:
   ALLOY_ENV: prod
   ALLOY_PROD_VERSION: $ALLOY_PROD_VERSION
   NPM_PACKAGE_VERSION: $NPM_PACKAGE_VERSION
+  PACKAGE_VERSION: $PACKAGE_VERSION
 testcafe:
-  version: 3.6.0
-nodeVersion: "22"
-speed: 1
-pageLoadTimeout: 3000
+  version: "package.json"
+nodeVersion: "$NODE_VERSION"
+speed: 0.5
+pageLoadTimeout: 8000
 disablePageCaching: true
 timeZone: New_York
-timeout: 3600
 npm:
   dependencies:
     - node-fetch
@@ -32,11 +33,13 @@ npm:
     - "@adobe/reactor-object-assign"
 suites:
   - name: "Firefox"
+    smartRetry:
+      failedOnly: true
     browserName: "firefox"
     browserVersion: "latest"
     src:
       - "test/functional/specs/**/*.js"
-    platformName: "macOS 13"
+    platformName: "Windows 11"
     excludedTestFiles:
       [
         "test/functional/specs/Data Collector/C81182.js",

.sauce/prod-safari.yml

@@ -2,19 +2,21 @@ apiVersion: v1alpha
 kind: testcafe
 sauce:
   region: us-west-1
+  retries: 3
 metadata:
   name: Alloy Prod
 rootDir: ./
 env:
   ALLOY_ENV: prod
   ALLOY_PROD_VERSION: $ALLOY_PROD_VERSION
   NPM_PACKAGE_VERSION: $NPM_PACKAGE_VERSION
+  PACKAGE_VERSION: $PACKAGE_VERSION
 testcafe:
   version: "package.json"
 nodeVersion: "$NODE_VERSION"
-speed: 1
-pageLoadTimeout: 3000
-disablePageCaching: true
+speed: 0.5
+pageLoadTimeout: 8000
+disablePageCaching: false
 timeZone: New_York
 npm:
   dependencies:
@@ -31,6 +33,8 @@ npm:
     - "@adobe/reactor-object-assign"
 suites:
   - name: "Safari"
+    smartRetry:
+      failedOnly: true
     browserName: "safari"
     browserVersion: "latest"
     src:

karma.saucelabs.conf.cjs

@@ -19,29 +19,32 @@ module.exports = (config) => {
       base: "SauceLabs",
       browserName: "chrome",
       browserVersion: "latest",
-      platform: "Windows 11",
+      platformName: "Windows 11",
+      "sauce:options": {
+        tunnelIdentifier: process.env.SAUCE_TUNNEL_ID,
+        headless: true,
+      },
     },
     sl_safariW3C: {
       base: "SauceLabs",
       browserName: "safari",
       browserVersion: "latest",
-      platform: "macOS 13",
+      platformName: "macOS 13",
+      "sauce:options": {
+        tunnelIdentifier: process.env.SAUCE_TUNNEL_ID,
+        headless: true,
+      },
     },
     sl_firefoxW3C: {
       base: "SauceLabs",
       browserName: "firefox",
-      platformName: "Windows 11",
       browserVersion: "latest",
+      platformName: "Windows 11",
       "sauce:options": {
-        geckodriverVersion: "0.34.0",
+        tunnelIdentifier: process.env.SAUCE_TUNNEL_ID,
+        headless: true,
       },
     },
-    sl_edgeW3C: {
-      base: "SauceLabs",
-      browserName: "microsoftedge",
-      browserVersion: "latest",
-      platform: "Windows 11",
-    },
   };
 
   config.set({
@@ -50,9 +53,12 @@ module.exports = (config) => {
     concurrency: 10,
     colors: true,
     sauceLabs: {
-      screenResolution: "800x600",
+      testName: "Alloy Unit Tests",
       build: `GH #${process.env.BUILD_NUMBER} (${process.env.BUILD_ID})`,
-      tunnelIdentifier: process.env.JOB_NUMBER,
+      tunnelIdentifier: process.env.SAUCE_TUNNEL_ID,
+      screenResolution: "1280x1024",
+      recordVideo: false,
+      recordScreenshots: false,
     },
     plugins: [
       "karma-jasmine",