diff --git a/.github/scripts/handle-test-failures.js b/.github/scripts/handle-test-failures.js new file mode 100644 index 000000000..1b4fad423 --- /dev/null +++ b/.github/scripts/handle-test-failures.js @@ -0,0 +1,53 @@ +module.exports = async ({ github, context, browser }) => { + const fs = require('fs'); + const path = require('path'); + + // Read the test results + const testResultsDir = './test-results'; + const files = fs.readdirSync(testResultsDir); + + for (const file of files) { + const content = fs.readFileSync(path.join(testResultsDir, file), 'utf8'); + let result; + try { + result = JSON.parse(content); + } catch (error) { + console.error(`Error parsing JSON from ${file}: ${error}`); + continue; + } + + if (result.status === 'failed') { + const testName = result.name; + const sauceLabsLink = `https://app.saucelabs.com/tests/${result.id}`; + + // Search for an existing issue + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: ['test-failure', browser] + }); + + const existingIssue = issues.data.find(issue => issue.title.includes(testName)); + + if (existingIssue) { + // Update existing issue + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existingIssue.number, + body: `Test failed again in the latest run.\nBrowser: ${browser}\nRun: ${context.runId}\n[View on SauceLabs](${sauceLabsLink})` + }); + } else { + // Create new issue + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `Test Failure: ${testName} (${browser})`, + body: `The test "${testName}" failed in the ${browser} browser.\n\nRun: ${context.runId}\n[View on SauceLabs](${sauceLabsLink})`, + labels: ['test-failure', browser] + }); + } + } + } + }; \ No newline at end of file diff --git a/.github/workflows/dev.yaml b/.github/workflows/dev.yaml index 09e86ed2b..af03becbf 100644 --- a/.github/workflows/dev.yaml +++ b/.github/workflows/dev.yaml @@ -9,6 +9,7 @@ env: SAUCE_CAPABILITIES_OVERRIDES_PATH: "sauceLabsCapabilities.json" EDGE_BASE_PATH: ee-pre-prd ALLOY_ENV: int + SAUCE_TUNNEL_ID: github-action-tunnel jobs: linting: diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index ceaca60e6..903f16e2c 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -143,22 +143,14 @@ jobs: echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV fi - - name: Notify Slack on Failure + - name: Handle Test Failures if: failure() - uses: 8398a7/action-slack@v3 + uses: actions/github-script@v7 with: - status: ${{ job.status }} - fields: repo,message,commit,author,action,eventName,ref,workflow - text: | - Alloy Prod Firefox Tests Failed - Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const handleTestFailures = require('./.github/scripts/handle-test-failures.js'); + await handleTestFailures({ github, context, browser: 'safari' }); alloy-prod-e2e-firefox: name: Firefox @@ -244,22 +236,14 @@ jobs: echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV fi - - name: Notify Slack on Failure + - name: Handle Test Failures if: failure() - uses: 8398a7/action-slack@v3 + uses: actions/github-script@v7 with: - status: ${{ job.status }} - fields: repo,message,commit,author,action,eventName,ref,workflow - text: | - Alloy Prod Firefox Tests Failed - Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const handleTestFailures = require('./.github/scripts/handle-test-failures.js'); + await handleTestFailures({ github, context, browser: 'firefox' }); alloy-prod-e2e-chrome: name: Chrome @@ -328,19 +312,11 @@ jobs: ALLOY_ENV=prod config-file: ./.sauce/prod-chrome.yml - - name: Notify Slack on Failure + - name: Handle Test Failures if: failure() - uses: 8398a7/action-slack@v3 + uses: actions/github-script@v7 with: - status: ${{ job.status }} - fields: repo,message,commit,author,action,eventName,ref,workflow - text: | - Alloy Prod Firefox Tests Failed - Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const handleTestFailures = require('./.github/scripts/handle-test-failures.js'); + await handleTestFailures({ github, context, browser: 'chrome' }); diff --git a/.sauce/dev.yml b/.sauce/dev.yml index a2fe3ef8c..946ad5ff5 100644 --- a/.sauce/dev.yml +++ b/.sauce/dev.yml @@ -14,7 +14,7 @@ testcafe: version: 3.6.0 nodeVersion: "22" speed: 1 -pageLoadTimeout: 3000 +pageLoadTimeout: 5000 disablePageCaching: true timeZone: New_York npm: @@ -26,22 +26,8 @@ npm: - "@adobe/reactor-cookie" - "@adobe/reactor-query-string" suites: - - name: "Edge" - browserName: "microsoftedge" - headless: true - browserVersion: "119" - src: - - "test/functional/specs/**/*.js" - platformName: "Windows 11" - excludedTestFiles: - [ - "test/functional/specs/Personalization/C782718.js", - "test/functional/specs/Privacy/C5594870.js", - "test/functional/specs/Personalization/C8631577.js", - ] - name: "Safari" browserName: "safari" - headless: true browserVersion: "latest" src: - "test/functional/specs/**/*.js" diff --git a/.sauce/prod-chrome.yml b/.sauce/prod-chrome.yml index f3751bc5c..3ce8139c8 100644 --- a/.sauce/prod-chrome.yml +++ b/.sauce/prod-chrome.yml @@ -2,6 +2,7 @@ apiVersion: v1alpha kind: testcafe sauce: region: us-west-1 + retries: 3 metadata: name: Alloy Prod rootDir: ./ @@ -9,12 +10,13 @@ env: ALLOY_ENV: prod ALLOY_PROD_VERSION: $ALLOY_PROD_VERSION NPM_PACKAGE_VERSION: $NPM_PACKAGE_VERSION + PACKAGE_VERSION: $PACKAGE_VERSION testcafe: version: "package.json" nodeVersion: "$NODE_VERSION" -speed: 1 -pageLoadTimeout: 3000 -disablePageCaching: true +speed: 0.5 +pageLoadTimeout: 8000 +disablePageCaching: false timeZone: New_York npm: dependencies: @@ -31,6 +33,8 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Chrome" + smartRetry: + failedOnly: true browserName: "chrome" browserVersion: "latest" src: diff --git a/.sauce/prod-firefox.yml b/.sauce/prod-firefox.yml index e7a108f4b..8fe63e058 100644 --- a/.sauce/prod-firefox.yml +++ b/.sauce/prod-firefox.yml @@ -2,6 +2,7 @@ apiVersion: v1alpha kind: testcafe sauce: region: us-west-1 + retries: 3 metadata: name: Alloy Prod rootDir: ./ @@ -9,14 +10,14 @@ env: ALLOY_ENV: prod ALLOY_PROD_VERSION: $ALLOY_PROD_VERSION NPM_PACKAGE_VERSION: $NPM_PACKAGE_VERSION + PACKAGE_VERSION: $PACKAGE_VERSION testcafe: - version: 3.6.0 -nodeVersion: "22" -speed: 1 -pageLoadTimeout: 3000 + version: "package.json" +nodeVersion: "$NODE_VERSION" +speed: 0.5 +pageLoadTimeout: 8000 disablePageCaching: true timeZone: New_York -timeout: 3600 npm: dependencies: - node-fetch @@ -32,11 +33,13 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Firefox" + smartRetry: + failedOnly: true browserName: "firefox" browserVersion: "latest" src: - "test/functional/specs/**/*.js" - platformName: "macOS 13" + platformName: "Windows 11" excludedTestFiles: [ "test/functional/specs/Data Collector/C81182.js", diff --git a/.sauce/prod-safari.yml b/.sauce/prod-safari.yml index cfd356222..4e25e5998 100644 --- a/.sauce/prod-safari.yml +++ b/.sauce/prod-safari.yml @@ -2,6 +2,7 @@ apiVersion: v1alpha kind: testcafe sauce: region: us-west-1 + retries: 3 metadata: name: Alloy Prod rootDir: ./ @@ -9,12 +10,13 @@ env: ALLOY_ENV: prod ALLOY_PROD_VERSION: $ALLOY_PROD_VERSION NPM_PACKAGE_VERSION: $NPM_PACKAGE_VERSION + PACKAGE_VERSION: $PACKAGE_VERSION testcafe: version: "package.json" nodeVersion: "$NODE_VERSION" -speed: 1 -pageLoadTimeout: 3000 -disablePageCaching: true +speed: 0.5 +pageLoadTimeout: 8000 +disablePageCaching: false timeZone: New_York npm: dependencies: @@ -31,6 +33,8 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Safari" + smartRetry: + failedOnly: true browserName: "safari" browserVersion: "latest" src: diff --git a/karma.saucelabs.conf.cjs b/karma.saucelabs.conf.cjs index 205d13a2c..4929ebcaf 100644 --- a/karma.saucelabs.conf.cjs +++ b/karma.saucelabs.conf.cjs @@ -19,29 +19,32 @@ module.exports = (config) => { base: "SauceLabs", browserName: "chrome", browserVersion: "latest", - platform: "Windows 11", + platformName: "Windows 11", + "sauce:options": { + tunnelIdentifier: process.env.SAUCE_TUNNEL_ID, + headless: true, + }, }, sl_safariW3C: { base: "SauceLabs", browserName: "safari", browserVersion: "latest", - platform: "macOS 13", + platformName: "macOS 13", + "sauce:options": { + tunnelIdentifier: process.env.SAUCE_TUNNEL_ID, + headless: true, + }, }, sl_firefoxW3C: { base: "SauceLabs", browserName: "firefox", - platformName: "Windows 11", browserVersion: "latest", + platformName: "Windows 11", "sauce:options": { - geckodriverVersion: "0.34.0", + tunnelIdentifier: process.env.SAUCE_TUNNEL_ID, + headless: true, }, }, - sl_edgeW3C: { - base: "SauceLabs", - browserName: "microsoftedge", - browserVersion: "latest", - platform: "Windows 11", - }, }; config.set({ @@ -50,9 +53,12 @@ module.exports = (config) => { concurrency: 10, colors: true, sauceLabs: { - screenResolution: "800x600", + testName: "Alloy Unit Tests", build: `GH #${process.env.BUILD_NUMBER} (${process.env.BUILD_ID})`, - tunnelIdentifier: process.env.JOB_NUMBER, + tunnelIdentifier: process.env.SAUCE_TUNNEL_ID, + screenResolution: "1280x1024", + recordVideo: false, + recordScreenshots: false, }, plugins: [ "karma-jasmine",