From b24d25482baa757398210aa384d40f63a8bc6d23 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Fri, 13 Sep 2024 12:43:50 -0400 Subject: [PATCH 01/21] Debug C8085776. --- test/functional/specs/Migration/C8085776.js | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/test/functional/specs/Migration/C8085776.js b/test/functional/specs/Migration/C8085776.js index 723a8265d..04077ffa1 100644 --- a/test/functional/specs/Migration/C8085776.js +++ b/test/functional/specs/Migration/C8085776.js @@ -56,12 +56,27 @@ test( "C8085776: At.js 2.x to Web SDK - Assert same session ID, edge cluster are used " + "for both of the requests interact and delivery API", async () => { + // Wait for a short period to allow for asynchronous requests + await t.wait(2000); + + const requestCount = await networkLogger.targetDeliveryEndpointLogs.count( + () => true, + ); + console.log(`Number of Target delivery requests: ${requestCount}`); + await t - .expect(networkLogger.targetDeliveryEndpointLogs.count(() => true)) - .eql(1); + .expect(requestCount) + .gte(1, "Expected at least one Target delivery request"); + // Get delivery API request const deliveryRequest = networkLogger.targetDeliveryEndpointLogs.requests[0]; + + if (!deliveryRequest) { + console.log("No Target delivery request found"); + await t.expect(false).ok("No Target delivery request was logged"); + } + await responseStatus(networkLogger.targetDeliveryEndpointLogs, [200, 207]); const { searchParams } = new URL(deliveryRequest.request.url); // Extract the session ID from the request query params From e8486e31ee948ef69e934852d6fb354a78c855f2 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Fri, 13 Sep 2024 12:54:44 -0400 Subject: [PATCH 02/21] Remove edge browser. --- .sauce/dev.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.sauce/dev.yml b/.sauce/dev.yml index a2fe3ef8c..9e363dcd6 100644 --- a/.sauce/dev.yml +++ b/.sauce/dev.yml @@ -26,19 +26,6 @@ npm: - "@adobe/reactor-cookie" - "@adobe/reactor-query-string" suites: - - name: "Edge" - browserName: "microsoftedge" - headless: true - browserVersion: "119" - src: - - "test/functional/specs/**/*.js" - platformName: "Windows 11" - excludedTestFiles: - [ - "test/functional/specs/Personalization/C782718.js", - "test/functional/specs/Privacy/C5594870.js", - "test/functional/specs/Personalization/C8631577.js", - ] - name: "Safari" browserName: "safari" headless: true From a0aadca5583670a32597c98e517e7ee1a736929a Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Fri, 13 Sep 2024 13:05:34 -0400 Subject: [PATCH 03/21] Remove edge and add additional time before assertion. --- test/functional/specs/Migration/C8085776.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/functional/specs/Migration/C8085776.js b/test/functional/specs/Migration/C8085776.js index 04077ffa1..b007ce24c 100644 --- a/test/functional/specs/Migration/C8085776.js +++ b/test/functional/specs/Migration/C8085776.js @@ -56,14 +56,18 @@ test( "C8085776: At.js 2.x to Web SDK - Assert same session ID, edge cluster are used " + "for both of the requests interact and delivery API", async () => { - // Wait for a short period to allow for asynchronous requests - await t.wait(2000); + // Increase wait time + await t.wait(5000); const requestCount = await networkLogger.targetDeliveryEndpointLogs.count( () => true, ); console.log(`Number of Target delivery requests: ${requestCount}`); + // Log the contents of the requests for debugging + const requests = networkLogger.targetDeliveryEndpointLogs.requests; + console.log("Target delivery requests:", JSON.stringify(requests, null, 2)); + await t .expect(requestCount) .gte(1, "Expected at least one Target delivery request"); From 3923878a9f6795705962e1a439bf39b94c1b24f0 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Wed, 18 Sep 2024 15:18:12 -0400 Subject: [PATCH 04/21] Debug unit test dev workflow. --- karma.saucelabs.conf.cjs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/karma.saucelabs.conf.cjs b/karma.saucelabs.conf.cjs index 205d13a2c..89f093940 100644 --- a/karma.saucelabs.conf.cjs +++ b/karma.saucelabs.conf.cjs @@ -19,29 +19,29 @@ module.exports = (config) => { base: "SauceLabs", browserName: "chrome", browserVersion: "latest", - platform: "Windows 11", + platformName: "Windows 11", + "sauce:options": { + headless: true, + }, }, sl_safariW3C: { base: "SauceLabs", browserName: "safari", browserVersion: "latest", - platform: "macOS 13", + platformName: "macOS 13", + "sauce:options": { + headless: true, + }, }, sl_firefoxW3C: { base: "SauceLabs", browserName: "firefox", - platformName: "Windows 11", browserVersion: "latest", + platformName: "Windows 11", "sauce:options": { - geckodriverVersion: "0.34.0", + headless: true, }, }, - sl_edgeW3C: { - base: "SauceLabs", - browserName: "microsoftedge", - browserVersion: "latest", - platform: "Windows 11", - }, }; config.set({ From 0388402af044dd4b5e9f7bce8ffc2335a953ea5e Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Thu, 19 Sep 2024 10:52:03 -0400 Subject: [PATCH 05/21] Debug unit test runner. --- .github/workflows/dev.yaml | 1 + karma.saucelabs.conf.cjs | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev.yaml b/.github/workflows/dev.yaml index 09e86ed2b..af03becbf 100644 --- a/.github/workflows/dev.yaml +++ b/.github/workflows/dev.yaml @@ -9,6 +9,7 @@ env: SAUCE_CAPABILITIES_OVERRIDES_PATH: "sauceLabsCapabilities.json" EDGE_BASE_PATH: ee-pre-prd ALLOY_ENV: int + SAUCE_TUNNEL_ID: github-action-tunnel jobs: linting: diff --git a/karma.saucelabs.conf.cjs b/karma.saucelabs.conf.cjs index 89f093940..4929ebcaf 100644 --- a/karma.saucelabs.conf.cjs +++ b/karma.saucelabs.conf.cjs @@ -21,6 +21,7 @@ module.exports = (config) => { browserVersion: "latest", platformName: "Windows 11", "sauce:options": { + tunnelIdentifier: process.env.SAUCE_TUNNEL_ID, headless: true, }, }, @@ -30,6 +31,7 @@ module.exports = (config) => { browserVersion: "latest", platformName: "macOS 13", "sauce:options": { + tunnelIdentifier: process.env.SAUCE_TUNNEL_ID, headless: true, }, }, @@ -39,6 +41,7 @@ module.exports = (config) => { browserVersion: "latest", platformName: "Windows 11", "sauce:options": { + tunnelIdentifier: process.env.SAUCE_TUNNEL_ID, headless: true, }, }, @@ -50,9 +53,12 @@ module.exports = (config) => { concurrency: 10, colors: true, sauceLabs: { - screenResolution: "800x600", + testName: "Alloy Unit Tests", build: `GH #${process.env.BUILD_NUMBER} (${process.env.BUILD_ID})`, - tunnelIdentifier: process.env.JOB_NUMBER, + tunnelIdentifier: process.env.SAUCE_TUNNEL_ID, + screenResolution: "1280x1024", + recordVideo: false, + recordScreenshots: false, }, plugins: [ "karma-jasmine", From 389e22e338db1b9708f17bc3d8d0ce36d111e94d Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Thu, 19 Sep 2024 10:56:33 -0400 Subject: [PATCH 06/21] Increase timeout for functional tests. --- .sauce/dev.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.sauce/dev.yml b/.sauce/dev.yml index 9e363dcd6..946ad5ff5 100644 --- a/.sauce/dev.yml +++ b/.sauce/dev.yml @@ -14,7 +14,7 @@ testcafe: version: 3.6.0 nodeVersion: "22" speed: 1 -pageLoadTimeout: 3000 +pageLoadTimeout: 5000 disablePageCaching: true timeZone: New_York npm: @@ -28,7 +28,6 @@ npm: suites: - name: "Safari" browserName: "safari" - headless: true browserVersion: "latest" src: - "test/functional/specs/**/*.js" From 7549fb43b1fb6eeca1dc16c4babd3548e5fb4a11 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Thu, 19 Sep 2024 11:10:21 -0400 Subject: [PATCH 07/21] Remove debug logs from C8085776. --- test/functional/specs/Migration/C8085776.js | 23 ++------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/test/functional/specs/Migration/C8085776.js b/test/functional/specs/Migration/C8085776.js index b007ce24c..723a8265d 100644 --- a/test/functional/specs/Migration/C8085776.js +++ b/test/functional/specs/Migration/C8085776.js @@ -56,31 +56,12 @@ test( "C8085776: At.js 2.x to Web SDK - Assert same session ID, edge cluster are used " + "for both of the requests interact and delivery API", async () => { - // Increase wait time - await t.wait(5000); - - const requestCount = await networkLogger.targetDeliveryEndpointLogs.count( - () => true, - ); - console.log(`Number of Target delivery requests: ${requestCount}`); - - // Log the contents of the requests for debugging - const requests = networkLogger.targetDeliveryEndpointLogs.requests; - console.log("Target delivery requests:", JSON.stringify(requests, null, 2)); - await t - .expect(requestCount) - .gte(1, "Expected at least one Target delivery request"); - + .expect(networkLogger.targetDeliveryEndpointLogs.count(() => true)) + .eql(1); // Get delivery API request const deliveryRequest = networkLogger.targetDeliveryEndpointLogs.requests[0]; - - if (!deliveryRequest) { - console.log("No Target delivery request found"); - await t.expect(false).ok("No Target delivery request was logged"); - } - await responseStatus(networkLogger.targetDeliveryEndpointLogs, [200, 207]); const { searchParams } = new URL(deliveryRequest.request.url); // Extract the session ID from the request query params From f0a425a60402c9dcfe83bd4d7cdb7a4e0770c684 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Thu, 26 Sep 2024 11:20:01 -0400 Subject: [PATCH 08/21] Update workflow to explicitly pass PACKAGE_VERSION to test environment --- .sauce/prod-chrome.yml | 1 + .sauce/prod-firefox.yml | 6 +++--- .sauce/prod-safari.yml | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.sauce/prod-chrome.yml b/.sauce/prod-chrome.yml index f3751bc5c..1238201ee 100644 --- a/.sauce/prod-chrome.yml +++ b/.sauce/prod-chrome.yml @@ -9,6 +9,7 @@ env: ALLOY_ENV: prod ALLOY_PROD_VERSION: $ALLOY_PROD_VERSION NPM_PACKAGE_VERSION: $NPM_PACKAGE_VERSION + PACKAGE_VERSION: $PACKAGE_VERSION testcafe: version: "package.json" nodeVersion: "$NODE_VERSION" diff --git a/.sauce/prod-firefox.yml b/.sauce/prod-firefox.yml index e7a108f4b..7370f19d3 100644 --- a/.sauce/prod-firefox.yml +++ b/.sauce/prod-firefox.yml @@ -9,14 +9,14 @@ env: ALLOY_ENV: prod ALLOY_PROD_VERSION: $ALLOY_PROD_VERSION NPM_PACKAGE_VERSION: $NPM_PACKAGE_VERSION + PACKAGE_VERSION: $PACKAGE_VERSION testcafe: - version: 3.6.0 -nodeVersion: "22" + version: "package.json" +nodeVersion: "$NODE_VERSION" speed: 1 pageLoadTimeout: 3000 disablePageCaching: true timeZone: New_York -timeout: 3600 npm: dependencies: - node-fetch diff --git a/.sauce/prod-safari.yml b/.sauce/prod-safari.yml index cfd356222..75658e715 100644 --- a/.sauce/prod-safari.yml +++ b/.sauce/prod-safari.yml @@ -9,6 +9,7 @@ env: ALLOY_ENV: prod ALLOY_PROD_VERSION: $ALLOY_PROD_VERSION NPM_PACKAGE_VERSION: $NPM_PACKAGE_VERSION + PACKAGE_VERSION: $PACKAGE_VERSION testcafe: version: "package.json" nodeVersion: "$NODE_VERSION" From 529aafc0bc5d60d24db140817294513158076942 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Thu, 26 Sep 2024 13:14:02 -0400 Subject: [PATCH 09/21] Add debug info C2589 for getLibraryInfo firefox issue. --- .github/workflows/prod.yml | 360 ++++++++++----------- test/functional/specs/LibraryInfo/C2589.js | 2 + 2 files changed, 182 insertions(+), 180 deletions(-) diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index ceaca60e6..63169cd79 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -59,106 +59,106 @@ jobs: core.setOutput("matrixInput", JSON.stringify(matrixInput)); console.log("matrixInput: ", matrixInput); - alloy-prod-e2e-safari: - name: Safari - needs: get-testing-tags - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - max-parallel: 10 - fail-fast: false - matrix: ${{ fromJSON(needs.get-testing-tags.outputs.matrixInput) }} - continue-on-error: true - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ matrix.tag.tag }} - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - - name: Cache dependencies - uses: actions/cache@v4 - id: npm-cache - with: - path: "**/node_modules" - key: ${{ runner.os }}-modules-${{ hashFiles('**/package-lock.json') }}-${{ secrets.NPM_CACHE_VERSION }} - - name: Install dependencies - if: steps.npm-cache.outputs.cache-hit != 'true' - run: npm ci - - name: Store Alloy version - uses: nyaa8/package-version@v1 - - name: Download sauce configs - uses: actions/download-artifact@v4 - with: - name: sauce-config - path: .sauce - - name: Download sauce ignore - uses: actions/download-artifact@v4 - with: - name: sauce-ignore - path: . - - name: Build - run: | - echo "PACKAGE_VERSION=$(npm pkg get version | tr -d '\"')" >> $GITHUB_ENV - npm run test:functional:build:prod - env: - ALLOY_PROD_VERSION: ${{ env.PACKAGE_VERSION }} - BASE_CODE_MIN: true - NPM_PACKAGE_PROD: true - - name: Update Sauce Config with Node - run: | - sed -i 's/"$NODE_VERSION"/"${{ matrix.tag.nodeVersion }}"/g' .sauce/prod-safari.yml - - name: Run Functional Tests - id: saucectl - uses: saucelabs/saucectl-run-action@v4.3.0 - env: - GITHUB_TOKEN: ${{ github.token }} - with: - sauce-username: ${{ secrets.SAUCE_USERNAME }} - sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} - env: | - SAUCE_USERNAME=${{ secrets.SAUCE_USERNAME }} - SAUCE_ACCESS_KEY=${{ secrets.SAUCE_ACCESS_KEY }} - ALLOY_PROD_VERSION=${{ env.PACKAGE_VERSION }} - NPM_PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} - ALLOY_ENV=prod - config-file: ./.sauce/prod-safari.yml + # alloy-prod-e2e-safari: + # name: Safari + # needs: get-testing-tags + # runs-on: ubuntu-latest + # timeout-minutes: 60 + # strategy: + # max-parallel: 10 + # fail-fast: false + # matrix: ${{ fromJSON(needs.get-testing-tags.outputs.matrixInput) }} + # continue-on-error: true + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + # with: + # ref: ${{ matrix.tag.tag }} + # - name: Set up Node.js + # uses: actions/setup-node@v4 + # with: + # node-version: 22 + # - name: Cache dependencies + # uses: actions/cache@v4 + # id: npm-cache + # with: + # path: "**/node_modules" + # key: ${{ runner.os }}-modules-${{ hashFiles('**/package-lock.json') }}-${{ secrets.NPM_CACHE_VERSION }} + # - name: Install dependencies + # if: steps.npm-cache.outputs.cache-hit != 'true' + # run: npm ci + # - name: Store Alloy version + # uses: nyaa8/package-version@v1 + # - name: Download sauce configs + # uses: actions/download-artifact@v4 + # with: + # name: sauce-config + # path: .sauce + # - name: Download sauce ignore + # uses: actions/download-artifact@v4 + # with: + # name: sauce-ignore + # path: . + # - name: Build + # run: | + # echo "PACKAGE_VERSION=$(npm pkg get version | tr -d '\"')" >> $GITHUB_ENV + # npm run test:functional:build:prod + # env: + # ALLOY_PROD_VERSION: ${{ env.PACKAGE_VERSION }} + # BASE_CODE_MIN: true + # NPM_PACKAGE_PROD: true + # - name: Update Sauce Config with Node + # run: | + # sed -i 's/"$NODE_VERSION"/"${{ matrix.tag.nodeVersion }}"/g' .sauce/prod-safari.yml + # - name: Run Functional Tests + # id: saucectl + # uses: saucelabs/saucectl-run-action@v4.3.0 + # env: + # GITHUB_TOKEN: ${{ github.token }} + # with: + # sauce-username: ${{ secrets.SAUCE_USERNAME }} + # sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} + # env: | + # SAUCE_USERNAME=${{ secrets.SAUCE_USERNAME }} + # SAUCE_ACCESS_KEY=${{ secrets.SAUCE_ACCESS_KEY }} + # ALLOY_PROD_VERSION=${{ env.PACKAGE_VERSION }} + # NPM_PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} + # ALLOY_ENV=prod + # config-file: ./.sauce/prod-safari.yml - - name: Get Saucelabs Test Results - if: failure() - id: saucelabs-results - env: - SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} - SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} - run: | - mkdir -p test-results - if [ -d "./test-results" ] && [ "$(ls -A ./test-results)" ]; then - cp -r ./test-results/* ./test-results/ - echo "TEST_RESULTS<> $GITHUB_ENV - cat ./test-results/* >> $GITHUB_ENV - echo "EOF" >> $GITHUB_ENV - else - echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV - fi + # - name: Get Saucelabs Test Results + # if: failure() + # id: saucelabs-results + # env: + # SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + # SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + # run: | + # mkdir -p test-results + # if [ -d "./test-results" ] && [ "$(ls -A ./test-results)" ]; then + # cp -r ./test-results/* ./test-results/ + # echo "TEST_RESULTS<> $GITHUB_ENV + # cat ./test-results/* >> $GITHUB_ENV + # echo "EOF" >> $GITHUB_ENV + # else + # echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV + # fi - - name: Notify Slack on Failure - if: failure() - uses: 8398a7/action-slack@v3 - with: - status: ${{ job.status }} - fields: repo,message,commit,author,action,eventName,ref,workflow - text: | - Alloy Prod Firefox Tests Failed - Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + # - name: Notify Slack on Failure + # if: failure() + # uses: 8398a7/action-slack@v3 + # with: + # status: ${{ job.status }} + # fields: repo,message,commit,author,action,eventName,ref,workflow + # text: | + # Alloy Prod Firefox Tests Failed + # Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} + # Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} + # Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} + # Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} + # Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} alloy-prod-e2e-firefox: name: Firefox @@ -261,86 +261,86 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - alloy-prod-e2e-chrome: - name: Chrome - needs: get-testing-tags - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - max-parallel: 10 - fail-fast: false - matrix: ${{ fromJSON(needs.get-testing-tags.outputs.matrixInput) }} - continue-on-error: true - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - ref: ${{ matrix.tag.tag }} - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 22 - - name: Cache dependencies - uses: actions/cache@v4 - id: npm-cache - with: - path: "**/node_modules" - key: ${{ runner.os }}-modules-${{ hashFiles('**/package-lock.json') }}-${{ secrets.NPM_CACHE_VERSION }} - - name: Install dependencies - if: steps.npm-cache.outputs.cache-hit != 'true' - run: npm ci - - name: Store Alloy version - uses: nyaa8/package-version@v1 - - name: Download sauce configs - uses: actions/download-artifact@v4 - with: - name: sauce-config - path: .sauce - - name: Download sauce ignore - uses: actions/download-artifact@v4 - with: - name: sauce-ignore - path: . - - name: Build - run: | - echo "PACKAGE_VERSION=$(npm pkg get version | tr -d '\"')" >> $GITHUB_ENV - npm run test:functional:build:prod - env: - ALLOY_PROD_VERSION: ${{ env.PACKAGE_VERSION }} - BASE_CODE_MIN: true - NPM_PACKAGE_PROD: true - - name: Update Sauce Config with Node - run: | - sed -i 's/"$NODE_VERSION"/"${{ matrix.tag.nodeVersion }}"/g' .sauce/prod-chrome.yml - - name: Run Functional Tests - id: saucectl - uses: saucelabs/saucectl-run-action@v4.3.0 - env: - GITHUB_TOKEN: ${{ github.token }} - with: - sauce-username: ${{ secrets.SAUCE_USERNAME }} - sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} - env: | - SAUCE_USERNAME=${{ secrets.SAUCE_USERNAME }} - SAUCE_ACCESS_KEY=${{ secrets.SAUCE_ACCESS_KEY }} - ALLOY_PROD_VERSION=${{ env.PACKAGE_VERSION }} - NPM_PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} - ALLOY_ENV=prod - config-file: ./.sauce/prod-chrome.yml + # alloy-prod-e2e-chrome: + # name: Chrome + # needs: get-testing-tags + # runs-on: ubuntu-latest + # timeout-minutes: 60 + # strategy: + # max-parallel: 10 + # fail-fast: false + # matrix: ${{ fromJSON(needs.get-testing-tags.outputs.matrixInput) }} + # continue-on-error: true + # steps: + # - name: Checkout code + # uses: actions/checkout@v4 + # with: + # ref: ${{ matrix.tag.tag }} + # - name: Set up Node.js + # uses: actions/setup-node@v4 + # with: + # node-version: 22 + # - name: Cache dependencies + # uses: actions/cache@v4 + # id: npm-cache + # with: + # path: "**/node_modules" + # key: ${{ runner.os }}-modules-${{ hashFiles('**/package-lock.json') }}-${{ secrets.NPM_CACHE_VERSION }} + # - name: Install dependencies + # if: steps.npm-cache.outputs.cache-hit != 'true' + # run: npm ci + # - name: Store Alloy version + # uses: nyaa8/package-version@v1 + # - name: Download sauce configs + # uses: actions/download-artifact@v4 + # with: + # name: sauce-config + # path: .sauce + # - name: Download sauce ignore + # uses: actions/download-artifact@v4 + # with: + # name: sauce-ignore + # path: . + # - name: Build + # run: | + # echo "PACKAGE_VERSION=$(npm pkg get version | tr -d '\"')" >> $GITHUB_ENV + # npm run test:functional:build:prod + # env: + # ALLOY_PROD_VERSION: ${{ env.PACKAGE_VERSION }} + # BASE_CODE_MIN: true + # NPM_PACKAGE_PROD: true + # - name: Update Sauce Config with Node + # run: | + # sed -i 's/"$NODE_VERSION"/"${{ matrix.tag.nodeVersion }}"/g' .sauce/prod-chrome.yml + # - name: Run Functional Tests + # id: saucectl + # uses: saucelabs/saucectl-run-action@v4.3.0 + # env: + # GITHUB_TOKEN: ${{ github.token }} + # with: + # sauce-username: ${{ secrets.SAUCE_USERNAME }} + # sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} + # env: | + # SAUCE_USERNAME=${{ secrets.SAUCE_USERNAME }} + # SAUCE_ACCESS_KEY=${{ secrets.SAUCE_ACCESS_KEY }} + # ALLOY_PROD_VERSION=${{ env.PACKAGE_VERSION }} + # NPM_PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} + # ALLOY_ENV=prod + # config-file: ./.sauce/prod-chrome.yml - - name: Notify Slack on Failure - if: failure() - uses: 8398a7/action-slack@v3 - with: - status: ${{ job.status }} - fields: repo,message,commit,author,action,eventName,ref,workflow - text: | - Alloy Prod Firefox Tests Failed - Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + # - name: Notify Slack on Failure + # if: failure() + # uses: 8398a7/action-slack@v3 + # with: + # status: ${{ job.status }} + # fields: repo,message,commit,author,action,eventName,ref,workflow + # text: | + # Alloy Prod Firefox Tests Failed + # Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} + # Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} + # Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} + # Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} + # Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/test/functional/specs/LibraryInfo/C2589.js b/test/functional/specs/LibraryInfo/C2589.js index 26aa80fdb..a3f888d29 100644 --- a/test/functional/specs/LibraryInfo/C2589.js +++ b/test/functional/specs/LibraryInfo/C2589.js @@ -89,9 +89,11 @@ test("C2589: getLibraryInfo command returns library information.", async () => { }, }; + console.log("Current Version:", currentVersion); // Add this line for debugging const alloy = createAlloyProxy(); await alloy.configure(debugEnabledConfig); const { libraryInfo } = await alloy.getLibraryInfo(); + console.log("Library Info Version:", libraryInfo.version); // Add this line for debugging delete libraryInfo.configs.edgeBasePath; await t.expect(libraryInfo.version).eql(currentVersion); await t.expect(libraryInfo.commands).eql(currentCommand); From 11e6df7149ed1fab71d47baa27fe33fb71081f3d Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Thu, 26 Sep 2024 13:26:31 -0400 Subject: [PATCH 10/21] Run firefox test on Windows to debug library info bug. --- .sauce/prod-firefox.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.sauce/prod-firefox.yml b/.sauce/prod-firefox.yml index 7370f19d3..eceb5b9e4 100644 --- a/.sauce/prod-firefox.yml +++ b/.sauce/prod-firefox.yml @@ -36,7 +36,7 @@ suites: browserVersion: "latest" src: - "test/functional/specs/**/*.js" - platformName: "macOS 13" + platformName: "Windows 11" excludedTestFiles: [ "test/functional/specs/Data Collector/C81182.js", From 0efdb9297a3fc544c065567f34181df8fd58ec3e Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Thu, 26 Sep 2024 13:49:23 -0400 Subject: [PATCH 11/21] Firefox get libraryinfo fixed, reenable chrome and safari tests. --- .github/workflows/prod.yml | 360 ++++++++++----------- test/functional/specs/LibraryInfo/C2589.js | 2 - 2 files changed, 180 insertions(+), 182 deletions(-) diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index 63169cd79..ceaca60e6 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -59,106 +59,106 @@ jobs: core.setOutput("matrixInput", JSON.stringify(matrixInput)); console.log("matrixInput: ", matrixInput); - # alloy-prod-e2e-safari: - # name: Safari - # needs: get-testing-tags - # runs-on: ubuntu-latest - # timeout-minutes: 60 - # strategy: - # max-parallel: 10 - # fail-fast: false - # matrix: ${{ fromJSON(needs.get-testing-tags.outputs.matrixInput) }} - # continue-on-error: true - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - # with: - # ref: ${{ matrix.tag.tag }} - # - name: Set up Node.js - # uses: actions/setup-node@v4 - # with: - # node-version: 22 - # - name: Cache dependencies - # uses: actions/cache@v4 - # id: npm-cache - # with: - # path: "**/node_modules" - # key: ${{ runner.os }}-modules-${{ hashFiles('**/package-lock.json') }}-${{ secrets.NPM_CACHE_VERSION }} - # - name: Install dependencies - # if: steps.npm-cache.outputs.cache-hit != 'true' - # run: npm ci - # - name: Store Alloy version - # uses: nyaa8/package-version@v1 - # - name: Download sauce configs - # uses: actions/download-artifact@v4 - # with: - # name: sauce-config - # path: .sauce - # - name: Download sauce ignore - # uses: actions/download-artifact@v4 - # with: - # name: sauce-ignore - # path: . - # - name: Build - # run: | - # echo "PACKAGE_VERSION=$(npm pkg get version | tr -d '\"')" >> $GITHUB_ENV - # npm run test:functional:build:prod - # env: - # ALLOY_PROD_VERSION: ${{ env.PACKAGE_VERSION }} - # BASE_CODE_MIN: true - # NPM_PACKAGE_PROD: true - # - name: Update Sauce Config with Node - # run: | - # sed -i 's/"$NODE_VERSION"/"${{ matrix.tag.nodeVersion }}"/g' .sauce/prod-safari.yml - # - name: Run Functional Tests - # id: saucectl - # uses: saucelabs/saucectl-run-action@v4.3.0 - # env: - # GITHUB_TOKEN: ${{ github.token }} - # with: - # sauce-username: ${{ secrets.SAUCE_USERNAME }} - # sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} - # env: | - # SAUCE_USERNAME=${{ secrets.SAUCE_USERNAME }} - # SAUCE_ACCESS_KEY=${{ secrets.SAUCE_ACCESS_KEY }} - # ALLOY_PROD_VERSION=${{ env.PACKAGE_VERSION }} - # NPM_PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} - # ALLOY_ENV=prod - # config-file: ./.sauce/prod-safari.yml + alloy-prod-e2e-safari: + name: Safari + needs: get-testing-tags + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + max-parallel: 10 + fail-fast: false + matrix: ${{ fromJSON(needs.get-testing-tags.outputs.matrixInput) }} + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ matrix.tag.tag }} + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Cache dependencies + uses: actions/cache@v4 + id: npm-cache + with: + path: "**/node_modules" + key: ${{ runner.os }}-modules-${{ hashFiles('**/package-lock.json') }}-${{ secrets.NPM_CACHE_VERSION }} + - name: Install dependencies + if: steps.npm-cache.outputs.cache-hit != 'true' + run: npm ci + - name: Store Alloy version + uses: nyaa8/package-version@v1 + - name: Download sauce configs + uses: actions/download-artifact@v4 + with: + name: sauce-config + path: .sauce + - name: Download sauce ignore + uses: actions/download-artifact@v4 + with: + name: sauce-ignore + path: . + - name: Build + run: | + echo "PACKAGE_VERSION=$(npm pkg get version | tr -d '\"')" >> $GITHUB_ENV + npm run test:functional:build:prod + env: + ALLOY_PROD_VERSION: ${{ env.PACKAGE_VERSION }} + BASE_CODE_MIN: true + NPM_PACKAGE_PROD: true + - name: Update Sauce Config with Node + run: | + sed -i 's/"$NODE_VERSION"/"${{ matrix.tag.nodeVersion }}"/g' .sauce/prod-safari.yml + - name: Run Functional Tests + id: saucectl + uses: saucelabs/saucectl-run-action@v4.3.0 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + sauce-username: ${{ secrets.SAUCE_USERNAME }} + sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} + env: | + SAUCE_USERNAME=${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY=${{ secrets.SAUCE_ACCESS_KEY }} + ALLOY_PROD_VERSION=${{ env.PACKAGE_VERSION }} + NPM_PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} + ALLOY_ENV=prod + config-file: ./.sauce/prod-safari.yml - # - name: Get Saucelabs Test Results - # if: failure() - # id: saucelabs-results - # env: - # SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} - # SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} - # run: | - # mkdir -p test-results - # if [ -d "./test-results" ] && [ "$(ls -A ./test-results)" ]; then - # cp -r ./test-results/* ./test-results/ - # echo "TEST_RESULTS<> $GITHUB_ENV - # cat ./test-results/* >> $GITHUB_ENV - # echo "EOF" >> $GITHUB_ENV - # else - # echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV - # fi + - name: Get Saucelabs Test Results + if: failure() + id: saucelabs-results + env: + SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} + run: | + mkdir -p test-results + if [ -d "./test-results" ] && [ "$(ls -A ./test-results)" ]; then + cp -r ./test-results/* ./test-results/ + echo "TEST_RESULTS<> $GITHUB_ENV + cat ./test-results/* >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + else + echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV + fi - # - name: Notify Slack on Failure - # if: failure() - # uses: 8398a7/action-slack@v3 - # with: - # status: ${{ job.status }} - # fields: repo,message,commit,author,action,eventName,ref,workflow - # text: | - # Alloy Prod Firefox Tests Failed - # Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - # Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - # Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - # Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - # Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + - name: Notify Slack on Failure + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow + text: | + Alloy Prod Firefox Tests Failed + Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} + Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} + Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} + Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} + Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} alloy-prod-e2e-firefox: name: Firefox @@ -261,86 +261,86 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - # alloy-prod-e2e-chrome: - # name: Chrome - # needs: get-testing-tags - # runs-on: ubuntu-latest - # timeout-minutes: 60 - # strategy: - # max-parallel: 10 - # fail-fast: false - # matrix: ${{ fromJSON(needs.get-testing-tags.outputs.matrixInput) }} - # continue-on-error: true - # steps: - # - name: Checkout code - # uses: actions/checkout@v4 - # with: - # ref: ${{ matrix.tag.tag }} - # - name: Set up Node.js - # uses: actions/setup-node@v4 - # with: - # node-version: 22 - # - name: Cache dependencies - # uses: actions/cache@v4 - # id: npm-cache - # with: - # path: "**/node_modules" - # key: ${{ runner.os }}-modules-${{ hashFiles('**/package-lock.json') }}-${{ secrets.NPM_CACHE_VERSION }} - # - name: Install dependencies - # if: steps.npm-cache.outputs.cache-hit != 'true' - # run: npm ci - # - name: Store Alloy version - # uses: nyaa8/package-version@v1 - # - name: Download sauce configs - # uses: actions/download-artifact@v4 - # with: - # name: sauce-config - # path: .sauce - # - name: Download sauce ignore - # uses: actions/download-artifact@v4 - # with: - # name: sauce-ignore - # path: . - # - name: Build - # run: | - # echo "PACKAGE_VERSION=$(npm pkg get version | tr -d '\"')" >> $GITHUB_ENV - # npm run test:functional:build:prod - # env: - # ALLOY_PROD_VERSION: ${{ env.PACKAGE_VERSION }} - # BASE_CODE_MIN: true - # NPM_PACKAGE_PROD: true - # - name: Update Sauce Config with Node - # run: | - # sed -i 's/"$NODE_VERSION"/"${{ matrix.tag.nodeVersion }}"/g' .sauce/prod-chrome.yml - # - name: Run Functional Tests - # id: saucectl - # uses: saucelabs/saucectl-run-action@v4.3.0 - # env: - # GITHUB_TOKEN: ${{ github.token }} - # with: - # sauce-username: ${{ secrets.SAUCE_USERNAME }} - # sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} - # env: | - # SAUCE_USERNAME=${{ secrets.SAUCE_USERNAME }} - # SAUCE_ACCESS_KEY=${{ secrets.SAUCE_ACCESS_KEY }} - # ALLOY_PROD_VERSION=${{ env.PACKAGE_VERSION }} - # NPM_PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} - # ALLOY_ENV=prod - # config-file: ./.sauce/prod-chrome.yml + alloy-prod-e2e-chrome: + name: Chrome + needs: get-testing-tags + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + max-parallel: 10 + fail-fast: false + matrix: ${{ fromJSON(needs.get-testing-tags.outputs.matrixInput) }} + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ matrix.tag.tag }} + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Cache dependencies + uses: actions/cache@v4 + id: npm-cache + with: + path: "**/node_modules" + key: ${{ runner.os }}-modules-${{ hashFiles('**/package-lock.json') }}-${{ secrets.NPM_CACHE_VERSION }} + - name: Install dependencies + if: steps.npm-cache.outputs.cache-hit != 'true' + run: npm ci + - name: Store Alloy version + uses: nyaa8/package-version@v1 + - name: Download sauce configs + uses: actions/download-artifact@v4 + with: + name: sauce-config + path: .sauce + - name: Download sauce ignore + uses: actions/download-artifact@v4 + with: + name: sauce-ignore + path: . + - name: Build + run: | + echo "PACKAGE_VERSION=$(npm pkg get version | tr -d '\"')" >> $GITHUB_ENV + npm run test:functional:build:prod + env: + ALLOY_PROD_VERSION: ${{ env.PACKAGE_VERSION }} + BASE_CODE_MIN: true + NPM_PACKAGE_PROD: true + - name: Update Sauce Config with Node + run: | + sed -i 's/"$NODE_VERSION"/"${{ matrix.tag.nodeVersion }}"/g' .sauce/prod-chrome.yml + - name: Run Functional Tests + id: saucectl + uses: saucelabs/saucectl-run-action@v4.3.0 + env: + GITHUB_TOKEN: ${{ github.token }} + with: + sauce-username: ${{ secrets.SAUCE_USERNAME }} + sauce-access-key: ${{ secrets.SAUCE_ACCESS_KEY }} + env: | + SAUCE_USERNAME=${{ secrets.SAUCE_USERNAME }} + SAUCE_ACCESS_KEY=${{ secrets.SAUCE_ACCESS_KEY }} + ALLOY_PROD_VERSION=${{ env.PACKAGE_VERSION }} + NPM_PACKAGE_VERSION=${{ env.PACKAGE_VERSION }} + ALLOY_ENV=prod + config-file: ./.sauce/prod-chrome.yml - # - name: Notify Slack on Failure - # if: failure() - # uses: 8398a7/action-slack@v3 - # with: - # status: ${{ job.status }} - # fields: repo,message,commit,author,action,eventName,ref,workflow - # text: | - # Alloy Prod Firefox Tests Failed - # Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - # Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - # Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - # Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - # Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + - name: Notify Slack on Failure + if: failure() + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow + text: | + Alloy Prod Firefox Tests Failed + Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} + Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} + Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} + Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} + Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file diff --git a/test/functional/specs/LibraryInfo/C2589.js b/test/functional/specs/LibraryInfo/C2589.js index a3f888d29..26aa80fdb 100644 --- a/test/functional/specs/LibraryInfo/C2589.js +++ b/test/functional/specs/LibraryInfo/C2589.js @@ -89,11 +89,9 @@ test("C2589: getLibraryInfo command returns library information.", async () => { }, }; - console.log("Current Version:", currentVersion); // Add this line for debugging const alloy = createAlloyProxy(); await alloy.configure(debugEnabledConfig); const { libraryInfo } = await alloy.getLibraryInfo(); - console.log("Library Info Version:", libraryInfo.version); // Add this line for debugging delete libraryInfo.configs.edgeBasePath; await t.expect(libraryInfo.version).eql(currentVersion); await t.expect(libraryInfo.commands).eql(currentCommand); From c1f628d000059de07c72b5bf62624d5601b40bc1 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Thu, 26 Sep 2024 14:54:04 -0400 Subject: [PATCH 12/21] Ensure that the mbox/json API request is being made. --- .sauce/prod-safari.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.sauce/prod-safari.yml b/.sauce/prod-safari.yml index 75658e715..00630f49b 100644 --- a/.sauce/prod-safari.yml +++ b/.sauce/prod-safari.yml @@ -13,8 +13,8 @@ env: testcafe: version: "package.json" nodeVersion: "$NODE_VERSION" -speed: 1 -pageLoadTimeout: 3000 +speed: 0.5 +pageLoadTimeout: 10000 disablePageCaching: true timeZone: New_York npm: From 1819b4fe8cbee0e39fa4c06f61fe56763480e3e0 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Thu, 26 Sep 2024 15:38:13 -0400 Subject: [PATCH 13/21] Enable quarantineMode to debug safari error. --- .sauce/prod-safari.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.sauce/prod-safari.yml b/.sauce/prod-safari.yml index 00630f49b..3d3b90fde 100644 --- a/.sauce/prod-safari.yml +++ b/.sauce/prod-safari.yml @@ -14,9 +14,11 @@ testcafe: version: "package.json" nodeVersion: "$NODE_VERSION" speed: 0.5 -pageLoadTimeout: 10000 +pageLoadTimeout: 8000 disablePageCaching: true timeZone: New_York +quarantineMode: true +retries: 3 npm: dependencies: - node-fetch From 3b8fe4792c8869324a425572df183c565b3c0f84 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Thu, 26 Sep 2024 15:59:02 -0400 Subject: [PATCH 14/21] Enable quarantineMode to debug safari error. --- .sauce/prod-safari.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.sauce/prod-safari.yml b/.sauce/prod-safari.yml index 3d3b90fde..c3e8bf00b 100644 --- a/.sauce/prod-safari.yml +++ b/.sauce/prod-safari.yml @@ -17,8 +17,6 @@ speed: 0.5 pageLoadTimeout: 8000 disablePageCaching: true timeZone: New_York -quarantineMode: true -retries: 3 npm: dependencies: - node-fetch @@ -34,6 +32,9 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Safari" + quarantineMode: + attemptLimit: 3 + successThreshold: 1 browserName: "safari" browserVersion: "latest" src: From 42077ee435ccf128adcc4fdc48717401b8c05235 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Fri, 27 Sep 2024 10:44:38 -0400 Subject: [PATCH 15/21] Safari failure fixed, removing quarantineMode. --- .sauce/prod-safari.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.sauce/prod-safari.yml b/.sauce/prod-safari.yml index c3e8bf00b..37d184583 100644 --- a/.sauce/prod-safari.yml +++ b/.sauce/prod-safari.yml @@ -32,9 +32,6 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Safari" - quarantineMode: - attemptLimit: 3 - successThreshold: 1 browserName: "safari" browserVersion: "latest" src: From 89b0983b9e8a1e2c695ec42669fe3472ca7c3046 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Fri, 27 Sep 2024 11:04:59 -0400 Subject: [PATCH 16/21] Enable 3 attemps with failure logs. --- .sauce/prod-chrome.yml | 3 +++ .sauce/prod-firefox.yml | 3 +++ .sauce/prod-safari.yml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/.sauce/prod-chrome.yml b/.sauce/prod-chrome.yml index 1238201ee..e27868fb5 100644 --- a/.sauce/prod-chrome.yml +++ b/.sauce/prod-chrome.yml @@ -32,6 +32,9 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Chrome" + quarantineMode: + attemptLimit: 3 + successThreshold: 1 browserName: "chrome" browserVersion: "latest" src: diff --git a/.sauce/prod-firefox.yml b/.sauce/prod-firefox.yml index eceb5b9e4..86cc48784 100644 --- a/.sauce/prod-firefox.yml +++ b/.sauce/prod-firefox.yml @@ -32,6 +32,9 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Firefox" + quarantineMode: + attemptLimit: 3 + successThreshold: 1 browserName: "firefox" browserVersion: "latest" src: diff --git a/.sauce/prod-safari.yml b/.sauce/prod-safari.yml index 37d184583..c3e8bf00b 100644 --- a/.sauce/prod-safari.yml +++ b/.sauce/prod-safari.yml @@ -32,6 +32,9 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Safari" + quarantineMode: + attemptLimit: 3 + successThreshold: 1 browserName: "safari" browserVersion: "latest" src: From 9b44fee44b8d2137175717b3a0474958d6167a5b Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Fri, 27 Sep 2024 12:05:52 -0400 Subject: [PATCH 17/21] Remove quarantineMode from the browser configs. --- .sauce/prod-chrome.yml | 7 ++----- .sauce/prod-firefox.yml | 7 ++----- .sauce/prod-safari.yml | 9 +++------ 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/.sauce/prod-chrome.yml b/.sauce/prod-chrome.yml index e27868fb5..c9977b9ec 100644 --- a/.sauce/prod-chrome.yml +++ b/.sauce/prod-chrome.yml @@ -13,8 +13,8 @@ env: testcafe: version: "package.json" nodeVersion: "$NODE_VERSION" -speed: 1 -pageLoadTimeout: 3000 +speed: 0.5 +pageLoadTimeout: 5000 disablePageCaching: true timeZone: New_York npm: @@ -32,9 +32,6 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Chrome" - quarantineMode: - attemptLimit: 3 - successThreshold: 1 browserName: "chrome" browserVersion: "latest" src: diff --git a/.sauce/prod-firefox.yml b/.sauce/prod-firefox.yml index 86cc48784..8aed04365 100644 --- a/.sauce/prod-firefox.yml +++ b/.sauce/prod-firefox.yml @@ -13,8 +13,8 @@ env: testcafe: version: "package.json" nodeVersion: "$NODE_VERSION" -speed: 1 -pageLoadTimeout: 3000 +speed: 0.5 +pageLoadTimeout: 8000 disablePageCaching: true timeZone: New_York npm: @@ -32,9 +32,6 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Firefox" - quarantineMode: - attemptLimit: 3 - successThreshold: 1 browserName: "firefox" browserVersion: "latest" src: diff --git a/.sauce/prod-safari.yml b/.sauce/prod-safari.yml index c3e8bf00b..1f37ffdea 100644 --- a/.sauce/prod-safari.yml +++ b/.sauce/prod-safari.yml @@ -13,9 +13,9 @@ env: testcafe: version: "package.json" nodeVersion: "$NODE_VERSION" -speed: 0.5 -pageLoadTimeout: 8000 -disablePageCaching: true +speed: 0.4 +pageLoadTimeout: 10000 +disablePageCaching: false timeZone: New_York npm: dependencies: @@ -32,9 +32,6 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Safari" - quarantineMode: - attemptLimit: 3 - successThreshold: 1 browserName: "safari" browserVersion: "latest" src: From 02b369c27a658f3ded9c32c5be7c8fd7a459192a Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Fri, 27 Sep 2024 12:31:39 -0400 Subject: [PATCH 18/21] Enable quarantineMode for safari. --- .sauce/prod-safari.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.sauce/prod-safari.yml b/.sauce/prod-safari.yml index 1f37ffdea..9351c2e47 100644 --- a/.sauce/prod-safari.yml +++ b/.sauce/prod-safari.yml @@ -13,10 +13,11 @@ env: testcafe: version: "package.json" nodeVersion: "$NODE_VERSION" -speed: 0.4 -pageLoadTimeout: 10000 +speed: 0.5 +pageLoadTimeout: 8000 disablePageCaching: false timeZone: New_York +retries: 3 npm: dependencies: - node-fetch @@ -37,6 +38,9 @@ suites: src: - "test/functional/specs/**/*.js" platformName: "macOS 13" + quarantineMode: + attemptLimit: 3 + successThreshold: 1 excludedTestFiles: [ "test/functional/specs/Privacy/C1472435.js", From cc42837563028dd396fb2018c705302c54325777 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Mon, 30 Sep 2024 12:57:49 -0400 Subject: [PATCH 19/21] Update config for Chrome to debug intermittent failures. --- .sauce/prod-chrome.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.sauce/prod-chrome.yml b/.sauce/prod-chrome.yml index c9977b9ec..b7f1b61cd 100644 --- a/.sauce/prod-chrome.yml +++ b/.sauce/prod-chrome.yml @@ -14,8 +14,8 @@ testcafe: version: "package.json" nodeVersion: "$NODE_VERSION" speed: 0.5 -pageLoadTimeout: 5000 -disablePageCaching: true +pageLoadTimeout: 8000 +disablePageCaching: false timeZone: New_York npm: dependencies: From a14c2046a07ff1fd22e54114936446d00f7cd077 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Mon, 30 Sep 2024 14:57:04 -0400 Subject: [PATCH 20/21] Set smartRetry on failed tests. --- .sauce/prod-chrome.yml | 3 +++ .sauce/prod-firefox.yml | 3 +++ .sauce/prod-safari.yml | 7 +++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.sauce/prod-chrome.yml b/.sauce/prod-chrome.yml index b7f1b61cd..3ce8139c8 100644 --- a/.sauce/prod-chrome.yml +++ b/.sauce/prod-chrome.yml @@ -2,6 +2,7 @@ apiVersion: v1alpha kind: testcafe sauce: region: us-west-1 + retries: 3 metadata: name: Alloy Prod rootDir: ./ @@ -32,6 +33,8 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Chrome" + smartRetry: + failedOnly: true browserName: "chrome" browserVersion: "latest" src: diff --git a/.sauce/prod-firefox.yml b/.sauce/prod-firefox.yml index 8aed04365..8fe63e058 100644 --- a/.sauce/prod-firefox.yml +++ b/.sauce/prod-firefox.yml @@ -2,6 +2,7 @@ apiVersion: v1alpha kind: testcafe sauce: region: us-west-1 + retries: 3 metadata: name: Alloy Prod rootDir: ./ @@ -32,6 +33,8 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Firefox" + smartRetry: + failedOnly: true browserName: "firefox" browserVersion: "latest" src: diff --git a/.sauce/prod-safari.yml b/.sauce/prod-safari.yml index 9351c2e47..4e25e5998 100644 --- a/.sauce/prod-safari.yml +++ b/.sauce/prod-safari.yml @@ -2,6 +2,7 @@ apiVersion: v1alpha kind: testcafe sauce: region: us-west-1 + retries: 3 metadata: name: Alloy Prod rootDir: ./ @@ -17,7 +18,6 @@ speed: 0.5 pageLoadTimeout: 8000 disablePageCaching: false timeZone: New_York -retries: 3 npm: dependencies: - node-fetch @@ -33,14 +33,13 @@ npm: - "@adobe/reactor-object-assign" suites: - name: "Safari" + smartRetry: + failedOnly: true browserName: "safari" browserVersion: "latest" src: - "test/functional/specs/**/*.js" platformName: "macOS 13" - quarantineMode: - attemptLimit: 3 - successThreshold: 1 excludedTestFiles: [ "test/functional/specs/Privacy/C1472435.js", From e63377205c465db355559ee399bac585ca7a0e17 Mon Sep 17 00:00:00 2001 From: Shamiul Mowla Date: Mon, 30 Sep 2024 15:12:03 -0400 Subject: [PATCH 21/21] Create a github issue when a test fails. --- .github/scripts/handle-test-failures.js | 53 ++++++++++++++++++++++ .github/workflows/prod.yml | 60 ++++++++----------------- 2 files changed, 71 insertions(+), 42 deletions(-) create mode 100644 .github/scripts/handle-test-failures.js diff --git a/.github/scripts/handle-test-failures.js b/.github/scripts/handle-test-failures.js new file mode 100644 index 000000000..1b4fad423 --- /dev/null +++ b/.github/scripts/handle-test-failures.js @@ -0,0 +1,53 @@ +module.exports = async ({ github, context, browser }) => { + const fs = require('fs'); + const path = require('path'); + + // Read the test results + const testResultsDir = './test-results'; + const files = fs.readdirSync(testResultsDir); + + for (const file of files) { + const content = fs.readFileSync(path.join(testResultsDir, file), 'utf8'); + let result; + try { + result = JSON.parse(content); + } catch (error) { + console.error(`Error parsing JSON from ${file}: ${error}`); + continue; + } + + if (result.status === 'failed') { + const testName = result.name; + const sauceLabsLink = `https://app.saucelabs.com/tests/${result.id}`; + + // Search for an existing issue + const issues = await github.rest.issues.listForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + labels: ['test-failure', browser] + }); + + const existingIssue = issues.data.find(issue => issue.title.includes(testName)); + + if (existingIssue) { + // Update existing issue + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: existingIssue.number, + body: `Test failed again in the latest run.\nBrowser: ${browser}\nRun: ${context.runId}\n[View on SauceLabs](${sauceLabsLink})` + }); + } else { + // Create new issue + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `Test Failure: ${testName} (${browser})`, + body: `The test "${testName}" failed in the ${browser} browser.\n\nRun: ${context.runId}\n[View on SauceLabs](${sauceLabsLink})`, + labels: ['test-failure', browser] + }); + } + } + } + }; \ No newline at end of file diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index ceaca60e6..903f16e2c 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -143,22 +143,14 @@ jobs: echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV fi - - name: Notify Slack on Failure + - name: Handle Test Failures if: failure() - uses: 8398a7/action-slack@v3 + uses: actions/github-script@v7 with: - status: ${{ job.status }} - fields: repo,message,commit,author,action,eventName,ref,workflow - text: | - Alloy Prod Firefox Tests Failed - Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const handleTestFailures = require('./.github/scripts/handle-test-failures.js'); + await handleTestFailures({ github, context, browser: 'safari' }); alloy-prod-e2e-firefox: name: Firefox @@ -244,22 +236,14 @@ jobs: echo "TEST_RESULTS=No test results found in ./test-results/" >> $GITHUB_ENV fi - - name: Notify Slack on Failure + - name: Handle Test Failures if: failure() - uses: 8398a7/action-slack@v3 + uses: actions/github-script@v7 with: - status: ${{ job.status }} - fields: repo,message,commit,author,action,eventName,ref,workflow - text: | - Alloy Prod Firefox Tests Failed - Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const handleTestFailures = require('./.github/scripts/handle-test-failures.js'); + await handleTestFailures({ github, context, browser: 'firefox' }); alloy-prod-e2e-chrome: name: Chrome @@ -328,19 +312,11 @@ jobs: ALLOY_ENV=prod config-file: ./.sauce/prod-chrome.yml - - name: Notify Slack on Failure + - name: Handle Test Failures if: failure() - uses: 8398a7/action-slack@v3 + uses: actions/github-script@v7 with: - status: ${{ job.status }} - fields: repo,message,commit,author,action,eventName,ref,workflow - text: | - Alloy Prod Firefox Tests Failed - Job ID: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - Build: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').build }} - Status: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').status }} - Error: ${{ fromJson(steps.saucelabs-results.outputs.results || '{}').error }} - Full Results: https://app.saucelabs.com/tests/${{ fromJson(steps.saucelabs-results.outputs.results || '{}').id }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} \ No newline at end of file + github-token: ${{secrets.GITHUB_TOKEN}} + script: | + const handleTestFailures = require('./.github/scripts/handle-test-failures.js'); + await handleTestFailures({ github, context, browser: 'chrome' });