From dc8ff85012f50afc20d1d6525b70c161b8db7df1 Mon Sep 17 00:00:00 2001 From: KrijnL Date: Mon, 26 Aug 2024 16:01:39 +0200 Subject: [PATCH] chore: add ctrf reporter and run e2e tests after deployment --- .github/workflows/deploy.yml | 8 + .github/workflows/e2e.yml | 46 ++++- e2e/.gitignore | 1 + e2e/package-lock.json | 324 ++++++++++++++++++++++++++++++++++- e2e/package.json | 4 +- e2e/playwright.config.ts | 5 +- 6 files changed, 382 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fd95dff..fe2f727 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -76,3 +76,11 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - name: Pause to let invalidation complete + run: | + sleep 180 + + Run_e2e_Tests: + needs: Invalidate_Cloudfront + uses: ./.github/workflows/e2e.yml + secrets: inherit diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6efb0df..98ccb7a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -2,10 +2,11 @@ name: E2E tests on: workflow_call: workflow_dispatch: - push: - branches: [ main, feat/workflow-for-e2e ] + schedule: + - cron: "30 9 * * *" pull_request: - branches: [ main ] + types: [ opened, synchronize ] + jobs: test: timeout-minutes: 60 @@ -41,3 +42,42 @@ jobs: e2e/playwright-report/ e2e/test-results/ retention-days: 10 + - name: Run CTRF results + if: always() + run: | + npx github-actions-ctrf tests ctrf/ctrf-report.json --pr-comment + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Set test result outputs + id: set_test_outputs + if: always() + run: | + echo "TEST_COUNT=`jq '.results.summary.passed' < ./ctrf/ctrf-report.json`" >> $GITHUB_OUTPUT + echo "PASSED_TESTS=`jq '.results.summary.passed' < ./ctrf/ctrf-report.json`" >> $GITHUB_OUTPUT + echo "FAILED_TESTS=`jq '.results.summary.failed' < ./ctrf/ctrf-report.json`" >> $GITHUB_OUTPUT + + - name: Add github branch name & SHA to environment + if: always() + run: | + if [ "$GITHUB_HEAD_REF" != "" ]; then + echo "GITHUB_BRANCH_NAME=$(echo $GITHUB_HEAD_REF)" >> $GITHUB_ENV + else + echo "GITHUB_BRANCH_NAME=$(echo $GITHUB_REF_NAME)" >> $GITHUB_ENV + fi + echo "GITHUB_COMMIT_SHA=${GITHUB_SHA:0:10}" >> $GITHUB_ENV + + + - name: Slack Notification on failure + uses: rtCamp/action-slack-notify@v2 + if: github.event_name !== 'pull_request' && steps.set_test_outputs.outputs.FAILED_TESTS > 0 + env: + SLACK_CHANNEL: engineering-alerts + SLACK_COLOR: failure + SLACK_MESSAGE: " + \n + :warning: ${{ steps.set_test_outputs.outputs.FAILED_TESTS }} out of ${{ steps.set_test_outputs.outputs.TEST_COUNT }} e2e tests failed for luzmo showcases \n" + SLACK_TITLE: ":git: luzmo-official/flex-showcases - ${{ env.GITHUB_BRANCH_NAME }} - `${{ env.GITHUB_COMMIT_SHA }}`" + SLACK_WEBHOOK: ${{ secrets.SLACK_ALERTS_WEBHOOK }} + MSG_MINIMAL: true + SLACK_FOOTER: "" + diff --git a/e2e/.gitignore b/e2e/.gitignore index a71674a..82f0985 100644 --- a/e2e/.gitignore +++ b/e2e/.gitignore @@ -2,3 +2,4 @@ /playwright-report/ /blob-report/ /playwright/.cache/ +/ctrf/ diff --git a/e2e/package-lock.json b/e2e/package-lock.json index f0156f5..28f216f 100644 --- a/e2e/package-lock.json +++ b/e2e/package-lock.json @@ -10,7 +10,38 @@ "license": "ISC", "devDependencies": { "@playwright/test": "^1.46.1", - "@types/node": "^22.5.0" + "@types/node": "^22.5.0", + "github-actions-ctrf": "^0.0.25", + "playwright-ctrf-json-reporter": "^0.0.15" + } + }, + "node_modules/@actions/core": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.10.1.tgz", + "integrity": "sha512-3lBR9EDAY+iYIpTnTIXmWcNbX3T2kCkAEQGIQx4NVQ0575nk2k3GRZDTPQG+vVtS2izSLmINlxXf0uLtnrTP+g==", + "dev": true, + "dependencies": { + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" + } + }, + "node_modules/@actions/http-client": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.2.3.tgz", + "integrity": "sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==", + "dev": true, + "dependencies": { + "tunnel": "^0.0.6", + "undici": "^5.25.4" + } + }, + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "dev": true, + "engines": { + "node": ">=14" } }, "node_modules/@playwright/test": { @@ -37,6 +68,77 @@ "undici-types": "~6.19.2" } }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/escalade": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", + "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/fsevents": { "version": "2.3.2", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", @@ -51,6 +153,74 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/github-actions-ctrf": { + "version": "0.0.25", + "resolved": "https://registry.npmjs.org/github-actions-ctrf/-/github-actions-ctrf-0.0.25.tgz", + "integrity": "sha512-8WGmKbRGVwD81navmBLA5/9NJEBl1oh6m6zoUAEB5ZFFwGARLNmzbh5JAQNu6VnpdqxzIvKXKfF53qrvUeC8Nw==", + "dev": true, + "dependencies": { + "@actions/core": "^1.10.1", + "handlebars": "^4.7.8", + "yargs": "^17.7.2" + }, + "bin": { + "github-actions-ctrf": "dist/index.js" + } + }, + "node_modules/handlebars": { + "version": "4.7.8", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", + "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "dev": true, + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.2", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true + }, "node_modules/playwright": { "version": "1.46.1", "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.46.1.tgz", @@ -81,11 +251,163 @@ "node": ">=18" } }, + "node_modules/playwright-ctrf-json-reporter": { + "version": "0.0.15", + "resolved": "https://registry.npmjs.org/playwright-ctrf-json-reporter/-/playwright-ctrf-json-reporter-0.0.15.tgz", + "integrity": "sha512-VA+hCH+uc+7snY2s49eKjo95bkyFX7qP5Lk3Ttp0xtYoTIBEKVa9cRv8lmy4xo10TM02lePibcFV2/Q9pURziw==", + "dev": true + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", + "dev": true, + "engines": { + "node": ">=0.6.11 <=0.7.0 || >=0.7.3" + } + }, + "node_modules/uglify-js": { + "version": "3.19.2", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.2.tgz", + "integrity": "sha512-S8KA6DDI47nQXJSi2ctQ629YzwOVs+bQML6DAtvy0wgNdpi+0ySpQK0g2pxBq2xfF2z3YCscu7NNA8nXT9PlIQ==", + "dev": true, + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/undici": { + "version": "5.28.4", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz", + "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==", + "dev": true, + "dependencies": { + "@fastify/busboy": "^2.0.0" + }, + "engines": { + "node": ">=14.0" + } + }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", "dev": true + }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true, + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "dev": true + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "engines": { + "node": ">=12" + } } } } diff --git a/e2e/package.json b/e2e/package.json index e895cf2..68f4ca9 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -12,6 +12,8 @@ "description": "", "devDependencies": { "@playwright/test": "^1.46.1", - "@types/node": "^22.5.0" + "@types/node": "^22.5.0", + "github-actions-ctrf": "^0.0.25", + "playwright-ctrf-json-reporter": "^0.0.15" } } diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index 2b55e47..9754355 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -21,7 +21,10 @@ export default defineConfig({ /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: 'html', + reporter: [ + ['html'], + ['playwright-ctrf-json-reporter', {}], + ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */