From 1831a0a959f159afe7c51243e898b8aa9105a2b3 Mon Sep 17 00:00:00 2001 From: Veaceslav <118342408+vCaisim@users.noreply.github.com> Date: Tue, 1 Oct 2024 22:00:41 +0300 Subject: [PATCH] [CSR-1547] feat: Implementation for `api` and `cache` commands (#12) * refactor: support multiple commands * feat: add cache command * feat: cache command implementation * fix: make projectId mandatory for the api command * feat: add last-run preset handler * feat: add matrixIndex and matrixTotal options * fix: show user a friendlier message when the cache for ID was not found * chore: update README files [skip ci] --------- Co-authored-by: DJ Mountney --- README.md | 26 +- package-lock.json | 592 ++++++++++++++++-- packages/cmd/README.md | 45 +- packages/cmd/package.json | 17 +- packages/cmd/src/api/cache.ts | 67 ++ .../cmd/src/api/{run.ts => create-run.ts} | 24 +- packages/cmd/src/api/get-run.ts | 41 ++ packages/cmd/src/api/index.ts | 5 +- packages/cmd/src/bin/index.ts | 39 +- packages/cmd/src/bin/program.ts | 75 +-- packages/cmd/src/commands/api/get-run.ts | 22 + packages/cmd/src/commands/api/index.ts | 58 ++ packages/cmd/src/commands/api/options.ts | 43 ++ packages/cmd/src/commands/cache/get.ts | 29 + packages/cmd/src/commands/cache/index.ts | 85 +++ packages/cmd/src/commands/cache/options.ts | 74 +++ packages/cmd/src/commands/cache/set.ts | 29 + .../{bin => commands/upload}/cli-config.ts | 14 +- packages/cmd/src/commands/upload/index.ts | 60 ++ .../src/{bin => commands/upload}/options.ts | 23 +- .../src/{bin => commands/upload}/tmp-file.ts | 0 packages/cmd/src/commands/upload/upload.ts | 24 + packages/cmd/src/commands/utils.ts | 46 ++ packages/cmd/src/config/api/config.ts | 86 +++ packages/cmd/src/config/api/env.ts | 56 ++ packages/cmd/src/config/api/index.ts | 2 + packages/cmd/src/config/cache/config.ts | 85 +++ packages/cmd/src/config/cache/env.ts | 86 +++ packages/cmd/src/config/{ => cache}/index.ts | 0 packages/cmd/src/config/cache/options.ts | 33 + .../cmd/src/config/{ => upload}/config.ts | 55 +- packages/cmd/src/config/{ => upload}/env.ts | 32 +- packages/cmd/src/config/upload/index.ts | 3 + .../cmd/src/config/{ => upload}/options.ts | 24 +- packages/cmd/src/config/utils.ts | 77 +++ packages/cmd/src/env/ciProvider.ts | 21 +- packages/cmd/src/env/types.ts | 29 + packages/cmd/src/http/axios.ts | 34 + packages/cmd/src/http/client.ts | 27 +- packages/cmd/src/http/http.ts | 7 +- packages/cmd/src/http/httpConfig.ts | 3 + packages/cmd/src/http/httpErrors.ts | 2 +- packages/cmd/src/http/httpRetry.ts | 2 +- packages/cmd/src/index.ts | 236 +------ packages/cmd/src/lib/fs.ts | 84 +-- packages/cmd/src/lib/index.ts | 2 - packages/cmd/src/logger/logger.ts | 8 +- packages/cmd/src/services/api/index.ts | 57 ++ .../src/services/cache/__tests__/fs.spec.ts | 100 +++ packages/cmd/src/services/cache/fs.ts | 102 +++ packages/cmd/src/services/cache/get.ts | 82 +++ packages/cmd/src/services/cache/index.ts | 2 + packages/cmd/src/services/cache/lib.ts | 49 ++ packages/cmd/src/services/cache/network.ts | 127 ++++ packages/cmd/src/services/cache/presets.ts | 106 ++++ packages/cmd/src/services/cache/set.ts | 131 ++++ packages/cmd/src/services/index.ts | 3 + .../upload}/discovery/createScanner.ts | 0 .../{ => services/upload}/discovery/index.ts | 0 .../upload}/discovery/jest/args/args.ts | 0 .../upload}/discovery/jest/args/config.ts | 4 +- .../upload}/discovery/jest/args/index.ts | 0 .../upload}/discovery/jest/index.ts | 0 .../upload}/discovery/jest/reporter.ts | 4 +- .../upload}/discovery/jest/scanner.ts | 6 +- .../upload}/discovery/jest/utils.ts | 7 - .../upload}/discovery/jest/utils/fs.ts | 0 .../upload}/discovery/jest/utils/index.ts | 0 .../upload}/discovery/jest/utils/test.ts | 0 .../upload}/discovery/scanner.ts | 0 .../{ => services/upload}/discovery/types.ts | 2 - packages/cmd/src/services/upload/fs.ts | 80 +++ packages/cmd/src/services/upload/index.ts | 233 +++++++ .../cmd/src/{ => services/upload}/types.ts | 0 packages/cmd/tsconfig.json | 15 +- packages/cmd/tsup.config.ts | 6 +- packages/cmd/vitest.config.ts | 14 + turbo.json | 1 + 78 files changed, 2942 insertions(+), 621 deletions(-) create mode 100644 packages/cmd/src/api/cache.ts rename packages/cmd/src/api/{run.ts => create-run.ts} (74%) create mode 100644 packages/cmd/src/api/get-run.ts create mode 100644 packages/cmd/src/commands/api/get-run.ts create mode 100644 packages/cmd/src/commands/api/index.ts create mode 100644 packages/cmd/src/commands/api/options.ts create mode 100644 packages/cmd/src/commands/cache/get.ts create mode 100644 packages/cmd/src/commands/cache/index.ts create mode 100644 packages/cmd/src/commands/cache/options.ts create mode 100644 packages/cmd/src/commands/cache/set.ts rename packages/cmd/src/{bin => commands/upload}/cli-config.ts (65%) create mode 100644 packages/cmd/src/commands/upload/index.ts rename packages/cmd/src/{bin => commands/upload}/options.ts (73%) rename packages/cmd/src/{bin => commands/upload}/tmp-file.ts (100%) create mode 100644 packages/cmd/src/commands/upload/upload.ts create mode 100644 packages/cmd/src/commands/utils.ts create mode 100644 packages/cmd/src/config/api/config.ts create mode 100644 packages/cmd/src/config/api/env.ts create mode 100644 packages/cmd/src/config/api/index.ts create mode 100644 packages/cmd/src/config/cache/config.ts create mode 100644 packages/cmd/src/config/cache/env.ts rename packages/cmd/src/config/{ => cache}/index.ts (100%) create mode 100644 packages/cmd/src/config/cache/options.ts rename packages/cmd/src/config/{ => upload}/config.ts (57%) rename packages/cmd/src/config/{ => upload}/env.ts (58%) create mode 100644 packages/cmd/src/config/upload/index.ts rename packages/cmd/src/config/{ => upload}/options.ts (57%) create mode 100644 packages/cmd/src/config/utils.ts create mode 100644 packages/cmd/src/http/axios.ts create mode 100644 packages/cmd/src/services/api/index.ts create mode 100644 packages/cmd/src/services/cache/__tests__/fs.spec.ts create mode 100644 packages/cmd/src/services/cache/fs.ts create mode 100644 packages/cmd/src/services/cache/get.ts create mode 100644 packages/cmd/src/services/cache/index.ts create mode 100644 packages/cmd/src/services/cache/lib.ts create mode 100644 packages/cmd/src/services/cache/network.ts create mode 100644 packages/cmd/src/services/cache/presets.ts create mode 100644 packages/cmd/src/services/cache/set.ts create mode 100644 packages/cmd/src/services/index.ts rename packages/cmd/src/{ => services/upload}/discovery/createScanner.ts (100%) rename packages/cmd/src/{ => services/upload}/discovery/index.ts (100%) rename packages/cmd/src/{ => services/upload}/discovery/jest/args/args.ts (100%) rename packages/cmd/src/{ => services/upload}/discovery/jest/args/config.ts (97%) rename packages/cmd/src/{ => services/upload}/discovery/jest/args/index.ts (100%) rename packages/cmd/src/{ => services/upload}/discovery/jest/index.ts (100%) rename packages/cmd/src/{ => services/upload}/discovery/jest/reporter.ts (97%) rename packages/cmd/src/{ => services/upload}/discovery/jest/scanner.ts (91%) rename packages/cmd/src/{ => services/upload}/discovery/jest/utils.ts (73%) rename packages/cmd/src/{ => services/upload}/discovery/jest/utils/fs.ts (100%) rename packages/cmd/src/{ => services/upload}/discovery/jest/utils/index.ts (100%) rename packages/cmd/src/{ => services/upload}/discovery/jest/utils/test.ts (100%) rename packages/cmd/src/{ => services/upload}/discovery/scanner.ts (100%) rename packages/cmd/src/{ => services/upload}/discovery/types.ts (99%) create mode 100644 packages/cmd/src/services/upload/fs.ts create mode 100644 packages/cmd/src/services/upload/index.ts rename packages/cmd/src/{ => services/upload}/types.ts (100%) create mode 100644 packages/cmd/vitest.config.ts diff --git a/README.md b/README.md index f869951..a412d5f 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,10 @@ A monorepo which contains the following packages: - `@currents/jest` - a jest reporter that writes the test results to json files in a Currents friendly manner -- `@currents/cmd` - exposes `currents` command which is used to discover the full test suite and upload the test results into Currents Dashboard. +- `@currents/cmd` - exposes the `currents` command which is used to interact with Currents APIs. It includes the following commands: + - `upload` command - used to discover the full test suite and upload the test results into the Currents Dashboard + - `api` command - retrieves information about Currents entities + - `cache` command - provides a convenient way to store and receive test artifacts - `examples` - a private package used to test the implementation ## Testing locally @@ -33,7 +36,7 @@ For a custom path for the report directory, set an absolute path to the `reportD ### Uploading the results -Set the `projectId`, `recordKey` and optionlly the `ciBuildId`. Run `npx currents --help` for details. +Set the `projectId`, `recordKey` and optionlly the `ciBuildId`. Run `npx currents upload --help` for details. Run `npm run report` or `CURRENTS_API_URL=http://localhost:1234 CURRENTS_PROJECT_ID=xxx CURRENTS_RECORD_KEY=yyy npx currents upload` @@ -41,6 +44,25 @@ To enable the debug mode, prefix the command with `DEBUG=currents,currents:*` or To provide a custom report dir path, use `CURRENTS_REPORT_DIR` env variable or `--report-dir` option. +### Obtaining run information + +Run `CURRENTS_REST_API_URL=http://localhost:4000 CURRENTS_PROJECT_ID=xxx npx currents api get-run --api-key --ci-build-id --output run.json` + +Run `npx currents api --help` to see all available api commands. + +To explore additional examples and filtering options for receiving runs, you can utilize the `npx currents api get-run --help` command. + + +### Caching artifacts + +The `currents cache` command allows you to archive files from specified locations and save them under an ID in Currents storage. It also stores a meta file with configuration data. You can provide the ID manually or it can be generated based on CI environment variables (only GitHub and GitLab are supported). The files to archive can be defined using the `paths ` CLI option or predefined using a `preset`. + +To cache files, run `npx currents cache set --key --id --paths `. + +To download files, run `npx currents cache set --key --preset last-run`. + +For more examples and usage options, run `npx currents cache --help`. + ## Release ```sh diff --git a/package-lock.json b/package-lock.json index 7152946..27bb214 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1955,7 +1955,6 @@ }, "node_modules/@isaacs/cliui": { "version": "8.0.2", - "dev": true, "license": "ISC", "dependencies": { "string-width": "^5.1.2", @@ -1971,7 +1970,6 @@ }, "node_modules/@isaacs/cliui/node_modules/ansi-regex": { "version": "6.0.1", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -1982,7 +1980,6 @@ }, "node_modules/@isaacs/cliui/node_modules/ansi-styles": { "version": "6.2.1", - "dev": true, "license": "MIT", "engines": { "node": ">=12" @@ -1993,12 +1990,10 @@ }, "node_modules/@isaacs/cliui/node_modules/emoji-regex": { "version": "9.2.2", - "dev": true, "license": "MIT" }, "node_modules/@isaacs/cliui/node_modules/string-width": { "version": "5.1.2", - "dev": true, "license": "MIT", "dependencies": { "eastasianwidth": "^0.2.0", @@ -2014,7 +2009,6 @@ }, "node_modules/@isaacs/cliui/node_modules/strip-ansi": { "version": "7.1.0", - "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^6.0.1" @@ -2028,7 +2022,6 @@ }, "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { "version": "8.1.0", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^6.1.0", @@ -2651,7 +2644,6 @@ }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", - "dev": true, "license": "MIT", "optional": true, "engines": { @@ -2981,6 +2973,26 @@ "dev": true, "peer": true }, + "node_modules/@types/archiver": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/@types/archiver/-/archiver-6.0.2.tgz", + "integrity": "sha512-KmROQqbQzKGuaAbmK+ZcytkJ51+YqDa7NmbXjmtC5YBLSyQYo21YaUnQ3HbaPFKL1ooo6RQ6OPYPIDyxfpDDXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/readdir-glob": "*" + } + }, + "node_modules/@types/async-retry": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/@types/async-retry/-/async-retry-1.4.8.tgz", + "integrity": "sha512-Qup/B5PWLe86yI5I3av6ePGaeQrIHNKCwbsQotD6aHQ6YkHsMUxVZkZsmx/Ry3VZQ6uysHwTjQ7666+k6UjVJA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/retry": "*" + } + }, "node_modules/@types/babel__code-frame": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/@types/babel__code-frame/-/babel__code-frame-7.0.6.tgz", @@ -3126,6 +3138,33 @@ "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", "dev": true }, + "node_modules/@types/proxy-from-env": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@types/proxy-from-env/-/proxy-from-env-1.0.4.tgz", + "integrity": "sha512-TPR9/bCZAr3V1eHN4G3LD3OLicdJjqX1QRXWuNcCYgE66f/K8jO2ZRtHxI2D9MbnuUP6+qiKSS8eUHp6TFHGCw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/readdir-glob": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@types/readdir-glob/-/readdir-glob-1.1.5.tgz", + "integrity": "sha512-raiuEPUYqXu+nvtY2Pe8s8FEmZ3x5yAH4VkLdihcPdalvsHltomrRC9BzuStrJ9yk06470hS0Crw0f1pXqD+Hg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/retry": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.5.tgz", + "integrity": "sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/semver": { "version": "7.5.8", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.8.tgz", @@ -3142,6 +3181,16 @@ "integrity": "sha512-chhaNf2oKHlRkDGt+tiKE2Z5aJ6qalm7Z9rlLdBwmOiAAf09YQvvoLXjWK4HWPF1xU/fqvMgfNfpVoBscA/tKA==", "dev": true }, + "node_modules/@types/unzipper": { + "version": "0.10.10", + "resolved": "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.10.tgz", + "integrity": "sha512-jKJdNxhmCHTZsaKW5x0qjn6rB+gHk0w5VFbEKsw84i+RJqXZyfTmGnpjDcKqzMpjz7VVLsUBMtO5T3mVidpt0g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/uuid": { "version": "9.0.8", "dev": true, @@ -3365,6 +3414,18 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/acorn": { "version": "7.4.1", "license": "MIT", @@ -3416,8 +3477,6 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", - "dev": true, - "peer": true, "dependencies": { "debug": "^4.3.4" }, @@ -3505,6 +3564,118 @@ "node": ">= 8" } }, + "node_modules/archiver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-7.0.1.tgz", + "integrity": "sha512-ZcbTaIqJOfCc03QwD468Unz/5Ir8ATtvAHsK+FdXbDIbGfihqh9mrvdcYunQzqn4HrvWWaFyaxJhGZagaJJpPQ==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.2", + "async": "^3.2.4", + "buffer-crc32": "^1.0.0", + "readable-stream": "^4.0.0", + "readdir-glob": "^1.1.2", + "tar-stream": "^3.0.0", + "zip-stream": "^6.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-5.0.2.tgz", + "integrity": "sha512-wuLJMmIBQYCsGZgYLTy5FIB2pF6Lfb6cXMSF8Qywwk3t20zWnAi7zLcQFdKQmIB8wyZpY5ER38x08GbwtR2cLA==", + "license": "MIT", + "dependencies": { + "glob": "^10.0.0", + "graceful-fs": "^4.2.0", + "is-stream": "^2.0.1", + "lazystream": "^1.0.0", + "lodash": "^4.17.15", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/archiver-utils/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/archiver-utils/node_modules/glob": { + "version": "10.4.5", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.5.tgz", + "integrity": "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==", + "license": "ISC", + "dependencies": { + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/archiver-utils/node_modules/minimatch": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/archiver-utils/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/archiver/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/argparse": { "version": "1.0.10", "license": "MIT", @@ -3704,8 +3875,7 @@ "version": "1.3.3", "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "dev": true, - "peer": true, + "license": "MIT", "dependencies": { "retry": "0.13.1" } @@ -3764,6 +3934,12 @@ "dequal": "^2.0.3" } }, + "node_modules/b4a": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", + "integrity": "sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg==", + "license": "Apache-2.0" + }, "node_modules/babel-jest": { "version": "29.7.0", "license": "MIT", @@ -3899,11 +4075,17 @@ "version": "1.0.2", "license": "MIT" }, + "node_modules/bare-events": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.4.2.tgz", + "integrity": "sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q==", + "license": "Apache-2.0", + "optional": true + }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "dev": true, "funding": [ { "type": "github", @@ -3917,8 +4099,7 @@ "type": "consulting", "url": "https://feross.org/support" } - ], - "peer": true + ] }, "node_modules/basic-ftp": { "version": "5.0.5", @@ -4194,7 +4375,6 @@ "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "dev": true, "funding": [ { "type": "github", @@ -4209,12 +4389,20 @@ "url": "https://feross.org/support" } ], - "peer": true, "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, + "node_modules/buffer-crc32": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", + "integrity": "sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==", + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "license": "MIT" @@ -4567,6 +4755,38 @@ "dot-prop": "^5.1.0" } }, + "node_modules/compress-commons": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-6.0.2.tgz", + "integrity": "sha512-6FqVXeETqWPoGcfzrXb37E50NP0LXT8kAMu5ooZayhWWdgEY4lBEEcbQNXtkuKQsGduxiIcI4gOTsxTmuq/bSg==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "crc32-stream": "^6.0.0", + "is-stream": "^2.0.1", + "normalize-path": "^3.0.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/compress-commons/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/concat-map": { "version": "0.0.1", "license": "MIT" @@ -4925,6 +5145,12 @@ "url": "https://opencollective.com/core-js" } }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "license": "MIT" + }, "node_modules/cosmiconfig": { "version": "8.3.6", "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-8.3.6.tgz", @@ -4972,6 +5198,47 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/crc-32": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/crc-32/-/crc-32-1.2.2.tgz", + "integrity": "sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ==", + "license": "Apache-2.0", + "bin": { + "crc32": "bin/crc32.njs" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/crc32-stream": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-6.0.0.tgz", + "integrity": "sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==", + "license": "MIT", + "dependencies": { + "crc-32": "^1.2.0", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/crc32-stream/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "node_modules/create-jest": { "version": "29.7.0", "license": "MIT", @@ -5503,9 +5770,53 @@ "node": ">=12" } }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha512-asLFVfWWtJ90ZyOUHMqk7/S2w2guQKxUI2itj3d92ADHhxUSbCMGi1f1cBcJ7xM1To+pE/Khbwo1yuNbMEPKeA==", + "license": "BSD-3-Clause", + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", - "dev": true, "license": "MIT" }, "node_modules/electron-to-chromium": { @@ -6292,6 +6603,24 @@ "node": ">=0.10.0" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/examples": { "resolved": "examples", "link": true @@ -6369,6 +6698,12 @@ "version": "3.1.3", "license": "MIT" }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, "node_modules/fast-glob": { "version": "3.3.2", "license": "MIT", @@ -6547,7 +6882,6 @@ }, "node_modules/foreground-child": { "version": "3.2.1", - "dev": true, "license": "ISC", "dependencies": { "cross-spawn": "^7.0.0", @@ -6562,7 +6896,6 @@ }, "node_modules/foreground-child/node_modules/signal-exit": { "version": "4.1.0", - "dev": true, "license": "ISC", "engines": { "node": ">=14" @@ -7130,8 +7463,6 @@ "version": "7.0.5", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", - "dev": true, - "peer": true, "dependencies": { "agent-base": "^7.0.2", "debug": "4" @@ -7164,7 +7495,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "dev": true, "funding": [ { "type": "github", @@ -7178,8 +7508,7 @@ "type": "consulting", "url": "https://feross.org/support" } - ], - "peer": true + ] }, "node_modules/ignore": { "version": "5.3.1", @@ -8181,7 +8510,6 @@ }, "node_modules/jackspeak": { "version": "3.4.0", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "@isaacs/cliui": "^8.0.2" @@ -8880,6 +9208,54 @@ "node": "> 0.8" } }, + "node_modules/lazystream": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.1.tgz", + "integrity": "sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==", + "license": "MIT", + "dependencies": { + "readable-stream": "^2.0.5" + }, + "engines": { + "node": ">= 0.6.3" + } + }, + "node_modules/lazystream/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/lazystream/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/lazystream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, "node_modules/leven": { "version": "3.1.0", "license": "MIT", @@ -9219,7 +9595,6 @@ }, "node_modules/minipass": { "version": "7.1.2", - "dev": true, "license": "ISC", "engines": { "node": ">=16 || 14 >=14.17" @@ -9873,6 +10248,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/package-json-from-dist": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz", + "integrity": "sha512-dATvCeZN/8wQsGywez1mzHtTlP22H8OEfPrVMLNr4/eGa+ijtLn/6M5f0dY8UKNrC2O9UCU6SSoG3qRKnt7STw==", + "license": "BlueOak-1.0.0" + }, "node_modules/package-json/node_modules/got": { "version": "12.6.1", "resolved": "https://registry.npmjs.org/got/-/got-12.6.1.tgz", @@ -9993,7 +10374,6 @@ }, "node_modules/path-scurry": { "version": "1.11.1", - "dev": true, "license": "BlueOak-1.0.0", "dependencies": { "lru-cache": "^10.2.0", @@ -10008,7 +10388,6 @@ }, "node_modules/path-scurry/node_modules/lru-cache": { "version": "10.2.2", - "dev": true, "license": "ISC", "engines": { "node": "14 || >=16.14" @@ -10204,6 +10583,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "license": "MIT", + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "license": "MIT" + }, "node_modules/progress": { "version": "2.0.3", "license": "MIT", @@ -10369,6 +10763,12 @@ ], "license": "MIT" }, + "node_modules/queue-tick": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/queue-tick/-/queue-tick-1.0.1.tgz", + "integrity": "sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag==", + "license": "MIT" + }, "node_modules/quick-lru": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", @@ -10628,6 +11028,36 @@ "node": ">= 6" } }, + "node_modules/readdir-glob": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/readdir-glob/-/readdir-glob-1.1.3.tgz", + "integrity": "sha512-v05I2k7xN8zXvPD9N+z/uhXPaj0sUFCe2rcWZIpBsqxfP7xXFQ0tipAd/wjj1YxWyWtUS5IDJpOG82JKt2EAVA==", + "license": "Apache-2.0", + "dependencies": { + "minimatch": "^5.1.0" + } + }, + "node_modules/readdir-glob/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "node_modules/readdir-glob/node_modules/minimatch": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", + "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/readdirp": { "version": "3.6.0", "dev": true, @@ -11124,8 +11554,6 @@ "version": "0.13.1", "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "dev": true, - "peer": true, "engines": { "node": ">= 4" } @@ -11329,7 +11757,6 @@ "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, "funding": [ { "type": "github", @@ -11679,11 +12106,24 @@ "node": ">= 0.4" } }, + "node_modules/streamx": { + "version": "2.20.1", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.20.1.tgz", + "integrity": "sha512-uTa0mU6WUC65iUvzKH4X9hEdvSW7rbPxPtwfWiLMSj3qTdQbAiUboZTxauKfpFuGIGa1C2BYijZ7wgdUXICJhA==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "queue-tick": "^1.0.1", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, "dependencies": { "safe-buffer": "~5.2.0" } @@ -11714,7 +12154,6 @@ "node_modules/string-width-cjs": { "name": "string-width", "version": "4.2.3", - "dev": true, "license": "MIT", "dependencies": { "emoji-regex": "^8.0.0", @@ -11805,7 +12244,6 @@ "node_modules/strip-ansi-cjs": { "name": "strip-ansi", "version": "6.0.1", - "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" @@ -11980,6 +12418,17 @@ "version": "1.0.0", "license": "MIT" }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, "node_modules/test-exclude": { "version": "6.0.0", "license": "ISC", @@ -11992,6 +12441,15 @@ "node": ">=8" } }, + "node_modules/text-decoder": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.0.tgz", + "integrity": "sha512-n1yg1mOj9DNpk3NeZOx7T6jchTbyJS3i3cucbNN6FcdPriMZx7NsgrGpWWdWZZGxD7ES1XB+3uoqHMgOKaN+fg==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, "node_modules/text-extensions": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-2.4.0.tgz", @@ -12495,6 +12953,25 @@ "node": ">=8" } }, + "node_modules/unzipper": { + "version": "0.12.3", + "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.12.3.tgz", + "integrity": "sha512-PZ8hTS+AqcGxsaQntl3IRBw65QrBI6lxzqDEL7IAo/XCEqRTKGfOX56Vea5TH9SZczRVxuzk1re04z/YjuYCJA==", + "license": "MIT", + "dependencies": { + "bluebird": "~3.7.2", + "duplexer2": "~0.1.4", + "fs-extra": "^11.2.0", + "graceful-fs": "^4.2.2", + "node-int64": "^0.4.0" + } + }, + "node_modules/unzipper/node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "license": "MIT" + }, "node_modules/update-browserslist-db": { "version": "1.0.16", "funding": [ @@ -12598,8 +13075,7 @@ "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", - "dev": true + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, "node_modules/uuid": { "version": "10.0.0", @@ -13180,7 +13656,6 @@ "node_modules/wrap-ansi-cjs": { "name": "wrap-ansi", "version": "7.0.0", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.0.0", @@ -13277,6 +13752,36 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zip-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-6.0.1.tgz", + "integrity": "sha512-zK7YHHz4ZXpW89AHXUPbQVGKI7uvkd3hzusTdotCg1UxyaVtg0zFJSTfW/Dq5f7OBBVnq6cZIaC8Ti4hb6dtCA==", + "license": "MIT", + "dependencies": { + "archiver-utils": "^5.0.0", + "compress-commons": "^6.0.2", + "readable-stream": "^4.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/zip-stream/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "license": "MIT", + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, "packages/cli": { "name": "@currents/cmd", "version": "0.0.0", @@ -13333,6 +13838,8 @@ "dependencies": { "@commander-js/extra-typings": "^11.1.0", "@currents/commit-info": "1.0.1-beta.0", + "archiver": "^7.0.1", + "async-retry": "^1.3.3", "axios": "^1.7.2", "axios-retry": "^4.4.0", "commander": "^11.1.0", @@ -13341,16 +13848,19 @@ "execa": "^8.0.1", "fs-extra": "^11.2.0", "getos": "^3.2.1", + "https-proxy-agent": "^7.0.4", "jest-cli": "^29.7.0", "jest-config": "^29.7.0", "lodash": "^4.17.21", "nanoid": "^3.3.4", "pretty-ms": "^7.0.1", + "proxy-from-env": "^1.1.0", "semver": "^7.6.0", "source-map-support": "^0.5.21", "tmp": "^0.2.3", "tmp-promise": "^3.0.3", - "ts-pattern": "^5.2.0" + "ts-pattern": "^5.2.0", + "unzipper": "^0.12.3" }, "bin": { "currents": "dist/bin/index.js", @@ -13360,13 +13870,17 @@ "@jest/reporters": "^29.7.0", "@jest/types": "^29.6.3", "@release-it/conventional-changelog": "^7.0.2", + "@types/archiver": "^6.0.2", + "@types/async-retry": "^1.4.8", "@types/debug": "^4.1.12", "@types/fs-extra": "^11.0.4", "@types/getos": "^3.0.4", "@types/lodash": "^4.17.5", "@types/node": "^20.14.2", + "@types/proxy-from-env": "^1.0.4", "@types/semver": "^7.5.8", "@types/tmp": "^0.2.6", + "@types/unzipper": "^0.10.10", "@types/uuid": "^9.0.8", "jest": "^29.7.0", "rimraf": "^5.0.7", diff --git a/packages/cmd/README.md b/packages/cmd/README.md index 69c54b7..31e7d8c 100644 --- a/packages/cmd/README.md +++ b/packages/cmd/README.md @@ -8,7 +8,7 @@ CLI tool for [Currents](https://currents.dev) - a cloud platform for debugging, npm install @currents/cmd --save-dev ``` -## Usage +## Upload test results - Generate test results with one of the [supported](https://docs.currents.dev) reporters - Upload the results @@ -19,7 +19,7 @@ npx currents upload --project-id=xxx --key=yyy ℹī¸ Get familiar with [CI Build ID 📖](https://docs.currents.dev/guides/ci-build-id) before using `currents` in CI. It is **important** to set the `CI Build ID` explicitly using `--ci-build-id` option, if you are using CI sharding or multiple CI machines to parallelize your tests. If not set explicitly, the `CI Build ID` will be set to a random value. -## Notes +### Notes Obtain `--project-id` and [--key](https://docs.currents.dev/guides/record-key) from https://app.currents.dev to identify the project and associate the results with your organization. @@ -28,7 +28,7 @@ Obtain `--project-id` and [--key](https://docs.currents.dev/guides/record-key) f - use `process.env.CURRENTS_REPORT_DIR` or `--reportDir` to read the results previously generated by one of supported reporters, otherwise - use the most recently created directory named as `.currents/[timestamp]-[uuidv4()]` in the current working directory -## Configuration +### Configuration Please note that all options apart from `--project-id` and `--key` are optional. @@ -46,6 +46,45 @@ Please note that all options apart from `--project-id` and `--key` are optional. The configuration is also available by running the CLI command with the `--help` argument. +## Use Currents API + +- Retrieve information about Currents resources in [JSON](https://docs.currents.dev/resources/api/api-resources) format + +ℹī¸ Please note that the command is experimental and was primarily built to obtain test run data in CI. Its functionality might be extended in the future. + +ℹī¸ The command requires the `--project-id` and [`--api-key`](https://docs.currents.dev/resources/api/api-keys#managing-the-api-keys) from [Currents](https://app.currents.dev) to authenticate the request and provide the required data. Alternatively, you can set the `CURRENTS_PROJECT_ID` and `CURRENTS_API_KEY` environment variables. + +**Supported operations:** + +- Obtain run data using the [CI Build ID 📖](https://docs.currents.dev/guides/ci-build-id) by running the following command: + + ```sh + npx currents api get-run --ci-build-id --output path/to/save/run.json + ``` + +- Obtain the last completed run data, filtered by `--branch` and/or `--tag`, by running the following command: + ```sh + npx currents api get-run --branch --tag + ``` + +For more examples and usage options, run `npx currents api --help`. + +## Cache test artifacts + +The `currents cache` command allows you to archive files from specified locations and save them under an ID in Currents storage. It also stores a meta file with configuration data. You can provide the ID manually or it can be generated based on CI environment variables (only GitHub and GitLab are supported). + +To cache files, use the following command: +```sh +npx currents cache set --key --id --paths +``` + +To download files, use the following command: +```sh +npx currents cache set --key --preset last-run +``` + +For more examples and usage options, run `npx currents cache --help`. + ## Troubleshooting Run the CLI command with the `--debug` argument or prefix it with `DEBUG="currents,currents:*"` to obtain detailed information about the command execution process. diff --git a/packages/cmd/package.json b/packages/cmd/package.json index c695871..faeed40 100644 --- a/packages/cmd/package.json +++ b/packages/cmd/package.json @@ -18,13 +18,17 @@ "@jest/reporters": "^29.7.0", "@jest/types": "^29.6.3", "@release-it/conventional-changelog": "^7.0.2", + "@types/archiver": "^6.0.2", + "@types/async-retry": "^1.4.8", "@types/debug": "^4.1.12", "@types/fs-extra": "^11.0.4", "@types/getos": "^3.0.4", "@types/lodash": "^4.17.5", "@types/node": "^20.14.2", + "@types/proxy-from-env": "^1.0.4", "@types/semver": "^7.5.8", "@types/tmp": "^0.2.6", + "@types/unzipper": "^0.10.10", "@types/uuid": "^9.0.8", "jest": "^29.7.0", "rimraf": "^5.0.7", @@ -35,6 +39,8 @@ "dependencies": { "@commander-js/extra-typings": "^11.1.0", "@currents/commit-info": "1.0.1-beta.0", + "archiver": "^7.0.1", + "async-retry": "^1.3.3", "axios": "^1.7.2", "axios-retry": "^4.4.0", "commander": "^11.1.0", @@ -45,14 +51,17 @@ "getos": "^3.2.1", "jest-cli": "^29.7.0", "jest-config": "^29.7.0", + "https-proxy-agent": "^7.0.4", "lodash": "^4.17.21", "nanoid": "^3.3.4", "pretty-ms": "^7.0.1", + "proxy-from-env": "^1.1.0", "semver": "^7.6.0", "source-map-support": "^0.5.21", "tmp": "^0.2.3", "tmp-promise": "^3.0.3", - "ts-pattern": "^5.2.0" + "ts-pattern": "^5.2.0", + "unzipper": "^0.12.3" }, "bin": { "currents": "./dist/bin/index.js", @@ -78,9 +87,9 @@ "types": "./dist/index.d.ts" }, "./discovery/jest": { - "import": "./dist/discovery/jest/reporter.js", - "require": "./dist/discovery/jest/reporter.js", - "types": "./dist/discovery/jest/reporter.d.ts" + "import": "./dist/services/upload/discovery/jest/reporter.js", + "require": "./dist/services/upload/discovery/jest/reporter.js", + "types": "./dist/services/upload/discovery/jest/reporter.d.ts" }, "./package.json": "./package.json" }, diff --git a/packages/cmd/src/api/cache.ts b/packages/cmd/src/api/cache.ts new file mode 100644 index 0000000..1107583 --- /dev/null +++ b/packages/cmd/src/api/cache.ts @@ -0,0 +1,67 @@ +import { debug as _debug } from "../debug"; +import { makeRequest } from "../http"; +import { ClientType } from "../http/client"; + +const debug = _debug.extend("api"); + +export type CacheRequestConfigParams = { + matrixIndex?: number; + matrixTotal?: number; +}; + +export type CacheRequestParams = { + recordKey: string; + ci: Record; + id?: string; + config?: CacheRequestConfigParams; +}; + +export type CacheCreationResponse = { + cacheId: string; + orgId: string; + uploadUrl: string; + metaUploadUrl: string; +}; + +export type CacheRetrievalResponse = { + cacheId: string; + orgId: string; + readUrl: string; + metaReadUrl: string; +}; + +export async function createCache(params: CacheRequestParams) { + try { + debug("Request params: %o", params); + + return makeRequest( + ClientType.API, + { + url: "cache/upload", + method: "POST", + data: params, + } + ).then((res) => res.data); + } catch (err) { + debug("Failed to create cache:", err); + throw err; + } +} + +export async function retrieveCache(params: CacheRequestParams) { + try { + debug("Request params: %o", params); + + return makeRequest( + ClientType.API, + { + url: "cache/download", + method: "POST", + data: params, + } + ).then((res) => res.data); + } catch (err) { + debug("Failed to retrieve cache:", err); + throw err; + } +} diff --git a/packages/cmd/src/api/run.ts b/packages/cmd/src/api/create-run.ts similarity index 74% rename from packages/cmd/src/api/run.ts rename to packages/cmd/src/api/create-run.ts index 784027f..81f40be 100644 --- a/packages/cmd/src/api/run.ts +++ b/packages/cmd/src/api/create-run.ts @@ -1,15 +1,16 @@ +import { Commit } from "@env/gitInfo"; +import { CiProvider, CiProviderData } from "@env/types"; +import { error } from "@logger"; import { promisify } from "node:util"; import { gzip } from "node:zlib"; -import { CurrentsConfig } from "../config"; +import { CurrentsConfig } from "../config/upload"; import { debug as _debug } from "../debug"; -import { FullTestSuite } from "../discovery"; -import { Commit } from "../env/gitInfo"; -import { CiProvider, CiProviderData } from "../env/types"; import { makeRequest } from "../http"; -import { InstanceReport } from "../types"; -import { error } from "../logger"; +import { FullTestSuite } from "../services/upload/discovery"; +import { InstanceReport } from "../services/upload/types"; +import { ClientType } from "../http/client"; -const debug = _debug.extend("run"); +const debug = _debug.extend("api"); const gzipPromise = promisify(gzip); export type Platform = { @@ -43,11 +44,12 @@ export type CI = { provider: CiProvider; }; -export type RunParams = { +export type CreateRunParams = { ciBuildId?: string; group: string; projectId: string; platform: Platform; + recordKey: string; machineId: string; framework: Framework; commit: Commit; @@ -58,7 +60,7 @@ export type RunParams = { config: RunCreationConfig; }; -export type RunResponse = { +export type CreateRunResponse = { runId: string; groupId: string; machineId: string; @@ -70,12 +72,12 @@ export type RunResponse = { warnings: any[]; }; -export async function createRun(params: RunParams) { +export async function createRun(params: CreateRunParams) { try { debug("Run params: %o", params); const data = await compressData(JSON.stringify(params)); - return makeRequest({ + return makeRequest(ClientType.API, { url: `v1/runs`, method: "POST", headers: { diff --git a/packages/cmd/src/api/get-run.ts b/packages/cmd/src/api/get-run.ts new file mode 100644 index 0000000..78447e2 --- /dev/null +++ b/packages/cmd/src/api/get-run.ts @@ -0,0 +1,41 @@ +import { debug as _debug } from "../debug"; +import { makeRequest } from "../http"; +import { ClientType } from "../http/client"; + +const debug = _debug.extend("api"); + +export type GetRunParams = { + projectId: string; + ciBuildId?: string; + branch?: string; + tag?: string[]; + pwLastRun?: boolean; +}; + +export type GetRunResponse< + T = { + pwLastRun?: unknown; + }, +> = { + data: T; + status: "OK"; +}; + +export async function getRun(apiKey: string, params: GetRunParams) { + try { + debug("Run params: %o", params); + + return makeRequest(ClientType.REST_API, { + url: `v1/runs/previous`, + params, + method: "GET", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${apiKey}`, + }, + }).then((res) => res.data); + } catch (err) { + debug("Failed to obtain run data:", err); + throw err; + } +} diff --git a/packages/cmd/src/api/index.ts b/packages/cmd/src/api/index.ts index 11d8ba3..f2414a5 100644 --- a/packages/cmd/src/api/index.ts +++ b/packages/cmd/src/api/index.ts @@ -1 +1,4 @@ -export * from "./run"; +export * from "./cache"; +export * from "./create-run"; +export * from "./get-run"; + diff --git a/packages/cmd/src/bin/index.ts b/packages/cmd/src/bin/index.ts index 53ae122..039aada 100644 --- a/packages/cmd/src/bin/index.ts +++ b/packages/cmd/src/bin/index.ts @@ -4,41 +4,10 @@ import "source-map-support/register"; import("dotenv/config"); -import { CommanderError } from "commander"; -import { getCurrentsConfig, setCurrentsConfig } from "../config"; -import { currentsReporter } from "../index"; -import { ValidationError } from "../lib"; -import { error, info, success } from "../logger"; -import { CLIManager } from "./cli-config"; +import { getProgram } from "./program"; -async function runScript() { - const cliManager = new CLIManager(); - setCurrentsConfig(cliManager.parsedConfig); - const config = getCurrentsConfig(); - - info("Currents config: %o", { - ...config, - recordKey: config?.recordKey ? "*****" : undefined, - }); - return currentsReporter(); +function runScript() { + getProgram().parse(); } -runScript() - .then(() => { - success("Script execution finished"); - process.exit(0); - }) - .catch((e) => { - if (e instanceof CommanderError) { - error(e.message); - process.exit(e.exitCode); - } - - if (e instanceof ValidationError) { - error(e.message); - process.exit(1); - } - - error("Script execution failed:", e); - process.exit(1); - }); +runScript(); diff --git a/packages/cmd/src/bin/program.ts b/packages/cmd/src/bin/program.ts index 8f9e52e..b35233a 100644 --- a/packages/cmd/src/bin/program.ts +++ b/packages/cmd/src/bin/program.ts @@ -1,70 +1,21 @@ import { Command } from "@commander-js/extra-typings"; -import chalk from "chalk"; +import { reporterVersion } from "@env/versions"; +import { getApiCommand } from "../commands/api"; +import { getCacheCommand } from "../commands/cache"; +import { getUploadCommand } from "../commands/upload"; -import { reporterVersion } from "../env/versions"; -import { dim } from "../logger"; -import { - ciBuildIdOption, - debugOption, - disableTitleTagsOption, - machineIdOption, - projectOption, - recordKeyOption, - removeTagOption, - reportDirOption, - tagOption, -} from "./options"; - -type CurrentsReporterCommand = Partial< - ReturnType["opts"]> ->; - -const NAME = "currents"; -export const getProgram = ( - command: Command<[], CurrentsReporterCommand> = getCurrentsReporterCommand() -) => command.version(reporterVersion); - -const currentsReporterExample = ` +const example = ` ---------------------------------------------------- 📖 Documentation: https://docs.currents.dev 🤙 Support: support@currents.dev ---------------------------------------------------- - -${chalk.bold("Examples")} - -Upload test results to Currents: -${dim(`${NAME} upload --key --project-id --ci-build-id `)} - -Upload test results to Currents, add tags "tagA", "tagB" to the recorded run: -${dim( - `${NAME} upload --key --project-id --ci-build-id --tag tagA --tag tagB` -)} - -Provide a custom path to the reports directory: -${dim( - `${NAME} upload --key --project-id --ci-build-id --report-dir ` -)} `; -export const getCurrentsReporterCommand = () => { - const command = new Command() - .name(NAME) - .command("upload") - .showHelpAfterError("(add --help for additional information)") - .allowUnknownOption() - .description( - `Upload test results generated by Currents reporters to https://currents.dev -${currentsReporterExample}` - ) - .addOption(ciBuildIdOption) - .addOption(recordKeyOption) - .addOption(projectOption) - .addOption(tagOption) - .addOption(removeTagOption) - .addOption(disableTitleTagsOption) - .addOption(machineIdOption) - .addOption(debugOption) - .addOption(reportDirOption); - - return command; -}; +const NAME = "currents"; +export const getProgram = () => + new Command(NAME) + .version(reporterVersion) + .description(`Currents CLI ${example}`) + .addCommand(getUploadCommand(NAME), { isDefault: true }) + .addCommand(getCacheCommand(NAME)) + .addCommand(getApiCommand(NAME)); diff --git a/packages/cmd/src/commands/api/get-run.ts b/packages/cmd/src/commands/api/get-run.ts new file mode 100644 index 0000000..452bb19 --- /dev/null +++ b/packages/cmd/src/commands/api/get-run.ts @@ -0,0 +1,22 @@ +import { debug as _debug } from "@debug"; +import { getRunCommand } from "."; +import { + getAPIGetRunCommandConfig, + setAPIGetRunCommandConfig, +} from "../../config/api"; +import { handleGetRun } from "../../services"; +import { commandHandler } from "../utils"; + +const debug = _debug.extend("cli"); + +export async function getRunHandler( + options: ReturnType["opts"]> +) { + await commandHandler(async (opts) => { + setAPIGetRunCommandConfig(opts); + const config = getAPIGetRunCommandConfig(); + + debug("Config: %o", config); + await handleGetRun(); + }, options); +} diff --git a/packages/cmd/src/commands/api/index.ts b/packages/cmd/src/commands/api/index.ts new file mode 100644 index 0000000..94bd5ec --- /dev/null +++ b/packages/cmd/src/commands/api/index.ts @@ -0,0 +1,58 @@ +import { Command } from "@commander-js/extra-typings"; +import { dim } from "@logger"; +import chalk from "chalk"; +import { getRunHandler } from "./get-run"; +import { + apiKeyOption, + branchOption, + ciBuildIdOption, + debugOption, + outputOption, + projectOption, + pwLastRunOption, + tagOption, +} from "./options"; + +const COMMAND_NAME = "api"; +const getExample = (name: string) => ` + +${chalk.bold("Examples")} + +Obtain run data by --ci-build-id: +${dim(`${name} ${COMMAND_NAME} get-run --api-key --ci-build-id `)} + +Obtain most recent run data by filters: +${dim(`${name} ${COMMAND_NAME} get-run --api-key --project-id --branch --tag tagA,tagB`)} + +Obtain last run data that matches the Playwright "test-results/.last-run.json": +${dim(`${name} ${COMMAND_NAME} get-run --api-key --ci-build-id --pw-last-run --output `)} + +`; + +export const getApiCommand = (name: string) => { + const command = new Command() + .command(COMMAND_NAME) + .showHelpAfterError("(add --help for additional information)") + .allowUnknownOption() + .addCommand(getRunCommand(name)); + + return command; +}; + +export const getRunCommand = (name: string) => { + const command = new Command() + .name("get-run") + .description(`Receive information from Currents API ${getExample(name)}`) + .allowUnknownOption() + .addOption(apiKeyOption) + .addOption(debugOption) + .addOption(ciBuildIdOption) + .addOption(projectOption) + .addOption(branchOption) + .addOption(tagOption) + .addOption(outputOption) + .addOption(pwLastRunOption) + .action(getRunHandler); + + return command; +}; diff --git a/packages/cmd/src/commands/api/options.ts b/packages/cmd/src/commands/api/options.ts new file mode 100644 index 0000000..6997a10 --- /dev/null +++ b/packages/cmd/src/commands/api/options.ts @@ -0,0 +1,43 @@ +import { Option } from "@commander-js/extra-typings"; +import { configKeys } from "../../config/api"; +import { getEnvironmentVariableName } from "../../config/utils"; +import { parseCommaSeparatedList } from "../utils"; + +export const apiKeyOption = new Option( + "--api-key ", + "API key from Currents dashboard for authentication" +).env(getEnvironmentVariableName(configKeys, "apiKey")); + +export const outputOption = new Option( + "-o, --output ", + "Path to the file where output will be written" +).env(getEnvironmentVariableName(configKeys, "output")); + +export const ciBuildIdOption = new Option( + "--ci-build-id ", + "Unique identifier for the run" +); + +export const projectOption = new Option( + "-p, --project-id ", + "Project ID from Currents associated with the run" +).env(getEnvironmentVariableName(configKeys, "projectId")); + +export const tagOption = new Option( + "-t, --tag ", + "Comma-separated list of tags for the run" +).argParser(parseCommaSeparatedList); + +export const debugOption = new Option("--debug", "Enable debug logging") + .env(getEnvironmentVariableName(configKeys, "debug")) + .default(false); + +export const branchOption = new Option( + "-b, --branch ", + "Branch name for the recorded run" +); + +export const pwLastRunOption = new Option( + "--pw-last-run", + 'Generate output formatted for Playwright ".last-run.json"' +); diff --git a/packages/cmd/src/commands/cache/get.ts b/packages/cmd/src/commands/cache/get.ts new file mode 100644 index 0000000..427d8fd --- /dev/null +++ b/packages/cmd/src/commands/cache/get.ts @@ -0,0 +1,29 @@ +import { debug as _debug } from "@debug"; +import { getCacheGetCommand } from "."; +import { + cacheGetCommandOptsToConfig, + getCacheCommandConfig, + setCacheGetCommandConfig, +} from "../../config/cache"; +import { handleGetCache } from "../../services"; +import { commandHandler } from "../utils"; + +const debug = _debug.extend("cli"); + +export type CacheGetCommandOpts = ReturnType< + ReturnType["opts"] +>; + +export async function getCacheGetHandler(options: CacheGetCommandOpts) { + await commandHandler(async (opts) => { + setCacheGetCommandConfig(cacheGetCommandOptsToConfig(opts)); + const config = getCacheCommandConfig(); + + debug("Config: %o", { + ...config.values, + recordKey: config.values?.recordKey ? "*****" : undefined, + }); + + await handleGetCache(); + }, options); +} diff --git a/packages/cmd/src/commands/cache/index.ts b/packages/cmd/src/commands/cache/index.ts new file mode 100644 index 0000000..51e1044 --- /dev/null +++ b/packages/cmd/src/commands/cache/index.ts @@ -0,0 +1,85 @@ +import { Command } from "@commander-js/extra-typings"; +import { dim } from "@logger"; +import chalk from "chalk"; +import { getCacheGetHandler } from "./get"; +import { + debugOption, + idOption, + matrixIndexOption, + matrixTotalOption, + outputDirOption, + pathsOption, + presetOption, + presetOutputOption, + pwOutputDirOption, + recordKeyOption, +} from "./options"; +import { getCacheSetHandler } from "./set"; + +const COMMAND_NAME = "cache"; +const getExample = (name: string) => ` + +${chalk.bold("Examples")} + +Save files to the cache under a specific ID: +${dim(`${name} ${COMMAND_NAME} set --key --id --paths `)} + +Retrieve files from the cache saved under a specific ID: +${dim(`${name} ${COMMAND_NAME} get --key --id `)} + +Store the last run data in the cache: +${dim(`${name} ${COMMAND_NAME} set --key --preset last-run`)} + +Retrieve the last run data from the cache: +${dim(`${name} ${COMMAND_NAME} get --key --preset last-run`)} + +Retrieve the last run data from the cache and save it to a custom directory: +${dim(`${name} ${COMMAND_NAME} get --key --preset last-run --output-dir `)} + +`; + +export const getCacheCommand = (name: string) => { + const command = new Command() + .command(COMMAND_NAME) + .description(`Cache data to Currents ${getExample(name)}`) + .showHelpAfterError("(add --help for additional information)") + .allowUnknownOption() + .addCommand(getCacheSetCommand()) + .addCommand(getCacheGetCommand()); + + return command; +}; + +export const getCacheSetCommand = () => { + const command = new Command() + .name("set") + .allowUnknownOption() + .addOption(recordKeyOption) + .addOption(idOption) + .addOption(presetOption) + .addOption(pathsOption) + .addOption(debugOption) + .addOption(pwOutputDirOption) + .addOption(matrixIndexOption) + .addOption(matrixTotalOption) + .action(getCacheSetHandler); + + return command; +}; + +export const getCacheGetCommand = () => { + const command = new Command() + .name("get") + .allowUnknownOption() + .addOption(recordKeyOption) + .addOption(idOption) + .addOption(presetOption) + .addOption(outputDirOption) + .addOption(presetOutputOption) + .addOption(debugOption) + .addOption(matrixIndexOption) + .addOption(matrixTotalOption) + .action(getCacheGetHandler); + + return command; +}; diff --git a/packages/cmd/src/commands/cache/options.ts b/packages/cmd/src/commands/cache/options.ts new file mode 100644 index 0000000..5cbe955 --- /dev/null +++ b/packages/cmd/src/commands/cache/options.ts @@ -0,0 +1,74 @@ +import { InvalidArgumentError, Option } from "@commander-js/extra-typings"; +import { configKeys } from "../../config/cache"; +import { getEnvironmentVariableName } from "../../config/utils"; +import { parseCommaSeparatedList } from "../utils"; + +export const recordKeyOption = new Option( + "-k, --key ", + "Your secret Record Key obtained from Currents" +).env(getEnvironmentVariableName(configKeys, "recordKey")); + +export const debugOption = new Option("--debug", "Enable debug logging") + .env(getEnvironmentVariableName(configKeys, "debug")) + .default(false); + +export const idOption = new Option( + "--id ", + "The ID the data is saved under in the cache" +); + +export const pathsOption = new Option( + "--paths ", + "Comma-separated list of paths to cache" +).argParser(parseCommaSeparatedList); + +export enum PRESETS { + lastRun = "last-run", +} + +export const presetOption = new Option( + "--preset ", + 'A set of predefined options. Use "last-run" to get the last run data' +).choices(Object.values(PRESETS)); + +export const outputDirOption = new Option( + "--output-dir ", + "Path to the directory where output will be written" +); + +export const pwOutputDirOption = new Option( + "--pw-output-dir ", + "Directory for artifacts produced by Playwright tests" +).default("test-results"); + +export const PRESET_OUTPUT_PATH = ".currents_env"; +export const presetOutputOption = new Option( + "--preset-output ", + "Path to the file containing the preset output" +).default(PRESET_OUTPUT_PATH); + +export const matrixIndexOption = new Option( + "--matrix-index ", + "The index of the matrix to use" +) + .default(1) + .argParser(validatePositiveInteger); + +export const matrixTotalOption = new Option( + "--matrix-total ", + "The total number of matrices available" +) + .default(1) + .argParser(validatePositiveInteger); + +function validatePositiveInteger(value: string) { + const parsedValue = parseInt(value, 10); + if ( + isNaN(parsedValue) || + parsedValue <= 0 || + !Number.isInteger(parsedValue) + ) { + throw new InvalidArgumentError("A positive integer is expected."); + } + return parsedValue; +} diff --git a/packages/cmd/src/commands/cache/set.ts b/packages/cmd/src/commands/cache/set.ts new file mode 100644 index 0000000..749f0af --- /dev/null +++ b/packages/cmd/src/commands/cache/set.ts @@ -0,0 +1,29 @@ +import { debug as _debug } from "@debug"; +import { getCacheSetCommand } from "."; +import { + cacheSetCommandOptsToConfig, + getCacheCommandConfig, + setCacheSetCommandConfig, +} from "../../config/cache"; +import { handleSetCache } from "../../services"; +import { commandHandler } from "../utils"; + +const debug = _debug.extend("cli"); + +export type CacheSetCommandOpts = ReturnType< + ReturnType["opts"] +>; + +export async function getCacheSetHandler(options: CacheSetCommandOpts) { + await commandHandler(async (opts) => { + setCacheSetCommandConfig(cacheSetCommandOptsToConfig(opts)); + const config = getCacheCommandConfig(); + + debug("Config: %o", { + ...config.values, + recordKey: config.values?.recordKey ? "*****" : undefined, + }); + + await handleSetCache(); + }, options); +} diff --git a/packages/cmd/src/bin/cli-config.ts b/packages/cmd/src/commands/upload/cli-config.ts similarity index 65% rename from packages/cmd/src/bin/cli-config.ts rename to packages/cmd/src/commands/upload/cli-config.ts index cbc80a0..5b253ea 100644 --- a/packages/cmd/src/bin/cli-config.ts +++ b/packages/cmd/src/commands/upload/cli-config.ts @@ -1,23 +1,17 @@ +import { debug as _debug } from "@debug"; import fs from "fs"; -import { CLIOptions, CurrentsConfig, cliOptionsToConfig } from "../config"; -import { debug as _debug } from "../debug"; -import { getProgram, getCurrentsReporterCommand } from "./program"; +import { CLIOptions, cliOptionsToConfig, CurrentsConfig } from "../../config/upload"; import { createTempFile } from "./tmp-file"; const debug = _debug.extend("cli"); -export type CurrentsProgram = ReturnType; - export class CLIManager { - program: CurrentsProgram; cliOptions: CLIOptions; parsedConfig: Partial; configFilePath: string | null = null; - constructor() { - this.program = getProgram(getCurrentsReporterCommand()); - - this.cliOptions = this.program.parse().opts(); + constructor(opts: CLIOptions) { + this.cliOptions = opts; debug("CLI options: %o", this.cliOptions); this.parsedConfig = cliOptionsToConfig(this.cliOptions); diff --git a/packages/cmd/src/commands/upload/index.ts b/packages/cmd/src/commands/upload/index.ts new file mode 100644 index 0000000..bbaf36b --- /dev/null +++ b/packages/cmd/src/commands/upload/index.ts @@ -0,0 +1,60 @@ +import { Command } from "@commander-js/extra-typings"; +import chalk from "chalk"; +import { dim } from "@logger"; +import { + ciBuildIdOption, + debugOption, + disableTitleTagsOption, + machineIdOption, + projectOption, + recordKeyOption, + removeTagOption, + reportDirOption, + tagOption, +} from "./options"; +import { uploadHandler } from "./upload"; + +const COMMAND_NAME = "upload"; + +const getExample = (name: string) => ` + +${chalk.bold("Examples")} + +Upload test results to Currents: +${dim(`${name} ${COMMAND_NAME} --key --project-id --ci-build-id `)} + +Upload test results to Currents, add tags "tagA", "tagB" to the recorded run: +${dim( + `${name} ${COMMAND_NAME} --key --project-id --ci-build-id --tag tagA --tag tagB` +)} + +Provide a custom path to the reports directory: +${dim( + `${name} ${COMMAND_NAME} --key --project-id --ci-build-id --report-dir ` +)} + +`; + +export const getUploadCommand = (name: string) => { + const command = new Command() + .name(COMMAND_NAME) + .command(COMMAND_NAME) + .showHelpAfterError("(add --help for additional information)") + .allowUnknownOption() + .description( + `Upload test results generated by Currents reporters to https://currents.dev +${getExample(name)}` + ) + .addOption(ciBuildIdOption) + .addOption(recordKeyOption) + .addOption(projectOption) + .addOption(tagOption) + .addOption(removeTagOption) + .addOption(disableTitleTagsOption) + .addOption(machineIdOption) + .addOption(debugOption) + .addOption(reportDirOption) + .action((options) => uploadHandler(options)); + + return command; +}; diff --git a/packages/cmd/src/bin/options.ts b/packages/cmd/src/commands/upload/options.ts similarity index 73% rename from packages/cmd/src/bin/options.ts rename to packages/cmd/src/commands/upload/options.ts index ce7b7c2..a9fb3d4 100644 --- a/packages/cmd/src/bin/options.ts +++ b/packages/cmd/src/commands/upload/options.ts @@ -1,21 +1,23 @@ import { Option } from "@commander-js/extra-typings"; -import { getEnvironmentVariableName } from "../config"; +import { configKeys } from "../../config/upload"; +import { getEnvironmentVariableName } from "../../config/utils"; +import { parseCommaSeparatedList } from "../utils"; export const ciBuildIdOption = new Option( "--ci-build-id ", "the unique identifier for the recorded build (run)" -).env(getEnvironmentVariableName("ciBuildId")); +).env(getEnvironmentVariableName(configKeys, "ciBuildId")); export const recordKeyOption = new Option( "-k, --key ", "your secret Record Key obtained from Currents" -).env(getEnvironmentVariableName("recordKey")); +).env(getEnvironmentVariableName(configKeys, "recordKey")); export const projectOption = new Option( "-p, --project-id ", "the project ID for results reporting obtained from Currents" -).env(getEnvironmentVariableName("projectId")); +).env(getEnvironmentVariableName(configKeys, "projectId")); export const tagOption = new Option( "-t, --tag ", @@ -35,20 +37,13 @@ export const disableTitleTagsOption = new Option( export const machineIdOption = new Option( "--machine-id ", "unique identifier of the machine running the tests. If not provided, it will be generated automatically. See: https://docs.currents.dev/?q=machineId" -).env(getEnvironmentVariableName("machineId")); +).env(getEnvironmentVariableName(configKeys, "machineId")); export const reportDirOption = new Option( "--report-dir ", "explicit path to the report directory" -).env(getEnvironmentVariableName("reportDir")); +).env(getEnvironmentVariableName(configKeys, "reportDir")); export const debugOption = new Option("--debug", "enable debug logs") - .env(getEnvironmentVariableName("debug")) + .env(getEnvironmentVariableName(configKeys, "debug")) .default(false); - -function parseCommaSeparatedList(value: string, previous: string[] = []) { - if (value) { - return previous.concat(value.split(",").map((t) => t.trim())); - } - return previous; -} diff --git a/packages/cmd/src/bin/tmp-file.ts b/packages/cmd/src/commands/upload/tmp-file.ts similarity index 100% rename from packages/cmd/src/bin/tmp-file.ts rename to packages/cmd/src/commands/upload/tmp-file.ts diff --git a/packages/cmd/src/commands/upload/upload.ts b/packages/cmd/src/commands/upload/upload.ts new file mode 100644 index 0000000..b4522da --- /dev/null +++ b/packages/cmd/src/commands/upload/upload.ts @@ -0,0 +1,24 @@ +import { info } from "@logger"; +import { getUploadCommand } from "."; + +import { getCurrentsConfig, setCurrentsConfig } from "../../config/upload"; +import { handleCurrentsReport } from "../../services"; +import { commandHandler } from "../utils"; +import { CLIManager } from "./cli-config"; + +export async function uploadHandler( + options: ReturnType["opts"]> +) { + await commandHandler(async (opts) => { + const cliManager = new CLIManager(opts); + setCurrentsConfig(cliManager.parsedConfig); + const config = getCurrentsConfig(); + + info("Currents config: %o", { + ...config, + recordKey: config?.recordKey ? "*****" : undefined, + }); + + await handleCurrentsReport(); + }, options); +} diff --git a/packages/cmd/src/commands/utils.ts b/packages/cmd/src/commands/utils.ts new file mode 100644 index 0000000..f279cbd --- /dev/null +++ b/packages/cmd/src/commands/utils.ts @@ -0,0 +1,46 @@ +import { CommanderError } from "@commander-js/extra-typings"; +import { ValidationError } from "@lib"; +import { error } from "@logger"; +import { isAxiosError } from "axios"; +import { enableDebug } from "../debug"; + +export function parseCommaSeparatedList( + value: string, + previous: string[] = [] +) { + if (value) { + return previous.concat(value.split(",").map((t) => t.trim())); + } + return previous; +} + +export async function commandHandler>( + action: (options: T) => Promise, + options: T +) { + try { + if (options.debug) { + enableDebug(); + } + await action(options); + process.exit(0); + } catch (e) { + if (e instanceof CommanderError) { + error(e.message); + process.exit(e.exitCode); + } + + if (e instanceof ValidationError) { + error(e.message); + process.exit(1); + } + + if (isAxiosError(e)) { + error(e.message); + process.exit(1); + } + + error("Script execution failed: %o", e); + process.exit(1); + } +} diff --git a/packages/cmd/src/config/api/config.ts b/packages/cmd/src/config/api/config.ts new file mode 100644 index 0000000..bf0b30f --- /dev/null +++ b/packages/cmd/src/config/api/config.ts @@ -0,0 +1,86 @@ +import { debug as _debug } from "@debug"; + +import { getValidatedConfig } from "../utils"; +import { configKeys, getEnvVariables } from "./env"; + +const debug = _debug.extend("config"); + +export type APICommandConfig = { + /** + * API key for authentication with the Currents API. + * For more information on managing API keys, visit: + * https://docs.currents.dev/resources/api/api-keys#managing-the-api-keys + */ + apiKey: string; + + /** + * Identifier for the project where the test run is recorded. + */ + projectId: string; + + /** + * Enable or disable debug logging. + */ + debug?: boolean; +}; + +export type APIGetRunCommandConfig = { + /** + * Identifier for the build associated with the test run. + * Refer to: https://currents.dev/readme/guides/ci-build-id for more details. + */ + ciBuildId?: string; + + /** + * The branch of the project for which the test run was created. + */ + branch?: string; + + /** + * List of tags associated with the test run. + */ + tag?: string[]; + + /** + * Flag indicating whether to return the "LastRunResponse" data. + * + * type LastRunResponse = { + * status: 'failed' | 'passed'; + * failedTests: string[]; + * }; + * + * Default is false if not specified. + */ + pwLastRun?: boolean; + + /** + * File path to which the output should be written. + * If not specified, the output will be written to the console. + */ + output?: string; +}; + +type MandatoryAPICommandConfigKeys = "apiKey" | "projectId"; + +const mandatoryConfigKeys: MandatoryAPICommandConfigKeys[] = [ + "apiKey", + "projectId", +]; + +let _config: (APICommandConfig & APIGetRunCommandConfig) | null = null; + +export function setAPIGetRunCommandConfig( + options?: Partial +) { + _config = getValidatedConfig( + configKeys, + mandatoryConfigKeys, + getEnvVariables, + options + ); + debug("Resolved config: %o", _config); +} + +export function getAPIGetRunCommandConfig() { + return _config; +} diff --git a/packages/cmd/src/config/api/env.ts b/packages/cmd/src/config/api/env.ts new file mode 100644 index 0000000..eb4fe28 --- /dev/null +++ b/packages/cmd/src/config/api/env.ts @@ -0,0 +1,56 @@ +import { APICommandConfig } from "./config"; + +const apiCommandConfigKeys = { + apiKey: { + name: "Api Key", + env: "CURRENTS_API_KEY", + cli: "--api-key", + }, + debug: { + name: "Debug", + env: "CURRENTS_DEBUG", + cli: "--debug", + }, +} as const; + +const apiGetRunCommandConfigKeys = { + branch: { + name: "Run Branch", + cli: "--branch", + }, + output: { + name: "Output Path", + env: "CURRENTS_OUTPUT", + cli: "--output", + }, + ciBuildId: { + name: "CI Build ID", + cli: "--ci-build-id", + }, + projectId: { + name: "Project ID", + env: "CURRENTS_PROJECT_ID", + cli: "--project-id", + }, + tag: { + name: "Run Tag", + cli: "--tag", + }, +} as const; + +export const configKeys = { + ...apiCommandConfigKeys, + ...apiGetRunCommandConfigKeys, +} as const; + +export function getEnvVariables(): Partial< + Record< + keyof APICommandConfig, + string | string[] | boolean | number | undefined + > +> { + return { + apiKey: process.env[configKeys.apiKey.env], + debug: !!process.env[configKeys.debug.env], + }; +} diff --git a/packages/cmd/src/config/api/index.ts b/packages/cmd/src/config/api/index.ts new file mode 100644 index 0000000..55623fe --- /dev/null +++ b/packages/cmd/src/config/api/index.ts @@ -0,0 +1,2 @@ +export * from "./config"; +export * from "./env"; diff --git a/packages/cmd/src/config/cache/config.ts b/packages/cmd/src/config/cache/config.ts new file mode 100644 index 0000000..322f5dd --- /dev/null +++ b/packages/cmd/src/config/cache/config.ts @@ -0,0 +1,85 @@ +import { debug as _debug } from "@debug"; + +import { getValidatedConfig } from "../utils"; +import { configKeys, getEnvVariables } from "./env"; + +const debug = _debug.extend("config"); + +export type CacheCommandConfig = { + /** + * The record key to be used to record the results on the remote dashboard. Read more: https://currents.dev/readme/guides/record-key + */ + recordKey: string; + + /** + * Enable or disable debug logging. + */ + debug?: boolean; +}; + +type CommonConfig = { + matrixIndex: number; + matrixTotal: number; + id?: string; + preset?: string; +}; + +export type CacheSetCommandConfig = CacheCommandConfig & + CommonConfig & { + paths?: string[]; + pwOutputDir?: string; + }; + +export type CacheGetCommandConfig = CacheCommandConfig & + CommonConfig & { + outputDir?: string; + presetOutput?: string; + }; + +type MandatoryCacheCommandKeys = "recordKey"; + +const mandatoryConfigKeys: MandatoryCacheCommandKeys[] = ["recordKey"]; + +let _config: + | { + type: "SET_COMMAND_CONFIG"; + values: (CacheCommandConfig & CacheSetCommandConfig) | null; + } + | { + type: "GET_COMMAND_CONFIG"; + values: (CacheCommandConfig & CacheGetCommandConfig) | null; + }; + +export function setCacheSetCommandConfig( + options?: Partial> +) { + _config = { + type: "SET_COMMAND_CONFIG", + values: getValidatedConfig( + configKeys, + mandatoryConfigKeys, + getEnvVariables, + options + ), + }; + debug("Resolved config: %o", _config); +} + +export function setCacheGetCommandConfig( + options?: Partial> +) { + _config = { + type: "GET_COMMAND_CONFIG", + values: getValidatedConfig( + configKeys, + mandatoryConfigKeys, + getEnvVariables, + options + ), + }; + debug("Resolved config: %o", _config); +} + +export function getCacheCommandConfig() { + return _config; +} diff --git a/packages/cmd/src/config/cache/env.ts b/packages/cmd/src/config/cache/env.ts new file mode 100644 index 0000000..ccc0610 --- /dev/null +++ b/packages/cmd/src/config/cache/env.ts @@ -0,0 +1,86 @@ +import { CacheCommandConfig } from "./config"; + +const cacheCommandConfigKeys = { + recordKey: { + name: "Record Key", + env: "CURRENTS_RECORD_KEY", + cli: "--key", + }, + debug: { + name: "Debug", + env: "CURRENTS_DEBUG", + cli: "--debug", + }, +} as const; + +const cacheSetCommandConfigKeys = { + id: { + name: "Cache id", + cli: "--id", + }, + preset: { + name: "Preset", + cli: "--preset", + }, + pwOutputDir: { + name: "Playwright output directory", + cli: "--pw-output-dir", + }, + presetOutput: { + name: "Preset output path", + cli: "--preset-output", + }, + paths: { + name: "Paths to cache", + cli: "--paths", + }, + matrixIndex: { + name: "Matrix index", + cli: "--matrix-index", + }, + matrixTotal: { + name: "Matrix total", + cli: "--matrix-total", + }, +} as const; + +const cacheGetCommandConfigKeys = { + id: { + name: "Cache id", + cli: "--id", + }, + preset: { + name: "Preset", + cli: "--preset", + }, + outputDir: { + name: "Custom directory to write output", + cli: "--output-dir", + }, + matrixIndex: { + name: "Matrix index", + cli: "--matrix-index", + }, + matrixTotal: { + name: "Matrix total", + cli: "--matrix-total", + }, +} as const; + +export const configKeys = { + ...cacheCommandConfigKeys, + ...cacheSetCommandConfigKeys, + ...cacheGetCommandConfigKeys, +} as const; + +export function getEnvVariables(): Partial< + Record< + keyof CacheCommandConfig, + string | string[] | boolean | number | undefined + > +> { + return { + recordKey: process.env[configKeys.recordKey.env], + debug: process.env[configKeys.debug.env], + }; +} diff --git a/packages/cmd/src/config/index.ts b/packages/cmd/src/config/cache/index.ts similarity index 100% rename from packages/cmd/src/config/index.ts rename to packages/cmd/src/config/cache/index.ts diff --git a/packages/cmd/src/config/cache/options.ts b/packages/cmd/src/config/cache/options.ts new file mode 100644 index 0000000..ff727c6 --- /dev/null +++ b/packages/cmd/src/config/cache/options.ts @@ -0,0 +1,33 @@ +import { CacheGetCommandOpts } from "../../commands/cache/get"; +import { CacheSetCommandOpts } from "../../commands/cache/set"; +import { CacheGetCommandConfig, CacheSetCommandConfig } from "./config"; + +export function cacheGetCommandOptsToConfig( + options: CacheGetCommandOpts +): Partial { + return { + recordKey: options.key, + id: options.id, + preset: options.preset, + outputDir: options.outputDir, + presetOutput: options.presetOutput, + debug: options.debug, + matrixIndex: options.matrixIndex, + matrixTotal: options.matrixTotal, + }; +} + +export function cacheSetCommandOptsToConfig( + options: CacheSetCommandOpts +): Partial { + return { + recordKey: options.key, + id: options.id, + preset: options.preset, + pwOutputDir: options.pwOutputDir, + paths: options.paths, + debug: options.debug, + matrixIndex: options.matrixIndex, + matrixTotal: options.matrixTotal, + }; +} diff --git a/packages/cmd/src/config/config.ts b/packages/cmd/src/config/upload/config.ts similarity index 57% rename from packages/cmd/src/config/config.ts rename to packages/cmd/src/config/upload/config.ts index 4478294..006692f 100644 --- a/packages/cmd/src/config/config.ts +++ b/packages/cmd/src/config/upload/config.ts @@ -1,14 +1,7 @@ -import { debug as _debug } from "../debug"; -import { ValidationError } from "../lib/error"; -import { dim, error } from "../logger"; - -import { - getCLIOptionName, - getConfigName, - getEnvVariables, - getEnvironmentVariableName, -} from "./env"; -import { getCLIOptions } from "./options"; +import { debug as _debug } from "@debug"; + +import { getValidatedConfig } from "../utils"; +import { configKeys, getEnvVariables } from "./env"; const debug = _debug.extend("config"); @@ -69,45 +62,19 @@ const mandatoryConfigKeys: MandatoryCurrentsConfigKeys[] = [ let _config: CurrentsConfig | null = null; /** - * Precendence: env > cli > reporter config + * Precendence: env > reporter config * @param reporterOptions reporter config */ export function setCurrentsConfig(reporterOptions?: Partial) { - const result = { - ...removeUndefined(reporterOptions), - ...removeUndefined(getCLIOptions()), - ...removeUndefined(getEnvVariables()), - }; - - mandatoryConfigKeys.forEach((i) => { - if (!result[i]) { - error( - `${getConfigName( - i - )} is required for Currents Reporter. Use the following methods to set Currents Project ID: -- as environment variable: ${dim(getEnvironmentVariableName(i))} -- as CLI flag of the command: ${dim(getCLIOptionName(i))}` - ); - throw new ValidationError("Missing required config variable"); - } - }); - - _config = result as CurrentsConfig; + _config = getValidatedConfig( + configKeys, + mandatoryConfigKeys, + getEnvVariables, + reporterOptions + ); debug("Resolved Currents config: %o", _config); } -function removeUndefined(obj?: T): T { - return Object.entries(obj ?? {}).reduce((acc, [key, value]) => { - if (value === undefined) { - return acc; - } - return { - ...acc, - [key]: value, - }; - }, {} as T); -} - export function getCurrentsConfig() { return _config; } diff --git a/packages/cmd/src/config/env.ts b/packages/cmd/src/config/upload/env.ts similarity index 58% rename from packages/cmd/src/config/env.ts rename to packages/cmd/src/config/upload/env.ts index aa60a81..2985cdd 100644 --- a/packages/cmd/src/config/env.ts +++ b/packages/cmd/src/config/upload/env.ts @@ -1,6 +1,6 @@ import { CurrentsConfig } from "./config"; -export const configKey = { +export const configKeys = { debug: { name: "Debug", env: "CURRENTS_DEBUG", @@ -48,18 +48,6 @@ export const configKey = { }, } as const; -export function getEnvironmentVariableName(variable: keyof typeof configKey) { - return configKey[variable].env; -} - -export function getCLIOptionName(variable: keyof typeof configKey) { - return configKey[variable].cli; -} - -export function getConfigName(variable: keyof typeof configKey) { - return configKey[variable].name; -} - /** * Converts Environment variables to Currents config. * @returns @@ -68,15 +56,15 @@ export function getEnvVariables(): Partial< Record > { return { - projectId: process.env[configKey.projectId.env], - recordKey: process.env[configKey.recordKey.env], - ciBuildId: process.env[configKey.ciBuildId.env], - tag: process.env[configKey.tag.env] - ? process.env[configKey.tag.env]?.split(",").map((i) => i.trim()) + projectId: process.env[configKeys.projectId.env], + recordKey: process.env[configKeys.recordKey.env], + ciBuildId: process.env[configKeys.ciBuildId.env], + tag: process.env[configKeys.tag.env] + ? process.env[configKeys.tag.env]?.split(",").map((i) => i.trim()) : undefined, - disableTitleTags: !!process.env[configKey.disableTitleTags.env], - removeTitleTags: !!process.env[configKey.removeTitleTags.env], - debug: !!process.env[configKey.debug.env], - machineId: process.env[configKey.machineId.env], + disableTitleTags: !!process.env[configKeys.disableTitleTags.env], + removeTitleTags: !!process.env[configKeys.removeTitleTags.env], + debug: !!process.env[configKeys.debug.env], + machineId: process.env[configKeys.machineId.env], }; } diff --git a/packages/cmd/src/config/upload/index.ts b/packages/cmd/src/config/upload/index.ts new file mode 100644 index 0000000..b2d445e --- /dev/null +++ b/packages/cmd/src/config/upload/index.ts @@ -0,0 +1,3 @@ +export * from "./config"; +export * from "./env"; +export * from "./options"; diff --git a/packages/cmd/src/config/options.ts b/packages/cmd/src/config/upload/options.ts similarity index 57% rename from packages/cmd/src/config/options.ts rename to packages/cmd/src/config/upload/options.ts index 6556e53..c05f24f 100644 --- a/packages/cmd/src/config/options.ts +++ b/packages/cmd/src/config/upload/options.ts @@ -1,12 +1,11 @@ import { Command } from "@commander-js/extra-typings"; -import fs from "fs"; -import { getProgram } from "../bin/program"; -import { debug } from "../debug"; - +import { getUploadCommand } from "../../commands/upload"; import { CurrentsConfig } from "./config"; type ExtractSecondTags = T extends Command ? U : never; -export type CLIOptions = ExtractSecondTags>; +export type CLIOptions = ExtractSecondTags< + ReturnType +>; /** * Converts CLI options to Currents config. @@ -29,18 +28,3 @@ export function cliOptionsToConfig( reportDir: cliOptions.reportDir, }; } - -export function getCLIOptions() { - if (process.env.CURRENTS_REPORTER_CONFIG_PATH) { - try { - const result: Partial = JSON.parse( - fs.readFileSync(process.env.CURRENTS_REPORTER_CONFIG_PATH).toString() - ); - debug("CLI options from file: %o", result); - return result; - } catch (error) { - return {}; - } - } - return {}; -} diff --git a/packages/cmd/src/config/utils.ts b/packages/cmd/src/config/utils.ts new file mode 100644 index 0000000..31679ce --- /dev/null +++ b/packages/cmd/src/config/utils.ts @@ -0,0 +1,77 @@ +import { debug as _debug } from "@debug"; +import { ValidationError } from "@lib/error"; +import { dim, error } from "@logger"; + +type ConfigKeys = Record< + string, + { + name: string; + cli: string; + env?: string; + } +>; + +export function removeUndefined(obj?: T): T { + return Object.entries(obj ?? {}).reduce((acc, [key, value]) => { + if (value === undefined) { + return acc; + } + return { + ...acc, + [key]: value, + }; + }, {} as T); +} + +export function getEnvironmentVariableName( + configKeys: T, + variable: keyof T +) { + return "env" in configKeys[variable] && !!configKeys[variable].env + ? configKeys[variable].env + : ""; +} + +export function getCLIOptionName( + configKeys: T, + variable: keyof T +) { + return configKeys[variable].cli; +} + +export function getConfigName( + configKeys: T, + variable: keyof T +) { + return configKeys[variable].name; +} + +export function getValidatedConfig( + configKeys: T, + mandatoryKeys: (keyof R)[], + getEnvVariables: () => Partial< + Record + >, + options?: Partial +) { + const result = { + ...removeUndefined(options), + ...removeUndefined(getEnvVariables()), + }; + + mandatoryKeys.forEach((i) => { + if (!result[i]) { + error( + `${getConfigName( + configKeys, + i as string + )} is required for Currents Reporter. Use the following methods to set the value: +- as environment variable: ${dim(getEnvironmentVariableName(configKeys, i as string))} +- as CLI flag of the command: ${dim(getCLIOptionName(configKeys, i as string))}` + ); + throw new ValidationError("Missing required config variable"); + } + }); + + return result as R; +} diff --git a/packages/cmd/src/env/ciProvider.ts b/packages/cmd/src/env/ciProvider.ts index 5a11e75..1ba1007 100644 --- a/packages/cmd/src/env/ciProvider.ts +++ b/packages/cmd/src/env/ciProvider.ts @@ -27,6 +27,7 @@ SOFTWARE. // import debugFn from "debug"; // @ts-ignore +import { userFacingNanoid } from "@lib/nano"; import { camelCase, chain, @@ -38,7 +39,6 @@ import { some, transform, } from "lodash"; -import { userFacingNanoid } from "../lib/nano"; import { debug as _debug } from "../debug"; import { CiProvider, CiProviderData } from "./types"; @@ -298,12 +298,12 @@ const _providerCiParams = (): ProviderCiParamsRes => { ]), // https://help.github.com/en/actions/automating-your-workflow-with-github-actions/using-environment-variables#default-environment-variables githubActions: extract([ - "GITHUB_WORKFLOW", - "GITHUB_ACTION", - "GITHUB_EVENT_NAME", - "GITHUB_RUN_ID", - "GITHUB_RUN_ATTEMPT", - "GITHUB_REPOSITORY", + 'GITHUB_WORKFLOW', + 'GITHUB_ACTION', + 'GITHUB_EVENT_NAME', + 'GITHUB_RUN_ID', + 'GITHUB_RUN_ATTEMPT', + 'GITHUB_REPOSITORY', ]), // see https://docs.gitlab.com/ee/ci/variables/ gitlab: extract([ @@ -315,6 +315,9 @@ const _providerCiParams = (): ProviderCiParamsRes => { "CI_JOB_ID", "CI_JOB_URL", "CI_JOB_NAME", + // matrix information + "CI_NODE_INDEX", + "CI_NODE_TOTAL", // other information "GITLAB_HOST", "CI_PROJECT_ID", @@ -322,6 +325,8 @@ const _providerCiParams = (): ProviderCiParamsRes => { "CI_REPOSITORY_URL", "CI_ENVIRONMENT_URL", "CI_DEFAULT_BRANCH", + // custom variables + "RUN_ATTEMPT", // custom param that we ourselves sometimes add for retrying jobs // for PRs: https://gitlab.com/gitlab-org/gitlab-ce/issues/23902 ]), // https://docs.gocd.org/current/faq/dev_use_current_revision_in_build.html#standard-gocd-environment-variables @@ -694,7 +699,7 @@ export function getCommitParams() { return _get(_providerCommitParams); } -export function getCI(explicitCiBuildId: string | undefined) { +export function getCI(explicitCiBuildId?: string | undefined) { const params = getCiParams(); const provider = getCiProvider(); const ciBuildId = getCIBuildId(explicitCiBuildId, provider); diff --git a/packages/cmd/src/env/types.ts b/packages/cmd/src/env/types.ts index 7551f24..e9818f8 100644 --- a/packages/cmd/src/env/types.ts +++ b/packages/cmd/src/env/types.ts @@ -13,3 +13,32 @@ export type CiProviderData = { runAttempt?: string; ghaEventData?: GhaEventData | null; }; + +export type GithubActionsParams = { + githubWorkflow: string; + githubAction: string; + githubEventName: string; + githubRunId: string; + githubRunAttempt: string; + githubRepository: string; +}; + + +export type GitLabParams = { + gitlabCi: string; + ciPipelineId: string; + ciPipelineUrl: string; + ciBuildId: string; + ciJobId: string; + ciJobUrl: string; + ciJobName: string; + gitlabHost: string; + ciProjectId: string; + ciProjectUrl: string; + ciRepositoryUrl: string; + ciEnvironmentUrl: string; + ciDefaultBranch: string; + ciNodeIndex?: string; + ciNodeTotal?: string; + runAttempt?: string; +}; \ No newline at end of file diff --git a/packages/cmd/src/http/axios.ts b/packages/cmd/src/http/axios.ts new file mode 100644 index 0000000..1e4ba4a --- /dev/null +++ b/packages/cmd/src/http/axios.ts @@ -0,0 +1,34 @@ +import axios, { CreateAxiosDefaults } from "axios"; +import { HttpsProxyAgent } from "https-proxy-agent"; +import proxyFromEnv from "proxy-from-env"; + +import { debug as _debug } from "../debug"; + +const debug = _debug.extend("axios"); + +export function getAxios(config: CreateAxiosDefaults = {}) { + const instance = axios.create(config); + + instance.interceptors.request.use((config) => { + const _config = { ...config }; + const uri = instance.getUri(config); + const proxyURL = proxyFromEnv.getProxyForUrl(uri); + if (proxyURL) { + debug("Using HTTP proxy %s for %s ", proxyURL, uri); + _config.proxy = false; + _config.httpsAgent = getHTTPSProxyAgent(proxyURL); + } + return _config; + }); + return instance; +} + +let _httpsProxyAgent: null | HttpsProxyAgent = null; +function getHTTPSProxyAgent(url: string) { + if (_httpsProxyAgent) { + return _httpsProxyAgent; + } + + _httpsProxyAgent = new HttpsProxyAgent(url); + return _httpsProxyAgent; +} diff --git a/packages/cmd/src/http/client.ts b/packages/cmd/src/http/client.ts index 32c5c8e..e45be20 100644 --- a/packages/cmd/src/http/client.ts +++ b/packages/cmd/src/http/client.ts @@ -1,10 +1,9 @@ import axios, { AxiosInstance } from "axios"; import axiosRetry from "axios-retry"; import _ from "lodash"; -import { nanoid } from "nanoid"; import { debug as _debug } from "../debug"; -import { getAPIBaseUrl, getTimeout } from "./httpConfig"; +import { getAPIBaseUrl, getRestAPIBaseUrl, getTimeout } from "./httpConfig"; import { getDelay, getMaxRetries, @@ -14,11 +13,19 @@ import { const debug = _debug.extend("http"); -let _client: AxiosInstance | null = null; +export enum ClientType { + API = "api", + REST_API = "restApi", +} + +let _clients: Record = { + [ClientType.API]: null, + [ClientType.REST_API]: null, +}; -export function createClient() { +export function createClient(type: ClientType) { const client = axios.create({ - baseURL: getAPIBaseUrl(), + baseURL: type === ClientType.API ? getAPIBaseUrl() : getRestAPIBaseUrl(), // Setting no timeout means axios will wait forever for a response // and the actual timeout will be handled by the underlying network // stack @@ -80,12 +87,12 @@ export function createClient() { return client; } -export function getClient() { - if (_client) { - return _client; +export function getClient(type: ClientType): AxiosInstance { + if (_clients[type] === null) { + _clients[type] = createClient(type); } - _client = createClient(); - return _client; + + return _clients[type]; } function getNetworkRequestDebugData(data: { diff --git a/packages/cmd/src/http/http.ts b/packages/cmd/src/http/http.ts index 55582a4..261dfa6 100644 --- a/packages/cmd/src/http/http.ts +++ b/packages/cmd/src/http/http.ts @@ -2,17 +2,20 @@ import { AxiosError, AxiosRequestConfig, AxiosResponse } from "axios"; import _ from "lodash"; import { debug as _debug } from "../debug"; -import { getClient } from "./client"; +import { ClientType, getClient } from "./client"; import { handleHTTPError } from "./httpErrors"; const debug = _debug.extend("http"); export async function makeRequest( + clientType: ClientType, config: AxiosRequestConfig, _getClient = getClient ) { try { - const res = await _getClient().request>(config); + const res = await _getClient(clientType).request>( + config + ); debug("network response: %o", { ..._.omit(res, "request", "config"), url: res.config.url, diff --git a/packages/cmd/src/http/httpConfig.ts b/packages/cmd/src/http/httpConfig.ts index f1a0f5b..8891323 100644 --- a/packages/cmd/src/http/httpConfig.ts +++ b/packages/cmd/src/http/httpConfig.ts @@ -1,4 +1,7 @@ export const getAPIBaseUrl = () => process.env.CURRENTS_API_URL ?? "https://cy.currents.dev"; +export const getRestAPIBaseUrl = () => + process.env.CURRENTS_REST_API_URL ?? "https://api.currents.dev"; + export const getTimeout = () => 30000; diff --git a/packages/cmd/src/http/httpErrors.ts b/packages/cmd/src/http/httpErrors.ts index ed1a637..19a42b3 100644 --- a/packages/cmd/src/http/httpErrors.ts +++ b/packages/cmd/src/http/httpErrors.ts @@ -1,7 +1,7 @@ +import * as log from "@logger"; import { AxiosError, isAxiosError } from "axios"; import _ from "lodash"; import { P, match } from "ts-pattern"; -import * as log from "../logger"; export function handleHTTPError(error: unknown) { match(error) diff --git a/packages/cmd/src/http/httpRetry.ts b/packages/cmd/src/http/httpRetry.ts index aaa1f23..be36cb2 100644 --- a/packages/cmd/src/http/httpRetry.ts +++ b/packages/cmd/src/http/httpRetry.ts @@ -1,8 +1,8 @@ import { AxiosError, AxiosRequestConfig, isAxiosError } from "axios"; import prettyMilliseconds from "pretty-ms"; +import { warn } from "@logger"; import { debug as _debug } from "../debug"; -import { warn } from "../logger"; const debug = _debug.extend("http"); diff --git a/packages/cmd/src/index.ts b/packages/cmd/src/index.ts index 8eacf0f..add7769 100644 --- a/packages/cmd/src/index.ts +++ b/packages/cmd/src/index.ts @@ -1,234 +1,2 @@ -import { mapValues } from "lodash"; -import path from "path"; -import semver from "semver"; -import { Framework, RunCreationConfig, createRun as createRunApi } from "./api"; -import { getCurrentsConfig } from "./config"; -import { debug, enableDebug, setTraceFilePath } from "./debug"; -import { FullTestSuite, createScanner } from "./discovery"; -import { getCI } from "./env/ciProvider"; -import { getGitInfo } from "./env/gitInfo"; -import { getPlatformInfo } from "./env/platform"; -import { reporterVersion } from "./env/versions"; -import { - checkPathExists, - getInstanceReportList, - nanoid, - readJsonFile, - resolveReportOptions, - writeFileAsync, -} from "./lib"; -import { info, warn } from "./logger"; -import { InstanceReport, ReportConfig, UploadMarkerInfo } from "./types"; - -export async function currentsReporter() { - const currentsConfig = getCurrentsConfig(); - if (!currentsConfig) { - throw new Error("Currents config is missing!"); - } - - if (currentsConfig.debug) { - enableDebug(); - } - - const reportOptions = await resolveReportOptions(currentsConfig); - // set the trace file path - setTraceFilePath(getTraceFilePath(reportOptions.reportDir)); - - debug("Reporter options: %o", reportOptions); - - info("Report directory: %s", reportOptions.reportDir); - - const config = await readJsonFile(reportOptions.configFilePath); - debug("Report config: %o", config); - - const instanceReportList = await getInstanceReportList( - reportOptions.reportDir - ); - debug( - "Found %d instance results in the reportDir: %s", - instanceReportList.length, - reportOptions.reportDir - ); - - const markerFilePath = getMarkerFilePath(reportOptions.reportDir); - const markerFileExists = await checkPathExists(markerFilePath); - - const fullTestSuiteFilePath = getFullTestSuiteFilePath( - reportOptions.reportDir - ); - - let fullTestSuite: FullTestSuite | null = null; - if (markerFileExists) { - const markerInfo = await readJsonFile(markerFilePath); - warn("Marker file detected. The report was already uploaded: %o", { - runUrl: markerInfo.response.runUrl, - isoDate: markerInfo.isoDate, - }); - - const fullTestSuiteFileExists = await checkPathExists( - fullTestSuiteFilePath - ); - - if (fullTestSuiteFileExists) { - fullTestSuite = await readJsonFile(fullTestSuiteFilePath); - debug("Full test suite file detected: %s", fullTestSuiteFilePath); - } - } - - if (!fullTestSuite) { - const scanner = createScanner(config); - fullTestSuite = await scanner.getFullTestSuite(); - - if (isEmptyTestSuite(fullTestSuite)) { - throw new Error("Failed to discover the full test suite!"); - } - - await writeFileAsync(fullTestSuiteFilePath, JSON.stringify(fullTestSuite)); - } else { - debug("The discovery stage was skipped"); - } - - const defaultGroup = - fullTestSuite.length === 1 ? fullTestSuite[0].name : null; - - const runCreationConfig: RunCreationConfig = { - currents: currentsConfig, - }; - - const framework: Framework = { - type: config.framework, - version: config.frameworkVersion, - clientVersion: reporterVersion, - }; - - const machineId = nanoid.userFacingNanoid(); - const ci = getCI(currentsConfig.ciBuildId); - - const instancesByGroup: Record = {}; - for await (const instanceReport of instanceReportList) { - const report = await readJsonFile(instanceReport); - if (!instancesByGroup[report.groupId]) { - instancesByGroup[report.groupId] = []; - } - instancesByGroup[report.groupId].push(report); - } - - for await (const key of Object.keys(instancesByGroup)) { - let instances = instancesByGroup[key]; - let group = key; - - if (defaultGroup) { - debug( - "Default group found: %s, overwriting the group in the results", - defaultGroup - ); - - group = defaultGroup; - instances = instances.map((i) => ({ - ...i, - groupId: defaultGroup, - })); - } - - try { - const response = await createRun({ - ci, - group, - instances, - fullTestSuite, - config: runCreationConfig, - machineId, - framework, - }); - - debug("Api response: %o", response); - - info("[%s] Run created: %s", group, response.runUrl); - - const markerInfo = { - response, - isoDate: new Date().toISOString(), - }; - - await writeFileAsync(markerFilePath, JSON.stringify(markerInfo)); - - debug( - "Marker file %s: %s", - markerFileExists ? "overwritten" : "created", - markerFilePath - ); - } catch (_) { - throw new Error("Failed to upload the results to the dashboard"); - } - } -} - -async function createRun({ - ci, - group, - instances, - fullTestSuite, - config, - machineId, - framework, -}: { - ci: ReturnType; - group: string; - instances: InstanceReport[]; - fullTestSuite: FullTestSuite; - config: RunCreationConfig; - machineId: string; - framework: Framework; -}) { - const commit = await getGitInfo(); - const platformInfo = await getPlatformInfo(); - const browserInfo = { - browserName: "node", - browserVersion: semver.coerce(process.version)?.version, - }; - - const { currents } = config; - - const platform = { ...browserInfo, ...platformInfo }; - const payload = { - platform, - ci, - commit, - group, - fullTestSuite, - config, - projectId: currents.projectId, - recordKey: currents.recordKey, - ciBuildId: ci.ciBuildId.value ?? undefined, - tags: currents.tag ?? [], - machineId: currents.machineId ?? machineId, - framework, - instances, - }; - - debug( - "Creating run: %o", - mapValues(payload, (v, k) => (k === "recordKey" ? "******" : v)) - ); - - return createRunApi(payload); -} - -function getMarkerFilePath(reportDir: string) { - return path.join(reportDir, "upload.marker.json"); -} - -function getFullTestSuiteFilePath(reportDir: string) { - return path.join(reportDir, "fullTestSuite.json"); -} - -function getTraceFilePath(reportDir: string) { - return path.join(reportDir, `.debug-${new Date().toISOString()}.log`); -} - -function isEmptyTestSuite(testSuite: FullTestSuite) { - return ( - testSuite.length === 0 || - testSuite.some((project) => project.tests.length === 0) - ); -} +export { handleCurrentsReport } from "./services/upload"; +export type * from "./services/upload/types"; diff --git a/packages/cmd/src/lib/fs.ts b/packages/cmd/src/lib/fs.ts index 8961079..f8678ee 100644 --- a/packages/cmd/src/lib/fs.ts +++ b/packages/cmd/src/lib/fs.ts @@ -1,81 +1,6 @@ +import { error } from "@logger"; import fs from "fs-extra"; -import path, { join, resolve } from "path"; -import { error } from "../logger"; -import { ReportOptions } from "../types"; - -export async function resolveReportOptions( - options?: ReportOptions -): Promise> { - const reportDir = await findReportDir(options?.reportDir); - - if (!reportDir) { - throw new Error("Failed to find the report dir"); - } - - return { - reportDir, - configFilePath: - options?.configFilePath ?? path.join(reportDir, "config.json"), - }; -} - -async function findReportDir(reportDir?: string): Promise { - if (reportDir) { - await checkPathExists(reportDir); - return reportDir; - } - - return getLastCreatedDirectory(join(process.cwd(), ".currents")); -} - -async function getLastCreatedDirectory(dir: string): Promise { - const entries = await fs.readdir(dir); - let latestDir: { name: string; birthtime: Date } | null = null; - - for (const entry of entries) { - const entryPath = path.join(dir, entry); - const stat = await fs.stat(entryPath); - - if (stat.isDirectory()) { - if (!latestDir || stat.birthtime > latestDir.birthtime) { - latestDir = { name: entry, birthtime: stat.birthtime }; - } - } - } - - return latestDir ? resolve(dir, latestDir.name) : null; -} - -export async function checkPathExists(path: string): Promise { - try { - const exists = await fs.pathExists(path); - return exists; - } catch (err) { - error("Error checking if path exists:", error); - return false; - } -} - -export async function getInstanceReportList(reportDir: string) { - const instancesDir = path.join(reportDir, "instances"); - return getAllFilePaths(instancesDir); -} - -async function getAllFilePaths(dir: string): Promise { - const files = await fs.readdir(dir); - const filePaths: string[] = []; - - for (const file of files) { - const filePath = path.join(dir, file); - const stat = await fs.stat(filePath); - - if (stat.isFile()) { - filePaths.push(filePath); - } - } - - return filePaths; -} +import { dirname } from "path"; export async function readJsonFile(filePath: string): Promise { try { @@ -96,3 +21,8 @@ export async function writeFileAsync(filePath: string, content: string) { throw err; } } + +export async function ensurePathExists(filePath: string): Promise { + const dir = dirname(filePath); + await fs.ensureDir(dir); +} diff --git a/packages/cmd/src/lib/index.ts b/packages/cmd/src/lib/index.ts index 5f3b411..5d56350 100644 --- a/packages/cmd/src/lib/index.ts +++ b/packages/cmd/src/lib/index.ts @@ -1,5 +1,3 @@ -import { nanoid } from "nanoid"; - export * from "./error"; export * from "./execa"; export * from "./fs"; diff --git a/packages/cmd/src/logger/logger.ts b/packages/cmd/src/logger/logger.ts index 66ac247..e624544 100644 --- a/packages/cmd/src/logger/logger.ts +++ b/packages/cmd/src/logger/logger.ts @@ -11,6 +11,12 @@ const log = (...args: unknown[]) => { console.log(stringToRender); }; +const _error = (...args: unknown[]) => { + const stringToRender = util.format(...args); + addToLogDrain(stringToRender); + console.error(stringToRender); +}; + export const info = log; export const warn = (...args: unknown[]) => { @@ -38,7 +44,7 @@ export const error = (...args: unknown[]) => { const msg = util.format(...args); errors.push(msg); debug("ERROR: ", msg); - return log(chalk.bgRed.white(" ERROR "), msg); + return _error(chalk.bgRed.white(" ERROR "), msg); }; export const title = (...args: unknown[]) => diff --git a/packages/cmd/src/services/api/index.ts b/packages/cmd/src/services/api/index.ts new file mode 100644 index 0000000..99ec8df --- /dev/null +++ b/packages/cmd/src/services/api/index.ts @@ -0,0 +1,57 @@ +import { debug } from "@debug"; +import { ensurePathExists } from "@lib"; +import { writeFile } from "fs/promises"; +import { getRun } from "../../api"; +import { + APICommandConfig, + APIGetRunCommandConfig, + getAPIGetRunCommandConfig, +} from "../../config/api"; +import { info } from "../../logger"; + +export async function handleGetRun() { + try { + const config = getAPIGetRunCommandConfig(); + if (!config) { + throw new Error("Config is missing!"); + } + + const params = config.ciBuildId + ? { + projectId: config.projectId, + ciBuildId: config.ciBuildId, + pwLastRun: config.pwLastRun, + } + : { + projectId: config.projectId, + branch: config.branch, + tag: config.tag, + pwLastRun: config.pwLastRun, + }; + + const result = await getRun(config.apiKey, params); + await handleOutput( + config.pwLastRun && result.data.pwLastRun + ? result.data.pwLastRun + : result.data, + config + ); + } catch (e) { + debug("Failed to get run data"); + throw e; + } +} + +async function handleOutput( + result: unknown, + config: APICommandConfig & APIGetRunCommandConfig +) { + const data = JSON.stringify(result, null, 2); + + if (config.output) { + await ensurePathExists(config.output); + await writeFile(config.output, data, "utf-8"); + } else { + info(data); + } +} diff --git a/packages/cmd/src/services/cache/__tests__/fs.spec.ts b/packages/cmd/src/services/cache/__tests__/fs.spec.ts new file mode 100644 index 0000000..fdaa63c --- /dev/null +++ b/packages/cmd/src/services/cache/__tests__/fs.spec.ts @@ -0,0 +1,100 @@ +import fs from "fs-extra"; +import path from "path"; +import { Readable } from "stream"; +import unzipper from "unzipper"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { zipFilesToBuffer } from "../fs"; + +describe("zipFilesToBuffer", () => { + const testDir = path.join(__dirname, "test-files"); + + const file1Path = path.join(testDir, "file1.txt"); + const file2Path = path.join(testDir, "file2.txt"); + const subdirPath = path.join(testDir, "subdir"); + const file3Path = path.join(subdirPath, "file3.txt"); + + beforeAll(async () => { + await fs.ensureDir(testDir); + await fs.writeFile(file1Path, ""); + await fs.writeFile(file2Path, ""); + await fs.ensureDir(subdirPath); + await fs.writeFile(file3Path, ""); + }); + + afterAll(async () => { + await fs.remove(testDir); + }); + + it("should zip a single file", async () => { + const buffer = await zipFilesToBuffer([file1Path]); + const entries = await unzipBuffer(buffer); + expect(entries.length).toBe(1); + }); + + it("should zip multiple files", async () => { + const buffer = await zipFilesToBuffer([file1Path, file2Path]); + const entries = await unzipBuffer(buffer); + expect(entries.length).toBe(2); + }); + + it("should zip a directory", async () => { + const buffer = await zipFilesToBuffer([testDir]); + const entries = await unzipBuffer(buffer); + const entryNames = entries.map((entry) => entry.path); + + expect(entryNames).toContain(path.relative(process.cwd(), file1Path)); + expect(entryNames).toContain(path.relative(process.cwd(), file2Path)); + expect(entryNames).toContain(path.relative(process.cwd(), file3Path)); + expect(entryNames).toContain( + path.relative(process.cwd(), subdirPath) + "/" + ); // "/" is added by archiver for directories + }); + + it("should zip mixed files and directories", async () => { + const buffer = await zipFilesToBuffer([file1Path, subdirPath]); + const entries = await unzipBuffer(buffer); + const entryNames = entries.map((entry) => entry.path); + + expect(entryNames).toContain(path.relative(process.cwd(), file1Path)); + expect(entryNames).toContain(path.relative(process.cwd(), file3Path)); // inside subdir + }); + + it("should handle empty input", async () => { + const buffer = await zipFilesToBuffer([]); + expect(buffer).toBeDefined(); + const zip = Readable.from(buffer).pipe(unzipper.Parse()); + const entries = []; + await new Promise((resolve) => { + zip.on("entry", (entry) => entries.push(entry)); + zip.on("close", resolve); + }); + expect(entries.length).toBe(0); + }); + + it("should throw an error for non-existing paths", async () => { + await expect(zipFilesToBuffer(["non-existing-file.txt"])).rejects.toThrow(); + }); + + it("should throw an error if zip size exceeds limit", async () => { + const largeFilePath = path.join(testDir, "largeFile.txt"); + await fs.writeFile(largeFilePath, "A".repeat(1_000_000)); // on "data" event is fired after + await expect(zipFilesToBuffer([largeFilePath], 500)).rejects.toThrow( + "Zip size exceeded the limit" + ); + }); +}); + +const unzipBuffer = async (buffer: Buffer) => { + const entries: any[] = []; + const zip = Readable.from(buffer).pipe(unzipper.Parse()); + + zip.on("entry", (entry) => { + entries.push(entry); + }); + + await new Promise((resolve) => { + zip.on("close", resolve); + }); + + return entries; +}; diff --git a/packages/cmd/src/services/cache/fs.ts b/packages/cmd/src/services/cache/fs.ts new file mode 100644 index 0000000..897914b --- /dev/null +++ b/packages/cmd/src/services/cache/fs.ts @@ -0,0 +1,102 @@ +import Archiver from "archiver"; +import fs from "fs-extra"; +import path from "path"; +import unzipper from "unzipper"; +import { warn } from "../../logger"; + +const MAX_ZIP_SIZE = 50 * 1024 * 1024; // 50MB + +/** + * Adds files and directories to an archive while preserving their original folder structure. + * + * This function supports various types of paths, including individual files, entire directories, + * and files located within subdirectories. When the archive is extracted, the original organization + * will be restored, making navigation straightforward. + * + * Paths based on a specific starting point will be stored relative to that point, helping to avoid + * naming conflicts and maintaining the logical arrangement of files and directories. + */ +export async function zipFilesToBuffer( + filePaths: string[], + maxSize: number = MAX_ZIP_SIZE +): Promise { + return new Promise((resolve, reject) => { + const archive = Archiver("zip", { zlib: { level: 9 } }); + const chunks: Buffer[] = []; + let totalSize = 0; + + archive.on("data", (chunk) => { + chunks.push(chunk); + totalSize += chunk.length; + if (totalSize > maxSize) { + reject(new Error(`Zip size exceeded the limit of ${maxSize} bytes`)); + } + }); + + archive.on("warning", (err) => { + if (err.code === "ENOENT") { + warn(err); + } else { + reject(err); + } + }); + + archive.on("error", (err) => reject(err)); + + archive.on("end", () => { + const buffer = Buffer.concat(chunks); + resolve(buffer); + }); + + const processFile = async (filePath: string) => { + const baseDir = process.cwd(); + const normalized = path.normalize(filePath); + const relativePath = path.relative(baseDir, normalized); + const stats = await fs.stat(filePath); + if (stats.isDirectory()) { + archive.directory(filePath, relativePath); + } else { + archive.file(filePath, { + name: relativePath, + stats, + }); + } + }; + + const processFiles = async (filePaths: string[]) => { + for (const filePath of filePaths) { + await processFile(filePath); + } + + await archive.finalize(); + }; + + processFiles(filePaths).catch(reject); + }); +} + +export async function unzipBuffer( + zipBuffer: Buffer, + outputDir: string +): Promise { + return unzipper.Open.buffer(zipBuffer).then((d) => + d.extract({ path: outputDir, concurrency: 3 }) + ); +} + +export function filterPaths(filePaths: string[]) { + const baseDir = process.cwd(); + return filePaths.filter((filePath) => { + const absolutePath = path.resolve(filePath); + const relativePath = path.relative(baseDir, absolutePath); + + if (filePath.startsWith("..") || path.isAbsolute(relativePath)) { + warn( + `Invalid path: "${filePath}". Path traversal detected. The path was skipped.` + ); + return false; + } + + return true; + }); +} diff --git a/packages/cmd/src/services/cache/get.ts b/packages/cmd/src/services/cache/get.ts new file mode 100644 index 0000000..2466648 --- /dev/null +++ b/packages/cmd/src/services/cache/get.ts @@ -0,0 +1,82 @@ +import { debug } from "@debug"; +import { isAxiosError } from "axios"; +import { retrieveCache } from "../../api"; +import { PRESETS } from "../../commands/cache/options"; +import { getCacheCommandConfig } from "../../config/cache"; +import { getCI } from "../../env/ciProvider"; +import { warnWithNoTrace } from "../../logger"; +import { unzipBuffer } from "./fs"; +import { MetaFile, warn } from "./lib"; +import { download } from "./network"; +import { handlePostLastRunPreset, handlePreLastRunPreset } from "./presets"; + +export async function handleGetCache() { + try { + const config = getCacheCommandConfig(); + if (config.type !== "GET_COMMAND_CONFIG" || !config.values) { + throw new Error("Config is missing!"); + } + + const { recordKey, id, preset, matrixIndex, matrixTotal } = config.values; + const outputDir = config.values.outputDir; + + const ci = getCI(); + + if (preset === PRESETS.lastRun) { + await handlePreLastRunPreset(config.values, ci); + } + + const result = await retrieveCache({ + recordKey, + ci, + id, + config: { + matrixIndex, + matrixTotal, + }, + }); + + try { + await handleArchiveDownload({ + readUrl: result.readUrl, + outputDir, + }); + + const meta = await handleMetaDownload(result.metaReadUrl); + + if (preset === PRESETS.lastRun) { + await handlePostLastRunPreset(config.values, ci, meta); + } + } catch (e) { + if (isAxiosError(e)) { + if (e.response?.status && e.response?.status < 500) { + warnWithNoTrace(`Cache with ID "${result.cacheId}" not found`); + return; + } + } + + throw e; + } + } catch (e) { + warn(e, "Failed to obtain cache"); + } +} + +async function handleArchiveDownload({ + readUrl, + outputDir, +}: { + readUrl: string; + outputDir?: string; +}) { + const buffer = await download(readUrl); + await unzipBuffer(buffer, outputDir || "."); + debug("Cache downloaded"); +} + +async function handleMetaDownload(readUrl: string) { + const buffer = await download(readUrl); + const meta = JSON.parse(buffer.toString("utf-8")) as MetaFile; + debug("Meta file: %O", meta); + return meta; +} diff --git a/packages/cmd/src/services/cache/index.ts b/packages/cmd/src/services/cache/index.ts new file mode 100644 index 0000000..c0718e0 --- /dev/null +++ b/packages/cmd/src/services/cache/index.ts @@ -0,0 +1,2 @@ +export * from "./get"; +export * from "./set"; diff --git a/packages/cmd/src/services/cache/lib.ts b/packages/cmd/src/services/cache/lib.ts new file mode 100644 index 0000000..fd81792 --- /dev/null +++ b/packages/cmd/src/services/cache/lib.ts @@ -0,0 +1,49 @@ +import path from "path"; +import { CacheSetCommandConfig } from "../../config/cache"; +import { warnWithNoTrace } from "../../logger"; +import { omit } from "lodash"; + +export type MetaFile = { + id: string; + orgId: string; + config: CacheSetCommandConfig; + paths: string[]; + ci: Record; + createdAt: string; +}; + +export function createMeta({ + config, + cacheId, + orgId, + paths, + ci, +}: { + config: CacheSetCommandConfig; + cacheId: string; + orgId: string; + paths: string[]; + ci: Record; +}) { + const meta = { + id: cacheId, + orgId, + config: omit(config, "recordKey"), + paths, + ci, + createdAt: new Date().toISOString(), + }; + + return Buffer.from(JSON.stringify(meta)); +} + +export const getLastRunFilePath = (output?: string) => + path.resolve(output ?? "test-results", ".last-run.json"); + +export function warn(error: unknown, msg: string) { + if (error instanceof Error) { + warnWithNoTrace("%s. %s.", msg, error.message); + } else { + warnWithNoTrace("%s. %s.", msg, "Unknown error"); + } +} diff --git a/packages/cmd/src/services/cache/network.ts b/packages/cmd/src/services/cache/network.ts new file mode 100644 index 0000000..c91dd30 --- /dev/null +++ b/packages/cmd/src/services/cache/network.ts @@ -0,0 +1,127 @@ +import retry from "async-retry"; +import { AxiosProgressEvent, isAxiosError, RawAxiosRequestConfig } from "axios"; +import { debug as _debug } from "../../debug"; +import { getAxios } from "../../http/axios"; +import { error, warn } from "../../logger"; + +const debug = _debug.extend("upload"); + +const UPLOAD_RETRY_COUNT = 5; + +export enum ContentType { + JSON = "application/json", + ZIP = "application/zip", +} + +export type BufferUpload = { + name: string | null; + buffer: Buffer; + uploadUrl: string; + contentType: string; +}; + +export async function sendBuffer( + upload: BufferUpload, + contentType: string, + onUploadProgress: RawAxiosRequestConfig["onUploadProgress"] +) { + debug("Uploading buffer %s", upload.name, { + buffer: Buffer.byteLength(upload.buffer), + }); + return send(upload.buffer, upload.uploadUrl, contentType, onUploadProgress); +} + +async function _send( + buffer: Buffer, + url: string, + contentType: string, + onUploadProgress: RawAxiosRequestConfig["onUploadProgress"] +) { + return getAxios().request({ + method: "put", + url, + data: buffer, + onUploadProgress, + headers: { + "Content-Disposition": `inline`, + "Content-Type": contentType, + }, + }); +} + +export async function download( + url: string, + onDownloadProgress?: RawAxiosRequestConfig["onDownloadProgress"] +): Promise { + try { + const response = await getAxios().get(url, { + responseType: "arraybuffer", + onDownloadProgress, + }); + + return Buffer.from(response.data); + } catch (error) { + if (isAxiosError(error)) { + debug("Failed to download %s: %s", url, error.message); + } + throw error; + } +} + +async function send(...args: Parameters) { + await retry( + async () => { + await _send(...args); + }, + { + retries: UPLOAD_RETRY_COUNT, + onRetry: (e: Error, retryCount: number) => { + debug( + "Upload failed %d out of %d attempts: %s", + retryCount, + UPLOAD_RETRY_COUNT, + e.message + ); + if (retryCount === UPLOAD_RETRY_COUNT) { + error(`Cannot upload after ${retryCount} times: ${e.message}`); + return; + } + warn( + `Upload failed ${retryCount} out of ${UPLOAD_RETRY_COUNT} attempts: ${e.message}` + ); + }, + } + ); +} + +export const getDefautUploadProgressHandler = + (label: string) => + ({ total, loaded }: AxiosProgressEvent) => { + () => { + debug( + "Uploading %s: %d / %d", + label, + bytesToMb(loaded), + bytesToMb(total ?? 0) + ); + }; + }; + +export const getDefaultDownloadProgressHandler = + (label: string) => + ({ loaded, total }: AxiosProgressEvent) => { + () => { + const percentCompleted = total ? Math.round((loaded * 100) / total) : 0; + debug( + "Downloaded %s: %d / %d (%d%)", + label, + bytesToMb(loaded), + bytesToMb(total ?? 0), + percentCompleted + ); + }; + }; + +function bytesToMb(bytes: number) { + return bytes / 1000 / 1000; +} diff --git a/packages/cmd/src/services/cache/presets.ts b/packages/cmd/src/services/cache/presets.ts new file mode 100644 index 0000000..3ec9381 --- /dev/null +++ b/packages/cmd/src/services/cache/presets.ts @@ -0,0 +1,106 @@ +import { debug } from "@debug"; +import _ from "lodash"; +import { PRESET_OUTPUT_PATH } from "../../commands/cache/options"; + +import { CacheGetCommandConfig } from "../../config/cache"; +import { getCI } from "../../env/ciProvider"; +import { GithubActionsParams, GitLabParams } from "../../env/types"; +import { writeFileAsync } from "../../lib"; +import { MetaFile } from "./lib"; + +export async function handlePreLastRunPreset( + config: CacheGetCommandConfig, + ci: ReturnType, +) { + switch (ci.provider) { + case "githubActions": + await dumpPWConfigForGHA(config, ci); + break; + case "gitlab": + await dumpPwConfigForGitlab(config, ci); + break; + default: + break; + } +} + +export async function handlePostLastRunPreset( + config: CacheGetCommandConfig, + ci: ReturnType, + meta: MetaFile, +) { + switch (ci.provider) { + case "gitlab": + await dumpPwConfigForGitlab(config, ci, meta); + break; + default: + break; + } +} + +async function dumpPwConfigForGitlab( + config: CacheGetCommandConfig, + ci: ReturnType, + meta: MetaFile | null = null, +) { + const ciParams = ci.params as GitLabParams; + const prevCiParams = meta?.ci.params as null | GitLabParams; + const prevRunAttempt = prevCiParams + ? parseIntSafe(prevCiParams.runAttempt, 0) + : 0; + const runAttempt = prevRunAttempt + 1; + const nodeIndex = parseIntSafe(ciParams.ciNodeIndex, 1); + const jobTotal = parseIntSafe(ciParams.ciNodeTotal, 1); + + const lastFailedOption = runAttempt > 1 ? "--last-failed" : ""; + + let shardOption = ""; + if (jobTotal > 1) { + shardOption = + runAttempt > 1 ? "--shard=1/1" : `--shard=${nodeIndex}/${jobTotal}`; + } + + const pwCliOptions = [lastFailedOption, shardOption] + .filter(Boolean) + .join(" "); + + await writeFileAsync( + config.presetOutput ?? PRESET_OUTPUT_PATH, + `EXTRA_PW_FLAGS="${pwCliOptions}" +EXTRA_PWCP_FLAGS="${lastFailedOption}" +RUN_ATTEMPT="${runAttempt}" +`, + ); +} + +async function dumpPWConfigForGHA( + config: CacheGetCommandConfig, + ci: ReturnType, +) { + const { matrixIndex, matrixTotal } = config; + const ciParams = ci.params as GithubActionsParams; + const runAttempt = parseIntSafe(ciParams.githubRunAttempt, 1); + + const lastFailedOption = runAttempt > 1 ? "--last-failed" : ""; + + let shardOption = ""; + if (matrixTotal > 1) { + shardOption = + runAttempt > 1 ? "--shard=1/1" : `--shard=${matrixIndex}/${matrixTotal}`; + } + + const pwCliOptions = [lastFailedOption, shardOption] + .filter(Boolean) + .join(" "); + const dumpPath = config.presetOutput ?? PRESET_OUTPUT_PATH; + await writeFileAsync(dumpPath, pwCliOptions); + debug('Dumped PW config: "%s" for GHA to %s', pwCliOptions, dumpPath); +} + +const parseIntSafe = ( + value: string | undefined, + defaultValue: number, +): number => { + const parsed = _.toNumber(value); + return _.isNaN(parsed) ? defaultValue : parsed; +}; diff --git a/packages/cmd/src/services/cache/set.ts b/packages/cmd/src/services/cache/set.ts new file mode 100644 index 0000000..0b80b74 --- /dev/null +++ b/packages/cmd/src/services/cache/set.ts @@ -0,0 +1,131 @@ +import { debug } from "@debug"; +import { createCache } from "../../api"; +import { PRESETS } from "../../commands/cache/options"; +import { getCacheCommandConfig } from "../../config/cache"; +import { getCI } from "../../env/ciProvider"; +import { filterPaths, zipFilesToBuffer } from "./fs"; +import { createMeta, getLastRunFilePath, warn } from "./lib"; +import { + ContentType, + getDefautUploadProgressHandler, + sendBuffer, +} from "./network"; + +export async function handleSetCache() { + try { + const config = getCacheCommandConfig(); + if (config.type !== "SET_COMMAND_CONFIG" || !config.values) { + throw new Error("Config is missing!"); + } + + const { recordKey, id, preset, pwOutputDir, matrixIndex, matrixTotal } = + config.values; + + const paths = config.values.paths ? filterPaths(config.values.paths) : []; + + const uploadPaths: string[] = []; + + if (paths && paths.length > 0) { + uploadPaths.push(...paths); + } + + const ci = getCI(); + + if (preset === PRESETS.lastRun) { + const lastRunPath = getLastRunFilePath(pwOutputDir); + uploadPaths.push(lastRunPath); + } + + if (uploadPaths.length === 0) { + throw new Error("No paths available to upload"); + } + + const result = await createCache({ + recordKey, + ci, + id, + config: { + matrixIndex, + matrixTotal, + }, + }); + + await handleArchiveUpload({ + archive: await zipFilesToBuffer(uploadPaths), + cacheId: result.cacheId, + uploadUrl: result.uploadUrl, + }); + + await handleMetaUpload({ + meta: createMeta({ + cacheId: result.cacheId, + config: config.values, + ci, + orgId: result.orgId, + paths: uploadPaths, + }), + cacheId: result.cacheId, + uploadUrl: result.metaUploadUrl, + }); + } catch (e) { + warn(e, "Failed to save cache"); + } +} + +async function handleArchiveUpload({ + archive, + cacheId, + uploadUrl, +}: { + archive: Buffer; + cacheId: string; + uploadUrl: string; +}) { + try { + const contentType = ContentType.ZIP; + await sendBuffer( + { + buffer: archive, + contentType, + name: cacheId, + uploadUrl, + }, + contentType, + getDefautUploadProgressHandler(cacheId) + ); + debug("Cache archive uploaded", { cacheId }); + } catch (error) { + debug("Failed to upload cache archive", error); + throw error; + } +} + +async function handleMetaUpload({ + meta, + cacheId, + uploadUrl, +}: { + meta: Buffer; + cacheId: string; + uploadUrl: string; +}) { + try { + const name = `${cacheId}_meta`; + const contentType = ContentType.JSON; + await sendBuffer( + { + buffer: meta, + contentType, + name, + uploadUrl, + }, + contentType, + getDefautUploadProgressHandler(name) + ); + + debug("Cache meta uploaded", { cacheId }); + } catch (error) { + debug("Failed to upload cache meta", error); + throw error; + } +} diff --git a/packages/cmd/src/services/index.ts b/packages/cmd/src/services/index.ts new file mode 100644 index 0000000..64d7b83 --- /dev/null +++ b/packages/cmd/src/services/index.ts @@ -0,0 +1,3 @@ +export { handleGetRun } from "./api"; +export { handleGetCache, handleSetCache } from "./cache"; +export { handleCurrentsReport } from "./upload"; diff --git a/packages/cmd/src/discovery/createScanner.ts b/packages/cmd/src/services/upload/discovery/createScanner.ts similarity index 100% rename from packages/cmd/src/discovery/createScanner.ts rename to packages/cmd/src/services/upload/discovery/createScanner.ts diff --git a/packages/cmd/src/discovery/index.ts b/packages/cmd/src/services/upload/discovery/index.ts similarity index 100% rename from packages/cmd/src/discovery/index.ts rename to packages/cmd/src/services/upload/discovery/index.ts diff --git a/packages/cmd/src/discovery/jest/args/args.ts b/packages/cmd/src/services/upload/discovery/jest/args/args.ts similarity index 100% rename from packages/cmd/src/discovery/jest/args/args.ts rename to packages/cmd/src/services/upload/discovery/jest/args/args.ts diff --git a/packages/cmd/src/discovery/jest/args/config.ts b/packages/cmd/src/services/upload/discovery/jest/args/config.ts similarity index 97% rename from packages/cmd/src/discovery/jest/args/config.ts rename to packages/cmd/src/services/upload/discovery/jest/args/config.ts index 1574f4b..01a43fb 100644 --- a/packages/cmd/src/discovery/jest/args/config.ts +++ b/packages/cmd/src/services/upload/discovery/jest/args/config.ts @@ -1,9 +1,9 @@ +import { debug as _debug } from "@debug"; +import { error } from "@logger"; import fs from "fs"; import { readInitialOptions } from "jest-config"; import { omit } from "lodash"; import path from "path"; -import { debug as _debug } from "../../../debug"; -import { error } from "../../../logger"; import { retryWithBackoff } from "../utils"; import { readFileContents } from "../utils/fs"; diff --git a/packages/cmd/src/discovery/jest/args/index.ts b/packages/cmd/src/services/upload/discovery/jest/args/index.ts similarity index 100% rename from packages/cmd/src/discovery/jest/args/index.ts rename to packages/cmd/src/services/upload/discovery/jest/args/index.ts diff --git a/packages/cmd/src/discovery/jest/index.ts b/packages/cmd/src/services/upload/discovery/jest/index.ts similarity index 100% rename from packages/cmd/src/discovery/jest/index.ts rename to packages/cmd/src/services/upload/discovery/jest/index.ts diff --git a/packages/cmd/src/discovery/jest/reporter.ts b/packages/cmd/src/services/upload/discovery/jest/reporter.ts similarity index 97% rename from packages/cmd/src/discovery/jest/reporter.ts rename to packages/cmd/src/services/upload/discovery/jest/reporter.ts index e16b72b..0188e69 100644 --- a/packages/cmd/src/discovery/jest/reporter.ts +++ b/packages/cmd/src/services/upload/discovery/jest/reporter.ts @@ -1,3 +1,4 @@ +import { debug as _debug } from "@debug"; import { AggregatedResult, Reporter, @@ -6,7 +7,6 @@ import { TestResult, } from "@jest/reporters"; import fs from "fs-extra"; -import { debug as _debug } from "../../debug"; import { getDefaultProjectId, getProjectId, @@ -16,7 +16,7 @@ import { testToSpecName, } from "./utils/test"; -import { dim, error } from "../../logger"; +import { dim, error } from "@logger"; import { FullSuiteProject, FullSuiteTest, FullTestSuite } from "../types"; const debug = _debug.extend("jest-discovery"); diff --git a/packages/cmd/src/discovery/jest/scanner.ts b/packages/cmd/src/services/upload/discovery/jest/scanner.ts similarity index 91% rename from packages/cmd/src/discovery/jest/scanner.ts rename to packages/cmd/src/services/upload/discovery/jest/scanner.ts index 19b923c..6277faa 100644 --- a/packages/cmd/src/discovery/jest/scanner.ts +++ b/packages/cmd/src/services/upload/discovery/jest/scanner.ts @@ -3,14 +3,14 @@ import fs from "fs-extra"; import { run } from "jest-cli"; import tmp from "tmp"; -import { debug as _debug } from "../../debug"; -import { readJsonFile } from "../../lib"; -import { dim, error } from "../../logger"; +import { debug as _debug } from "@debug"; +import { dim, error } from "@logger"; import { CLIArgs } from "../../types"; import { FullTestSuite } from "../types"; import { getCLIArgs } from "./args"; import { retryWithBackoff } from "./utils"; import { readFileContents } from "./utils/fs"; +import { readJsonFile } from "@lib"; const debug = _debug.extend("jest-discovery"); diff --git a/packages/cmd/src/discovery/jest/utils.ts b/packages/cmd/src/services/upload/discovery/jest/utils.ts similarity index 73% rename from packages/cmd/src/discovery/jest/utils.ts rename to packages/cmd/src/services/upload/discovery/jest/utils.ts index f886d5e..432707a 100644 --- a/packages/cmd/src/discovery/jest/utils.ts +++ b/packages/cmd/src/services/upload/discovery/jest/utils.ts @@ -1,6 +1,3 @@ -// import { debug } from "../debug"; - - export function retryWithBackoff( func: (...args: P) => T, backoffIntervals: number[] @@ -10,11 +7,8 @@ export function retryWithBackoff( let attempt = 0; const executeFunction = () => { attempt++; - // debug("Attempt %d to execute function with args: %o", attempt, args); - try { const result = func(...args); - // debug("Function with args %o executed at attempt %d", args, attempt); resolve(result); } catch (error) { if (attempt >= backoffIntervals.length) { @@ -26,7 +20,6 @@ export function retryWithBackoff( } }; - // Initial attempt to execute the function executeFunction(); }); }; diff --git a/packages/cmd/src/discovery/jest/utils/fs.ts b/packages/cmd/src/services/upload/discovery/jest/utils/fs.ts similarity index 100% rename from packages/cmd/src/discovery/jest/utils/fs.ts rename to packages/cmd/src/services/upload/discovery/jest/utils/fs.ts diff --git a/packages/cmd/src/discovery/jest/utils/index.ts b/packages/cmd/src/services/upload/discovery/jest/utils/index.ts similarity index 100% rename from packages/cmd/src/discovery/jest/utils/index.ts rename to packages/cmd/src/services/upload/discovery/jest/utils/index.ts diff --git a/packages/cmd/src/discovery/jest/utils/test.ts b/packages/cmd/src/services/upload/discovery/jest/utils/test.ts similarity index 100% rename from packages/cmd/src/discovery/jest/utils/test.ts rename to packages/cmd/src/services/upload/discovery/jest/utils/test.ts diff --git a/packages/cmd/src/discovery/scanner.ts b/packages/cmd/src/services/upload/discovery/scanner.ts similarity index 100% rename from packages/cmd/src/discovery/scanner.ts rename to packages/cmd/src/services/upload/discovery/scanner.ts diff --git a/packages/cmd/src/discovery/types.ts b/packages/cmd/src/services/upload/discovery/types.ts similarity index 99% rename from packages/cmd/src/discovery/types.ts rename to packages/cmd/src/services/upload/discovery/types.ts index 729220e..0cb101d 100644 --- a/packages/cmd/src/discovery/types.ts +++ b/packages/cmd/src/services/upload/discovery/types.ts @@ -12,5 +12,3 @@ export type FullSuiteTest = { tags: string[]; testId: string; }; - - diff --git a/packages/cmd/src/services/upload/fs.ts b/packages/cmd/src/services/upload/fs.ts new file mode 100644 index 0000000..bcd8edd --- /dev/null +++ b/packages/cmd/src/services/upload/fs.ts @@ -0,0 +1,80 @@ +import fs from "fs-extra"; +import path, { join, resolve } from "path"; +import { error } from "@logger"; +import { ReportOptions } from "./types"; + +export async function resolveReportOptions( + options?: ReportOptions +): Promise> { + const reportDir = await findReportDir(options?.reportDir); + + if (!reportDir) { + throw new Error("Failed to find the report dir"); + } + + return { + reportDir, + configFilePath: + options?.configFilePath ?? path.join(reportDir, "config.json"), + }; +} + +async function findReportDir(reportDir?: string): Promise { + if (reportDir) { + await checkPathExists(reportDir); + return reportDir; + } + + return getLastCreatedDirectory(join(process.cwd(), ".currents")); +} + +async function getLastCreatedDirectory(dir: string): Promise { + const entries = await fs.readdir(dir); + let latestDir: { name: string; birthtime: Date } | null = null; + + for (const entry of entries) { + const entryPath = path.join(dir, entry); + const stat = await fs.stat(entryPath); + + if (stat.isDirectory()) { + if (!latestDir || stat.birthtime > latestDir.birthtime) { + latestDir = { name: entry, birthtime: stat.birthtime }; + } + } + } + + return latestDir ? resolve(dir, latestDir.name) : null; +} + +export async function checkPathExists(path: string): Promise { + try { + const exists = await fs.pathExists(path); + return exists; + } catch (err) { + error("Error checking if path exists:", error); + return false; + } +} + +export async function getInstanceReportList(reportDir: string) { + const instancesDir = path.join(reportDir, "instances"); + return getAllFilePaths(instancesDir); +} + +async function getAllFilePaths(dir: string): Promise { + const files = await fs.readdir(dir); + const filePaths: string[] = []; + + for (const file of files) { + const filePath = path.join(dir, file); + const stat = await fs.stat(filePath); + + if (stat.isFile()) { + filePaths.push(filePath); + } + } + + return filePaths; +} + + diff --git a/packages/cmd/src/services/upload/index.ts b/packages/cmd/src/services/upload/index.ts new file mode 100644 index 0000000..a6267e6 --- /dev/null +++ b/packages/cmd/src/services/upload/index.ts @@ -0,0 +1,233 @@ +import { debug, setTraceFilePath } from "@debug"; +import { getCI } from "@env/ciProvider"; +import { getGitInfo } from "@env/gitInfo"; +import { getPlatformInfo } from "@env/platform"; +import { reporterVersion } from "@env/versions"; +import { nanoid, readJsonFile, writeFileAsync } from "@lib"; +import { info, warn } from "@logger"; +import { mapValues } from "lodash"; +import path from "path"; +import semver from "semver"; +import { + Framework, + RunCreationConfig, + createRun as createRunApi, +} from "../../api"; +import { getCurrentsConfig } from "../../config/upload"; +import { FullTestSuite, createScanner } from "./discovery"; +import { + checkPathExists, + getInstanceReportList, + resolveReportOptions, +} from "./fs"; +import { InstanceReport, ReportConfig, UploadMarkerInfo } from "./types"; + +export async function handleCurrentsReport() { + const currentsConfig = getCurrentsConfig(); + if (!currentsConfig) { + throw new Error("Currents config is missing!"); + } + + const reportOptions = await resolveReportOptions(currentsConfig); + // set the trace file path + setTraceFilePath(getTraceFilePath(reportOptions.reportDir)); + + debug("Reporter options: %o", reportOptions); + + info("Report directory: %s", reportOptions.reportDir); + + const config = await readJsonFile(reportOptions.configFilePath); + debug("Report config: %o", config); + + const instanceReportList = await getInstanceReportList( + reportOptions.reportDir + ); + debug( + "Found %d instance results in the reportDir: %s", + instanceReportList.length, + reportOptions.reportDir + ); + + const markerFilePath = getMarkerFilePath(reportOptions.reportDir); + const markerFileExists = await checkPathExists(markerFilePath); + + const fullTestSuiteFilePath = getFullTestSuiteFilePath( + reportOptions.reportDir + ); + + let fullTestSuite: FullTestSuite | null = null; + if (markerFileExists) { + const markerInfo = await readJsonFile(markerFilePath); + warn("Marker file detected. The report was already uploaded: %o", { + runUrl: markerInfo.response.runUrl, + isoDate: markerInfo.isoDate, + }); + + const fullTestSuiteFileExists = await checkPathExists( + fullTestSuiteFilePath + ); + + if (fullTestSuiteFileExists) { + fullTestSuite = await readJsonFile(fullTestSuiteFilePath); + debug("Full test suite file detected: %s", fullTestSuiteFilePath); + } + } + + if (!fullTestSuite) { + const scanner = createScanner(config); + fullTestSuite = await scanner.getFullTestSuite(); + + if (isEmptyTestSuite(fullTestSuite)) { + throw new Error("Failed to discover the full test suite!"); + } + + await writeFileAsync(fullTestSuiteFilePath, JSON.stringify(fullTestSuite)); + } else { + debug("The discovery stage was skipped"); + } + + const defaultGroup = + fullTestSuite.length === 1 ? fullTestSuite[0].name : null; + + const runCreationConfig: RunCreationConfig = { + currents: currentsConfig, + }; + + const framework: Framework = { + type: config.framework, + version: config.frameworkVersion, + clientVersion: reporterVersion, + }; + + const machineId = nanoid.userFacingNanoid(); + const ci = getCI(currentsConfig.ciBuildId); + + const instancesByGroup: Record = {}; + for await (const instanceReport of instanceReportList) { + const report = await readJsonFile(instanceReport); + if (!instancesByGroup[report.groupId]) { + instancesByGroup[report.groupId] = []; + } + instancesByGroup[report.groupId].push(report); + } + + for await (const key of Object.keys(instancesByGroup)) { + let instances = instancesByGroup[key]; + let group = key; + + if (defaultGroup) { + debug( + "Default group found: %s, overwriting the group in the results", + defaultGroup + ); + + group = defaultGroup; + instances = instances.map((i) => ({ + ...i, + groupId: defaultGroup, + })); + } + + try { + const response = await createRun({ + ci, + group, + instances, + fullTestSuite, + config: runCreationConfig, + machineId, + framework, + }); + + debug("Api response: %o", response); + + info("[%s] Run created: %s", group, response.runUrl); + + const markerInfo = { + response, + isoDate: new Date().toISOString(), + }; + + await writeFileAsync(markerFilePath, JSON.stringify(markerInfo)); + + debug( + "Marker file %s: %s", + markerFileExists ? "overwritten" : "created", + markerFilePath + ); + } catch (e) { + debug("Failed to upload the results to the dashboard"); + throw e; + } + } +} + +async function createRun({ + ci, + group, + instances, + fullTestSuite, + config, + machineId, + framework, +}: { + ci: ReturnType; + group: string; + instances: InstanceReport[]; + fullTestSuite: FullTestSuite; + config: RunCreationConfig; + machineId: string; + framework: Framework; +}) { + const commit = await getGitInfo(); + const platformInfo = await getPlatformInfo(); + const browserInfo = { + browserName: "node", + browserVersion: semver.coerce(process.version)?.version, + }; + + const { currents } = config; + + const platform = { ...browserInfo, ...platformInfo }; + const payload = { + platform, + ci, + commit, + group, + fullTestSuite, + config, + projectId: currents.projectId, + recordKey: currents.recordKey, + ciBuildId: ci.ciBuildId.value ?? undefined, + tags: currents.tag ?? [], + machineId: currents.machineId ?? machineId, + framework, + instances, + }; + + debug( + "Creating run: %o", + mapValues(payload, (v, k) => (k === "recordKey" ? "******" : v)) + ); + + return createRunApi(payload); +} + +function getMarkerFilePath(reportDir: string) { + return path.join(reportDir, "upload.marker.json"); +} + +function getFullTestSuiteFilePath(reportDir: string) { + return path.join(reportDir, "fullTestSuite.json"); +} + +function getTraceFilePath(reportDir: string) { + return path.join(reportDir, `.debug-${new Date().toISOString()}.log`); +} + +function isEmptyTestSuite(testSuite: FullTestSuite) { + return ( + testSuite.length === 0 || + testSuite.some((project) => project.tests.length === 0) + ); +} diff --git a/packages/cmd/src/types.ts b/packages/cmd/src/services/upload/types.ts similarity index 100% rename from packages/cmd/src/types.ts rename to packages/cmd/src/services/upload/types.ts diff --git a/packages/cmd/tsconfig.json b/packages/cmd/tsconfig.json index 8ea53e3..2df16da 100644 --- a/packages/cmd/tsconfig.json +++ b/packages/cmd/tsconfig.json @@ -1,4 +1,17 @@ { "extends": "../tsconfig.json", - "exclude": ["node_modules", "../../src/__tests__/*.ts", "dist"] + "exclude": ["node_modules", "../../src/__tests__/*.ts", "dist"], + "compilerOptions": { + "baseUrl": "src", + "paths": { + "@debug": ["debug"], + "@debug/*": ["debug/*"], + "@env": ["env"], + "@env/*": ["env/*"], + "@lib": ["lib"], + "@lib/*": ["lib/*"], + "@logger": ["logger"], + "@logger/*": ["logger/*"] + } + } } diff --git a/packages/cmd/tsup.config.ts b/packages/cmd/tsup.config.ts index b9cc803..15b6c1a 100644 --- a/packages/cmd/tsup.config.ts +++ b/packages/cmd/tsup.config.ts @@ -1,7 +1,11 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: ["src/index.ts", "src/bin/index.ts", "src/discovery/jest/reporter.ts"], + entry: [ + "src/index.ts", + "src/bin/index.ts", + "src/services/upload/discovery/jest/reporter.ts", + ], esbuildOptions: (options) => { options.legalComments = "linked"; }, diff --git a/packages/cmd/vitest.config.ts b/packages/cmd/vitest.config.ts new file mode 100644 index 0000000..c54968c --- /dev/null +++ b/packages/cmd/vitest.config.ts @@ -0,0 +1,14 @@ +import path from "path"; + +const config = { + resolve: { + alias: { + "@debug": path.resolve(__dirname, "./src/debug"), + "@env": path.resolve(__dirname, "./src/env"), + "@lib": path.resolve(__dirname, "./src/lib"), + "@logger": path.resolve(__dirname, "./src/logger"), + }, + }, +}; + +export default config; diff --git a/turbo.json b/turbo.json index 382ce48..d39ff6c 100644 --- a/turbo.json +++ b/turbo.json @@ -12,6 +12,7 @@ "DEBUG", "CURRENTS_RUN_URL_FILE", "CURRENTS_API_URL", + "CURRENTS_REST_API_URL", "CURRENTS_REMOTE_LOGGING", "TF_BUILD", "TF_BUILD_BUILDNUMBER",