diff --git a/package-lock.json b/package-lock.json index 7bf28b55..1f17f82c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,14 +1,15 @@ { "name": "@govtechsg/purple-hats", - "version": "0.10.7", + "version": "0.10.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@govtechsg/purple-hats", - "version": "0.10.7", + "version": "0.10.8", "license": "MIT", "dependencies": { + "@crawlee/core": "github:GovTechSG/crawlee-core", "@json2csv/node": "^7.0.3", "@napi-rs/canvas": "^0.1.53", "axe-core": "^4.9.1", @@ -27,6 +28,7 @@ "prettier": "^3.1.0", "print-message": "^3.0.1", "safe-regex": "^2.1.1", + "sync-request-curl": "^3.0.0", "typescript": "^5.4.5", "url": "^0.11.3", "validator": "^13.11.0", @@ -974,8 +976,8 @@ }, "node_modules/@crawlee/core": { "version": "3.8.1", - "resolved": "https://registry.npmjs.org/@crawlee/core/-/core-3.8.1.tgz", - "integrity": "sha512-bvA+U6xUH3vMnH0EkMkWxmeA3aC3gtDX8kj+OaLqGAYEWNxFO8mhmipwykzX4Oz5cDuoE05tEugMALp3LNDL+Q==", + "resolved": "git+ssh://git@github.com/GovTechSG/crawlee-core.git#464f08c915c18015a6e260ea50e0d2573f5419c6", + "license": "Apache-2.0", "dependencies": { "@apify/consts": "^2.20.0", "@apify/datastructures": "^2.0.0", @@ -2140,8 +2142,6 @@ "version": "1.0.11", "resolved": "https://registry.npmjs.org/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz", "integrity": "sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==", - "optional": true, - "peer": true, "dependencies": { "detect-libc": "^2.0.0", "https-proxy-agent": "^5.0.0", @@ -2161,8 +2161,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", - "optional": true, - "peer": true, "dependencies": { "semver": "^6.0.0" }, @@ -2177,18 +2175,14 @@ "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "optional": true, - "peer": true, "bin": { "semver": "bin/semver.js" } }, "node_modules/@mapbox/node-pre-gyp/node_modules/semver": { - "version": "7.6.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", - "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", - "optional": true, - "peer": true, + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", + "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", "bin": { "semver": "bin/semver.js" }, @@ -2215,6 +2209,21 @@ "@napi-rs/canvas-win32-x64-msvc": "0.1.53" } }, + "node_modules/@napi-rs/canvas-android-arm64": { + "version": "0.1.53", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-0.1.53.tgz", + "integrity": "sha512-2YhxfVsZguATlRWE0fZdTx35SE9+r5D7HV5GPNDataZOKmHf+zZ5//dspuuBSbOriQdoicaFrgXKCUqI0pK3WQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@napi-rs/canvas-darwin-arm64": { "version": "0.1.53", "resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-0.1.53.tgz", @@ -2245,6 +2254,21 @@ "node": ">= 10" } }, + "node_modules/@napi-rs/canvas-linux-arm-gnueabihf": { + "version": "0.1.53", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-0.1.53.tgz", + "integrity": "sha512-p9km/3C/loDxu3AvA8/vtpIS1BGMd/Ehkl2Iu/v/Gw8N/KUIt3HUvTS7AKApyVE28bxTfq96wJQjtcT8jzDncw==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@napi-rs/canvas-linux-arm64-gnu": { "version": "0.1.53", "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-0.1.53.tgz", @@ -2260,6 +2284,21 @@ "node": ">= 10" } }, + "node_modules/@napi-rs/canvas-linux-arm64-musl": { + "version": "0.1.53", + "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-0.1.53.tgz", + "integrity": "sha512-2N41U0X8RnrTKzpTtPv1ozlYkJtPsUdbfF3uP/KEd/BsULGd8Y8ghkGMS6CM+821au4ex0dPrWOOdT9wC1rSqQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@napi-rs/canvas-linux-x64-gnu": { "version": "0.1.53", "resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-0.1.53.tgz", @@ -2340,6 +2379,83 @@ "node": ">= 8" } }, + "node_modules/@npmcli/agent": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/@npmcli/agent/-/agent-2.2.2.tgz", + "integrity": "sha512-OrcNPXdpSl9UX7qPVRWbmWMCSXrcDa2M9DvrbOTj7ao1S4PlqVFYv9/yLKMkrJKZ/V5A/kDBC690or307i26Og==", + "dependencies": { + "agent-base": "^7.1.0", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.1", + "lru-cache": "^10.0.1", + "socks-proxy-agent": "^8.0.3" + }, + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/@npmcli/agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@npmcli/agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@npmcli/agent/node_modules/https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@npmcli/agent/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" + }, + "node_modules/@npmcli/fs": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-3.1.1.tgz", + "integrity": "sha512-q9CRWjpHCMIh5sVyefoD1cA7PkvILqCZsnSOEUUivORLjxCO/Irmue2DprETiNgEqktDBZaM1Bi+jrarx1XdCg==", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/@npmcli/fs/node_modules/semver": { + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", + "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -2823,9 +2939,7 @@ "node_modules/abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", - "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", - "optional": true, - "peer": true + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, "node_modules/acorn": { "version": "8.11.3", @@ -2883,6 +2997,18 @@ "node": ">= 6.0.0" } }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -2970,17 +3096,13 @@ "node_modules/aproba": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", - "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==", - "optional": true, - "peer": true + "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==" }, "node_modules/are-we-there-yet": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz", "integrity": "sha512-Ci/qENmwHnsYo9xKIcUJN5LeDKdJ6R1Z1j9V/J5wyq8nh/mYPEpIKJbBZXtZjG04HiK7zV/p6Vs9952MrMeUIw==", "deprecated": "This package is no longer supported.", - "optional": true, - "peer": true, "dependencies": { "delegates": "^1.0.0", "readable-stream": "^3.6.0" @@ -3387,6 +3509,33 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "node_modules/cacache": { + "version": "18.0.4", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-18.0.4.tgz", + "integrity": "sha512-B+L5iIa9mgcjLbliir2th36yEwPftrzteHYujzsx3dFP/31GCHcIeS8f5MGd80odLOjaOvSpU3EEAmRQptkxLQ==", + "dependencies": { + "@npmcli/fs": "^3.1.0", + "fs-minipass": "^3.0.0", + "glob": "^10.2.2", + "lru-cache": "^10.0.1", + "minipass": "^7.0.3", + "minipass-collect": "^2.0.1", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "p-map": "^4.0.0", + "ssri": "^10.0.0", + "tar": "^6.1.11", + "unique-filename": "^3.0.0" + }, + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/cacache/node_modules/lru-cache": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.4.3.tgz", + "integrity": "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==" + }, "node_modules/cacheable-lookup": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-7.0.0.tgz", @@ -3466,22 +3615,6 @@ } ] }, - "node_modules/canvas": { - "version": "2.11.2", - "resolved": "https://registry.npmjs.org/canvas/-/canvas-2.11.2.tgz", - "integrity": "sha512-ItanGBMrmRV7Py2Z+Xhs7cT+FNt5K0vPL4p9EZ/UX/Mu7hFbkxSjKF2KVtPwX7UYWp7dRKnrTvReflgrItJbdw==", - "hasInstallScript": true, - "optional": true, - "peer": true, - "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.0", - "nan": "^2.17.0", - "simple-get": "^3.0.3" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -3551,8 +3684,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", - "optional": true, - "peer": true, "engines": { "node": ">=10" } @@ -3578,6 +3709,14 @@ "integrity": "sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==", "dev": true }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "engines": { + "node": ">=6" + } + }, "node_modules/cli-cursor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", @@ -3699,8 +3838,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", - "optional": true, - "peer": true, "bin": { "color-support": "bin.js" } @@ -3752,9 +3889,7 @@ "node_modules/console-control-strings": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", - "optional": true, - "peer": true + "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==" }, "node_modules/content-type": { "version": "1.0.5", @@ -4060,16 +4195,12 @@ "node_modules/delegates": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", - "optional": true, - "peer": true + "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==" }, "node_modules/detect-libc": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", - "optional": true, - "peer": true, "engines": { "node": ">=8" } @@ -4214,6 +4345,21 @@ "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" }, + "node_modules/easy-libcurl": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/easy-libcurl/-/easy-libcurl-2.0.1.tgz", + "integrity": "sha512-m+5j8S8C4cCk+UIUA7TobGXoEI+vfBTCKsQ3MTQ+4eWczRF4CtHYgYd0MGuYP3em+ic2DtQ8YEcjwpWqbHP0Gg==", + "hasInstallScript": true, + "dependencies": { + "@mapbox/node-pre-gyp": "1.0.11", + "nan": "^2.18.0", + "node-gyp": "10.0.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.14" + } + }, "node_modules/ejs": { "version": "3.1.10", "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", @@ -4255,6 +4401,15 @@ "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, "node_modules/entities": { "version": "4.5.0", "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", @@ -4266,6 +4421,19 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "engines": { + "node": ">=6" + } + }, + "node_modules/err-code": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", + "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==" + }, "node_modules/error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -4908,6 +5076,11 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/exponential-backoff": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz", + "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw==" + }, "node_modules/external-editor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", @@ -5247,43 +5420,20 @@ } }, "node_modules/fs-minipass": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", - "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "optional": true, - "peer": true, - "dependencies": { - "minipass": "^3.0.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/fs-minipass/node_modules/minipass": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", - "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "optional": true, - "peer": true, + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-3.0.3.tgz", + "integrity": "sha512-XUBA9XClHbnJWSfBzjkm6RvPsyg3sryZt06BEQoXcF7EK/xpGaQYJgQKDJSUH5SGZ76Y7pFx1QBnXz09rU5Fbw==", "dependencies": { - "yallist": "^4.0.0" + "minipass": "^7.0.3" }, "engines": { - "node": ">=8" + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, - "node_modules/fs-minipass/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "optional": true, - "peer": true - }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "devOptional": true + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "node_modules/function-bind": { "version": "1.1.2", @@ -5325,8 +5475,6 @@ "resolved": "https://registry.npmjs.org/gauge/-/gauge-3.0.2.tgz", "integrity": "sha512-+5J6MS/5XksCuXq++uFRsnUd7Ovu1XenbeuIuNRJxYWjgQbPuFhT14lAvsWfqfAmnwluf1OwMjz39HjfLPci0Q==", "deprecated": "This package is no longer supported.", - "optional": true, - "peer": true, "dependencies": { "aproba": "^1.0.3 || ^2.0.0", "color-support": "^1.1.2", @@ -5345,9 +5493,7 @@ "node_modules/gauge/node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "optional": true, - "peer": true + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" }, "node_modules/generative-bayesian-network": { "version": "2.1.49", @@ -5716,9 +5862,7 @@ "node_modules/has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", - "optional": true, - "peer": true + "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==" }, "node_modules/hasown": { "version": "2.0.0", @@ -5913,16 +6057,22 @@ "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, "engines": { "node": ">=0.8.19" } }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "engines": { + "node": ">=8" + } + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "devOptional": true, "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -5983,6 +6133,23 @@ "node": ">= 0.4" } }, + "node_modules/ip-address": { + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-9.0.5.tgz", + "integrity": "sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==", + "dependencies": { + "jsbn": "1.1.0", + "sprintf-js": "^1.1.3" + }, + "engines": { + "node": ">= 12" + } + }, + "node_modules/ip-address/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==" + }, "node_modules/is-any-array": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-any-array/-/is-any-array-2.0.1.tgz", @@ -6121,6 +6288,11 @@ "node": ">=8" } }, + "node_modules/is-lambda": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz", + "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==" + }, "node_modules/is-negative-zero": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", @@ -7148,6 +7320,11 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha512-4bYVV3aAMtDTTu4+xsDYa6sy9GyJ69/amsu9sYF2zqjiEoZA5xJi3BrfX3uY+/IekIu7MwdObdbDWpoZdBv3/A==" + }, "node_modules/jsdom": { "version": "21.1.2", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-21.1.2.tgz", @@ -7473,6 +7650,36 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true }, + "node_modules/make-fetch-happen": { + "version": "13.0.1", + "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-13.0.1.tgz", + "integrity": "sha512-cKTUFc/rbKUd/9meOvgrpJ2WrNzymt6jfRDdwg5UCnVzv9dTpEj9JS5m3wtziXVCjluIXyL8pcaukYqezIzZQA==", + "dependencies": { + "@npmcli/agent": "^2.0.0", + "cacache": "^18.0.0", + "http-cache-semantics": "^4.1.1", + "is-lambda": "^1.0.1", + "minipass": "^7.0.2", + "minipass-fetch": "^3.0.0", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^0.6.3", + "proc-log": "^4.2.0", + "promise-retry": "^2.0.1", + "ssri": "^10.0.0" + }, + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/make-fetch-happen/node_modules/proc-log": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-4.2.0.tgz", + "integrity": "sha512-g8+OnU/L2v+wyiVK+D5fA34J7EH8jZ8DDlvwhRCMxmMj7UCBvxiO1mGeN+36JXIKF4zevU4kRBd8lVgG9vLelA==", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, "node_modules/makeerror": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", @@ -7584,12 +7791,118 @@ "node": ">=16 || 14 >=14.17" } }, + "node_modules/minipass-collect": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-2.0.1.tgz", + "integrity": "sha512-D7V8PO9oaz7PWGLbCACuI1qEOsq7UKfLotx/C0Aet43fCUB/wfQ7DYeq2oR/svFJGYDHPr38SHATeaj/ZoKHKw==", + "dependencies": { + "minipass": "^7.0.3" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/minipass-fetch": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-3.0.5.tgz", + "integrity": "sha512-2N8elDQAtSnFV0Dk7gt15KHsS0Fyz6CbYZ360h0WTYV1Ty46li3rAXVOQj1THMNLdmrD9Vt5pBPtWtVkpwGBqg==", + "dependencies": { + "minipass": "^7.0.3", + "minipass-sized": "^1.0.3", + "minizlib": "^2.1.2" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + }, + "optionalDependencies": { + "encoding": "^0.1.13" + } + }, + "node_modules/minipass-flush": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", + "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/minipass-flush/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-flush/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/minipass-pipeline": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", + "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-pipeline/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-pipeline/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, + "node_modules/minipass-sized": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz", + "integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/minipass-sized/node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" + }, "node_modules/minizlib": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", - "optional": true, - "peer": true, "dependencies": { "minipass": "^3.0.0", "yallist": "^4.0.0" @@ -7602,8 +7915,6 @@ "version": "3.3.6", "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "optional": true, - "peer": true, "dependencies": { "yallist": "^4.0.0" }, @@ -7614,16 +7925,12 @@ "node_modules/minizlib/node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "optional": true, - "peer": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/mkdirp": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "optional": true, - "peer": true, "bin": { "mkdirp": "bin/cmd.js" }, @@ -7690,9 +7997,7 @@ "node_modules/nan": { "version": "2.20.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.20.0.tgz", - "integrity": "sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==", - "optional": true, - "peer": true + "integrity": "sha512-bk3gXBZDGILuuo/6sKtr0DQmSThYHLtNCdSdXk9YkxD/jK6X2vmCyyXBBxyqZ4XcnzTyYEAThfX3DCEnLf6igw==" }, "node_modules/nanoid": { "version": "3.3.7", @@ -7717,12 +8022,18 @@ "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", "dev": true }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/node-fetch": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "optional": true, - "peer": true, "dependencies": { "whatwg-url": "^5.0.0" }, @@ -7741,28 +8052,78 @@ "node_modules/node-fetch/node_modules/tr46": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "optional": true, - "peer": true + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, "node_modules/node-fetch/node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "optional": true, - "peer": true + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, "node_modules/node-fetch/node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "optional": true, - "peer": true, "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, + "node_modules/node-gyp": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-10.0.1.tgz", + "integrity": "sha512-gg3/bHehQfZivQVfqIyy8wTdSymF9yTyP4CJifK73imyNMU8AIGQE2pUa7dNWfmMeG9cDVF2eehiRMv0LC1iAg==", + "dependencies": { + "env-paths": "^2.2.0", + "exponential-backoff": "^3.1.1", + "glob": "^10.3.10", + "graceful-fs": "^4.2.6", + "make-fetch-happen": "^13.0.0", + "nopt": "^7.0.0", + "proc-log": "^3.0.0", + "semver": "^7.3.5", + "tar": "^6.1.2", + "which": "^4.0.0" + }, + "bin": { + "node-gyp": "bin/node-gyp.js" + }, + "engines": { + "node": "^16.14.0 || >=18.0.0" + } + }, + "node_modules/node-gyp/node_modules/abbrev": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-2.0.0.tgz", + "integrity": "sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/node-gyp/node_modules/nopt": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-7.2.1.tgz", + "integrity": "sha512-taM24ViiimT/XntxbPyJQzCG+p4EKOpgD3mxFwW38mGjVUrfERQOeY4EDHjdnptttfHuHQXFx+lTP08Q+mLa/w==", + "dependencies": { + "abbrev": "^2.0.0" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/node-gyp/node_modules/semver": { + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", + "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/node-int64": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", @@ -7778,8 +8139,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", - "optional": true, - "peer": true, "dependencies": { "abbrev": "1" }, @@ -7827,8 +8186,6 @@ "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-5.0.1.tgz", "integrity": "sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==", "deprecated": "This package is no longer supported.", - "optional": true, - "peer": true, "dependencies": { "are-we-there-yet": "^2.0.0", "console-control-strings": "^1.1.0", @@ -7856,8 +8213,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "optional": true, - "peer": true, "engines": { "node": ">=0.10.0" } @@ -7961,7 +8316,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "devOptional": true, "dependencies": { "wrappy": "1" } @@ -8112,6 +8466,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -8193,7 +8561,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -8454,6 +8821,26 @@ "node": ">=8.0.0" } }, + "node_modules/proc-log": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/proc-log/-/proc-log-3.0.0.tgz", + "integrity": "sha512-++Vn7NS4Xf9NacaU9Xq3URUuqZETPsf8L4j5/ckhaRYsfPeRyzGw+iDjFhV/Jr3uNmTvvddEJFWh5R1gRgUH8A==", + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/promise-retry": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", + "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", + "dependencies": { + "err-code": "^2.0.2", + "retry": "^0.12.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/prompts": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", @@ -8741,7 +9128,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "devOptional": true, "dependencies": { "glob": "^7.1.3" }, @@ -8756,7 +9142,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "devOptional": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -8766,7 +9151,6 @@ "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "devOptional": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -8786,7 +9170,6 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "devOptional": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -8946,9 +9329,7 @@ "node_modules/set-blocking": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", - "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", - "optional": true, - "peer": true + "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" }, "node_modules/set-function-length": { "version": "1.2.2", @@ -9027,65 +9408,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "optional": true, - "peer": true - }, - "node_modules/simple-get": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.1.tgz", - "integrity": "sha512-CQ5LTKGfCpvE1K0n2us+kuMPbk/q0EKl82s4aheV9oXjFEz6W/Y7oQFVJuU6QG77hRT4Ghb5RURteF5vnWjupA==", - "optional": true, - "peer": true, - "dependencies": { - "decompress-response": "^4.2.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, - "node_modules/simple-get/node_modules/decompress-response": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", - "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", - "optional": true, - "peer": true, - "dependencies": { - "mimic-response": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/simple-get/node_modules/mimic-response": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", - "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", - "optional": true, - "peer": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/simple-swizzle": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", @@ -9114,6 +9436,52 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", + "dependencies": { + "ip-address": "^9.0.5", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz", + "integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==", + "dependencies": { + "agent-base": "^7.1.1", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/socks-proxy-agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -9150,6 +9518,17 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, + "node_modules/ssri": { + "version": "10.0.6", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-10.0.6.tgz", + "integrity": "sha512-MGrFH9Z4NP9Iyhqn16sDtBpRRNJ0Y2hNa6D65h736fVSaPCHr4DM4sWUNvVaSuC+0OBGhwsrydQwmgfg5LncqQ==", + "dependencies": { + "minipass": "^7.0.3" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, "node_modules/stack-trace": { "version": "0.0.10", "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", @@ -9382,6 +9761,14 @@ "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" }, + "node_modules/sync-request-curl": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/sync-request-curl/-/sync-request-curl-3.0.0.tgz", + "integrity": "sha512-d7Frl/Kbwa/qKvJ4CDmJ2m8MoYkyofpcyM2HZ2BR0qEKHW07jilC39O1z2GxMULVUCOSW4GKkX4EN1vvhI3KkQ==", + "dependencies": { + "easy-libcurl": "^2.0.1" + } + }, "node_modules/synckit": { "version": "0.8.8", "resolved": "https://registry.npmjs.org/synckit/-/synckit-0.8.8.tgz", @@ -9402,8 +9789,6 @@ "version": "6.2.1", "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==", - "optional": true, - "peer": true, "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", @@ -9416,12 +9801,32 @@ "node": ">=10" } }, + "node_modules/tar/node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/tar/node_modules/fs-minipass/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/tar/node_modules/minipass": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/minipass/-/minipass-5.0.0.tgz", "integrity": "sha512-3FnjYuehv9k6ovOEbyOswadCDPX1piCfhV8ncmYtHOjuPwylVWsghTLo7rabjC3Rx5xD4HDx8Wm1xnMF7S5qFQ==", - "optional": true, - "peer": true, "engines": { "node": ">=8" } @@ -9429,9 +9834,7 @@ "node_modules/tar/node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "optional": true, - "peer": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/test-exclude": { "version": "6.0.0", @@ -9882,6 +10285,28 @@ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" }, + "node_modules/unique-filename": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-3.0.0.tgz", + "integrity": "sha512-afXhuC55wkAmZ0P18QsVE6kp8JaxrEokN2HGIoIVv2ijHQd419H0+6EigAFcIzXeMIkcIkNBpB3L/DXB3cTS/g==", + "dependencies": { + "unique-slug": "^4.0.0" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, + "node_modules/unique-slug": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-4.0.0.tgz", + "integrity": "sha512-WrcA6AyEfqDX5bWige/4NQfPZMtASNVxdmWR76WESYQVAACSgWcR6e9i0mofqqBxYFtL4oAxPIptY73/0YE1DQ==", + "dependencies": { + "imurmurhash": "^0.1.4" + }, + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0" + } + }, "node_modules/universalify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", @@ -10114,8 +10539,6 @@ "version": "1.1.5", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", - "optional": true, - "peer": true, "dependencies": { "string-width": "^1.0.2 || 2 || 3 || 4" } @@ -10187,8 +10610,7 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "devOptional": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/write-file-atomic": { "version": "4.0.2", diff --git a/package.json b/package.json index 0705893f..3085af6c 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,13 @@ { "name": "@govtechsg/purple-hats", "main": "dist/npmIndex.js", - "version": "0.10.7", + "version": "0.10.8", "type": "module", "imports": { "#root/*.js": "./dist/*.js" }, "dependencies": { + "@crawlee/core": "github:GovTechSG/crawlee-core", "@json2csv/node": "^7.0.3", "@napi-rs/canvas": "^0.1.53", "axe-core": "^4.9.1", @@ -25,6 +26,7 @@ "prettier": "^3.1.0", "print-message": "^3.0.1", "safe-regex": "^2.1.1", + "sync-request-curl": "^3.0.0", "typescript": "^5.4.5", "url": "^0.11.3", "validator": "^13.11.0", diff --git a/src/cli.ts b/src/cli.ts index 42b92d3f..6083c95f 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -19,6 +19,7 @@ import { validateDirPath, validateFilePath, validateCustomFlowLabel, + parseHeaders, } from './constants/common.js'; import constants, { ScannerTypes } from './constants/constants.js'; import { cliOptions, messageOptions } from './constants/cliFunctions.js'; @@ -178,27 +179,13 @@ Usage: npm run cli -- -c -d -w -u OPTIONS`, return option; }) .coerce('m', option => { - const headerValues = option.split(', '); - const allHeaders = {}; - - headerValues.map((headerValue: string) => { - const headerValuePair = headerValue.split(/ (.*)/s); - if (headerValuePair.length < 2) { - printMessage( - [ - `Invalid value for authorisation request header. Please provide valid keywords in the format: "
". For multiple authentication headers, please provide the keywords in the format: "
, , ..." .`, - ], - messageOptions, - ); - process.exit(1); - } - allHeaders[headerValuePair[0]] = headerValuePair[1]; // {"header": "value", "header2": "value2", ...} - }); - - return allHeaders; + return parseHeaders(option); }) .check(argvs => { - if ((argvs.scanner === ScannerTypes.CUSTOM || argvs.scanner === ScannerTypes.LOCALFILE) && argvs.maxpages) { + if ( + (argvs.scanner === ScannerTypes.CUSTOM || argvs.scanner === ScannerTypes.LOCALFILE) && + argvs.maxpages + ) { throw new Error('-p or --maxpages is only available in website and sitemap scans.'); } return true; diff --git a/src/combine.ts b/src/combine.ts index 1dbdead3..3f2cfb6c 100644 --- a/src/combine.ts +++ b/src/combine.ts @@ -5,14 +5,13 @@ import crawlLocalFile from './crawlers/crawlLocalFile.js'; import crawlIntelligentSitemap from './crawlers/crawlIntelligentSitemap.js'; import { generateArtifacts } from './mergeAxeResults.js'; import { getHost, createAndUpdateResultsFolders, createDetailsAndLogs } from './utils.js'; -import { ScannerTypes,UrlsCrawled} from './constants/constants.js'; +import { ScannerTypes, UrlsCrawled } from './constants/constants.js'; import { getBlackListedPatterns, submitForm, urlWithoutAuth } from './constants/common.js'; import { consoleLogger, silentLogger } from './logs.js'; import runCustom from './crawlers/runCustom.js'; import { alertMessageOptions } from './constants/cliFunctions.js'; import { Data } from './index.js'; -import { fileURLToPath, pathToFileURL } from 'url'; - +import { pathToFileURL } from 'url'; // Class exports export class ViewportSettingsClass { @@ -21,7 +20,12 @@ export class ViewportSettingsClass { viewportWidth: number; playwrightDeviceDetailsObject: any; // You can replace 'any' with a more specific type if possible - constructor(deviceChosen: string, customDevice: string, viewportWidth: number, playwrightDeviceDetailsObject: any) { + constructor( + deviceChosen: string, + customDevice: string, + viewportWidth: number, + playwrightDeviceDetailsObject: any, + ) { this.deviceChosen = deviceChosen; this.customDevice = customDevice; this.viewportWidth = viewportWidth; @@ -29,8 +33,7 @@ export class ViewportSettingsClass { } } - -const combineRun = async (details:Data, deviceToScan:string) => { +const combineRun = async (details: Data, deviceToScan: string) => { const envDetails = { ...details }; const { @@ -57,18 +60,15 @@ const combineRun = async (details:Data, deviceToScan:string) => { customFlowLabel = 'Custom Flow', extraHTTPHeaders, safeMode, - zip + zip, } = envDetails; process.env.CRAWLEE_LOG_LEVEL = 'ERROR'; process.env.CRAWLEE_STORAGE_DIR = randomToken; - const host = - (type === ScannerTypes.SITEMAP || type === ScannerTypes.LOCALFILE) - ? '' - : getHost(url); + const host = type === ScannerTypes.SITEMAP || type === ScannerTypes.LOCALFILE ? '' : getHost(url); - let blacklistedPatterns:string[] | null = null; + let blacklistedPatterns: string[] | null = null; try { blacklistedPatterns = getBlackListedPatterns(blacklistedPatternsFilename); } catch (error) { @@ -78,8 +78,10 @@ const combineRun = async (details:Data, deviceToScan:string) => { } // remove basic-auth credentials from URL - let finalUrl = (!(type === ScannerTypes.SITEMAP|| type === ScannerTypes.LOCALFILE)) ? urlWithoutAuth(url) : new URL(pathToFileURL(url)); - + let finalUrl = !(type === ScannerTypes.SITEMAP || type === ScannerTypes.LOCALFILE) + ? urlWithoutAuth(url) + : new URL(pathToFileURL(url)); + //Use the string version of finalUrl to reduce logic at submitForm let finalUrlString = finalUrl.toString(); @@ -91,14 +93,14 @@ const combineRun = async (details:Data, deviceToScan:string) => { urlsCrawled: new UrlsCrawled(), }; - const viewportSettings:ViewportSettingsClass = new ViewportSettingsClass( + const viewportSettings: ViewportSettingsClass = new ViewportSettingsClass( deviceChosen, customDevice, viewportWidth, playwrightDeviceDetailsObject, ); - let urlsCrawledObj; + let urlsCrawledObj: UrlsCrawled; switch (type) { case ScannerTypes.CUSTOM: urlsCrawledObj = await runCustom( @@ -127,22 +129,22 @@ const combineRun = async (details:Data, deviceToScan:string) => { ); break; - case ScannerTypes.LOCALFILE: - urlsCrawledObj = await crawlLocalFile( - url, - randomToken, - host, - viewportSettings, - maxRequestsPerCrawl, - browser, - userDataDirectory, - specifiedMaxConcurrency, - fileTypes, - blacklistedPatterns, - includeScreenshots, - extraHTTPHeaders, - ); - break; + case ScannerTypes.LOCALFILE: + urlsCrawledObj = await crawlLocalFile( + url, + randomToken, + host, + viewportSettings, + maxRequestsPerCrawl, + browser, + userDataDirectory, + specifiedMaxConcurrency, + fileTypes, + blacklistedPatterns, + includeScreenshots, + extraHTTPHeaders, + ); + break; case ScannerTypes.INTELLIGENT: urlsCrawledObj = await crawlIntelligentSitemap( @@ -194,43 +196,43 @@ const combineRun = async (details:Data, deviceToScan:string) => { scanDetails.urlsCrawled = urlsCrawledObj; await createDetailsAndLogs(randomToken); if (scanDetails.urlsCrawled) { - if (scanDetails.urlsCrawled.scanned.length > 0) { - await createAndUpdateResultsFolders(randomToken); - const pagesNotScanned = [ - ...urlsCrawledObj.error, - ...urlsCrawledObj.invalid, - ...urlsCrawledObj.forbidden, - ]; - const basicFormHTMLSnippet = await generateArtifacts( - randomToken, - url, - type, - deviceToScan, - urlsCrawledObj.scanned, - pagesNotScanned, - customFlowLabel, - undefined, - scanDetails, - zip - ); - const [name, email] = nameEmail.split(':'); - - await submitForm( - browser, - userDataDirectory, - url, // scannedUrl - new URL(finalUrlString).href, //entryUrl - type, - email, - name, - JSON.stringify(basicFormHTMLSnippet), - urlsCrawledObj.scanned.length, - urlsCrawledObj.scannedRedirects.length, - pagesNotScanned.length, - metadata, - ); - } -}else { + if (scanDetails.urlsCrawled.scanned.length > 0) { + await createAndUpdateResultsFolders(randomToken); + const pagesNotScanned = [ + ...urlsCrawledObj.error, + ...urlsCrawledObj.invalid, + ...urlsCrawledObj.forbidden, + ]; + const basicFormHTMLSnippet = await generateArtifacts( + randomToken, + url, + type, + deviceToScan, + urlsCrawledObj.scanned, + pagesNotScanned, + customFlowLabel, + undefined, + scanDetails, + zip, + ); + const [name, email] = nameEmail.split(':'); + + await submitForm( + browser, + userDataDirectory, + url, // scannedUrl + new URL(finalUrlString).href, //entryUrl + type, + email, + name, + JSON.stringify(basicFormHTMLSnippet), + urlsCrawledObj.scanned.length, + urlsCrawledObj.scannedRedirects.length, + pagesNotScanned.length, + metadata, + ); + } + } else { printMessage([`No pages were scanned.`], alertMessageOptions); } }; diff --git a/src/constants/common.ts b/src/constants/common.ts index 92fc77e3..8871b1f3 100644 --- a/src/constants/common.ts +++ b/src/constants/common.ts @@ -6,7 +6,7 @@ import validator from 'validator'; import axios from 'axios'; import { JSDOM } from 'jsdom'; import * as cheerio from 'cheerio'; -import crawlee, { Request } from 'crawlee'; +import crawlee, { EnqueueStrategy, Request } from 'crawlee'; import { parseString } from 'xml2js'; import fs from 'fs'; import path from 'path'; @@ -53,9 +53,9 @@ export const validateDirPath = (dirPath: string): string => { }; export class RES { - status: number - url: string - content: string + status: number; + url: string; + content: string; constructor(res?: Partial) { if (res) { Object.assign(this, res); @@ -120,7 +120,9 @@ export const validateFilePath = (filePath: string, cliDir: string) => { } }; -export const getBlackListedPatterns = (blacklistedPatternsFilename: string|null): string[] | null=> { +export const getBlackListedPatterns = ( + blacklistedPatternsFilename: string | null, +): string[] | null => { let exclusionsFile = null; if (blacklistedPatternsFilename) { exclusionsFile = blacklistedPatternsFilename; @@ -233,7 +235,7 @@ export const getFileSitemap = (filePath: string): string | null => { const file = fs.readFileSync(filePath, 'utf8'); const isLocalFileScan = isSitemapContent(file); - return isLocalFileScan || (file != undefined) ? filePath : null; + return isLocalFileScan || file != undefined ? filePath : null; }; export const getUrlMessage = (scanner: ScannerTypes): string => { @@ -272,7 +274,11 @@ export const sanitizeUrlInput = (url: string): { isValid: boolean; url: string } } }; -const requestToUrl = async (url, isCustomFlow, extraHTTPHeaders) => { +const requestToUrl = async ( + url: string, + isCustomFlow: boolean, + extraHTTPHeaders: Record, +) => { // User-Agent is modified to emulate a browser to handle cases where some sites ban non browser agents, resulting in a 403 error const res = new RES(); const parsedUrl = new URL(url); @@ -531,14 +537,14 @@ export const checkUrl = async ( } if ( - res.status === constants.urlCheckStatuses.success.code && + res.status === constants.urlCheckStatuses.success.code && (scanner === ScannerTypes.SITEMAP || scanner === ScannerTypes.LOCALFILE) -) { + ) { const isSitemap = isSitemapContent(res.content); if (!isSitemap && scanner === ScannerTypes.LOCALFILE) { res.status = constants.urlCheckStatuses.notALocalFile.code; - } else if (!isSitemap){ + } else if (!isSitemap) { res.status = constants.urlCheckStatuses.notASitemap.code; } } @@ -547,6 +553,27 @@ export const checkUrl = async ( const isEmptyObject = (obj: Object): boolean => !Object.keys(obj).length; +export const parseHeaders = (header?: string): Record => { + // parse HTTP headers from string + if (!header) return {}; + const headerValues = header.split(', '); + const allHeaders = {}; + headerValues.map((headerValue: string) => { + const headerValuePair = headerValue.split(/ (.*)/s); + if (headerValuePair.length < 2) { + printMessage( + [ + `Invalid value for authorisation request header. Please provide valid keywords in the format: "
". For multiple authentication headers, please provide the keywords in the format: "
, , ..." .`, + ], + messageOptions, + ); + process.exit(1); + } + allHeaders[headerValuePair[0]] = headerValuePair[1]; // {"header": "value", "header2": "value2", ...} + }); + return allHeaders; +}; + export const prepareData = async (argv: Answers): Promise => { if (isEmptyObject(argv)) { throw Error('No inputs should be provided'); @@ -574,7 +601,7 @@ export const prepareData = async (argv: Answers): Promise => { followRobots, header, safeMode, - zip + zip, } = argv; // construct filename for scan results @@ -604,7 +631,8 @@ export const prepareData = async (argv: Answers): Promise => { viewportWidth, playwrightDeviceDetailsObject, maxRequestsPerCrawl: maxpages || constants.maxRequestsPerCrawl, - strategy, + strategy: + strategy === 'same-hostname' ? EnqueueStrategy.SameHostname : EnqueueStrategy.SameDomain, isLocalFileScan, browser: browserToRun, nameEmail, @@ -616,9 +644,9 @@ export const prepareData = async (argv: Answers): Promise => { includeScreenshots: !(additional === 'none'), metadata, followRobots, - extraHTTPHeaders: header, + extraHTTPHeaders: parseHeaders(header), safeMode, - zip + zip, }; }; @@ -629,7 +657,7 @@ export const getUrlsFromRobotsTxt = async (url: string, browserToRun: string): P if (constants.robotsTxtUrls[domain]) return; const robotsUrl = domain.concat('/robots.txt'); - let robotsTxt :string ; + let robotsTxt: string; try { if (proxy) { robotsTxt = await getRobotsTxtViaPlaywright(robotsUrl, browserToRun); @@ -639,7 +667,7 @@ export const getUrlsFromRobotsTxt = async (url: string, browserToRun: string): P } catch (e) { silentLogger.info(e); } - console.log("robotsTxt",robotsTxt) + console.log('robotsTxt', robotsTxt); if (!robotsTxt) { constants.robotsTxtUrls[domain] = {}; return; @@ -714,7 +742,7 @@ const getRobotsTxtViaAxios = async (robotsUrl: string): Promise => { }), }); - const robotsTxt = await (await instance.get(robotsUrl, { timeout: 2000 })).data as string; + const robotsTxt = (await (await instance.get(robotsUrl, { timeout: 2000 })).data) as string; return robotsTxt; }; @@ -766,14 +794,14 @@ export const getLinksFromSitemap = async ( ? (url = addBasicAuthCredentials(url, username, password)) : url; - url = convertPathToLocalFile(url); + url = convertPathToLocalFile(url); - let request; - try { - request = new Request({ url: url }); - } catch (e) { - console.log('Error creating request', e); - } + let request; + try { + request = new Request({ url: url }); + } catch (e) { + console.log('Error creating request', e); + } if (isUrlPdf(url)) { request.skipNavigation = true; } @@ -862,36 +890,36 @@ export const getLinksFromSitemap = async ( let parsedUrl; - if (scannedSitemaps.has(url)) { - // Skip processing if the sitemap has already been scanned - return; - } + if (scannedSitemaps.has(url)) { + // Skip processing if the sitemap has already been scanned + return; + } - scannedSitemaps.add(url); + scannedSitemaps.add(url); - // Convert file if its not local file path - url = convertLocalFileToPath(url) + // Convert file if its not local file path + url = convertLocalFileToPath(url); - // Check whether its a file path or a URL - if (isFilePath(url)) { - if (!fs.existsSync(url)) { - return; - } - parsedUrl = url; - } else if(isValidHttpUrl(url)){ - parsedUrl = new URL(url); - - if (parsedUrl.username !== '' && parsedUrl.password !== '') { - isBasicAuth = true; - username = decodeURIComponent(parsedUrl.username); - password = decodeURIComponent(parsedUrl.password); - parsedUrl.username = ''; - parsedUrl.password = ''; - } - } else{ + // Check whether its a file path or a URL + if (isFilePath(url)) { + if (!fs.existsSync(url)) { + return; + } + parsedUrl = url; + } else if (isValidHttpUrl(url)) { + parsedUrl = new URL(url); + + if (parsedUrl.username !== '' && parsedUrl.password !== '') { + isBasicAuth = true; + username = decodeURIComponent(parsedUrl.username); + password = decodeURIComponent(parsedUrl.password); + parsedUrl.username = ''; + parsedUrl.password = ''; + } + } else { printMessage([`Invalid Url/Filepath: ${url}`], messageOptions); return; - } + } const getDataUsingPlaywright = async () => { const browserContext = await constants.launcher.launchPersistentContext( @@ -946,9 +974,9 @@ export const getLinksFromSitemap = async ( password: password, }, }); - try{ - data = await (await instance.get(url, { timeout: 80000 })).data; - } catch(error){ + try { + data = await (await instance.get(url, { timeout: 80000 })).data; + } catch (error) { return; //to skip the error } } catch (error) { @@ -1027,7 +1055,7 @@ export const getLinksFromSitemap = async ( } const requestList = Object.values(urls); - + return requestList; }; @@ -1073,7 +1101,6 @@ export const getBrowserToRun = ( preferredBrowser: BrowserTypes, isCli = false, ): { browserToRun: BrowserTypes; clonedBrowserDataDir: string } => { - const platform = os.platform(); // Prioritise Chrome on Windows and Mac platforms if user does not specify a browser @@ -1345,10 +1372,9 @@ const cloneLocalStateFile = (options, destDir) => { }); const profileNamesRegex = /([^/\\]+)[/\\]Local State$/; - if (localState.length > 0) { let success = true; - + localState.forEach(dir => { const profileName = dir.match(profileNamesRegex)[1]; try { @@ -1772,20 +1798,20 @@ export const getPlaywrightLaunchOptions = (browser?: string): LaunchOptions => { return options; }; -export const urlWithoutAuth = (url: string): URL => { +export const urlWithoutAuth = (url: string): string => { const parsedUrl = new URL(url); parsedUrl.username = ''; parsedUrl.password = ''; - return parsedUrl; + return parsedUrl.toString(); }; export const waitForPageLoaded = async (page, timeout = 10000) => { return Promise.race([ - page.waitForLoadState('load'), - page.waitForLoadState('networkidle'), - new Promise((resolve) => setTimeout(resolve, timeout)) + page.waitForLoadState('load'), + page.waitForLoadState('networkidle'), + new Promise(resolve => setTimeout(resolve, timeout)), ]); -} +}; function isValidHttpUrl(urlString) { const pattern = /^(http|https):\/\/[^ "]+$/; @@ -1795,10 +1821,12 @@ function isValidHttpUrl(urlString) { export const isFilePath = (url: string): boolean => { const driveLetterPattern = /^[A-Z]:/i; const backslashPattern = /\\/; - return url.startsWith('file://') || - url.startsWith('/') || - driveLetterPattern.test(url) || - backslashPattern.test(url); + return ( + url.startsWith('file://') || + url.startsWith('/') || + driveLetterPattern.test(url) || + backslashPattern.test(url) + ); }; export function convertLocalFileToPath(url: string): string { @@ -1809,13 +1837,13 @@ export function convertLocalFileToPath(url: string): string { } export function convertPathToLocalFile(filePath: string): string { - if (filePath.startsWith("/")){ + if (filePath.startsWith('/')) { filePath = pathToFileURL(filePath).toString(); } return filePath; -} +} -export function convertToFilePath(fileUrl) { +export function convertToFilePath(fileUrl: string) { // Parse the file URL const parsedUrl = url.parse(fileUrl); // Decode the URL-encoded path diff --git a/src/constants/constants.ts b/src/constants/constants.ts index 87dec164..e471a5d6 100644 --- a/src/constants/constants.ts +++ b/src/constants/constants.ts @@ -184,13 +184,13 @@ export const basicAuthRegex = /^.*\/\/.*:.*@.*$/i; export const axeScript = path.join(__dirname, '../../node_modules/axe-core/axe.min.js'); export class UrlsCrawled { toScan: string[] = []; - scanned: { url: string; pageTitle: string }[] = []; + scanned: { url: string; actualUrl: string; pageTitle: string }[] = []; invalid: string[] = []; - scannedRedirects: string[] = []; - notScannedRedirects: string[] = []; + scannedRedirects: { fromUrl: string; toUrl: string }[] = []; + notScannedRedirects: { fromUrl: string; toUrl: string }[] = []; outOfDomain: string[] = []; blacklisted: string[] = []; - error: string[] = []; + error: { url: string }[] = []; exceededRequests: string[] = []; forbidden: string[] = []; userExcluded: string[] = []; diff --git a/src/crawlers/commonCrawlerFunc.ts b/src/crawlers/commonCrawlerFunc.ts index 1d57d916..529ae53c 100644 --- a/src/crawlers/commonCrawlerFunc.ts +++ b/src/crawlers/commonCrawlerFunc.ts @@ -194,7 +194,7 @@ export const runAxeScript = async ( return filterAxeResults(results, pageTitle, customFlowDetails); }; -export const createCrawleeSubFolders = async randomToken => { +export const createCrawleeSubFolders = async (randomToken:string): Promise<{dataset:crawlee.Dataset, requestQueue:crawlee.RequestQueue}> => { const dataset= await crawlee.Dataset.open(randomToken); const requestQueue = await crawlee.RequestQueue.open(randomToken); return { dataset, requestQueue }; diff --git a/src/crawlers/crawlDomain.ts b/src/crawlers/crawlDomain.ts index 13980673..cf2d33c5 100644 --- a/src/crawlers/crawlDomain.ts +++ b/src/crawlers/crawlDomain.ts @@ -1,7 +1,4 @@ -/* eslint-disable no-shadow */ -/* eslint-disable no-undef */ - -import crawlee from 'crawlee'; +import crawlee, { EnqueueStrategy } from 'crawlee'; import { createCrawleeSubFolders, preNavigationHooks, @@ -9,9 +6,10 @@ import { isUrlPdf, } from './commonCrawlerFunc.js'; import constants, { + UrlsCrawled, blackListedFileExtensions, guiInfoStatusTypes, - cssQuerySelectors + cssQuerySelectors, } from '../constants/constants.js'; import { getPlaywrightLaunchOptions, @@ -28,30 +26,35 @@ import { handlePdfDownload, runPdfScan, mapPdfScanResults, doPdfScreenshots } fr import fs from 'fs'; import { silentLogger, guiInfoLog } from '../logs.js'; import type { BrowserContext, ElementHandle, Frame, Page } from 'playwright'; +import request from 'sync-request-curl'; +import { ViewportSettingsClass } from '#root/combine.js'; +import type { EnqueueLinksOptions, RequestOptions } from 'crawlee'; +import type { BatchAddRequestsResult } from '@crawlee/types'; +import axios from 'axios'; const crawlDomain = async ( - url, - randomToken, - host, - viewportSettings, - maxRequestsPerCrawl, - browser, - userDataDirectory, - strategy, - specifiedMaxConcurrency, - fileTypes, - blacklistedPatterns, - includeScreenshots, - followRobots, - extraHTTPHeaders, - safeMode = false, // optional - fromCrawlIntelligentSitemap = false, //optional - datasetFromIntelligent = null, //optional - urlsCrawledFromIntelligent = null, //optional + url: string, + randomToken: string, + _host: string, + viewportSettings: ViewportSettingsClass, + maxRequestsPerCrawl: number, + browser: string, + userDataDirectory: string, + strategy: EnqueueStrategy, + specifiedMaxConcurrency: number, + fileTypes: string, + blacklistedPatterns: string[], + includeScreenshots: boolean, + followRobots: boolean, + extraHTTPHeaders: Record, + safeMode: boolean = false, // optional + fromCrawlIntelligentSitemap: boolean = false, // optional + datasetFromIntelligent: crawlee.Dataset = null, // optional + urlsCrawledFromIntelligent: UrlsCrawled = null, // optional ) => { - let dataset; - let urlsCrawled; - let requestQueue; + let dataset: crawlee.Dataset; + let urlsCrawled: UrlsCrawled; + let requestQueue: crawlee.RequestQueue; if (fromCrawlIntelligentSitemap) { dataset = datasetFromIntelligent; @@ -79,12 +82,12 @@ const crawlDomain = async ( let authHeader = ''; // Test basic auth and add auth header if auth exist - const parsedUrl = new URL(url); - let username:string; - let password:string; + const parsedUrl = new URL(url); + let username: string; + let password: string; if (parsedUrl.username !== '' && parsedUrl.password !== '') { isBasicAuth = true; - username= decodeURIComponent(parsedUrl.username); + username = decodeURIComponent(parsedUrl.username); password = decodeURIComponent(parsedUrl.password); // Create auth header @@ -110,22 +113,65 @@ const crawlDomain = async ( }); } - const enqueueProcess = async (page, enqueueLinks, browserContext) => { + const isProcessibleUrl = async (url: string): Promise => { + try { + const response = await axios.head(url, { headers: { Authorization: authHeader } }); + const contentType = response.headers['content-type'] || ''; + + if (!contentType.includes('text/html') && !contentType.includes('application/pdf')) { + silentLogger.info(`Skipping MIME type ${contentType} at URL ${url}`); + return false; + } + + // further check for zip files where the url ends with .zip + if (url.endsWith('.zip')) { + silentLogger.info(`Checking for zip file magic number at URL ${url}`); + // download first 4 bytes of file to check the magic number + const response = await axios.get(url, { + headers: { Range: 'bytes=0-3', Authorization: authHeader }, + }); + // check using startsWith because some server does not handle Range header and returns the whole file + if (response.data.startsWith('PK\x03\x04')) { + // PK\x03\x04 is the magic number for zip files + silentLogger.info(`Skipping zip file at URL ${url}`); + return false; + } else { + // print out the hex value of the first 4 bytes + silentLogger.info( + `Not skipping ${url} as it has magic number: ${response.data.slice(0, 4).toString('hex')}`, + ); + } + } + } catch (e) { + silentLogger.error(`Error checking the MIME type of ${url}: ${e.message}`); + // when failing to check the MIME type (e.g. need to go through proxy), let crawlee handle the request + return true; + } + return true; + }; + + const enqueueProcess = async ( + page: Page, + enqueueLinks: (options: EnqueueLinksOptions) => Promise, + browserContext: BrowserContext, + ) => { try { await enqueueLinks({ // set selector matches anchor elements with href but not contains # or starting with mailto: selector: 'a:not(a[href*="#"],a[href^="mailto:"])', strategy, requestQueue, - transformRequestFunction(req) { + transformRequestFunction: async (req: RequestOptions): Promise => { try { req.url = encodeURI(req.url); req.url = req.url.replace(/(?<=&|\?)utm_.*?(&|$)/gim, ''); + const processible = await isProcessibleUrl(req.url); + if (!processible) return null; } catch (e) { - silentLogger.info(e); + silentLogger.error(e); } - if (urlsCrawled.scanned.some(item => item.url.href === req.url)) { + if (urlsCrawled.scanned.some(item => item.url === req.url)) { req.skipNavigation = true; } if (isDisallowedInRobotsTxt(req.url)) return null; @@ -153,16 +199,19 @@ const crawlDomain = async ( } }; - const customEnqueueLinksByClickingElements = async (page: Page, browserContext: BrowserContext): Promise => { + const customEnqueueLinksByClickingElements = async ( + page: Page, + browserContext: BrowserContext, + ): Promise => { const initialPageUrl: string = page.url().toString(); const isExcluded = (newPageUrl: string): boolean => { - const isAlreadyScanned: boolean = urlsCrawled.scanned.some(item => item.url.href === newPageUrl); + const isAlreadyScanned: boolean = urlsCrawled.scanned.some(item => item.url === newPageUrl); const isBlacklistedUrl: boolean = isBlacklisted(newPageUrl); const isNotFollowStrategy: boolean = !isFollowStrategy(newPageUrl, initialPageUrl, strategy); return isAlreadyScanned || isBlacklistedUrl || isNotFollowStrategy; }; - const setPageListeners = (page: Page) : void => { + const setPageListeners = (page: Page): void => { // event listener to handle new page popups upon button click page.on('popup', async (newPage: Page) => { try { @@ -172,19 +221,16 @@ const crawlDomain = async ( url: encodeURI(newPageUrl), skipNavigation: isUrlPdf(encodeURI(newPage.url())), }); - } - else { + } else { try { await newPage.close(); - } - catch (e) { + } catch (e) { // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks. // Handles browser page object been closed. } } return; - } - catch (e) { + } catch (e) { // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks. // Handles browser page object been closed. } @@ -193,9 +239,11 @@ const crawlDomain = async ( // event listener to handle navigation to new url within same page upon element click page.on('framenavigated', async (newFrame: Frame) => { try { - if (newFrame.url() !== initialPageUrl && + if ( + newFrame.url() !== initialPageUrl && !isExcluded(newFrame.url()) && - !(newFrame.url() == 'about:blank')) { + !(newFrame.url() == 'about:blank') + ) { let newFrameUrl: string = newFrame.url().replace(/(?<=&|\?)utm_.*?(&|$)/gim, ''); await requestQueue.addRequest({ url: encodeURI(newFrameUrl), @@ -203,13 +251,11 @@ const crawlDomain = async ( }); } return; - } - catch (e) { + } catch (e) { // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks. // Handles browser page object been closed. } }); - }; setPageListeners(page); let currentElementIndex: number = 0; @@ -220,19 +266,19 @@ const crawlDomain = async ( if (page.url() != initialPageUrl) { try { await page.close(); - } - catch (e) { + } catch (e) { // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks. // Handles browser page object been closed. } - page = await browserContext.newPage() + page = await browserContext.newPage(); await page.goto(initialPageUrl, { waitUntil: 'domcontentloaded', }); setPageListeners(page); } - let selectedElementsString = cssQuerySelectors.join(", "); - const selectedElements: ElementHandle[] = await page.$$(selectedElementsString); + let selectedElementsString = cssQuerySelectors.join(', '); + const selectedElements: ElementHandle[] = + await page.$$(selectedElementsString); // edge case where there might be elements on page that appears intermittently if (currentElementIndex + 1 > selectedElements.length || !selectedElements) { break; @@ -241,14 +287,14 @@ const crawlDomain = async ( if (currentElementIndex + 1 == selectedElements.length) { isAllElementsHandled = true; } - let element: ElementHandle = selectedElements[currentElementIndex]; + let element: ElementHandle = + selectedElements[currentElementIndex]; currentElementIndex += 1; let newUrlFoundInElement: string = null; if (await element.isVisible()) { // Find url in html elements without clicking them await page .evaluate(element => { - //find href attribute let hrefUrl: string = element.getAttribute('href'); @@ -267,8 +313,7 @@ const crawlDomain = async ( try { // Check if newUrlFoundInElement is a valid absolute URL absoluteUrl = new URL(newUrlFoundInElement); - } - catch (e) { + } catch (e) { // If it's not a valid URL, treat it as a relative URL absoluteUrl = new URL(newUrlFoundInElement, baseUrl); } @@ -276,28 +321,27 @@ const crawlDomain = async ( } }); if (newUrlFoundInElement && !isExcluded(newUrlFoundInElement)) { - - let newUrlFoundInElementUrl: string = newUrlFoundInElement.replace(/(?<=&|\?)utm_.*?(&|$)/gim, ''); + let newUrlFoundInElementUrl: string = newUrlFoundInElement.replace( + /(?<=&|\?)utm_.*?(&|$)/gim, + '', + ); await requestQueue.addRequest({ url: encodeURI(newUrlFoundInElementUrl), skipNavigation: isUrlPdf(encodeURI(newUrlFoundInElement)), }); - } - else if (!newUrlFoundInElement) { + } else if (!newUrlFoundInElement) { try { // Find url in html elements by manually clicking them. New page navigation/popups will be handled by event listeners above await element.click(); await page.waitForTimeout(1000); // Add a delay of 1 second between each Element click - } - catch (e) { + } catch (e) { // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks. // Handles browser page object been closed. } } } - } - catch (e) { + } catch (e) { // No logging for this case as it is best effort to handle dynamic client-side JavaScript redirects and clicks. // Handles browser page object been closed. } @@ -305,7 +349,7 @@ const crawlDomain = async ( return; }; - const isBlacklisted = url => { + const isBlacklisted = (url: string) => { const blacklistedPatterns = getBlackListedPatterns(null); if (!blacklistedPatterns) { return false; @@ -360,32 +404,24 @@ const crawlDomain = async ( }, ] : [ - async ({ page, request }) => { + async ({ request }) => { request.url = encodeURI(request.url); preNavigationHooks(extraHTTPHeaders); }, ], requestHandlerTimeoutSecs: 90, // Allow each page to be processed by up from default 60 seconds - requestHandler: async ({ - page, - request, - response, - crawler, - sendRequest, - enqueueLinks, - }) => { + requestHandler: async ({ page, request, response, crawler, sendRequest, enqueueLinks }) => { const browserContext: BrowserContext = page.context(); try { // Set basic auth header if needed - if (isBasicAuth) - { + if (isBasicAuth) { await page.setExtraHTTPHeaders({ Authorization: authHeader, }); const currentUrl = new URL(request.url); currentUrl.username = username; currentUrl.password = password; - request.url = currentUrl.href; + request.url = currentUrl.href; } await waitForPageLoaded(page, 10000); @@ -410,7 +446,7 @@ const crawlDomain = async ( } // if URL has already been scanned - if (urlsCrawled.scanned.some(item => item.url.href === request.url)) { + if (urlsCrawled.scanned.some(item => item.url === request.url)) { await enqueueProcess(page, enqueueLinks, browserContext); return; } @@ -445,7 +481,7 @@ const crawlDomain = async ( const resHeaders = response ? response.headers() : {}; // Safely access response headers const contentType = resHeaders['content-type'] || ''; // Ensure contentType is defined - // whitelist html and pdf document types + // Skip non-HTML and non-PDF URLs if (!contentType.includes('text/html') && !contentType.includes('application/pdf')) { guiInfoLog(guiInfoStatusTypes.SKIPPED, { numScanned: urlsCrawled.scanned.length, @@ -475,7 +511,7 @@ const crawlDomain = async ( numScanned: urlsCrawled.scanned.length, urlScanned: request.url, }); - urlsCrawled.forbidden.push({ url: request.url }); + urlsCrawled.forbidden.push(request.url); return; } @@ -484,7 +520,7 @@ const crawlDomain = async ( numScanned: urlsCrawled.scanned.length, urlScanned: request.url, }); - urlsCrawled.invalid.push({ url: request.url }); + urlsCrawled.invalid.push(request.url); return; } @@ -506,11 +542,11 @@ const crawlDomain = async ( return; } - const results = await runAxeScript(includeScreenshots, page, randomToken,null); + const results = await runAxeScript(includeScreenshots, page, randomToken, null); if (isRedirected) { const isLoadedUrlInCrawledUrls = urlsCrawled.scanned.some( - item => (item.actualUrl || item.url.href) === request.loadedUrl, + item => (item.actualUrl || item.url) === request.loadedUrl, ); if (isLoadedUrlInCrawledUrls) { @@ -552,6 +588,7 @@ const crawlDomain = async ( }); urlsCrawled.scanned.push({ url: urlWithoutAuth(request.url), + actualUrl: request.url, pageTitle: results.pageTitle, }); await dataset.pushData(results); @@ -582,7 +619,9 @@ const crawlDomain = async ( await page.route('**/*', async route => { const interceptedRequest = route.request(); if (interceptedRequest.resourceType() === 'document') { - let interceptedRequestUrl = interceptedRequest.url().replace(/(?<=&|\?)utm_.*?(&|$)/gim, ''); + let interceptedRequestUrl = interceptedRequest + .url() + .replace(/(?<=&|\?)utm_.*?(&|$)/gim, ''); await requestQueue.addRequest({ url: interceptedRequestUrl, skipNavigation: isUrlPdf(interceptedRequest.url()), @@ -641,4 +680,5 @@ const crawlDomain = async ( return urlsCrawled; }; + export default crawlDomain; diff --git a/src/crawlers/pdfScanFunc.ts b/src/crawlers/pdfScanFunc.ts index abb8677d..010d8c4a 100644 --- a/src/crawlers/pdfScanFunc.ts +++ b/src/crawlers/pdfScanFunc.ts @@ -271,7 +271,11 @@ export const handlePdfDownload = (randomToken: string, pdfDownloads: Promise; safeMode: boolean; userDataDirectory?: string; zip?: string; diff --git a/src/npmIndex.ts b/src/npmIndex.ts index adf7e076..bb422372 100644 --- a/src/npmIndex.ts +++ b/src/npmIndex.ts @@ -30,7 +30,7 @@ export const init = async ( viewportSettings = { width: 1000, height: 660 }, // cypress' default viewport settings thresholds = { mustFix: undefined, goodToFix: undefined }, scanAboutMetadata = undefined, - zip = undefined + zip = undefined, ) => { console.log('Starting Purple A11y'); @@ -138,6 +138,7 @@ export const init = async ( }); urlsCrawled.scanned.push({ url: urlWithoutAuth(res.pageUrl).toString(), + actualUrl: 'tbd', pageTitle: `${pageIndex}: ${res.pageTitle}`, }); @@ -206,7 +207,7 @@ export const init = async ( testLabel, scanAboutMetadata, scanDetails, - zip + zip, ); await submitForm(