diff --git a/README.md b/README.md index a7fa27f..bdb820c 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Web scrapper built with [Puppeteer](https://developers.google.com/web/tools/pupp I made this because I needed to download a lot of resources and automating the process seemed a fun way to learn about web scraping. ## DISCLOSURE -This scrapper was developed based on the interface of a education institution that I'm part of. So it might not work on yours. +This scrapper was developed based on the interface of a education institution that I was part of. So it might not work on yours. ## Pre-Requisites Before installing the scrapper, you will need the following: @@ -23,20 +23,25 @@ You can download here: [https://nodejs.org/en/](https://nodejs.org/en/) - Build the project: `npx tsc` ## Run configuration -You can run the scrapper by simply using `npm run start` or `node ./dist/main.js`. +You can run the scrapper by simply using `npm run start` or `node ./dist/main.js`. Remember to pay attention to the command line for inputs. -There are some run configurations you can set up. Either via command line and/or configuration file (`scrapper-config.json`). +There are some run configurations you need to set up. Either via command line and/or configuration file (`scrapper-config.json`). There is no `scrapper-config.json` because private information might be present there, so create one based on the file: `template-scrapper-config.json`. ### Config -If there there is a option that is present in both command line and configuration file, the CLI flag and JSON parameter will be seperated by a `|`. +If there there is a option that is present in both command line and configuration file, the CLI flag and JSON parameter will be seperated by a `|`. The required parameters are `authorizeUrl`, `waitPageAfterLogin`, `modulesListPage`. -- `--username ` | `username` The username you use to login -- `--download-path ` | `downloadPath` The path where the resources will be downloaded(By default it's the `./downloads` folder located in the root of the project) -- `--no-headless` When this flag is on, Puppeteer will be executed with the option with the headless mode deactivated. Headless mode allows the scrapper to run without displaying the UI. + +- `--authorize-url` | `authorizeUrl` The usual moodle link you use to enter it. Include the protocol(e.g http, https, https://mymoodle.com) - `--wait-page-after-login | waitPageAfterLogin` - What page should the scrapper wait after authenticating. +- `--modules-list-page | modulesListPage` - What page contains the list of modules. This scrapper uses the Dashboard as basis to obtain the modules list. +- `--username ` | `username` The username you use to login. +- `--download-path ` | `downloadPath` The path where the resources will be downloaded(By default it's the `./downloads` folder located in the root of the project). + +- `--headless` | `headless` When this parameter is set to `true`, Puppeteer will be executed with the option with the [headless mode](https://pptr.dev/api/puppeteer.browserlaunchargumentoptions.headless/) activated. Headless mode allows the scrapper to run without displaying the UI. Default value is `false`. Optional. + - `--auth-method` | `authMethod` - It can have one of the following values: `"user-control"`, `"terminal-user-passw"`. Default `user-control` @@ -45,8 +50,8 @@ It's useful if you don't want to input your credentials in the terminal, to see Only disadvantage is that the UI will need to be displayed, which requires the headless mode to be disabled. -- - `terminal-user-passw` The username and password will be prompted by the terminal. Doing this way allows the scrapper/puppeteer to run in headless mode(or not with the flag `--no-headless`) +- - `terminal-user-passw` The username and password will be prompted by the terminal. Doing this way allows the scrapper/puppeteer to run in headless mode. + - - If you want to use `terminal-user-passw`, it can also be run with the shortcut: `npm run start:auth-terminal` -- - I prefer using `terminal-user-passw` because the UI does not provide much information, it will be mostly noise. Microsoft 2FA does not require extra input, just confirmation on the phone. diff --git a/package-lock.json b/package-lock.json index 0189326..764feac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,8 +1,742 @@ { - "name": "mooodle-scapper", - "version": "1.0.0", - "lockfileVersion": 1, + "name": "mooodle-resource-downloader", + "version": "0.4.0", + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "name": "mooodle-resource-downloader", + "version": "0.4.0", + "license": "MIT", + "dependencies": { + "axios": "^0.21.1", + "mime-types": "^2.1.28", + "prompts": "^2.4.0", + "puppeteer": "^15.5.0", + "qs": "^6.9.6" + }, + "devDependencies": { + "@types/mime-types": "^2.1.0", + "@types/prompts": "^2.0.9", + "@types/puppeteer": "^5.4.2", + "typescript": "^4.2.4" + } + }, + "node_modules/@types/mime-types": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/mime-types/-/mime-types-2.1.0.tgz", + "integrity": "sha1-nKUs2jY/aZxpRmwqbM2q2RPqenM=", + "dev": true + }, + "node_modules/@types/node": { + "version": "14.14.21", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.21.tgz", + "integrity": "sha512-cHYfKsnwllYhjOzuC5q1VpguABBeecUp24yFluHpn/BQaVxB1CuQ1FSRZCzrPxrkIfWISXV2LbeoBthLWg0+0A==", + "devOptional": true + }, + "node_modules/@types/prompts": { + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.9.tgz", + "integrity": "sha512-TORZP+FSjTYMWwKadftmqEn6bziN5RnfygehByGsjxoK5ydnClddtv6GikGWPvCm24oI+YBwck5WDxIIyNxUrA==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/puppeteer": { + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-5.4.2.tgz", + "integrity": "sha512-yjbHoKjZFOGqA6bIEI2dfBE5UPqU0YGWzP+ipDVP1iGzmlhksVKTBVZfT3Aj3wnvmcJ2PQ9zcncwOwyavmafBw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw==", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/axios": { + "version": "0.21.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz", + "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==", + "dependencies": { + "follow-redirects": "^1.10.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "engines": { + "node": "*" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "node_modules/cross-fetch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", + "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "dependencies": { + "node-fetch": "2.6.7" + } + }, + "node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1019158", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1019158.tgz", + "integrity": "sha512-wvq+KscQ7/6spEV7czhnZc9RM/woz1AY+/Vpd8/h2HFMwJSdTliu7f/yr1A6vDdJfKICZsShqsYpEQbdhg8AFQ==" + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "dependencies": { + "pend": "~1.2.0" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/follow-redirects": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.1.tgz", + "integrity": "sha512-SSG5xmZh1mkPGyKzjZP8zLjltIfpW32Y5QpdNJyjcfGxK3qo3NDDkZOZSFiGn1A6SclQxY9GzEwAHQ3dmYRWpg==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" + }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/https-proxy-agent": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", + "dependencies": { + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "engines": { + "node": ">=6" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mime-db": { + "version": "1.45.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.45.0.tgz", + "integrity": "sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w==", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.28", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.28.tgz", + "integrity": "sha512-0TO2yJ5YHYr7M2zzT7gDU1tbwHxEUWBCLt0lscSNpcdAfFyJOVEpRYNS7EXVcTLNj/25QO8gulHC5JtTzSE2UQ==", + "dependencies": { + "mime-db": "1.45.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/node-fetch": { + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/prompts": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.0.tgz", + "integrity": "sha512-awZAKrk3vN6CroQukBL+R9051a4R3zCZBlJm/HBfrSZ8iTpYix3VX1vU4mveiLpiwmOJT4wokTF9m6HUk4KqWQ==", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/puppeteer": { + "version": "15.5.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-15.5.0.tgz", + "integrity": "sha512-+vZPU8iBSdCx1Kn5hHas80fyo0TiVyMeqLGv/1dygX2HKhAZjO9YThadbRTCoTYq0yWw+w/CysldPsEekDtjDQ==", + "hasInstallScript": true, + "dependencies": { + "cross-fetch": "3.1.5", + "debug": "4.3.4", + "devtools-protocol": "0.0.1019158", + "extract-zip": "2.0.1", + "https-proxy-agent": "5.0.1", + "pkg-dir": "4.2.0", + "progress": "2.0.3", + "proxy-from-env": "1.1.0", + "rimraf": "3.0.2", + "tar-fs": "2.1.1", + "unbzip2-stream": "1.4.3", + "ws": "8.8.0" + }, + "engines": { + "node": ">=14.1.0" + } + }, + "node_modules/qs": { + "version": "6.9.6", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.6.tgz", + "integrity": "sha512-TIRk4aqYLNoJUbd+g2lEdz5kLWIuTMRagAXxl78Q0RiVjAOugHmeKNGdd3cwo/ktpf9aL9epCfFqWDEKysUlLQ==", + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==" + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" + }, + "node_modules/typescript": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.2.4.tgz", + "integrity": "sha512-V+evlYHZnQkaz8TRBuxTA92yZBPotr5H+WhQ7bD3hZUndx5tGOa1fuCgeSjxAzM1RiN5IzvadIXTVefuuwZCRg==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/unbzip2-stream": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", + "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", + "dependencies": { + "buffer": "^5.2.1", + "through": "^2.3.8" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + }, + "node_modules/ws": { + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.0.tgz", + "integrity": "sha512-JDAgSYQ1ksuwqfChJusw1LSJ8BizJ2e/vVu5Lxjq3YvNJNlROv1ui4i+c/kUUrPheBvQl4c5UbERhTwKa6QBJQ==", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + } + }, "dependencies": { "@types/mime-types": { "version": "2.1.0", @@ -13,7 +747,8 @@ "@types/node": { "version": "14.14.21", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.21.tgz", - "integrity": "sha512-cHYfKsnwllYhjOzuC5q1VpguABBeecUp24yFluHpn/BQaVxB1CuQ1FSRZCzrPxrkIfWISXV2LbeoBthLWg0+0A==" + "integrity": "sha512-cHYfKsnwllYhjOzuC5q1VpguABBeecUp24yFluHpn/BQaVxB1CuQ1FSRZCzrPxrkIfWISXV2LbeoBthLWg0+0A==", + "devOptional": true }, "@types/prompts": { "version": "2.0.9", @@ -34,18 +769,21 @@ } }, "@types/yauzl": { - "version": "2.9.1", - "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz", - "integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==", + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw==", "optional": true, "requires": { "@types/node": "*" } }, "agent-base": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", - "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==" + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4" + } }, "axios": { "version": "0.21.1", @@ -56,9 +794,9 @@ } }, "balanced-match": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, "base64-js": { "version": "1.5.1", @@ -66,9 +804,9 @@ "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, "bl": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", - "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "requires": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -96,7 +834,7 @@ "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==" }, "chownr": { "version": "1.1.4", @@ -106,20 +844,28 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + }, + "cross-fetch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", + "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "requires": { + "node-fetch": "2.6.7" + } }, "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "requires": { "ms": "2.1.2" } }, "devtools-protocol": { - "version": "0.0.818844", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.818844.tgz", - "integrity": "sha512-AD1hi7iVJ8OD0aMLQU5VK0XH9LDlA1+BcPIgrAxPfaibx2DbWucuyOhc4oyQCbnvDDO68nN6/LcKfqTP343Jjg==" + "version": "0.0.1019158", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1019158.tgz", + "integrity": "sha512-wvq+KscQ7/6spEV7czhnZc9RM/woz1AY+/Vpd8/h2HFMwJSdTliu7f/yr1A6vDdJfKICZsShqsYpEQbdhg8AFQ==" }, "end-of-stream": { "version": "1.4.4", @@ -143,7 +889,7 @@ "fd-slicer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", "requires": { "pend": "~1.2.0" } @@ -170,7 +916,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" }, "get-stream": { "version": "5.2.0", @@ -181,24 +927,24 @@ } }, "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", - "minimatch": "^3.0.4", + "minimatch": "^3.1.1", "once": "^1.3.0", "path-is-absolute": "^1.0.0" } }, "https-proxy-agent": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", - "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", "requires": { - "agent-base": "5", + "agent-base": "6", "debug": "4" } }, @@ -210,7 +956,7 @@ "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", "requires": { "once": "^1.3.0", "wrappy": "1" @@ -248,9 +994,9 @@ } }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" } @@ -266,14 +1012,17 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + "version": "2.6.7", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", + "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "requires": { + "whatwg-url": "^5.0.0" + } }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "requires": { "wrappy": "1" } @@ -307,12 +1056,12 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==" }, "pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" }, "pkg-dir": { "version": "4.2.0", @@ -351,22 +1100,22 @@ } }, "puppeteer": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-5.5.0.tgz", - "integrity": "sha512-OM8ZvTXAhfgFA7wBIIGlPQzvyEETzDjeRa4mZRCRHxYL+GNH5WAuYUQdja3rpWZvkX/JKqmuVgbsxDNsDFjMEg==", - "requires": { - "debug": "^4.1.0", - "devtools-protocol": "0.0.818844", - "extract-zip": "^2.0.0", - "https-proxy-agent": "^4.0.0", - "node-fetch": "^2.6.1", - "pkg-dir": "^4.2.0", - "progress": "^2.0.1", - "proxy-from-env": "^1.0.0", - "rimraf": "^3.0.2", - "tar-fs": "^2.0.0", - "unbzip2-stream": "^1.3.3", - "ws": "^7.2.3" + "version": "15.5.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-15.5.0.tgz", + "integrity": "sha512-+vZPU8iBSdCx1Kn5hHas80fyo0TiVyMeqLGv/1dygX2HKhAZjO9YThadbRTCoTYq0yWw+w/CysldPsEekDtjDQ==", + "requires": { + "cross-fetch": "3.1.5", + "debug": "4.3.4", + "devtools-protocol": "0.0.1019158", + "extract-zip": "2.0.1", + "https-proxy-agent": "5.0.1", + "pkg-dir": "4.2.0", + "progress": "2.0.3", + "proxy-from-env": "1.1.0", + "rimraf": "3.0.2", + "tar-fs": "2.1.1", + "unbzip2-stream": "1.4.3", + "ws": "8.8.0" } }, "qs": { @@ -436,7 +1185,12 @@ "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==" + }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, "typescript": { "version": "4.2.4", @@ -456,22 +1210,37 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" + }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" + }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "ws": { - "version": "7.4.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.2.tgz", - "integrity": "sha512-T4tewALS3+qsrpGI/8dqNMLIVdq/g/85U98HPMa6F0m6xTbvhXU6RCQLqPH3+SlomNV/LdY6RXEbBpMH6EOJnA==" + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.8.0.tgz", + "integrity": "sha512-JDAgSYQ1ksuwqfChJusw1LSJ8BizJ2e/vVu5Lxjq3YvNJNlROv1ui4i+c/kUUrPheBvQl4c5UbERhTwKa6QBJQ==", + "requires": {} }, "yauzl": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", "requires": { "buffer-crc32": "~0.2.3", "fd-slicer": "~1.1.0" diff --git a/package.json b/package.json index c531e5e..628f64e 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "axios": "^0.21.1", "mime-types": "^2.1.28", "prompts": "^2.4.0", - "puppeteer": "^5.5.0", + "puppeteer": "^15.5.0", "qs": "^6.9.6" }, "devDependencies": { diff --git a/src/IScrapperConfig.ts b/src/IScrapperConfig.ts index 95ab6f8..80037b2 100644 --- a/src/IScrapperConfig.ts +++ b/src/IScrapperConfig.ts @@ -7,5 +7,7 @@ export interface IScrapperConfig { downloadPath?: string; authorizeUrl?: string; waitPageAfterLogin?: string; + modulesListPage?: string; authMethod?: EnumAuthMethod; + headless?: boolean; } \ No newline at end of file diff --git a/src/main.ts b/src/main.ts index 841bd03..90b33db 100644 --- a/src/main.ts +++ b/src/main.ts @@ -37,28 +37,43 @@ async function execute() : Promise{ let defaultDownloadPath = __dirname + "/../downloads"; let downloadPathInConfig = scrapperConfig.downloadPath || undefined; let resourcesDownloadPath = getArgParam("--download-path") || downloadPathInConfig || defaultDownloadPath; + let fullResourcesDownloadPath = path.resolve(resourcesDownloadPath) let authMethod = getArgParam("--auth-method") || scrapperConfig.authMethod || EnumAuthMethod.USER_CONTROL; let authorizeUrl = getArgParam("--authorize-url") || scrapperConfig.authorizeUrl; let waitForPageAfterLogin = getArgParam("--wait-page-after-login") || scrapperConfig.waitPageAfterLogin; + let modulesListPage = getArgParam("--modules-list-page") || scrapperConfig.modulesListPage || undefined; + + if (!authorizeUrl) { throw new Error("Authorization URL not provided.") } - let isHeadless = true; - if (hasArg("--no-headless")) { - isHeadless = false; - } else if (authMethod === EnumAuthMethod.USER_CONTROL) { + let cliArgHeadless = getArgParam("--headless"); + let isHeadless = false; + if (cliArgHeadless && cliArgHeadless === "true") { + isHeadless = true + } else if (typeof scrapperConfig.headless === "boolean"){ + isHeadless = scrapperConfig.headless; + } + + if (authMethod === EnumAuthMethod.USER_CONTROL) { isHeadless = false; + console.log("Warning: disabling headless mode due to Auth Method being 'user-control'"); } + + const responseTimeout = 1800000; const browser = await puppeteer.launch({ headless: isHeadless, product: "chrome", - timeout: 90000 + timeout: responseTimeout }); const page = await browser.newPage(); + page.setDefaultTimeout(responseTimeout); + page.setDefaultNavigationTimeout(responseTimeout); + let authenticator: IAuthProcess | undefined = undefined; @@ -100,7 +115,8 @@ async function execute() : Promise{ authorizeUrl: authorizeUrl, username: userEmail, password: userPassword, - waitForPageAfterLogin: waitForPageAfterLogin + waitForPageAfterLogin: waitForPageAfterLogin, + modulesListPage: modulesListPage, }); } else { @@ -109,7 +125,8 @@ async function execute() : Promise{ authenticator = new UserControlAuth(page, { username: userEmail, authorizeUrl: authorizeUrl, - waitForPageAfterLogin: waitForPageAfterLogin + waitForPageAfterLogin: waitForPageAfterLogin, + modulesListPage: modulesListPage, }); } @@ -149,7 +166,7 @@ async function execute() : Promise{ let moduleSelection = await prompts({ type: 'text', name: 'moduleChoices', - message: "Select the modules by typing the numbers seperated by commas. E.g 1,5,2:" + message: "Select the modules by typing the numbers seperated by commas or by content of their names. E.g 1,5,2,Algorithms:" }) if (moduleSelection.moduleChoices === "exit") { throw new Error("REQUESTED_EXIT"); @@ -201,7 +218,7 @@ async function execute() : Promise{ console.log(" =>", moduleDoc.name); } - console.log("\nDownloads folder targeted at:", resourcesDownloadPath); + console.log("\nDownloads folder targeted at:", fullResourcesDownloadPath); let confirmChoiceInput = ""; while (["y", "n"].indexOf(confirmChoiceInput.toLowerCase()) === -1) { let promptInput = await prompts({ diff --git a/src/scrapper-navigator/authenticators/UserControlAuth.ts b/src/scrapper-navigator/authenticators/UserControlAuth.ts index 3135ff7..edfe081 100644 --- a/src/scrapper-navigator/authenticators/UserControlAuth.ts +++ b/src/scrapper-navigator/authenticators/UserControlAuth.ts @@ -5,7 +5,7 @@ import asyncTimeout from '../../utils/async-timeout'; export interface IUserControlAuthConfig { authorizeUrl: string; - goToPageAfterLogin?: string; + modulesListPage?: string; waitForPageAfterLogin?: string; username?: string; } @@ -34,8 +34,8 @@ export default class UserControlAuth implements IAuthProcess { await this.page.waitForResponse(config.waitForPageAfterLogin); //await this.procedureWaitForPageAfterLogin(this.page, config.waitForPageAfterLogin); } - if (config.goToPageAfterLogin) { - await this.page.goto(config.goToPageAfterLogin); + if (config.modulesListPage) { + await this.page.goto(config.modulesListPage); } console.log("MOODLE PAGE YEAH!"); } diff --git a/src/scrapper-navigator/authenticators/UserPasswAuth.ts b/src/scrapper-navigator/authenticators/UserPasswAuth.ts index f5974ae..31f273e 100644 --- a/src/scrapper-navigator/authenticators/UserPasswAuth.ts +++ b/src/scrapper-navigator/authenticators/UserPasswAuth.ts @@ -7,7 +7,7 @@ import asyncTimeout from '../../utils/async-timeout'; export interface IUserPasswAuthenticateConfig extends AuthenticateConfig { username: string; password: string; - goToPageAfterLogin?: string; + modulesListPage?: string; waitForPageAfterLogin?: string; } @@ -49,11 +49,11 @@ export default class UserPasswAuth implements IAuthProcess { console.log("[WARNING] If you're using 2 Factor Authentication, remember to check your phone for any prompts."); await this.page.goto(config.authorizeUrl); - await this.page.screenshot({path: './screenshots/login.png'}); + //await this.page.screenshot({path: './screenshots/login.png'}); console.log("Login Page entered") await this.page.type('[name="UserName"]', username); console.log("Username typed"); - await this.page.screenshot({path: './screenshots/email_inserted.png'}); + //await this.page.screenshot({path: './screenshots/email_inserted.png'}); await asyncTimeout(1000); await this.page.click("#nextButton"); await this.page.waitForSelector("#passwordInput", { @@ -61,10 +61,10 @@ export default class UserPasswAuth implements IAuthProcess { }); await asyncTimeout(1000); console.log("PAssword input is visible"); - await this.page.screenshot({path: './screenshots/submitted_username.png'}); - await this.page.type('#passwordInput', password); + //await this.page.screenshot({path: './screenshots/submitted_username.png'}); + //await this.page.type('#passwordInput', password); console.log("PAssword input inputted"); - await this.page.screenshot({path: './screenshots/password_filled.png'}); + //await this.page.screenshot({path: './screenshots/password_filled.png'}); await asyncTimeout(1000); await this.page.click("#submitButton"); console.log("Form submitted"); @@ -72,8 +72,8 @@ export default class UserPasswAuth implements IAuthProcess { if (config.waitForPageAfterLogin) { await this.page.waitForResponse(config.waitForPageAfterLogin); } - if (config.goToPageAfterLogin) { - await this.page.goto(config.goToPageAfterLogin); + if (config.modulesListPage) { + await this.page.goto(config.modulesListPage); } console.log("MOODLE PAGE YEAH!"); return; diff --git a/src/scrapper-navigator/file-resource.ts b/src/scrapper-navigator/file-resource.ts index 69a8602..4998b1f 100644 --- a/src/scrapper-navigator/file-resource.ts +++ b/src/scrapper-navigator/file-resource.ts @@ -1,5 +1,5 @@ -import { Page, Request} from "puppeteer"; -import axios, {AxiosRequestConfig} from 'axios'; +import { Page, HTTPRequest} from "puppeteer"; +import axios, {AxiosRequestConfig, Method as AxiosMethod} from 'axios'; import fs from 'fs'; import path from 'path'; import {replaceReservedDirectoryCharacters} from '../utils/string-utils' @@ -14,7 +14,8 @@ export interface FileResource { name: string; } -function goToAndIntercept(url: string, page: PageExtended) : Promise{ + +function goToAndIntercept(url: string, page: PageExtended) : Promise{ return new Promise((resolve, reject) => { @@ -40,7 +41,8 @@ export async function downloadFileResource(page: PageExtended, fileResource: Fil let cookies = await page.cookies(); - var dataCookies = await page._client.send('Network.getAllCookies'); + let client = await page.target().createCDPSession(); + var dataCookies = await client.send('Network.getAllCookies'); //console.log("dataCookies", dataCookies); @@ -65,7 +67,7 @@ export async function downloadFileResource(page: PageExtended, fileResource: Fil const options : AxiosRequestConfig = { responseType: 'arraybuffer', - method: xRequest.method(), + method: xRequest.method() as AxiosMethod, url: xRequest.url(), data: xRequest.postData(), headers: xRequest.headers(), diff --git a/src/scrapper-navigator/folder-form-resource.ts b/src/scrapper-navigator/folder-form-resource.ts index 7f97e83..becafdc 100644 --- a/src/scrapper-navigator/folder-form-resource.ts +++ b/src/scrapper-navigator/folder-form-resource.ts @@ -1,5 +1,5 @@ -import { Page, Request} from "puppeteer"; -import axios, {AxiosRequestConfig} from 'axios'; +import { Page, HTTPRequest} from "puppeteer"; +import axios, {AxiosRequestConfig, Method as AxiosMethod} from 'axios'; import getFileNameFromContentDisposition from '../utils/get-filename-from-content-disposition'; import fs from 'fs'; import path from 'path'; @@ -19,7 +19,7 @@ interface DownloadFolderFormResourceConfig { folderNameType: "resource_page_name" | "resource_download_name" } -function interceptButtonClick(buttonSelector: string, page: Page) : Promise{ +function interceptButtonClick(buttonSelector: string, page: Page) : Promise{ return new Promise((resolve, reject) => { page.once('request', interceptedRequest => { @@ -46,7 +46,7 @@ export async function downloadFolderFormResource(page: Page, folderFormResource: const options: AxiosRequestConfig = { responseType: "arraybuffer", - method: xRequest.method(), + method: xRequest.method() as AxiosMethod, url: folderFormResource.action, data: xRequest.postData(), headers: xRequest.headers(), diff --git a/src/scrapper-navigator/index.ts b/src/scrapper-navigator/index.ts index 644a0a9..4623b2c 100644 --- a/src/scrapper-navigator/index.ts +++ b/src/scrapper-navigator/index.ts @@ -1,5 +1,5 @@ import EventEmitter from 'events'; -import {Page, ElementHandle, Request} from 'puppeteer'; +import {Page, ElementHandle, HTTPRequest} from 'puppeteer'; import asyncTimeout from '../utils/async-timeout'; import {ModulePage} from './module-page'; import path from 'path'; @@ -41,15 +41,15 @@ export class ScrapperNavigator extends EventEmitter { async loadModulesPages(modulesListPageUrl?: string, loadSections: boolean = false) { if (modulesListPageUrl) { if (this.page.url() !== modulesListPageUrl) { - this.page.goto(modulesListPageUrl); + await this.page.goto(modulesListPageUrl); } } - await this.page.screenshot({path: path.join(__dirname, '/../../screenshots/dashboard_moodle_page.png')}); + //await this.page.screenshot({path: path.join(__dirname, '/../../screenshots/dashboard_moodle_page.png')}); await this.page.select("#coc-filterterm", "all") console.log("Selected All modules"); - await this.page.screenshot({path: path.join(__dirname, '/../../screenshots/selected_all.png')}); + //await this.page.screenshot({path: path.join(__dirname, '/../../screenshots/selected_all.png')}); @@ -68,7 +68,7 @@ export class ScrapperNavigator extends EventEmitter { }); }) - let modulesList = tempModulesList.map((e) => new ModulePage({name: e.name, url: e.url, sections: []})); + let modulesList = tempModulesList.map((e: ModuleLink) => new ModulePage({name: e.name, url: e.url, sections: []})); // Load Sections if (loadSections) { diff --git a/src/scrapper-navigator/module-page.ts b/src/scrapper-navigator/module-page.ts index ed3f9e6..b3a37e2 100644 --- a/src/scrapper-navigator/module-page.ts +++ b/src/scrapper-navigator/module-page.ts @@ -1,5 +1,5 @@ -import {Page, ElementHandle, Request} from 'puppeteer'; +import {Page, ElementHandle, HTTPRequest} from 'puppeteer'; import axios, { AxiosRequestConfig } from 'axios'; import path from 'path'; import fs from 'fs'; diff --git a/src/template-scrapper-config.json b/src/template-scrapper-config.json new file mode 100644 index 0000000..3c680d2 --- /dev/null +++ b/src/template-scrapper-config.json @@ -0,0 +1,8 @@ +{ + "username": "", + "downloadPath": "", + "authorizeUrl": "", + "waitPageAfterLogin": "", + "authMethod": "", + "headless": false +} \ No newline at end of file diff --git a/template-scrapper-config.json b/template-scrapper-config.json index d13b762..954abff 100644 --- a/template-scrapper-config.json +++ b/template-scrapper-config.json @@ -3,5 +3,7 @@ "downloadPath": "", "authorizeUrl": "", "waitPageAfterLogin": "", - "authMethod": "" + "modulesListPage": "", + "authMethod": "", + "headless": false } \ No newline at end of file