diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 0000000..a0de486 --- /dev/null +++ b/.eslintignore @@ -0,0 +1,2 @@ +website +lib diff --git a/.eslintrc b/.eslintrc new file mode 100644 index 0000000..6fb31c5 --- /dev/null +++ b/.eslintrc @@ -0,0 +1,12 @@ +{ + "parser": "eslint-plugin-typescript/parser", + "plugins": ["typescript", "no-only-tests"], + "rules": { + "no-unused-expressions": "error", + "no-console": "warn", + "typescript/no-unused-vars": "error", + "typescript/explicit-member-accessibility": "error", + "typescript/member-ordering": "error", + "no-only-tests/no-only-tests": "error" + } +} diff --git a/.travis.yml b/.travis.yml index 71fe839..41f6d25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,11 @@ jobs: node_js: '8' script: npm run lint + - stage: typecheck-test-lint + name: check formatting + node_js: '8' + script: npm run format:check + - stage: deploy-npm node_js: '8' script: skip diff --git a/README.md b/README.md index 4f59039..8d06afd 100644 --- a/README.md +++ b/README.md @@ -17,10 +17,10 @@ npm install scrape-pages ## Usage -lets download the five most recent images from nasa's image of the day archive +lets download the five most recent images from NASA's image of the day archive ```javascript -const ScrapePages = require('scrape-pages') +const { scraper } = require('scrape-pages') // create a config file const config = { scrape: { @@ -42,21 +42,22 @@ const config = { } } } +const options = { + folder: './downloads', + logLevel: 'info', + logFile: './nasa-download.log' +} // load the config into a new 'scraper' -const siteScraper = new ScrapePages(config) -// begin scraping -const emitter = siteScraper.run({ folder: './downloads' }) - -emitter.on('image:complete', (queryFor, { id }) => +const scraper = await scrape(config, options) +const { on, emit, query } = scraper +on('image:compete', id => { console.log('COMPLETED image', id) -) - -emitter.on('done', async queryFor => { +}) +on('done', () => { console.log('finished.') - const result = await queryFor({ scrapers: { images: ['filename'] } }) - console.log(result) - // [{ + const result = query({ scrapers: ['images'] }) + // result = [{ // images: [{ filename: 'img1.jpg' }, { filename: 'img2.jpg' }, ...] // }] }) @@ -66,41 +67,49 @@ For more real world examples, visit the [examples](examples) directory ## Documentation -Detailed usage documentation is coming, but for now, [typescript](https://www.typescriptlang.org/) typings -exist for the surface API. - -- for scraper config object documentation see [src/configuration/types.ts](src/configuration/types.ts) -- for runtime options documentation see [src/run-options/types.ts](src/run-options/types.ts) - The scraper instance created from a config object is meant to be reusable and cached. It only knows about the config object. `scraper.run` can be called multiple times, and, as long as different folders are provided, each run will work independently. `scraper.run` returns **emitter** -### emitter +### scrape -#### Listenable events +| param | type | required | type file | description | +| ------- | ---------------- | -------- | -------------------------------------------------------------- | ----------------------------- | +| config | `ConfigInit` | Yes | [src/settings/config/types.ts](src/settings/config/types.ts) | _what_ is being downloaded | +| options | `RunOptionsInit` | Yes | [src/settings/options/types.ts](src/settings/options/types.ts) | _how_ something is downloaded | -each event will return the **queryFor** function as its first argument -- `'done'`: when the scraper has completed -- `'error'`: when the scraper encounters an error (this also stops the scraper) -- `':progress'`: emits progress of download until completed -- `':queued'`: when a download is queued -- `':complete'`: when a download is completed +### scraper +The `scrape` function returns a promise which yeilds these utilities (`on`, `emit`, and `query`) -#### Emittable events +#### on +Listen for events from the scraper +| event | callback arguments | description | +| ---------------------- | --------------------- | ------------------------------------------ | +| `'done'` | queryFor | when the scraper has completed | +| `'error'` | error | if the scraper encounters an error | +| `':progress'` | queryFor, download id | emits progress of download until completed | +| `':queued'` | queryFor, download id | when a download is queued | +| `':complete'` | queryFor, download id | when a download is completed | -- '`useRateLimiter'`: pass a boolean to turn on or off the rate limit defined in the run options -- `'stop'`: emit this event to stop the crawler (note that any in progress promises will still complete) +#### emit -### queryFor +While the scraper is working, you can affect its behavior by emitting these events: +| event | arguments | description | +| --- | --- | --- | +| `'useRateLimiter'` | boolean | turn on or off the rate limit defined in the run options | +| `'stop'` | | stop the crawler (note that in progress requests will still complete) | + +each event will return the **queryFor** function as its first argument -This function is used to get data back out of the scraper whenever you need it. The function takes an object -with three keys: +#### query -- `scrapers`: `{ [name]: Array<'filename'|'parsedValue'> }` -- `groupBy?`: name of a scraper which will delineate the values in `scrapers` -- `stmtCacheKey?`: `Symbol` which helps the internal database cache queries. +This function is an argument in the emitter callback and is used to get data back out of the scraper whenever +you need it. These are its arguments: +| name | type | required | description | +| --- | --- | --- | --- | +| `scrapers` | `string[]` | Yes | scrapers who will return their filenames and parsed values, in order | +| `groupBy` | `string` | Yes | name of a scraper which will delineate the values in `scrapers` | ## Motivation diff --git a/custom.d.ts b/custom.d.ts index 8d21ac9..a195a9b 100644 --- a/custom.d.ts +++ b/custom.d.ts @@ -7,3 +7,10 @@ declare module 'flow-runtime' { const content: any export default content } + +type ArgumentTypes = F extends (...args: infer A) => any + ? A + : never + +type Nullable = T | null +type Voidable = T | void diff --git a/examples/__tests__/deviantart.unit.test.ts b/examples/__tests__/deviantart.unit.test.ts index 9919a74..9173300 100644 --- a/examples/__tests__/deviantart.unit.test.ts +++ b/examples/__tests__/deviantart.unit.test.ts @@ -1,5 +1,5 @@ import deviantartConfig from '../deviantart.config.json' -import { assertConfigType } from '../../src/configuration/site-traversal' +import { assertConfigType } from '../../src/settings/config' describe('deviantart config', () => { it('is properly typed', () => { diff --git a/examples/__tests__/nasa-image-of-the-day.unit.test.ts b/examples/__tests__/nasa-image-of-the-day.unit.test.ts index 256f461..63962b0 100644 --- a/examples/__tests__/nasa-image-of-the-day.unit.test.ts +++ b/examples/__tests__/nasa-image-of-the-day.unit.test.ts @@ -1,5 +1,5 @@ import nasaIotdConfig from '../nasa-image-of-the-day.config.json' -import { assertConfigType } from '../../src/configuration/site-traversal' +import { assertConfigType } from '../../src/settings/config' describe('nasa iotd config', () => { it('is properly typed', () => { diff --git a/examples/__tests__/simple-config.unit.test.ts b/examples/__tests__/simple-config.unit.test.ts index ca4e572..0b97ff2 100644 --- a/examples/__tests__/simple-config.unit.test.ts +++ b/examples/__tests__/simple-config.unit.test.ts @@ -1,4 +1,4 @@ -import { assertConfigType } from '../../src/configuration/site-traversal' +import { assertConfigType } from '../../src/settings/config' import * as testingConfigs from '../../testing/resources/testing-configs' describe('example simple config', () => { diff --git a/examples/__tests__/tumblr.unit.test.ts b/examples/__tests__/tumblr.unit.test.ts index eb8db73..92e3d1e 100644 --- a/examples/__tests__/tumblr.unit.test.ts +++ b/examples/__tests__/tumblr.unit.test.ts @@ -1,5 +1,5 @@ import tumblrConfig from '../tumblr.config.json' -import { assertConfigType } from '../../src/configuration/site-traversal' +import { assertConfigType } from '../../src/settings/config' describe('tumblr config', () => { it('is properly typed', () => { diff --git a/package-lock.json b/package-lock.json index 81372ca..0689200 100644 --- a/package-lock.json +++ b/package-lock.json @@ -893,14 +893,23 @@ } }, "@types/better-sqlite3": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-3.1.3.tgz", - "integrity": "sha512-nnoebG2LgbK1WQ0dn7jtJ+RQJXcC/0G9oajJA7T+tKq2HrL9KzjMDKfe6sqkigg5HLH08WE4LRK3QDDYXPO1+w==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-5.2.1.tgz", + "integrity": "sha512-S1q/Noh3Bc6gMTVEQAk33yxHSfojKRxxvvpynPW6dFjuArxCXzSeRsbi7/K4E8bN6/1jeI6OxWeRrSPyLUFOsQ==", "dev": true, "requires": { "@types/integer": "*" } }, + "@types/bunyan": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@types/bunyan/-/bunyan-1.8.5.tgz", + "integrity": "sha512-7n8ANtxh2c5A/NfCuv8cVtWcgSLdq76MQbtmbInpzXuPw4TSAReUJ+MGHK4m67I4zI3ynCJoABfaeHYJaYSeRg==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/chai": { "version": "4.1.7", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.1.7.tgz", @@ -975,263 +984,182 @@ "@types/node": "*" } }, + "@types/verror": { + "version": "1.10.3", + "resolved": "https://registry.npmjs.org/@types/verror/-/verror-1.10.3.tgz", + "integrity": "sha512-7Jz0MPsW2pWg5dJfEp9nJYI0RDCYfgjg2wIo5HfQ8vOJvUq0/BxT7Mv2wNQvkKBmV9uT++6KF3reMnLmh/0HrA==", + "dev": true + }, "@webassemblyjs/ast": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.5.13.tgz", - "integrity": "sha512-49nwvW/Hx9i+OYHg+mRhKZfAlqThr11Dqz8TsrvqGKMhdI2ijy3KBJOun2Z4770TPjrIJhR6KxChQIDaz8clDA==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.7.11.tgz", + "integrity": "sha512-ZEzy4vjvTzScC+SH8RBssQUawpaInUdMTYwYYLh54/s8TuT0gBLuyUnppKsVyZEi876VmmStKsUs28UxPgdvrA==", "dev": true, "requires": { - "@webassemblyjs/helper-module-context": "1.5.13", - "@webassemblyjs/helper-wasm-bytecode": "1.5.13", - "@webassemblyjs/wast-parser": "1.5.13", - "debug": "^3.1.0", - "mamacro": "^0.0.3" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - } + "@webassemblyjs/helper-module-context": "1.7.11", + "@webassemblyjs/helper-wasm-bytecode": "1.7.11", + "@webassemblyjs/wast-parser": "1.7.11" } }, "@webassemblyjs/floating-point-hex-parser": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.5.13.tgz", - "integrity": "sha512-vrvvB18Kh4uyghSKb0NTv+2WZx871WL2NzwMj61jcq2bXkyhRC+8Q0oD7JGVf0+5i/fKQYQSBCNMMsDMRVAMqA==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.7.11.tgz", + "integrity": "sha512-zY8dSNyYcgzNRNT666/zOoAyImshm3ycKdoLsyDw/Bwo6+/uktb7p4xyApuef1dwEBo/U/SYQzbGBvV+nru2Xg==", "dev": true }, "@webassemblyjs/helper-api-error": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.5.13.tgz", - "integrity": "sha512-dBh2CWYqjaDlvMmRP/kudxpdh30uXjIbpkLj9HQe+qtYlwvYjPRjdQXrq1cTAAOUSMTtzqbXIxEdEZmyKfcwsg==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.7.11.tgz", + "integrity": "sha512-7r1qXLmiglC+wPNkGuXCvkmalyEstKVwcueZRP2GNC2PAvxbLYwLLPr14rcdJaE4UtHxQKfFkuDFuv91ipqvXg==", "dev": true }, "@webassemblyjs/helper-buffer": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.5.13.tgz", - "integrity": "sha512-v7igWf1mHcpJNbn4m7e77XOAWXCDT76Xe7Is1VQFXc4K5jRcFrl9D0NrqM4XifQ0bXiuTSkTKMYqDxu5MhNljA==", - "dev": true, - "requires": { - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - } - } + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.7.11.tgz", + "integrity": "sha512-MynuervdylPPh3ix+mKZloTcL06P8tenNH3sx6s0qE8SLR6DdwnfgA7Hc9NSYeob2jrW5Vql6GVlsQzKQCa13w==", + "dev": true }, "@webassemblyjs/helper-code-frame": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-code-frame/-/helper-code-frame-1.5.13.tgz", - "integrity": "sha512-yN6ScQQDFCiAXnVctdVO/J5NQRbwyTbQzsGzEgXsAnrxhjp0xihh+nNHQTMrq5UhOqTb5LykpJAvEv9AT0jnAQ==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-code-frame/-/helper-code-frame-1.7.11.tgz", + "integrity": "sha512-T8ESC9KMXFTXA5urJcyor5cn6qWeZ4/zLPyWeEXZ03hj/x9weSokGNkVCdnhSabKGYWxElSdgJ+sFa9G/RdHNw==", "dev": true, "requires": { - "@webassemblyjs/wast-printer": "1.5.13" + "@webassemblyjs/wast-printer": "1.7.11" } }, "@webassemblyjs/helper-fsm": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-fsm/-/helper-fsm-1.5.13.tgz", - "integrity": "sha512-hSIKzbXjVMRvy3Jzhgu+vDd/aswJ+UMEnLRCkZDdknZO3Z9e6rp1DAs0tdLItjCFqkz9+0BeOPK/mk3eYvVzZg==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-fsm/-/helper-fsm-1.7.11.tgz", + "integrity": "sha512-nsAQWNP1+8Z6tkzdYlXT0kxfa2Z1tRTARd8wYnc/e3Zv3VydVVnaeePgqUzFrpkGUyhUUxOl5ML7f1NuT+gC0A==", "dev": true }, "@webassemblyjs/helper-module-context": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-module-context/-/helper-module-context-1.5.13.tgz", - "integrity": "sha512-zxJXULGPLB7r+k+wIlvGlXpT4CYppRz8fLUM/xobGHc9Z3T6qlmJD9ySJ2jknuktuuiR9AjnNpKYDECyaiX+QQ==", - "dev": true, - "requires": { - "debug": "^3.1.0", - "mamacro": "^0.0.3" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - } - } + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-module-context/-/helper-module-context-1.7.11.tgz", + "integrity": "sha512-JxfD5DX8Ygq4PvXDucq0M+sbUFA7BJAv/GGl9ITovqE+idGX+J3QSzJYz+LwQmL7fC3Rs+utvWoJxDb6pmC0qg==", + "dev": true }, "@webassemblyjs/helper-wasm-bytecode": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.5.13.tgz", - "integrity": "sha512-0n3SoNGLvbJIZPhtMFq0XmmnA/YmQBXaZKQZcW8maGKwLpVcgjNrxpFZHEOLKjXJYVN5Il8vSfG7nRX50Zn+aw==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.7.11.tgz", + "integrity": "sha512-cMXeVS9rhoXsI9LLL4tJxBgVD/KMOKXuFqYb5oCJ/opScWpkCMEz9EJtkonaNcnLv2R3K5jIeS4TRj/drde1JQ==", "dev": true }, "@webassemblyjs/helper-wasm-section": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.5.13.tgz", - "integrity": "sha512-IJ/goicOZ5TT1axZFSnlAtz4m8KEjYr12BNOANAwGFPKXM4byEDaMNXYowHMG0yKV9a397eU/NlibFaLwr1fbw==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.7.11.tgz", + "integrity": "sha512-8ZRY5iZbZdtNFE5UFunB8mmBEAbSI3guwbrsCl4fWdfRiAcvqQpeqd5KHhSWLL5wuxo53zcaGZDBU64qgn4I4Q==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.5.13", - "@webassemblyjs/helper-buffer": "1.5.13", - "@webassemblyjs/helper-wasm-bytecode": "1.5.13", - "@webassemblyjs/wasm-gen": "1.5.13", - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - } + "@webassemblyjs/ast": "1.7.11", + "@webassemblyjs/helper-buffer": "1.7.11", + "@webassemblyjs/helper-wasm-bytecode": "1.7.11", + "@webassemblyjs/wasm-gen": "1.7.11" } }, "@webassemblyjs/ieee754": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.5.13.tgz", - "integrity": "sha512-TseswvXEPpG5TCBKoLx9tT7+/GMACjC1ruo09j46ULRZWYm8XHpDWaosOjTnI7kr4SRJFzA6MWoUkAB+YCGKKg==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.7.11.tgz", + "integrity": "sha512-Mmqx/cS68K1tSrvRLtaV/Lp3NZWzXtOHUW2IvDvl2sihAwJh4ACE0eL6A8FvMyDG9abes3saB6dMimLOs+HMoQ==", "dev": true, "requires": { - "ieee754": "^1.1.11" + "@xtuc/ieee754": "^1.2.0" } }, "@webassemblyjs/leb128": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.5.13.tgz", - "integrity": "sha512-0NRMxrL+GG3eISGZBmLBLAVjphbN8Si15s7jzThaw1UE9e5BY1oH49/+MA1xBzxpf1OW5sf9OrPDOclk9wj2yg==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.7.11.tgz", + "integrity": "sha512-vuGmgZjjp3zjcerQg+JA+tGOncOnJLWVkt8Aze5eWQLwTQGNgVLcyOTqgSCxWTR4J42ijHbBxnuRaL1Rv7XMdw==", "dev": true, "requires": { - "long": "4.0.0" - }, - "dependencies": { - "long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", - "dev": true - } + "@xtuc/long": "4.2.1" } }, "@webassemblyjs/utf8": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.5.13.tgz", - "integrity": "sha512-Ve1ilU2N48Ew0lVGB8FqY7V7hXjaC4+PeZM+vDYxEd+R2iQ0q+Wb3Rw8v0Ri0+rxhoz6gVGsnQNb4FjRiEH/Ng==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.7.11.tgz", + "integrity": "sha512-C6GFkc7aErQIAH+BMrIdVSmW+6HSe20wg57HEC1uqJP8E/xpMjXqQUxkQw07MhNDSDcGpxI9G5JSNOQCqJk4sA==", "dev": true }, "@webassemblyjs/wasm-edit": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.5.13.tgz", - "integrity": "sha512-X7ZNW4+Hga4f2NmqENnHke2V/mGYK/xnybJSIXImt1ulxbCOEs/A+ZK/Km2jgihjyVxp/0z0hwIcxC6PrkWtgw==", - "dev": true, - "requires": { - "@webassemblyjs/ast": "1.5.13", - "@webassemblyjs/helper-buffer": "1.5.13", - "@webassemblyjs/helper-wasm-bytecode": "1.5.13", - "@webassemblyjs/helper-wasm-section": "1.5.13", - "@webassemblyjs/wasm-gen": "1.5.13", - "@webassemblyjs/wasm-opt": "1.5.13", - "@webassemblyjs/wasm-parser": "1.5.13", - "@webassemblyjs/wast-printer": "1.5.13", - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - } + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.7.11.tgz", + "integrity": "sha512-FUd97guNGsCZQgeTPKdgxJhBXkUbMTY6hFPf2Y4OedXd48H97J+sOY2Ltaq6WGVpIH8o/TGOVNiVz/SbpEMJGg==", + "dev": true, + "requires": { + "@webassemblyjs/ast": "1.7.11", + "@webassemblyjs/helper-buffer": "1.7.11", + "@webassemblyjs/helper-wasm-bytecode": "1.7.11", + "@webassemblyjs/helper-wasm-section": "1.7.11", + "@webassemblyjs/wasm-gen": "1.7.11", + "@webassemblyjs/wasm-opt": "1.7.11", + "@webassemblyjs/wasm-parser": "1.7.11", + "@webassemblyjs/wast-printer": "1.7.11" } }, "@webassemblyjs/wasm-gen": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.5.13.tgz", - "integrity": "sha512-yfv94Se8R73zmr8GAYzezFHc3lDwE/lBXQddSiIZEKZFuqy7yWtm3KMwA1uGbv5G1WphimJxboXHR80IgX1hQA==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.7.11.tgz", + "integrity": "sha512-U/KDYp7fgAZX5KPfq4NOupK/BmhDc5Kjy2GIqstMhvvdJRcER/kUsMThpWeRP8BMn4LXaKhSTggIJPOeYHwISA==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.5.13", - "@webassemblyjs/helper-wasm-bytecode": "1.5.13", - "@webassemblyjs/ieee754": "1.5.13", - "@webassemblyjs/leb128": "1.5.13", - "@webassemblyjs/utf8": "1.5.13" + "@webassemblyjs/ast": "1.7.11", + "@webassemblyjs/helper-wasm-bytecode": "1.7.11", + "@webassemblyjs/ieee754": "1.7.11", + "@webassemblyjs/leb128": "1.7.11", + "@webassemblyjs/utf8": "1.7.11" } }, "@webassemblyjs/wasm-opt": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.5.13.tgz", - "integrity": "sha512-IkXSkgzVhQ0QYAdIayuCWMmXSYx0dHGU8Ah/AxJf1gBvstMWVnzJnBwLsXLyD87VSBIcsqkmZ28dVb0mOC3oBg==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.7.11.tgz", + "integrity": "sha512-XynkOwQyiRidh0GLua7SkeHvAPXQV/RxsUeERILmAInZegApOUAIJfRuPYe2F7RcjOC9tW3Cb9juPvAC/sCqvg==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.5.13", - "@webassemblyjs/helper-buffer": "1.5.13", - "@webassemblyjs/wasm-gen": "1.5.13", - "@webassemblyjs/wasm-parser": "1.5.13", - "debug": "^3.1.0" - }, - "dependencies": { - "debug": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", - "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", - "dev": true, - "requires": { - "ms": "2.0.0" - } - } + "@webassemblyjs/ast": "1.7.11", + "@webassemblyjs/helper-buffer": "1.7.11", + "@webassemblyjs/wasm-gen": "1.7.11", + "@webassemblyjs/wasm-parser": "1.7.11" } }, "@webassemblyjs/wasm-parser": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.5.13.tgz", - "integrity": "sha512-XnYoIcu2iqq8/LrtmdnN3T+bRjqYFjRHqWbqK3osD/0r/Fcv4d9ecRzjVtC29ENEuNTK4mQ9yyxCBCbK8S/cpg==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.7.11.tgz", + "integrity": "sha512-6lmXRTrrZjYD8Ng8xRyvyXQJYUQKYSXhJqXOBLw24rdiXsHAOlvw5PhesjdcaMadU/pyPQOJ5dHreMjBxwnQKg==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.5.13", - "@webassemblyjs/helper-api-error": "1.5.13", - "@webassemblyjs/helper-wasm-bytecode": "1.5.13", - "@webassemblyjs/ieee754": "1.5.13", - "@webassemblyjs/leb128": "1.5.13", - "@webassemblyjs/utf8": "1.5.13" + "@webassemblyjs/ast": "1.7.11", + "@webassemblyjs/helper-api-error": "1.7.11", + "@webassemblyjs/helper-wasm-bytecode": "1.7.11", + "@webassemblyjs/ieee754": "1.7.11", + "@webassemblyjs/leb128": "1.7.11", + "@webassemblyjs/utf8": "1.7.11" } }, "@webassemblyjs/wast-parser": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-parser/-/wast-parser-1.5.13.tgz", - "integrity": "sha512-Lbz65T0LQ1LgzKiUytl34CwuhMNhaCLgrh0JW4rJBN6INnBB8NMwUfQM+FxTnLY9qJ+lHJL/gCM5xYhB9oWi4A==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-parser/-/wast-parser-1.7.11.tgz", + "integrity": "sha512-lEyVCg2np15tS+dm7+JJTNhNWq9yTZvi3qEhAIIOaofcYlUp0UR5/tVqOwa/gXYr3gjwSZqw+/lS9dscyLelbQ==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.5.13", - "@webassemblyjs/floating-point-hex-parser": "1.5.13", - "@webassemblyjs/helper-api-error": "1.5.13", - "@webassemblyjs/helper-code-frame": "1.5.13", - "@webassemblyjs/helper-fsm": "1.5.13", - "long": "^3.2.0", - "mamacro": "^0.0.3" + "@webassemblyjs/ast": "1.7.11", + "@webassemblyjs/floating-point-hex-parser": "1.7.11", + "@webassemblyjs/helper-api-error": "1.7.11", + "@webassemblyjs/helper-code-frame": "1.7.11", + "@webassemblyjs/helper-fsm": "1.7.11", + "@xtuc/long": "4.2.1" } }, "@webassemblyjs/wast-printer": { - "version": "1.5.13", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.5.13.tgz", - "integrity": "sha512-QcwogrdqcBh8Z+eUF8SG+ag5iwQSXxQJELBEHmLkk790wgQgnIMmntT2sMAMw53GiFNckArf5X0bsCA44j3lWQ==", + "version": "1.7.11", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.7.11.tgz", + "integrity": "sha512-m5vkAsuJ32QpkdkDOUPGSltrg8Cuk3KBx4YrmAGQwCZPRdUHXxG4phIOuuycLemHFr74sWL9Wthqss4fzdzSwg==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.5.13", - "@webassemblyjs/wast-parser": "1.5.13", - "long": "^3.2.0" + "@webassemblyjs/ast": "1.7.11", + "@webassemblyjs/wast-parser": "1.7.11", + "@xtuc/long": "4.2.1" } }, "@webpack-contrib/schema-utils": { @@ -1318,10 +1246,22 @@ } } }, + "@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "dev": true + }, + "@xtuc/long": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.1.tgz", + "integrity": "sha512-FZdkNBDqBRHKQ2MEbSC17xnPFOhZxeJ2YGSfr2BKf3sujG49Qe3bB+rGCwQfIaA7WHnGeGkSijX4FuBCdrzW/g==", + "dev": true + }, "acorn": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.1.tgz", - "integrity": "sha512-d+nbxBUGKg7Arpsvbnlq61mc12ek3EY8EQldM3GPAhWJ1UVxC6TDGbIvUMNU6obBX3i1+ptCIzV4vq0gFPEGVQ==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz", + "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw==", "dev": true }, "acorn-dynamic-import": { @@ -1333,6 +1273,30 @@ "acorn": "^5.0.0" } }, + "acorn-jsx": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.0.1.tgz", + "integrity": "sha512-HJ7CfNHrfJLlNTzIEUTj43LNWGkqpRLxm3YjAlcD0ACydk9XynzYsCBHxut+iqt+1aBXkx9UP/w/ZqMr13XIzg==", + "dev": true + }, + "ajv": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.6.2.tgz", + "integrity": "sha512-FBHEW6Jf5TB9MGBgUUA9XHkTbjXYfAUjY43ACMfmdMRHniyoMHjHjzD50OK8LGDWQwp4rWEsIq5kEqq7rvIM1g==", + "dev": true, + "requires": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ajv-errors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-1.0.1.tgz", + "integrity": "sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==", + "dev": true + }, "ajv-keywords": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.2.0.tgz", @@ -1376,6 +1340,15 @@ "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "dev": true }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "requires": { + "sprintf-js": "~1.0.2" + } + }, "arr-diff": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", @@ -1476,6 +1449,11 @@ } } }, + "assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" + }, "assertion-error": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", @@ -1488,6 +1466,12 @@ "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=", "dev": true }, + "astral-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", + "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", + "dev": true + }, "async": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/async/-/async-2.6.1.tgz", @@ -1605,12 +1589,12 @@ "dev": true }, "better-sqlite3": { - "version": "4.1.4", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-4.1.4.tgz", - "integrity": "sha512-Y11HN9PQ9YUeKFMrmiHyOLAKElk2ATJzBZJvuzNwTMxoS7vUEEyLnUCtcBFqViLwbomr0RQwp2MBy/ogxF50PA==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-5.2.1.tgz", + "integrity": "sha512-FAqPi+jUl6VBBDZpIZPtva+VcYAxw3sbXKjImNBPvK1TmU6bCuaPn42XuNk6VL6Vhjl5KBZpeIesxJlDqTFBqA==", "requires": { - "bindings": "^1.3.0", - "integer": "^1.0.5" + "integer": "^2.1.0", + "tar": "^4.4.6" } }, "big.js": { @@ -1625,11 +1609,6 @@ "integrity": "sha1-RqoXUftqL5PuXmibsQh9SxTGwgU=", "dev": true }, - "bindings": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.0.tgz", - "integrity": "sha512-DpLh5EzMR2kzvX1KIlVC0VkC3iZtHKTgdtZ0a3pglBZdaQFjt5S9g9xd1lE+YvXyfd6mtCeRnrUfOLYiTMlNSw==" - }, "bluebird": { "version": "3.5.1", "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.1.tgz", @@ -1813,6 +1792,18 @@ "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=", "dev": true }, + "bunyan": { + "version": "1.8.12", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz", + "integrity": "sha1-8VDw9nSKvdcq6uhPBEA74u8RN5c=", + "dev": true, + "requires": { + "dtrace-provider": "~0.8", + "moment": "^2.10.6", + "mv": "~2", + "safe-json-stringify": "~1" + } + }, "cacache": { "version": "10.0.4", "resolved": "https://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz", @@ -1851,6 +1842,21 @@ "unset-value": "^1.0.0" } }, + "caller-path": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/caller-path/-/caller-path-0.1.0.tgz", + "integrity": "sha1-lAhe9jWB7NPaqSREqP6U6CV3dR8=", + "dev": true, + "requires": { + "callsites": "^0.2.0" + } + }, + "callsites": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-0.2.0.tgz", + "integrity": "sha1-r6uWJikQp/M8GaV3WCXGnzTjUMo=", + "dev": true + }, "camelcase": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", @@ -1968,6 +1974,12 @@ "safe-buffer": "^5.0.1" } }, + "circular-json": { + "version": "0.3.3", + "resolved": "https://registry.npmjs.org/circular-json/-/circular-json-0.3.3.tgz", + "integrity": "sha512-UZK3NBx2Mca+b5LsG7bY183pHWt5Y1xts4P3Pz7ENTwGVnJOUWbRb3ocjvX7hx9tq/yTAdclXm9sZ38gNuem4A==", + "dev": true + }, "class-utils": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", @@ -1991,12 +2003,6 @@ } } }, - "clean-terminal-webpack-plugin": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/clean-terminal-webpack-plugin/-/clean-terminal-webpack-plugin-1.1.0.tgz", - "integrity": "sha512-aobm7IUmpaZi4g7yFeSyrGne1GBUF0Bw47cSj/xaFA/l8Zm/rM529v/HJKIjMqplbvzX/hepBIY4Cq4YtyyCrw==", - "dev": true - }, "cli-cursor": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", @@ -2322,6 +2328,12 @@ "integrity": "sha1-9dJgKStmDghO/0zbyfCK0yR0SLU=", "dev": true }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "dev": true + }, "defaults": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", @@ -2435,6 +2447,15 @@ } } }, + "doctrine": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-2.1.0.tgz", + "integrity": "sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==", + "dev": true, + "requires": { + "esutils": "^2.0.2" + } + }, "dom-serializer": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", @@ -2479,6 +2500,16 @@ "domelementtype": "1" } }, + "dtrace-provider": { + "version": "0.8.7", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.7.tgz", + "integrity": "sha1-3JObTT4GIM/gwc2APQ0tftBP/QQ=", + "dev": true, + "optional": true, + "requires": { + "nan": "^2.10.0" + } + }, "duplexify": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.6.0.tgz", @@ -2622,6 +2653,124 @@ "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, + "eslint": { + "version": "5.11.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.11.1.tgz", + "integrity": "sha512-gOKhM8JwlFOc2acbOrkYR05NW8M6DCMSvfcJiBB5NDxRE1gv8kbvxKaC9u69e6ZGEMWXcswA/7eKR229cEIpvg==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "ajv": "^6.5.3", + "chalk": "^2.1.0", + "cross-spawn": "^6.0.5", + "debug": "^4.0.1", + "doctrine": "^2.1.0", + "eslint-scope": "^4.0.0", + "eslint-utils": "^1.3.1", + "eslint-visitor-keys": "^1.0.0", + "espree": "^5.0.0", + "esquery": "^1.0.1", + "esutils": "^2.0.2", + "file-entry-cache": "^2.0.0", + "functional-red-black-tree": "^1.0.1", + "glob": "^7.1.2", + "globals": "^11.7.0", + "ignore": "^4.0.6", + "imurmurhash": "^0.1.4", + "inquirer": "^6.1.0", + "js-yaml": "^3.12.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.5", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.2", + "path-is-inside": "^1.0.2", + "pluralize": "^7.0.0", + "progress": "^2.0.0", + "regexpp": "^2.0.1", + "require-uncached": "^1.0.3", + "semver": "^5.5.1", + "strip-ansi": "^4.0.0", + "strip-json-comments": "^2.0.1", + "table": "^5.0.2", + "text-table": "^0.2.0" + }, + "dependencies": { + "ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dev": true, + "requires": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "requires": { + "ms": "^2.1.1" + } + }, + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "dev": true + }, + "semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==", + "dev": true + }, + "strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "requires": { + "ansi-regex": "^3.0.0" + } + } + } + }, + "eslint-plugin-no-only-tests": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-no-only-tests/-/eslint-plugin-no-only-tests-2.1.0.tgz", + "integrity": "sha512-T02dNNDj7sKJNvH7YLKqgv4+BDupxKG4OgadF0AecDHrYTb9hlosxqCgZbFKt28C7Ueof6ziCtEh6rnPvN4YYA==", + "dev": true + }, + "eslint-plugin-typescript": { + "version": "1.0.0-rc.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-typescript/-/eslint-plugin-typescript-1.0.0-rc.1.tgz", + "integrity": "sha512-K45mNfd6Ru3ItHyLpwkQgjCgTMLcLvmsJmH+81yKxqFZRn0BQ6doM9TAovNXvMn2ZryXmBbXWeGhYftU2exh8w==", + "dev": true, + "requires": { + "requireindex": "^1.2.0", + "typescript-eslint-parser": "21.0.2" + } + }, "eslint-scope": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.0.tgz", @@ -2632,6 +2781,52 @@ "estraverse": "^4.1.1" } }, + "eslint-utils": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.3.1.tgz", + "integrity": "sha512-Z7YjnIldX+2XMcjr7ZkgEsOj/bREONV60qYeB/bjMAqqqZ4zxKyWX+BOUkdmRmA9riiIPVvo5x86m5elviOk0Q==", + "dev": true + }, + "eslint-visitor-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", + "integrity": "sha512-qzm/XxIbxm/FHyH341ZrbnMUpe+5Bocte9xkmFMzPMjRaZMcXww+MpBptFvtU+79L362nqiLhekCxCxDPaUMBQ==", + "dev": true + }, + "espree": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.0.tgz", + "integrity": "sha512-1MpUfwsdS9MMoN7ZXqAr9e9UKdVHDcvrJpyx7mm1WuQlx/ygErEQBzgi5Nh5qBHIoYweprhtMkTCb9GhcAIcsA==", + "dev": true, + "requires": { + "acorn": "^6.0.2", + "acorn-jsx": "^5.0.0", + "eslint-visitor-keys": "^1.0.0" + }, + "dependencies": { + "acorn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.0.5.tgz", + "integrity": "sha512-i33Zgp3XWtmZBMNvCr4azvOFeWVw1Rk6p3hfi3LUDvIFraOMywb1kAtrbi+med14m4Xfpqm3zRZMT+c0FNE7kg==", + "dev": true + } + } + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "esquery": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.0.1.tgz", + "integrity": "sha512-SmiyZ5zIWH9VM+SRUReLS5Q8a7GxtRdxEBVZpm98rJM7Sb+A9DVCndXfkeFUd3byderg+EbDkfnevfCwynWaNA==", + "dev": true, + "requires": { + "estraverse": "^4.0.0" + } + }, "esrecurse": { "version": "4.2.1", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", @@ -2827,6 +3022,17 @@ } } }, + "extsprintf": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.4.0.tgz", + "integrity": "sha1-4mifjzVvrWLMplo6kcXfX5VRaS8=" + }, + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", + "dev": true + }, "fast-equals": { "version": "1.6.1", "resolved": "https://registry.npmjs.org/fast-equals/-/fast-equals-1.6.1.tgz", @@ -2839,6 +3045,12 @@ "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", "dev": true }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "dev": true + }, "fclone": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/fclone/-/fclone-1.0.11.tgz", @@ -2860,6 +3072,16 @@ "escape-string-regexp": "^1.0.5" } }, + "file-entry-cache": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-2.0.0.tgz", + "integrity": "sha1-w5KZDD5oR4PYOLjISkXYoEhFg2E=", + "dev": true, + "requires": { + "flat-cache": "^1.2.1", + "object-assign": "^4.0.1" + } + }, "fill-range": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", @@ -2903,8 +3125,20 @@ "locate-path": "^2.0.0" } }, - "flow-runtime": { - "version": "0.17.0", + "flat-cache": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-1.3.4.tgz", + "integrity": "sha512-VwyB3Lkgacfik2vhqR4uv2rvebqmDvFu4jlN/C1RzWoJEo8I7z4Q404oiqYCkq41mni8EzQnm95emU9seckwtg==", + "dev": true, + "requires": { + "circular-json": "^0.3.1", + "graceful-fs": "^4.1.2", + "rimraf": "~2.6.2", + "write": "^0.2.1" + } + }, + "flow-runtime": { + "version": "0.17.0", "resolved": "https://registry.npmjs.org/flow-runtime/-/flow-runtime-0.17.0.tgz", "integrity": "sha512-x621HugMPrtU68ddiRX0TkNRif9PS6ml3oeP6oo0k+Kv2issqCmifC4ZX59XnLUh9dfmBcj4GoZJXu4oe8L28Q==" }, @@ -2949,6 +3183,14 @@ "readable-stream": "^2.0.0" } }, + "fs-minipass": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.5.tgz", + "integrity": "sha512-JhBl0skXjUPCFH7x6x61gQxrKyXsxB5gcgePLZCwfyCGGsTISMoIeObbrvVeP6Xmyaudw4TT43qV2Gz+iyd2oQ==", + "requires": { + "minipass": "^2.2.1" + } + }, "fs-write-stream-atomic": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz", @@ -3501,6 +3743,12 @@ "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", "dev": true }, + "functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "dev": true + }, "get-caller-file": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz", @@ -3679,9 +3927,9 @@ } }, "hash.js": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.5.tgz", - "integrity": "sha512-eWI5HG9Np+eHV1KQhisXWwM+4EPPYe5dFX1UZZH7k/E3JzDEazVH+VGlZi6R94ZqImq+A3D1mCEtrFIfg/E7sA==", + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", "dev": true, "requires": { "inherits": "^2.0.3", @@ -3858,12 +4106,9 @@ } }, "integer": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/integer/-/integer-1.0.5.tgz", - "integrity": "sha512-3jqqAHL1gwgl3y0YzFLU0E1fECboNMz9RON8ycqRr3P/cAZN6kcMWWn35UybkYNuB1VjhpSf5WnAqmN0WpKP1Q==", - "requires": { - "bindings": "^1.3.0" - } + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/integer/-/integer-2.1.0.tgz", + "integrity": "sha512-vBtiSgrEiNocWvvZX1RVfeOKa2mCHLZQ2p9nkQkQZ/BvEiY+6CcUz0eyjvIiewjJoeNidzg2I+tpPJvpyspL1w==" }, "interpret": { "version": "1.1.0", @@ -4105,6 +4350,16 @@ "integrity": "sha1-mGbfOVECEw449/mWvOtlRDIJwls=", "dev": true }, + "js-yaml": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.12.0.tgz", + "integrity": "sha512-PIt2cnwmPfL4hKNwqeiuz4bKfnzHTBv6HyVgjahA6mPLwPDzjDWrplJBMjHUFxku/N3FlmrbyPclad+I+4mJ3A==", + "dev": true, + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "jsesc": { "version": "2.5.2", "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", @@ -4117,6 +4372,18 @@ "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", "dev": true }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "dev": true + }, "json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", @@ -4150,10 +4417,20 @@ "invert-kv": "^1.0.0" } }, + "levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "dev": true, + "requires": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + } + }, "loader-runner": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.3.0.tgz", - "integrity": "sha1-9IKuqC1UPgeSFwDVpG7yb9rGuKI=", + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.3.1.tgz", + "integrity": "sha512-By6ZFY7ETWOc9RFaAIb23IjJVcM4dvJC/N57nmdz9RSkMXvAXGI7SyVlAw3v8vjtDRlqThgVDVmTnr9fqMlxkw==", "dev": true }, "loader-utils": { @@ -4188,6 +4465,12 @@ "integrity": "sha1-gteb/zCmfEAF/9XiUVMArZyk168=", "dev": true }, + "lodash.unescape": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.unescape/-/lodash.unescape-4.0.1.tgz", + "integrity": "sha1-vyJJiGzlFM2hEvrpIYzcBlIR/Jw=", + "dev": true + }, "log-symbols": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-2.2.0.tgz", @@ -4234,12 +4517,6 @@ "object.assign": "^4.1.0" } }, - "long": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/long/-/long-3.2.0.tgz", - "integrity": "sha1-2CG3E4yhy1gcFymQ7xTbIAtcR0s=", - "dev": true - }, "loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -4276,12 +4553,6 @@ } } }, - "mamacro": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/mamacro/-/mamacro-0.0.3.tgz", - "integrity": "sha512-qMEwh+UujcQ+kbz3T6V+wAmO2U8veoq2w+3wY8MquqwVA3jChfwY+Tk52GZKDfACEPjuZ7r2oJLejwpt8jtwTA==", - "dev": true - }, "map-cache": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", @@ -4298,13 +4569,14 @@ } }, "md5.js": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.4.tgz", - "integrity": "sha1-6b296UogpawYsENA/Fdk1bCdkB0=", + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", + "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==", "dev": true, "requires": { "hash-base": "^3.0.0", - "inherits": "^2.0.1" + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" } }, "mem": { @@ -4393,6 +4665,30 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=" }, + "minipass": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz", + "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==", + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + }, + "dependencies": { + "yallist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + } + } + }, + "minizlib": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.2.1.tgz", + "integrity": "sha512-7+4oTUOWKg7AuL3vloEWekXY2/D20cevzsrNT2kGWm+39J9hGTCBv8VI5Pm5lXZ/o3/mdR4f8rflAPhnQb8mPA==", + "requires": { + "minipass": "^2.2.1" + } + }, "mississippi": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-2.0.0.tgz", @@ -4436,7 +4732,6 @@ "version": "0.5.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", - "dev": true, "requires": { "minimist": "0.0.8" } @@ -4528,6 +4823,13 @@ } } }, + "moment": { + "version": "2.23.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.23.0.tgz", + "integrity": "sha512-3IE39bHVqFbWWaPOMHZF98Q9c3LDKGTmypMiTM2QygGXXElkFWIH7GxfmlwmY2vwa+wmNsoYZmG2iusf1ZjJoA==", + "dev": true, + "optional": true + }, "move-concurrently": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", @@ -4554,6 +4856,44 @@ "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", "dev": true }, + "mv": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", + "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=", + "dev": true, + "optional": true, + "requires": { + "mkdirp": "~0.5.1", + "ncp": "~2.0.0", + "rimraf": "~2.4.0" + }, + "dependencies": { + "glob": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", + "dev": true, + "optional": true, + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "rimraf": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", + "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=", + "dev": true, + "optional": true, + "requires": { + "glob": "^6.0.1" + } + } + } + }, "nan": { "version": "2.10.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.10.0.tgz", @@ -4580,10 +4920,23 @@ "to-regex": "^3.0.1" } }, + "natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "dev": true + }, + "ncp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", + "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=", + "dev": true, + "optional": true + }, "neo-async": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.5.1.tgz", - "integrity": "sha512-3KL3fvuRkZ7s4IFOMfztb7zJp3QaVWnBeGoJlgB38XnCRPj/0tLzzLG5IB8NYOHbJ8g8UGrgZv44GLDk6CxTxA==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.0.tgz", + "integrity": "sha512-MFh0d/Wa7vkKO3Y3LlacqAEeHK0mckVqzDieUKTT+KGxi+zIpeVsFxymkIiRpbpDziHc290Xr9A1O4Om7otoRA==", "dev": true }, "next-tick": { @@ -4856,6 +5209,12 @@ "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", "dev": true }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "dev": true + }, "object-copy": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", @@ -4958,6 +5317,28 @@ "wordwrap": "~0.0.2" } }, + "optionator": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", + "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", + "dev": true, + "requires": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.4", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "wordwrap": "~1.0.0" + }, + "dependencies": { + "wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", + "dev": true + } + } + }, "ora": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/ora/-/ora-3.0.0.tgz", @@ -5066,9 +5447,9 @@ "dev": true }, "pako": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.6.tgz", - "integrity": "sha512-lQe48YPsMJAig+yngZ87Lus+NF+3mtu7DVOBu6b/gHO1YpKwIj5AWjZ/TOS7i46HD/UixzWb1zeWDZfGZ3iYcg==", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.7.tgz", + "integrity": "sha512-3HNK5tW4x8o5mO8RuHZp3Ydw9icZXx0RANAOMzlMzx7LVXhMJ4mo3MOBpzyd7r/+RUu8BmndP47LXT+vzjtWcQ==", "dev": true }, "parallel-transform": { @@ -5132,6 +5513,12 @@ "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, + "path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=", + "dev": true + }, "path-key": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", @@ -5151,9 +5538,9 @@ "dev": true }, "pbkdf2": { - "version": "3.0.16", - "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.0.16.tgz", - "integrity": "sha512-y4CXP3thSxqf7c0qmOF+9UeOTrifiVTIM+u7NWlq+PRsHbr7r7dpCmvzrZxa96JJUNi0Y5w9VqG5ZNeCVMoDcA==", + "version": "3.0.17", + "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.0.17.tgz", + "integrity": "sha512-U/il5MsrZp7mGg3mSQfn742na2T+1/vHDCG5/iTI3X9MKUuYUZVLQhyRsg06mCgDBTd57TxzgZt7P+fYfjRLtA==", "dev": true, "requires": { "create-hash": "^1.1.2", @@ -5187,12 +5574,24 @@ "find-up": "^2.1.0" } }, + "pluralize": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pluralize/-/pluralize-7.0.0.tgz", + "integrity": "sha512-ARhBOdzS3e41FbkW/XWrTEtukqqLoK5+Z/4UeDaLuSW+39JPeFgs4gCGqsrJHVZX0fUrx//4OF0K1CUGwlIFow==", + "dev": true + }, "posix-character-classes": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=", "dev": true }, + "prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "dev": true + }, "prettier": { "version": "1.14.2", "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.14.2.tgz", @@ -5252,16 +5651,17 @@ "dev": true }, "public-encrypt": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.2.tgz", - "integrity": "sha512-4kJ5Esocg8X3h8YgJsKAuoesBgB7mqH3eowiDzMUPKiRDDE7E/BqqZD1hnTByIaAFiwAw246YEltSq7tdrOH0Q==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", + "integrity": "sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==", "dev": true, "requires": { "bn.js": "^4.1.0", "browserify-rsa": "^4.0.0", "create-hash": "^1.1.0", "parse-asn1": "^5.0.0", - "randombytes": "^2.0.1" + "randombytes": "^2.0.1", + "safe-buffer": "^5.1.2" } }, "pump": { @@ -5399,6 +5799,12 @@ "safe-regex": "^1.1.0" } }, + "regexpp": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", + "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==", + "dev": true + }, "regexpu-core": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-4.4.0.tgz", @@ -5466,6 +5872,30 @@ "integrity": "sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE=", "dev": true }, + "require-uncached": { + "version": "1.0.3", + "resolved": "http://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz", + "integrity": "sha1-Tg1W1slmL9MeQwEcS5WqSZVUIdM=", + "dev": true, + "requires": { + "caller-path": "^0.1.0", + "resolve-from": "^1.0.0" + }, + "dependencies": { + "resolve-from": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-1.0.1.tgz", + "integrity": "sha1-Jsv+k10a7uq7Kbw/5a6wHpPUQiY=", + "dev": true + } + } + }, + "requireindex": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/requireindex/-/requireindex-1.2.0.tgz", + "integrity": "sha512-L9jEkOi3ASd9PYit2cwRfyppc9NoABujTP8/5gFcbERmo5jUoAKovIC3fsF17pkTnGsrByysqX+Kxd2OTNI1ww==", + "dev": true + }, "resolve": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.9.0.tgz", @@ -5569,6 +5999,13 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, + "safe-json-stringify": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz", + "integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==", + "dev": true, + "optional": true + }, "safe-regex": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", @@ -5592,6 +6029,16 @@ "truncate-utf8-bytes": "^1.0.0" } }, + "schema-utils": { + "version": "0.4.7", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.7.tgz", + "integrity": "sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==", + "dev": true, + "requires": { + "ajv": "^6.1.0", + "ajv-keywords": "^3.1.0" + } + }, "semver": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/semver/-/semver-5.5.0.tgz", @@ -5693,6 +6140,17 @@ "integrity": "sha1-xB8vbDn8FtHNF61LXYlhFK5HDVU=", "dev": true }, + "slice-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.0.0.tgz", + "integrity": "sha512-4j2WTWjp3GsZ+AOagyzVbzp4vWGtZ0hEZ/gDY/uTvm6MTxUfTUIsnMIFb1bn8o0RuXiqUw15H1bue8f22Vw2oQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", + "is-fullwidth-code-point": "^2.0.0" + } + }, "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", @@ -5886,6 +6344,12 @@ "extend-shallow": "^3.0.0" } }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "http://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, "ssri": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/ssri/-/ssri-5.3.0.tgz", @@ -6016,6 +6480,12 @@ "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", "dev": true }, + "strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "dev": true + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -6025,12 +6495,50 @@ "has-flag": "^3.0.0" } }, + "table": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/table/-/table-5.1.1.tgz", + "integrity": "sha512-NUjapYb/qd4PeFW03HnAuOJ7OMcBkJlqeClWxeNlQ0lXGSb52oZXGzkO0/I0ARegQ2eUT1g2VDJH0eUxDRcHmw==", + "dev": true, + "requires": { + "ajv": "^6.6.1", + "lodash": "^4.17.11", + "slice-ansi": "2.0.0", + "string-width": "^2.1.1" + } + }, "tapable": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.0.0.tgz", "integrity": "sha512-dQRhbNQkRnaqauC7WqSJ21EEksgT0fYZX2lqXzGkpo8JNig9zGZTYoMGvyI2nWmXlE2VSVXVDu7wLVGu/mQEsg==", "dev": true }, + "tar": { + "version": "4.4.8", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.8.tgz", + "integrity": "sha512-LzHF64s5chPQQS0IYBn9IN5h3i98c12bo4NCO7e0sGM2llXQ3p2FGC5sdENN4cTW48O915Sh+x+EXx7XW96xYQ==", + "requires": { + "chownr": "^1.1.1", + "fs-minipass": "^1.2.5", + "minipass": "^2.3.4", + "minizlib": "^1.1.1", + "mkdirp": "^0.5.0", + "safe-buffer": "^5.1.2", + "yallist": "^3.0.2" + }, + "dependencies": { + "chownr": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz", + "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==" + }, + "yallist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==" + } + } + }, "terser": { "version": "3.8.1", "resolved": "https://registry.npmjs.org/terser/-/terser-3.8.1.tgz", @@ -6348,6 +6856,15 @@ "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=", "dev": true }, + "type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "dev": true, + "requires": { + "prelude-ls": "~1.1.2" + } + }, "type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", @@ -6365,6 +6882,27 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.1.1.tgz", "integrity": "sha512-Veu0w4dTc/9wlWNf2jeRInNodKlcdLgemvPsrNpfu5Pq39sgfFjvIIgTsvUHCoLBnMhPoUA+tFxsXjU6VexVRQ==" }, + "typescript-eslint-parser": { + "version": "21.0.2", + "resolved": "https://registry.npmjs.org/typescript-eslint-parser/-/typescript-eslint-parser-21.0.2.tgz", + "integrity": "sha512-u+pj4RVJBr4eTzj0n5npoXD/oRthvfUCjSKndhNI714MG0mQq2DJw5WP7qmonRNIFgmZuvdDOH3BHm9iOjIAfg==", + "dev": true, + "requires": { + "eslint-scope": "^4.0.0", + "eslint-visitor-keys": "^1.0.0", + "typescript-estree": "5.3.0" + } + }, + "typescript-estree": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/typescript-estree/-/typescript-estree-5.3.0.tgz", + "integrity": "sha512-Vu0KmYdSCkpae+J48wsFC1ti19Hq3Wi/lODUaE+uesc3gzqhWbZ5itWbsjylLVbjNW4K41RqDzSfnaYNbmEiMQ==", + "dev": true, + "requires": { + "lodash.unescape": "4.0.1", + "semver": "5.5.0" + } + }, "uglify-js": { "version": "3.4.9", "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.4.9.tgz", @@ -6383,74 +6921,6 @@ } } }, - "uglifyjs-webpack-plugin": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/uglifyjs-webpack-plugin/-/uglifyjs-webpack-plugin-1.2.7.tgz", - "integrity": "sha512-1VicfKhCYHLS8m1DCApqBhoulnASsEoJ/BvpUpP4zoNAPpKzdH+ghk0olGJMmwX2/jprK2j3hAHdUbczBSy2FA==", - "dev": true, - "requires": { - "cacache": "^10.0.4", - "find-cache-dir": "^1.0.0", - "schema-utils": "^0.4.5", - "serialize-javascript": "^1.4.0", - "source-map": "^0.6.1", - "uglify-es": "^3.3.4", - "webpack-sources": "^1.1.0", - "worker-farm": "^1.5.2" - }, - "dependencies": { - "ajv": { - "version": "6.5.4", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.5.4.tgz", - "integrity": "sha512-4Wyjt8+t6YszqaXnLDfMmG/8AlO5Zbcsy3ATHncCzjW/NoPzAId8AK6749Ybjmdt+kUY1gP60fCu46oDxPv/mg==", - "dev": true, - "requires": { - "fast-deep-equal": "^2.0.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "commander": { - "version": "2.13.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.13.0.tgz", - "integrity": "sha512-MVuS359B+YzaWqjCL/c+22gfryv+mCBPHAv3zyVI2GN8EY6IRP8VwtasXn8jyyhvvq84R4ImN1OKRtcbIasjYA==", - "dev": true - }, - "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", - "dev": true - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "schema-utils": { - "version": "0.4.7", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.7.tgz", - "integrity": "sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==", - "dev": true, - "requires": { - "ajv": "^6.1.0", - "ajv-keywords": "^3.1.0" - } - }, - "uglify-es": { - "version": "3.3.9", - "resolved": "https://registry.npmjs.org/uglify-es/-/uglify-es-3.3.9.tgz", - "integrity": "sha512-r+MU0rfv4L/0eeW3xZrd16t4NZfK8Ld4SWVglYBb7ez5uXFWHuVRs6xCTrf1yirs9a4j4Y27nn7SRfO6v67XsQ==", - "dev": true, - "requires": { - "commander": "~2.13.0", - "source-map": "~0.6.1" - } - } - } - }, "unicode-canonical-property-names-ecmascript": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-1.0.4.tgz", @@ -6668,6 +7138,16 @@ "spdx-expression-parse": "^3.0.0" } }, + "verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "requires": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, "vm-browserify": { "version": "0.0.4", "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-0.0.4.tgz", @@ -6697,16 +7177,15 @@ } }, "webpack": { - "version": "4.16.5", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.16.5.tgz", - "integrity": "sha512-i5cHYHonzSc1zBuwB5MSzW4v9cScZFbprkHK8ZgzPDCRkQXGGpYzPmJhbus5bOrZ0tXTcQp+xyImRSvKb0b+Kw==", + "version": "4.28.3", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.28.3.tgz", + "integrity": "sha512-vLZN9k5I7Nr/XB1IDG9GbZB4yQd1sPuvufMFgJkx0b31fi2LD97KQIjwjxE7xytdruAYfu5S0FLBLjdxmwGJCg==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.5.13", - "@webassemblyjs/helper-module-context": "1.5.13", - "@webassemblyjs/wasm-edit": "1.5.13", - "@webassemblyjs/wasm-opt": "1.5.13", - "@webassemblyjs/wasm-parser": "1.5.13", + "@webassemblyjs/ast": "1.7.11", + "@webassemblyjs/helper-module-context": "1.7.11", + "@webassemblyjs/wasm-edit": "1.7.11", + "@webassemblyjs/wasm-parser": "1.7.11", "acorn": "^5.6.2", "acorn-dynamic-import": "^3.0.0", "ajv": "^6.1.0", @@ -6723,45 +7202,240 @@ "neo-async": "^2.5.0", "node-libs-browser": "^2.0.0", "schema-utils": "^0.4.4", - "tapable": "^1.0.0", - "uglifyjs-webpack-plugin": "^1.2.4", + "tapable": "^1.1.0", + "terser-webpack-plugin": "^1.1.0", "watchpack": "^1.5.0", - "webpack-sources": "^1.0.1" + "webpack-sources": "^1.3.0" }, "dependencies": { - "ajv": { - "version": "6.5.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.5.2.tgz", - "integrity": "sha512-hOs7GfvI6tUI1LfZddH82ky6mOMyTuY0mk7kE2pWpmhhUSkumzaTO5vbVwij39MdwPQWCV4Zv57Eo06NtL/GVA==", + "bluebird": { + "version": "3.5.3", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.5.3.tgz", + "integrity": "sha512-/qKPUQlaW1OyR51WeCPBvRnAlnZFUJkCSG5HzGnuIqhgyJtF+T94lFnn33eiazjRm2LAHVy2guNnaq48X9SJuw==", + "dev": true + }, + "cacache": { + "version": "11.3.2", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-11.3.2.tgz", + "integrity": "sha512-E0zP4EPGDOaT2chM08Als91eYnf8Z+eH1awwwVsngUmgppfM5jjJ8l3z5vO5p5w/I3LsiXawb1sW0VY65pQABg==", "dev": true, "requires": { - "fast-deep-equal": "^2.0.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.1" + "bluebird": "^3.5.3", + "chownr": "^1.1.1", + "figgy-pudding": "^3.5.1", + "glob": "^7.1.3", + "graceful-fs": "^4.1.15", + "lru-cache": "^5.1.1", + "mississippi": "^3.0.0", + "mkdirp": "^0.5.1", + "move-concurrently": "^1.0.1", + "promise-inflight": "^1.0.1", + "rimraf": "^2.6.2", + "ssri": "^6.0.1", + "unique-filename": "^1.1.1", + "y18n": "^4.0.0" } }, - "fast-deep-equal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", - "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", + "chownr": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz", + "integrity": "sha512-j38EvO5+LHX84jlo6h4UzmOwi0UgW61WRyPtJz4qaadK5eY3BTS5TY/S1Stc3Uk2lIM6TPevAlULiEJwie860g==", "dev": true }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "figgy-pudding": { + "version": "3.5.1", + "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.1.tgz", + "integrity": "sha512-vNKxJHTEKNThjfrdJwHc7brvM6eVevuO5nTj6ez8ZQ1qbXTvGthucRF7S4vf2cr71QVnT70V34v0S1DyQsti0w==", + "dev": true + }, + "find-cache-dir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.0.0.tgz", + "integrity": "sha512-LDUY6V1Xs5eFskUVYtIwatojt6+9xC9Chnlk/jYOOvn3FAFfSaWddxahDGyNHh0b2dMXa6YW2m0tk8TdVaXHlA==", + "dev": true, + "requires": { + "commondir": "^1.0.1", + "make-dir": "^1.0.0", + "pkg-dir": "^3.0.0" + } + }, + "find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "requires": { + "locate-path": "^3.0.0" + } + }, + "glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "graceful-fs": { + "version": "4.1.15", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", + "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==", "dev": true }, - "schema-utils": { - "version": "0.4.7", - "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-0.4.7.tgz", - "integrity": "sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==", + "locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", "dev": true, "requires": { - "ajv": "^6.1.0", - "ajv-keywords": "^3.1.0" + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" } + }, + "lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "requires": { + "yallist": "^3.0.2" + } + }, + "mississippi": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz", + "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==", + "dev": true, + "requires": { + "concat-stream": "^1.5.0", + "duplexify": "^3.4.2", + "end-of-stream": "^1.1.0", + "flush-write-stream": "^1.0.0", + "from2": "^2.1.0", + "parallel-transform": "^1.1.0", + "pump": "^3.0.0", + "pumpify": "^1.3.3", + "stream-each": "^1.1.0", + "through2": "^2.0.0" + } + }, + "p-limit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz", + "integrity": "sha512-NhURkNcrVB+8hNfLuysU8enY5xn2KXphsHBaC2YmRNTZRc7RWusw6apSpdEj3jo4CMb6W9nrF6tTnsJsJeyu6g==", + "dev": true, + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "requires": { + "p-limit": "^2.0.0" + } + }, + "p-try": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.0.0.tgz", + "integrity": "sha512-hMp0onDKIajHfIkdRk3P4CdCmErkYAxxDtP3Wx/4nZ3aGlau2VKh3mZpcuFkH27WQkL/3WBCPOktzA9ZOAnMQQ==", + "dev": true + }, + "pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, + "requires": { + "find-up": "^3.0.0" + } + }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "ssri": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.1.tgz", + "integrity": "sha512-3Wge10hNcT1Kur4PDFwEieXSCMCJs/7WvSACcrMYrNp+b8kDL1/0wJch5Ni2WrtwEa2IO8OsVfeKIciKCDx/QA==", + "dev": true, + "requires": { + "figgy-pudding": "^3.5.1" + } + }, + "tapable": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.1.tgz", + "integrity": "sha512-9I2ydhj8Z9veORCw5PRm4u9uebCn0mcCa6scWoNcbZ6dAtoo2618u9UUzxgmsCOreJpqDDuv61LvwofW7hLcBA==", + "dev": true + }, + "terser-webpack-plugin": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.2.1.tgz", + "integrity": "sha512-GGSt+gbT0oKcMDmPx4SRSfJPE1XaN3kQRWG4ghxKQw9cn5G9x6aCKSsgYdvyM0na9NJ4Drv0RG6jbBByZ5CMjw==", + "dev": true, + "requires": { + "cacache": "^11.0.2", + "find-cache-dir": "^2.0.0", + "schema-utils": "^1.0.0", + "serialize-javascript": "^1.4.0", + "source-map": "^0.6.1", + "terser": "^3.8.1", + "webpack-sources": "^1.1.0", + "worker-farm": "^1.5.2" + }, + "dependencies": { + "schema-utils": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", + "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==", + "dev": true, + "requires": { + "ajv": "^6.1.0", + "ajv-errors": "^1.0.0", + "ajv-keywords": "^3.1.0" + } + } + } + }, + "unique-filename": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", + "integrity": "sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==", + "dev": true, + "requires": { + "unique-slug": "^2.0.0" + } + }, + "webpack-sources": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.3.0.tgz", + "integrity": "sha512-OiVgSrbGu7NEnEvQJJgdSFPl2qWKkWq5lHMhgiToIiN9w34EBnjYzSYs+VbL5KoYiLNtFFa7BZIKxRED3I32pA==", + "dev": true, + "requires": { + "source-list-map": "^2.0.0", + "source-map": "~0.6.1" + } + }, + "yallist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz", + "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==", + "dev": true } } }, @@ -7066,6 +7740,15 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, + "write": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/write/-/write-0.2.1.tgz", + "integrity": "sha1-X8A4KOJkzqP+kUVUdvejxWbLB1c=", + "dev": true, + "requires": { + "mkdirp": "^0.5.1" + } + }, "xregexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.0.0.tgz", diff --git a/package.json b/package.json index 11c90e4..1d33926 100644 --- a/package.json +++ b/package.json @@ -12,16 +12,16 @@ "build": "run-s clean compile:runtime compile", "build:watch": "webpack --mode=development --watch", "compile": "webpack --mode=production", - "compile:runtime": "tsr --compilerOptions '{\"outDir\": \"src/configuration/site-traversal/runtime/\"}' src/configuration/site-traversal/assert.ts && tsr --compilerOptions '{\"outDir\": \"src/configuration/run-options/runtime\"}' src/configuration/run-options/assert.ts", + "compile:runtime": "tsr --tsConfig src/settings/config/runtime/tsconfig.runtime.json src/settings/config/assert.ts && tsr --tsConfig src/settings/options/runtime/tsconfig.runtime.json src/settings/options/assert.ts", "clean": "rimraf lib", - "lint": "git ls-files '*.ts' '*.json' | xargs prettier --list-different", - "lint:fix": "git ls-files '*.ts' '*.json' | xargs prettier --write", + "lint": "eslint '**/*.ts'", + "format": "git ls-files '*.ts' '*.json' | xargs prettier --write", + "format:check": "git ls-files '*.ts' '*.json' | xargs prettier --list-different", "typecheck": "tsc --noEmit --project tsconfig.json", "typecheck:watch": "npm run typecheck -- --watch", - "test": "mocha-webpack", - "test:watch": "npm run test -- --watch", - "test:unit": "npm run test **/*.unit.test.ts", - "test:functional": "npm run test **/*functional.test.ts" + "test:watch": "mocha-webpack --grep node_modules -i **/*.test.ts --watch", + "test:unit": "mocha-webpack **/*.unit.test.ts", + "test:functional": "mocha-webpack **/*functional.test.ts" }, "repository": { "type": "git", @@ -38,7 +38,8 @@ "@babel/preset-env": "^7.2.3", "@babel/preset-typescript": "^7.1.0", "@babel/register": "^7.0.0", - "@types/better-sqlite3": "^3.1.3", + "@types/better-sqlite3": "^5.2.1", + "@types/bunyan": "^1.8.5", "@types/chai": "^4.1.7", "@types/cheerio": "^0.22.9", "@types/handlebars": "^4.0.39", @@ -48,11 +49,15 @@ "@types/node-fetch": "^2.1.2", "@types/sanitize-filename": "^1.1.28", "@types/source-map-support": "^0.4.1", + "@types/verror": "^1.10.3", "babel-loader": "^8.0.4", + "bunyan": "^1.8.12", "chai": "^4.2.0", "chai-exclude": "^1.0.12", - "clean-terminal-webpack-plugin": "^1.1.0", "copy-webpack-plugin": "^4.5.2", + "eslint": "^5.11.1", + "eslint-plugin-no-only-tests": "^2.1.0", + "eslint-plugin-typescript": "^1.0.0-rc.1", "lodash": "^4.17.11", "mocha": "^5.2.0", "mocha-webpack": "^2.0.0-beta.0", @@ -64,18 +69,19 @@ "rxjs-marbles": "^4.3.5", "terser-webpack-plugin": "^1.0.0", "typescript": "^3.1.1", - "webpack": "^4.12.0", + "webpack": "^4.28.3", "webpack-cli": "^3.0.6", "webpack-node-externals": "^1.7.2" }, "dependencies": { - "better-sqlite3": "^4.1.4", + "better-sqlite3": "^5.2.1", "cheerio": "^1.0.0-rc.2", "handlebars": "^4.0.12", "node-fetch": "^2.1.2", "rxjs": "^6.3.3", "sanitize-filename": "^1.6.1", "source-map-support": "^0.5.6", - "ts-runtime": "^0.2.0" + "ts-runtime": "^0.2.0", + "verror": "^1.10.0" } } diff --git a/src/configuration/run-options/assert.ts b/src/configuration/run-options/assert.ts deleted file mode 100644 index 6813147..0000000 --- a/src/configuration/run-options/assert.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { RunOptionsInit } from './types' - -export const assertOptionsType = (options: RunOptionsInit) => {} diff --git a/src/configuration/run-options/normalize.ts b/src/configuration/run-options/normalize.ts deleted file mode 100644 index 35d0cb8..0000000 --- a/src/configuration/run-options/normalize.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { resolve } from 'path' -import { makeFlatConfig } from '../site-traversal/make-flat-config' -import { assertOptionsType } from './' -// type imports -import { RunOptionsInit, FlatRunOptions } from './types' -import { Config } from '../site-traversal/types' - -const assertValidInput = (config: Config, runParams: RunOptionsInit) => { - const configInputKeys = Object.keys(config.input) - const runParamsInputKeys = runParams.input ? Object.keys(runParams.input) : [] - if (configInputKeys.length < runParamsInputKeys.length) { - const missingKeys = runParamsInputKeys - .filter(key => !configInputKeys.includes(key)) - .join() - throw new Error(`Invalid input! Options has extra key(s) [${missingKeys}]`) - } else if (configInputKeys.length > runParamsInputKeys.length) { - const missingKeys = configInputKeys - .filter(key => !runParamsInputKeys.includes(key)) - .join() - throw new Error(`Invalid input! Config is missing key(s) [${missingKeys}]`) - } -} - -const normalizeOptions = (config: Config, runParams: RunOptionsInit) => { - assertOptionsType(runParams) - - const flatConfig = makeFlatConfig(config) - const { optionsEach = {}, ...globalOptions } = runParams - - assertValidInput(config, runParams) - - const defaults = { - input: {}, - cache: true, - useLimiter: true, - downloadPriority: 0, - ...globalOptions // user preferences for all things override - } - - const options: FlatRunOptions = Object.values(flatConfig).reduce( - (acc: FlatRunOptions, scraperConfig) => { - const { name } = scraperConfig - const scraperOptions = optionsEach[name] - acc[name] = { - ...defaults, - folder: resolve(defaults.folder, name), - ...scraperOptions - } - return acc - }, - {} - ) - - return options -} -export { normalizeOptions } diff --git a/src/configuration/run-options/runtime/assert.js b/src/configuration/run-options/runtime/assert.js deleted file mode 100644 index 248b51a..0000000 --- a/src/configuration/run-options/runtime/assert.js +++ /dev/null @@ -1,5 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -var lib_1 = require("ts-runtime/lib"); -var types_1 = require("./types"); -exports.assertOptionsType = lib_1.default.annotate(function (options) { var _optionsType = lib_1.default.nullable(lib_1.default.ref(types_1.RunOptionsInit)); lib_1.default.param("options", _optionsType).assert(options); }, lib_1.default.function(lib_1.default.param("options", lib_1.default.nullable(lib_1.default.ref(types_1.RunOptionsInit))), lib_1.default.return(lib_1.default.any()))); diff --git a/src/configuration/run-options/runtime/types.js b/src/configuration/run-options/runtime/types.js deleted file mode 100644 index e9cb0a3..0000000 --- a/src/configuration/run-options/runtime/types.js +++ /dev/null @@ -1,11 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -var lib_1 = require("ts-runtime/lib"); -var Input = lib_1.default.type("Input", lib_1.default.object(lib_1.default.indexer("inputName", lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.number()), lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.boolean())))))); -var OptionsAny = lib_1.default.type("OptionsAny", lib_1.default.object(lib_1.default.property("cache", lib_1.default.nullable(lib_1.default.boolean()), true))); -var ScraperOptionsInit = lib_1.default.type("ScraperOptionsInit", lib_1.default.intersect(lib_1.default.ref(OptionsAny), lib_1.default.object(lib_1.default.property("downloadPriority", lib_1.default.nullable(lib_1.default.number()), true)))); -var ScraperOptions = lib_1.default.type("ScraperOptions", lib_1.default.intersect(lib_1.default.ref(OptionsAny), lib_1.default.object(lib_1.default.property("cache", lib_1.default.nullable(lib_1.default.boolean())), lib_1.default.property("downloadPriority", lib_1.default.nullable(lib_1.default.number()))))); -exports.Parallelism = lib_1.default.type("Parallelism", lib_1.default.object(lib_1.default.property("maxConcurrent", lib_1.default.nullable(lib_1.default.number()), true), lib_1.default.property("rateLimit", lib_1.default.object(lib_1.default.property("rate", lib_1.default.nullable(lib_1.default.number())), lib_1.default.property("limit", lib_1.default.nullable(lib_1.default.number()))), true))); -exports.RunOptionsInit = lib_1.default.type("RunOptionsInit", lib_1.default.intersect(lib_1.default.ref(OptionsAny), lib_1.default.ref(exports.Parallelism), lib_1.default.object(lib_1.default.property("input", lib_1.default.nullable(lib_1.default.ref(Input)), true), lib_1.default.property("folder", lib_1.default.nullable(lib_1.default.string())), lib_1.default.property("optionsEach", lib_1.default.object(lib_1.default.indexer("name", lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.ref(ScraperOptionsInit)))), true)))); -exports.RunOptions = lib_1.default.type("RunOptions", lib_1.default.intersect(lib_1.default.ref(ScraperOptions), lib_1.default.object(lib_1.default.property("input", lib_1.default.nullable(lib_1.default.ref(Input))), lib_1.default.property("folder", lib_1.default.nullable(lib_1.default.string()))))); -exports.FlatRunOptions = lib_1.default.type("FlatRunOptions", lib_1.default.object(lib_1.default.indexer("name", lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.ref(exports.RunOptions))))); diff --git a/src/configuration/run-options/types.ts b/src/configuration/run-options/types.ts deleted file mode 100644 index b28baf8..0000000 --- a/src/configuration/run-options/types.ts +++ /dev/null @@ -1,39 +0,0 @@ -type Input = { [inputName: string]: number | string | boolean } - -interface OptionsAny { - cache?: boolean -} - -interface ScraperOptionsInit extends OptionsAny { - downloadPriority?: number -} -interface ScraperOptions extends OptionsAny { - cache: boolean - downloadPriority: number -} - -export interface Parallelism { - maxConcurrent?: number - rateLimit?: { - rate: number - limit: number - } -} - -export interface RunOptionsInit extends OptionsAny, Parallelism { - input?: Input - folder: string - cleanFolder?: boolean - optionsEach?: { - [name: string]: ScraperOptionsInit - } -} - -export interface RunOptions extends ScraperOptions { - input: Input - folder: string -} - -export type FlatRunOptions = { - [name: string]: RunOptions -} diff --git a/src/emitter.ts b/src/emitter.ts deleted file mode 100644 index 8c8598c..0000000 --- a/src/emitter.ts +++ /dev/null @@ -1,71 +0,0 @@ -import EventEmitter from 'events' -import * as Rx from 'rxjs' -import { makeFlatConfig } from './configuration/site-traversal/make-flat-config' -import { Config } from './configuration/site-traversal/types' - -class ScrapeEmitter { - private _emitter: EventEmitter - private store: any - - forScraper: { - [name: string]: { - emitQueuedDownload: (id: number) => void - emitProgress: (id: number, progress: number) => void - emitCompletedDownload: (id: number) => void - } - } = {} - - constructor(config: Config, store: any) { - // TODO replace w/ store.queryFor - this._emitter = new EventEmitter() - this.store = store - - const flatConfig = makeFlatConfig(config) - for (const name of Object.keys(flatConfig)) { - this.forScraper[name] = { - emitQueuedDownload: id => { - this.emitter.emit(`${name}:queued`, this.store.queryFor, { id }) - this.emitter.emit('queued', this.store.queryFor, { name, id }) - }, - emitProgress: (id, progress) => { - this.emitter.emit(`${name}:progress`, this.store.queryFor, { - id, - progress - }) - }, - emitCompletedDownload: id => { - this.emitter.emit(`${name}:complete`, this.store.queryFor, { id }) - this.emitter.emit('complete', this.store.queryFor, { name, id }) - } - } - } - } - - hasListenerFor(eventName: string) { - return Boolean(this.emitter.listenerCount(eventName)) - } - - emitDone() { - this.emitter.emit('done', this.store.queryFor) - } - - emitError(error: Error) { - if (this.emitter.listenerCount('error')) { - this.emitter.emit('error', error) - } else { - throw error - } - } - - get emitter() { - return this._emitter - } - - getRxEventStream = (eventName: string) => - Rx.fromEvent(this.emitter, eventName) - - onStop(cb: () => any) { - this.emitter.on('stop', cb) - } -} -export default ScrapeEmitter diff --git a/src/index.ts b/src/index.ts index 0b4bf2e..51ad1ee 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ import 'source-map-support/register' -import PageScraper from './scraper' -export default PageScraper +export { scrape } from './scraper' +export * from './settings/external-utils' diff --git a/src/logger.ts b/src/logger.ts deleted file mode 100644 index 39c0fa0..0000000 --- a/src/logger.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { tap } from 'rxjs/operators' - -const logLevelConstants = { - DEBUG: 3, - INFO: 2, - WARN: 1, - ERROR: 0 -} -type LogConstantsType = typeof logLevelConstants -export type LogType = keyof LogConstantsType - -class Logger { - permittedLogLevel: LogType - - debug: (...args: any[]) => void - info: (...args: any[]) => void - warn: (...args: any[]) => void - error: (...args: any[]) => void - tap: (...args: any[]) => void - - constructor({ logLevel }: { logLevel: LogType }) { - this.permittedLogLevel = logLevel - // setup loggers - this.debug = this.isPermitted('DEBUG') ? this._log('DEBUG') : () => {} - this.info = this.isPermitted('INFO') ? this._log() : () => {} - this.warn = this.isPermitted('WARN') ? this._log('WARN') : () => {} - this.error = this.isPermitted('ERROR') - ? this._log('ERRO', console.error) - : () => {} - this.tap = this.isPermitted('DEBUG') - ? (name = 'TAP') => tap(this._log(name)) - : () => tap - } - - isPermitted = (logLevel: LogType): boolean => - logLevelConstants[logLevel] <= logLevelConstants[this.permittedLogLevel] - - _log = (prefix?: string, logger = console.log) => - prefix - ? (...messages: any[]) => { - logger(prefix, ...messages) - } - : (...messages: any[]) => { - logger(...messages) - } -} - -export default Logger diff --git a/src/scraper/index.ts b/src/scraper/index.ts index 6f82853..4b9053e 100644 --- a/src/scraper/index.ts +++ b/src/scraper/index.ts @@ -1,105 +1,60 @@ -import * as Rx from 'rxjs' -import Emitter from '../emitter' -import Store from '../store' -import Logger from '../logger' -import Queue from '../queue' -import scraper from './scrape-step' -import { mkdirp, mkdir, rmrf } from '../util/fs' -import { normalizeConfig } from '../configuration/site-traversal' -import { normalizeOptions } from '../configuration/run-options' +import { initTools } from '../tools' +import { ScrapeStep } from './scrape-step' +import { mkdirp, rmrf } from '../util/fs' +import { normalizeConfig } from '../settings/config' +import { normalizeOptions } from '../settings/options' // type imports -import { LogType } from '../logger' -import { Config, ConfigInit } from '../configuration/site-traversal/types' -import { - RunOptionsInit, - FlatRunOptions -} from '../configuration/run-options/types' -import { Dependencies } from './types' - -class ScrapePages { - config: Config - configuredScraper: ReturnType - scrapingScheme: ReturnType> - // dependencies - store: Store - emitter: Emitter - logger: Logger - queue: Queue - - constructor(config: ConfigInit) { - this.config = normalizeConfig(config) - this.configuredScraper = scraper(this.config.scrape) - } - - initResources = async ( - runParams: RunOptionsInit, - flatRunParams: FlatRunOptions - ) => { - if (runParams.cleanFolder) { - this.logger.info(`Cleaning ${runParams.folder}`) - await rmrf(runParams.folder) - } - - this.logger.info('Making folders.') - await mkdirp(runParams.folder) - for (const { folder } of Object.values(flatRunParams)) await mkdirp(folder) - - this.logger.info('Setting up SQLite database.') - // syncronous db creation - this.store.init(runParams) - - this.logger.info('Begin downloading with inputs', runParams.input) - } - - // TODO add parsable input for this first parse step - initDependencies = (runParams: RunOptionsInit, logLevel: LogType) => { - const flatRunParams = normalizeOptions(this.config, runParams) - - this.store = new Store(this.config) - this.emitter = new Emitter(this.config, this.store) - this.logger = new Logger({ logLevel }) - const rateLimiterEventStream = this.emitter.getRxEventStream( - 'useRateLimiter' - ) as Rx.Observable // deal with incoming values on this event as truthy or falsey - this.queue = new Queue(runParams, flatRunParams, rateLimiterEventStream) - - this.scrapingScheme = this.configuredScraper(flatRunParams, { - queue: this.queue, - emitter: this.emitter, - logger: this.logger, - store: this.store - }) - - return Rx.concat( - this.initResources(runParams, flatRunParams), - this.scrapingScheme([{ parsedValue: '' }]) - ) - } - - run = (runParams: RunOptionsInit, logLevel: LogType = 'ERROR') => { - const scrapingObservable = this.initDependencies(runParams, logLevel) - const subscription = scrapingObservable.subscribe( - undefined, - error => { - this.emitter.emitError(error) - subscription.unsubscribe() - this.queue.closeQueue() - }, - () => { - // TODO add timer to show how long it took - this.logger.info('Done!') - this.queue.closeQueue() - this.emitter.emitDone() - } - ) +import { Config, ConfigInit } from '../settings/config/types' +import { OptionsInit, FlatOptions } from '../settings/options/types' + +const initFolders = async ( + config: Config, + optionsInit: OptionsInit, + flatOptions: FlatOptions +) => { + if (optionsInit.cleanFolder) await rmrf(optionsInit.folder) + + await mkdirp(optionsInit.folder) + for (const { folder } of flatOptions.values()) await mkdirp(folder) +} - this.emitter.onStop(() => { - this.queue.closeQueue() +export const scrape = async ( + configInit: ConfigInit, + optionsInit: OptionsInit +) => { + const config = normalizeConfig(configInit) + const flatOptions = normalizeOptions(config, optionsInit) + await initFolders(config, optionsInit, flatOptions) + const tools = initTools(config, optionsInit, flatOptions) + // create the observable + const scrapingScheme = new ScrapeStep(config.scrape, flatOptions, tools) + const scrapingObservable = scrapingScheme.run([{ parsedValue: '' }]) + // start running the observable + const { emitter, queue, logger, store } = tools + const subscription = scrapingObservable.subscribe({ + error: (error: Error) => { + emitter.emit.error(error) subscription.unsubscribe() - this.logger.info('Exited manually.') - }) - return this.emitter.emitter + queue.closeQueue() + }, + complete: () => { + // TODO add timer to show how long it took + queue.closeQueue() + emitter.emit.done() + logger.info('Done!') + } + }) + + emitter.on.stop(() => { + logger.info('Exiting manually.') + queue.closeQueue() + subscription.unsubscribe() + logger.info('Done!') + emitter.emit.done() + }) + return { + on: emitter.getBoundOn(), + emit: emitter.getBoundEmit(), + query: store.query } } - -export default ScrapePages diff --git a/src/scraper/scrape-step/downloader/abstract.ts b/src/scraper/scrape-step/downloader/abstract.ts new file mode 100644 index 0000000..08e73ef --- /dev/null +++ b/src/scraper/scrape-step/downloader/abstract.ts @@ -0,0 +1,51 @@ +import { ScrapeConfig } from '../../../settings/config/types' +import { Options } from '../../../settings/options/types' +import { Tools } from '../../../tools' + +export type DownloadParams = { + parentId?: number + incrementIndex: number + value?: string +} +type RetrieveValue = { downloadValue?: string; filename?: string } +/** + * base abstract class which other downloaders derive from + */ +export abstract class AbstractDownloader { + protected config: ScrapeConfig + protected options: Options + protected tools: Tools + + public constructor(config: ScrapeConfig, options: Options, tools: Tools) { + Object.assign(this, { config, options, tools }) + } + public run = async (downloadParams: DownloadParams) => { + const downloadData = this.constructDownload(downloadParams) + const downloadId = this.tools.store.qs.insertQueuedDownload( + this.config.name, + downloadParams, + downloadData + ) + this.tools.emitter.scraper(this.config.name).emit.queued(downloadId) + const { downloadValue, filename } = await this.retrieve( + downloadId, + downloadData + ) + + return { + downloadId, + // downloadValue conditional is for identity parser, + // essentially, if there was no data that went into a download, then nothing important was recieved + downloadValue: downloadData ? downloadValue : downloadParams.value, + filename + } + } + // implement these methods + protected abstract constructDownload( + downloadParams: DownloadParams + ): DownloadData + protected abstract retrieve( + downloadId: number, + downloadParams: DownloadData + ): RetrieveValue | Promise +} diff --git a/src/scraper/scrape-step/downloader/implementations/http.ts b/src/scraper/scrape-step/downloader/implementations/http.ts new file mode 100644 index 0000000..f8ff975 --- /dev/null +++ b/src/scraper/scrape-step/downloader/implementations/http.ts @@ -0,0 +1,164 @@ +import fetch, * as Fetch from 'node-fetch' +import { createWriteStream } from 'fs' +import path from 'path' +import { mkdirp, sanitizeFilename } from '../../../../util/fs' + +import { AbstractDownloader, DownloadParams } from '../abstract' +import { compileTemplate } from '../../../../util/handlebars' +// type imports +import { URL } from 'url' +import { ScrapeConfig } from '../../../../settings/config/types' +import { Options } from '../../../../settings/options/types' +import { Tools } from '../../../../tools' +import { DownloadConfig } from '../../../../settings/config/types' + +type Headers = { [header: string]: string } +type DownloadData = [ + string, + { headers: Headers; method: DownloadConfig['method'] } +] +type FetchFunction = ( + downloadId: number, + DownloadData: DownloadData +) => Promise<{ + downloadValue?: string + filename?: string +}> + +/** + * downloader pertaining to all http/https requests + */ +export class Downloader extends AbstractDownloader { + private urlTemplate: ReturnType + private headerTemplates: Map> + private fetcher: FetchFunction + + public constructor(config: ScrapeConfig, options: Options, tools: Tools) { + super(config, options, tools) + // set templates + this.urlTemplate = compileTemplate(config.download!.urlTemplate) + this.headerTemplates = new Map() + Object.entries(config.download!.headerTemplates).forEach( + ([key, templateStr]) => + this.headerTemplates.set(key, compileTemplate(templateStr)) + ) + // choose fetcher + // TODO change this to manual options option ONLY + const shouldDownloadToMemory = + this.config.scrapeEach.length || this.config.parse + const shouldDownloadToFile = this.options.cache + if (shouldDownloadToMemory && shouldDownloadToFile) { + this.fetcher = this.downloadToFileAndMemory + } else if (shouldDownloadToMemory) { + this.fetcher = this.downloadToMemoryOnly + } else { + this.fetcher = this.downloadToFileOnly + } + } + + protected constructDownload = ({ + value, + incrementIndex: index + }: DownloadParams): DownloadData => { + const templateVals = { ...this.options.input, value, index } + // construct url + const url = new URL(this.urlTemplate(templateVals)).toString() + // construct headers + const headers: Headers = {} + for (const [key, template] of this.headerTemplates) { + headers[key] = template(templateVals) + } + return [url, { headers, method: this.config.download!.method }] + } + + protected retrieve = (downloadId: number, downloadData: DownloadData) => { + return this.fetcher(downloadId, downloadData) + } + + private verifyResponseOk = (response: Fetch.Response, url: string) => { + if (!response.ok) { + throw new Error(`status ${response.status} for ${url}`) + } + } + private downloadToFileAndMemory: FetchFunction = async ( + downloadId, + [url, fetchOptions] + ) => { + const downloadFolder = path.resolve( + this.options.folder, + downloadId.toString() + ) + const filename = path.resolve(downloadFolder, sanitizeFilename(url)) + + const response = await this.tools.queue.add( + () => fetch(url, fetchOptions), + this.options.downloadPriority + ) + await mkdirp(downloadFolder) + this.verifyResponseOk(response, url) + const dest = createWriteStream(filename) + const buffers: Buffer[] = [] + + const buffer = await new Promise((resolve, reject) => { + response.body.pipe(dest) + response.body.on('error', error => reject(error)) + response.body.on('data', chunk => buffers.push(chunk)) + this.tools.emitter + .scraper(this.config.name) + .emit.progress(downloadId, response) + dest.on('error', error => reject(error)) + dest.on('close', () => resolve(Buffer.concat(buffers))) + }) + return { + downloadValue: buffer.toString(), + filename + } + } + private downloadToFileOnly: FetchFunction = async ( + downloadId, + [url, fetchOptions] + ) => { + const downloadFolder = path.resolve( + this.options.folder, + downloadId.toString() + ) + const filename = path.resolve(downloadFolder, sanitizeFilename(url)) + + const response = await this.tools.queue.add( + () => fetch(url.toString(), fetchOptions), + this.options.downloadPriority + ) + await mkdirp(downloadFolder) + await new Promise((resolve, reject) => { + this.verifyResponseOk(response, url) + const dest = createWriteStream(filename) + response.body.pipe(dest) + this.tools.emitter + .scraper(this.config.name) + .emit.progress(downloadId, response) + response.body.on('error', error => reject(error)) + dest.on('error', error => reject(error)) + dest.on('close', resolve) + }) + return { + downloadValue: undefined, + filename + } + } + private downloadToMemoryOnly: FetchFunction = ( + downloadId, + [url, fetchOptions] + ) => + this.tools.queue + .add(() => fetch(url, fetchOptions), this.options.downloadPriority) + .then(response => { + this.verifyResponseOk(response, url) + this.tools.emitter + .scraper(this.config.name) + .emit.progress(downloadId, response) + return response.text() + }) + .then(downloadValue => ({ + downloadValue + })) +} diff --git a/src/scraper/scrape-step/downloader/implementations/identity.ts b/src/scraper/scrape-step/downloader/implementations/identity.ts new file mode 100644 index 0000000..6aee1d1 --- /dev/null +++ b/src/scraper/scrape-step/downloader/implementations/identity.ts @@ -0,0 +1,16 @@ +import { AbstractDownloader } from '../abstract' + +type DownloadData = void +/** + * identitiy downloader, does nothing and passes value through itself + */ +export class Downloader extends AbstractDownloader { + protected insertDownloadData = false + + protected constructDownload = (): DownloadData => {} + + protected retrieve = () => ({ + downloadValue: undefined, + filename: undefined + }) +} diff --git a/src/scraper/scrape-step/downloader/index.ts b/src/scraper/scrape-step/downloader/index.ts index 81b0378..45a26a6 100644 --- a/src/scraper/scrape-step/downloader/index.ts +++ b/src/scraper/scrape-step/downloader/index.ts @@ -1,36 +1,18 @@ -import { downloader as urlDownloader } from './variations/url-downloader' -import { downloader as identityDownloader } from './variations/identity-downloader' +import { Downloader as IdentityDownloader } from './implementations/identity' +import { Downloader as HttpDownloader } from './implementations/http' // type imports -import { ScrapeConfig } from '../../../configuration/site-traversal/types' -import { RunOptions } from '../../../configuration/run-options/types' -import { Dependencies } from '../../types' +import { ScrapeConfig } from '../../../settings/config/types' +import { Options } from '../../../settings/options/types' +import { Tools } from '../../../tools' -export type DownloadParams = { - parentId?: number - scrapeNextIndex: number - incrementIndex: number - value?: string +export const downloaderClassFactory = ( + config: ScrapeConfig, + options: Options, + tools: Tools +) => { + // TODO use type guards + if (config.download) return new HttpDownloader(config, options, tools) + else return new IdentityDownloader(config, options, tools) } -type ReturnType = - | Promise<{ - downloadValue: string - downloadId: number - filename?: string - }> - | { - downloadValue: string - downloadId: number - filename: null - } -export type DownloaderType = ( - config: ScrapeConfig -) => ( - runParams: RunOptions, - dependencies: Dependencies -) => (downloadParams: DownloadParams) => ReturnType - -export default (config: ScrapeConfig) => { - if (config.download) return urlDownloader(config) - else return identityDownloader(config) -} +export type DownloaderClass = IdentityDownloader | HttpDownloader diff --git a/src/scraper/scrape-step/downloader/variations/identity-downloader.ts b/src/scraper/scrape-step/downloader/variations/identity-downloader.ts deleted file mode 100644 index cdde6da..0000000 --- a/src/scraper/scrape-step/downloader/variations/identity-downloader.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { DownloaderType } from '../' - -export const downloader: DownloaderType = config => ( - runParams, - { emitter, store } -) => ({ parentId, scrapeNextIndex, incrementIndex, value }) => { - const downloadId = store.qs.insertQueuedDownload({ - scraper: config.name, - parentId, - scrapeNextIndex, - incrementIndex - }) - emitter.forScraper[config.name].emitQueuedDownload(downloadId) - return { downloadValue: value, downloadId, filename: null } -} diff --git a/src/scraper/scrape-step/downloader/variations/url-downloader/construct-url.ts b/src/scraper/scrape-step/downloader/variations/url-downloader/construct-url.ts deleted file mode 100644 index cbf6383..0000000 --- a/src/scraper/scrape-step/downloader/variations/url-downloader/construct-url.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { URL } from 'url' -import { compileTemplate } from '../../../../../util/handlebars' -// type imports -import { ScrapeConfig } from '../../../../../configuration/site-traversal/types' -import { RunOptions } from '../../../../../configuration/run-options/types' -import { Dependencies } from '../../../../types' -import { DownloadParams } from '../../' - -const getTemplateVars = ( - config: ScrapeConfig, - { input }: RunOptions, - { value, incrementIndex }: DownloadParams -) => { - const index = incrementIndex - const templateVals = { ...input, value, index } - return templateVals -} -type TemplateVars = ReturnType - -// TODO memoize compileTemplates -const constructUrl = (config: ScrapeConfig, templateVals: TemplateVars) => { - const populatedUriString = compileTemplate(config.download.urlTemplate)( - templateVals - ) - try { - return new URL(populatedUriString) - } catch (e) { - throw new Error(`cannot create url from "${populatedUriString}"`) - } -} - -const constructHeaders = (config: ScrapeConfig, templateVals: TemplateVars) => { - const headers: { [HeaderName: string]: string } = {} - for (const [key, val] of Object.entries(config.download.headerTemplates)) { - headers[key] = compileTemplate(val)(templateVals) - } - return headers -} - -const constructCookieString = ( - config: ScrapeConfig, - templateVals: TemplateVars -) => { - let cookies = '' - for (const [key, val] of Object.entries(config.download.cookieTemplates)) { - const populatedVal = compileTemplate(val)(templateVals) - cookies += `${key}=${populatedVal};` - } - return cookies -} - -const constructFetch = ( - config: ScrapeConfig, - runParams: RunOptions, - downloadParams: DownloadParams -) => { - const templateVals = getTemplateVars(config, runParams, downloadParams) - const url = constructUrl(config, templateVals) - const headers = constructHeaders(config, templateVals) - const cookies = constructCookieString(config, templateVals) - if (cookies) { - headers['cookie'] = headers['cookie'] - ? headers['cookie'] + cookies - : cookies - } - return { - url, - fetchOptions: { - headers, - method: config.download.method - } - } -} -export type ConstructedFetch = ReturnType - -export { constructFetch } diff --git a/src/scraper/scrape-step/downloader/variations/url-downloader/fetchers.ts b/src/scraper/scrape-step/downloader/variations/url-downloader/fetchers.ts deleted file mode 100644 index c44f25c..0000000 --- a/src/scraper/scrape-step/downloader/variations/url-downloader/fetchers.ts +++ /dev/null @@ -1,134 +0,0 @@ -import fetch, * as Fetch from 'node-fetch' -import { URL } from 'url' -import { createWriteStream } from 'fs' -import { resolve } from 'path' -import sanitize from 'sanitize-filename' -import { readFile, exists, mkdirp } from '../../../../../util/fs' -// type imports -import { ScrapeConfig } from '../../../../../configuration/site-traversal/types' -import { RunOptions } from '../../../../../configuration/run-options/types' -import { Dependencies } from '../../../../types' -import { ConstructedFetch } from './construct-url' - -const sanitizeUrl = (url: URL) => sanitize(url.toString(), { replacement: '_' }) - -const verifyResponseOk = ( - name: ScrapeConfig['name'], - response: Fetch.Response, - url: URL -) => { - if (!response.ok) { - throw new Error( - `scraper '${name}' status ${response.status} for ${url.toString()}` - ) - } -} - -const emitProgressIfListenersAttached = ( - emitter: Dependencies['emitter'], - response: Fetch.Response, - name: ScrapeConfig['name'], - downloadId: number -) => { - if (emitter.hasListenerFor(`${name}:progress`)) { - const contentLength = parseInt(response.headers.get('content-length')) - let bytesLength = 0 - response.body.on('data', chunk => { - bytesLength += chunk.length - emitter.forScraper[name].emitProgress( - downloadId, - bytesLength / contentLength - ) - }) - } -} - -interface FetchParams extends ConstructedFetch { - downloadId: number -} -type Fetcher = ( - config: ScrapeConfig, - runParams: RunOptions, - dependencies: Dependencies, - fetchParams: FetchParams -) => Promise<{ - downloadValue?: string - filename?: string -}> -export const downloadToFileAndMemory: Fetcher = async ( - { name }, - { folder, downloadPriority }, - { queue, emitter }, - { downloadId, url, fetchOptions } -) => { - const downloadFolder = resolve(folder, downloadId.toString()) - const filename = resolve(downloadFolder, sanitizeUrl(url)) - - const response = await queue.add( - () => fetch(url.toString(), fetchOptions), - downloadPriority - ) - await mkdirp(downloadFolder) - verifyResponseOk(name, response, url) - const contentLength = response.headers.get('content-length') - const hasProgressListener = emitter.hasListenerFor(`${name}:progress`) - const dest = createWriteStream(filename) - const buffers: Buffer[] = [] - - const buffer = await new Promise((resolve, reject) => { - response.body.pipe(dest) - response.body.on('error', error => reject(error)) - response.body.on('data', chunk => buffers.push(chunk)) - emitProgressIfListenersAttached(emitter, response, name, downloadId) - dest.on('error', error => reject(error)) - dest.on('close', () => resolve(Buffer.concat(buffers))) - }) - return { - downloadValue: buffer.toString(), - filename - } -} -export const downloadToFileOnly: Fetcher = async ( - { name }, - { folder, downloadPriority }, - { queue, emitter }, - { downloadId, url, fetchOptions } -) => { - const downloadFolder = resolve(folder, downloadId.toString()) - const filename = resolve(downloadFolder, sanitizeUrl(url)) - - const response = await queue.add( - () => fetch(url.toString(), fetchOptions), - downloadPriority - ) - await mkdirp(downloadFolder) - const buffer = await new Promise((resolve, reject) => { - verifyResponseOk(name, response, url) - const dest = createWriteStream(filename) - response.body.pipe(dest) - emitProgressIfListenersAttached(emitter, response, name, downloadId) - response.body.on('error', error => reject(error)) - dest.on('error', error => reject(error)) - dest.on('close', resolve) - }) - return { - filename - } -} - -export const downloadToMemoryOnly: Fetcher = ( - { name }, - { downloadPriority }, - { queue, emitter }, - { downloadId, url, fetchOptions } -) => - queue - .add(() => fetch(url.toString(), fetchOptions), downloadPriority) - .then(response => { - verifyResponseOk(name, response, url) - emitProgressIfListenersAttached(emitter, response, name, downloadId) - return response.text() - }) - .then(downloadValue => ({ - downloadValue - })) diff --git a/src/scraper/scrape-step/downloader/variations/url-downloader/index.ts b/src/scraper/scrape-step/downloader/variations/url-downloader/index.ts deleted file mode 100644 index 328751a..0000000 --- a/src/scraper/scrape-step/downloader/variations/url-downloader/index.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { resolve, basename } from 'path' -import { constructFetch } from './construct-url' -import { - downloadToFileAndMemory, - downloadToMemoryOnly, - downloadToFileOnly -} from './fetchers' -import { DownloaderType } from '../../' - -export const downloader: DownloaderType = config => ( - runParams, - dependencies -) => { - const { store, queue, emitter } = dependencies - - const shouldDownloadToMemory = Boolean( - config.scrapeEach.length || config.parse - ) - const shouldDownloadToFile = runParams.cache - - const fetcher = - shouldDownloadToMemory && shouldDownloadToFile - ? downloadToFileAndMemory - : shouldDownloadToMemory - ? downloadToMemoryOnly - : downloadToFileOnly - - return async downloadParams => { - const { value, parentId, scrapeNextIndex, incrementIndex } = downloadParams - const { url, fetchOptions } = constructFetch( - config, - runParams, - downloadParams - ) - const downloadId = store.qs.insertQueuedDownload({ - scraper: config.name, - parentId, - scrapeNextIndex, - incrementIndex, - url: url.toString() - }) - - emitter.forScraper[config.name].emitQueuedDownload(downloadId) - const { downloadValue, filename } = await fetcher( - config, - runParams, - dependencies, - { - downloadId, - url, - fetchOptions - } - ) - return { downloadValue, downloadId, filename } - } -} diff --git a/src/scraper/scrape-step/incrementer/index.ts b/src/scraper/scrape-step/incrementer/index.ts index 19d41eb..bad8755 100644 --- a/src/scraper/scrape-step/incrementer/index.ts +++ b/src/scraper/scrape-step/incrementer/index.ts @@ -1,19 +1,23 @@ import * as Rx from 'rxjs' import * as ops from 'rxjs/operators' -import { fromAsyncGenerator } from '../../../util/rxjs/observables' -import { ScrapeConfig } from '../../../configuration/site-traversal/types' +import { ScrapeConfig } from '../../../settings/config/types' import { ParsedValue } from '../' import { whileLoopObservable } from '../../../util/rxjs/observables/while-loop' +import { ScrapeStep, IdentityScrapeStep } from '../' -const incrementUntilEmptyParse = ( +type OkToIncrementWhileLoop = ( parsedValues: ParsedValue[], incrementIndex: number +) => boolean + +const incrementUntilEmptyParse: OkToIncrementWhileLoop = ( + parsedValues ): boolean => !!parsedValues.length -const incrementUntilNumericIndex = (incrementUntil: number) => ( - parsedValues: ParsedValue[], - incrementIndex: number -): boolean => incrementUntil >= incrementIndex +const incrementUntilNumericIndex = ( + incrementUntil: number +): OkToIncrementWhileLoop => (parsedValues, incrementIndex): boolean => + incrementUntil >= incrementIndex const incrementAlways = () => true @@ -25,8 +29,7 @@ const throwAnyError = (e: Error) => Rx.throwError(e) export type DownloadParseFunction = ( parsedValueWithId: ParsedValue, - incrementIndex: number, - scrapeNextIndex?: number + incrementIndex: number ) => Promise type StatefulVars = { @@ -35,15 +38,12 @@ type StatefulVars = { nextPromises: Promise<{}>[] } -const incrementer = ({ name, incrementUntil }: ScrapeConfig) => { - const okToIncrementWhileLoop = - incrementUntil === 'empty-parse' - ? incrementUntilEmptyParse - : incrementUntil === 'failed-download' - ? incrementAlways // failed download is handled in the try catch - : incrementUntilNumericIndex(incrementUntil) - - const okToIncrementScrapeNext = +const incrementer = ( + { incrementUntil }: ScrapeConfig, + asyncFunction: DownloadParseFunction, + scrapeNextChild: ScrapeStep | IdentityScrapeStep +) => { + const okToIncrementWhileLoop: OkToIncrementWhileLoop = incrementUntil === 'empty-parse' ? incrementUntilEmptyParse : incrementUntil === 'failed-download' @@ -53,41 +53,29 @@ const incrementer = ({ name, incrementUntil }: ScrapeConfig) => { const ignoreFetchError = incrementUntil === 'failed-download' ? catchFetchError : throwAnyError - return ( - asyncFunction: DownloadParseFunction, - scrapeNextChild: ( - parsedValues: ParsedValue[] - ) => Rx.Observable - ) => { - return (parsedValueWithId: ParsedValue): Rx.Observable => - whileLoopObservable( - asyncFunction, - okToIncrementWhileLoop, - parsedValueWithId - ).pipe( - ops.catchError(ignoreFetchError), - ops.flatMap((parsedValues, incrementIndex) => - Rx.of(parsedValues).pipe( - ops.expand(parsedValues => - scrapeNextChild(parsedValues).pipe( - ops.flatMap((parsedValues, scrapeNextIndex) => - // TODO get proper scrape next index for asyncFunction - parsedValues.map(parsedValue => - asyncFunction( - parsedValue, - incrementIndex, - scrapeNextIndex + 1 - ) - ) - ), - ops.mergeAll(), - ops.takeWhile(incrementUntilEmptyParse) - ) + return (parsedValueWithId: ParsedValue): Rx.Observable => + whileLoopObservable( + asyncFunction, + okToIncrementWhileLoop, + parsedValueWithId + ).pipe( + ops.catchError(ignoreFetchError), + ops.flatMap((parsedValues, incrementIndex) => + Rx.of(parsedValues).pipe( + ops.expand(parsedValues => + scrapeNextChild.run(parsedValues).pipe( + ops.flatMap(parsedValues => + parsedValues.map(parsedValue => + asyncFunction(parsedValue, incrementIndex) + ) + ), + ops.mergeAll(), + ops.takeWhile(incrementUntilEmptyParse) ) ) ) ) - } + ) } -export default incrementer +export { incrementer } diff --git a/src/scraper/scrape-step/index.ts b/src/scraper/scrape-step/index.ts index 23e2659..b0c124f 100644 --- a/src/scraper/scrape-step/index.ts +++ b/src/scraper/scrape-step/index.ts @@ -1,102 +1,135 @@ import * as Rx from 'rxjs' import * as ops from 'rxjs/operators' -import setDownloaderConfig from './downloader' -import setParserConfig from './parser' -import incrementer from './incrementer' -import { mkdirpSync } from '../../util/fs' +import { downloaderClassFactory } from './downloader' +import { parserClassFactory } from './parser' +import { incrementer } from './incrementer' +import { wrapError } from '../../util/error' // type imports -import { ScrapeConfig } from '../../configuration/site-traversal/types' -import { FlatRunOptions } from '../../configuration/run-options/types' -import { Dependencies } from '../types' -import { SelectedRow as ParsedValueWithId } from '../../store/queries/select-parsed-values' +import { ScraperName, ScrapeConfig } from '../../settings/config/types' +import { FlatOptions } from '../../settings/options/types' +import { Tools } from '../../tools' +import { SelectedRow as ParsedValueWithId } from '../../tools/store/queries/select-parsed-values' import { DownloadParseFunction } from './incrementer' +import { DownloaderClass } from './downloader' +import { ParserClass } from './parser' -type ScrapeValue = (parentValues: any[]) => Rx.Observable -type ScrapeStep = ( - config: ScrapeConfig -) => (flatRunParams: FlatRunOptions, dependencies: Dependencies) => ScrapeValue type InputValue = { parsedValue: string id?: number // nonexistent } export type ParsedValue = InputValue | ParsedValueWithId -// init setup -const scraper = (config: ScrapeConfig) => { - const setDownloaderOptions = setDownloaderConfig(config) - const setParserOptions = setParserConfig(config) - const getIncrementObservable = incrementer(config) - const childrenSetup = config.scrapeEach.map(scrapeConfig => - scraper(scrapeConfig) - ) +abstract class AbstractScrapeStep { + public abstract run: ( + parsedValues: ParsedValue[] + ) => Rx.Observable +} +class IdentityScrapeStep extends AbstractScrapeStep { + public run: typeof AbstractScrapeStep.prototype.run = () => Rx.empty() +} +class ScrapeStep extends AbstractScrapeStep { + private scraperName: ScraperName + private config: ScrapeConfig + private flatOptions: FlatOptions + private tools: Tools + private scraperLogger: ReturnType - // run setup - return (flatRunParams: FlatRunOptions, dependencies: Dependencies) => { - const runParams = flatRunParams[config.name] - const downloader = setDownloaderOptions(runParams, dependencies) - const parser = setParserOptions() + private downloader: DownloaderClass + private parser: ParserClass + private incrementObservableFunction: ReturnType + private children: ScrapeStep[] - const { queue, store, emitter } = dependencies - // simpler to keep this synchronous - mkdirpSync(runParams.folder) - const children = childrenSetup.map(child => - child(flatRunParams, dependencies) + public constructor( + config: ScrapeConfig, + flatOptions: FlatOptions, + tools: Tools + ) { + super() + this.scraperName = config.name + this.config = config + this.flatOptions = flatOptions + this.tools = tools + + // const getIncrementObservable = incrementer(config) + this.children = config.scrapeEach.map( + scrapeConfig => new ScrapeStep(scrapeConfig, flatOptions, tools) ) + const scraperOptions = flatOptions.get(this.scraperName)! + this.downloader = downloaderClassFactory(config, scraperOptions, tools) + this.parser = parserClassFactory(config, scraperOptions, tools) + + this.scraperLogger = tools.logger.scraper(this.scraperName)! const scrapeNextChild = config.scrapeNext - ? scraper(config.scrapeNext)(flatRunParams, dependencies) - : (parentValues: ParsedValue[]) => Rx.empty() + ? new ScrapeStep(config.scrapeNext, flatOptions, tools) + : new IdentityScrapeStep() + this.incrementObservableFunction = incrementer( + config, + this.downloadParseFunction, + scrapeNextChild + ) + } + + public run: typeof AbstractScrapeStep.prototype.run = ( + parentValues: ParsedValue[] + ): Rx.Observable => + Rx.from(parentValues).pipe( + ops.flatMap(this.incrementObservableFunction), + ops.catchError(wrapError(`scraper '${this.scraperName}'`)), + ops.flatMap( + parsedValues => + this.children.length + ? this.children.map(child => child.run(parsedValues)) + : [Rx.of(parsedValues)] + ), + ops.mergeAll() + ) - const downloadParseFunction: DownloadParseFunction = async ( - { parsedValue: value, id: parentId }, + private downloadParseFunction: DownloadParseFunction = async ( + { parsedValue: value, id: parentId }, + incrementIndex + ) => { + const { store, emitter } = this.tools + const { id: downloadId } = store.qs.selectCompletedDownload({ incrementIndex, - scrapeNextIndex = 0 - ) => { - const { id: downloadId } = store.qs.selectCompletedDownload({ - incrementIndex, - parentId, - scraper: config.name - }) - if (downloadId) { - const parsedValuesWithId = store.qs.selectParsedValues(downloadId) - return parsedValuesWithId - } else { - const { downloadValue, downloadId, filename } = await downloader({ + parentId, + scraper: this.scraperName + }) + if (downloadId) { + const parsedValuesWithId = store.qs.selectParsedValues(downloadId) + this.scraperLogger.info( + { parsedValuesWithId, downloadId }, + 'loaded cached values' + ) + return parsedValuesWithId + } else { + const { downloadValue, downloadId, filename } = await this.downloader.run( + { incrementIndex, - scrapeNextIndex: 0, parentId, value - }) - const parsedValues = parser(downloadValue) + } + ) + const parsedValues = this.parser.run(downloadValue) - const parsedValuesWithId = store.asTransaction(() => { - store.qs.updateDownloadToComplete({ downloadId, filename }) - store.qs.insertBatchParsedValues({ - name: config.name, - parentId, - downloadId, - parsedValues - }) - emitter.forScraper[config.name].emitCompletedDownload(downloadId) - return store.qs.selectParsedValues(downloadId) - })() - return parsedValuesWithId - } - } + store.transaction(() => { + store.qs.updateDownloadToComplete({ downloadId, filename }) + store.qs.insertBatchParsedValues({ + name: this.scraperName, + parentId, + downloadId, + parsedValues + }) + emitter.scraper(this.scraperName).emit.completed(downloadId) + })() + const parsedValuesWithId = store.qs.selectParsedValues(downloadId) - // called per each value - return (parentValues: ParsedValue[]): Rx.Observable => - Rx.from(parentValues).pipe( - ops.flatMap( - getIncrementObservable(downloadParseFunction, scrapeNextChild) - ), - ops.flatMap( - parsedValues => - children.length - ? children.map(child => child(parsedValues)) - : [Rx.of(parsedValues)] - ), - ops.mergeAll() + this.scraperLogger.info( + { parsedValuesWithId, downloadId }, + 'inserted new values' ) + return parsedValuesWithId + } } } -export default scraper + +export { ScrapeStep, IdentityScrapeStep } diff --git a/src/scraper/scrape-step/parser/abstract.ts b/src/scraper/scrape-step/parser/abstract.ts new file mode 100644 index 0000000..9d7e080 --- /dev/null +++ b/src/scraper/scrape-step/parser/abstract.ts @@ -0,0 +1,32 @@ +import { ScrapeConfig } from '../../../settings/config/types' +import { Options } from '../../../settings/options/types' +import { Tools } from '../../../tools' + +export type ParserValues = string[] | [undefined | string] +/** + * base abstract class which other downloaders derive from + */ +export abstract class AbstractParser { + protected config: ScrapeConfig + protected options: Options + protected tools: Tools + protected selector: string + protected attribute: string + + public constructor(config: ScrapeConfig, options: Options, tools: Tools) { + Object.assign(this, { config, options, tools, ...config.parse }) + } + public run = (value?: string) => { + /** TODO + * ok I need to re-evaluate what an undefined passed value means + * images should be undefined IF I MANUALLY SAY SO + * but that will stop the flow from happening + * + * any time we say dont store the value, it will stop the flow + * AND stop the scraper (unless we pass down a null value) + * but storing images as text is a waste of energy + */ + return this.parse(value) + } + protected abstract parse: (value?: string) => ParserValues +} diff --git a/src/scraper/scrape-step/parser/implementations/html.ts b/src/scraper/scrape-step/parser/implementations/html.ts new file mode 100644 index 0000000..0ff65ea --- /dev/null +++ b/src/scraper/scrape-step/parser/implementations/html.ts @@ -0,0 +1,35 @@ +import cheerio from 'cheerio' +import { AbstractParser } from '../abstract' +// type imports +import { ScrapeConfig } from '../../../../settings/config/types' +import { Options } from '../../../../settings/options/types' +import { Tools } from '../../../../tools' + +export class Parser extends AbstractParser { + private parser: (value: string) => string[] + + public constructor(config: ScrapeConfig, options: Options, tools: Tools) { + super(config, options, tools) + this.parser = this.attribute ? this.selectAttrVals : this.selectTextVals + } + protected parse = (value: string) => this.parser(value) + + private selectTextVals = (value: string) => { + const $ = cheerio.load(value) + const values: string[] = [] + const selection = $(this.selector) + selection.each(function() { + values.push($(this).text()) + }) + return values + } + private selectAttrVals = (value: string) => { + const $ = cheerio.load(value) + const values: string[] = [] + const selection = $(this.selector) + selection.attr(this.attribute, (i: number, attributeVal: string) => { + if (attributeVal) values.push(attributeVal) + }) + return values + } +} diff --git a/src/scraper/scrape-step/parser/implementations/identity.ts b/src/scraper/scrape-step/parser/implementations/identity.ts new file mode 100644 index 0000000..f1cfb34 --- /dev/null +++ b/src/scraper/scrape-step/parser/implementations/identity.ts @@ -0,0 +1,5 @@ +import { AbstractParser, ParserValues } from '../abstract' + +export class Parser extends AbstractParser { + public parse = (value?: string): ParserValues => [value] +} diff --git a/src/scraper/scrape-step/parser/implementations/json.ts b/src/scraper/scrape-step/parser/implementations/json.ts new file mode 100644 index 0000000..01f05ee --- /dev/null +++ b/src/scraper/scrape-step/parser/implementations/json.ts @@ -0,0 +1,5 @@ +import { AbstractParser } from '../abstract' + +export class Parser extends AbstractParser { + public parse = (value: string) => [value] +} diff --git a/src/scraper/scrape-step/parser/index.ts b/src/scraper/scrape-step/parser/index.ts index d6d45cd..174f560 100644 --- a/src/scraper/scrape-step/parser/index.ts +++ b/src/scraper/scrape-step/parser/index.ts @@ -1,19 +1,25 @@ -import { parser as htmlParser } from './variations/html-parser' -import { parser as jsonParser } from './variations/json-parser' -import { parser as identityParser } from './variations/identity-parser' -import { ScrapeConfig } from '../../../configuration/site-traversal/types' +import { Parser as HtmlParser } from './implementations/html' +import { Parser as JsonParser } from './implementations/json' +import { Parser as IdentityParser } from './implementations/identity' +// type imports +import { ScrapeConfig } from '../../../settings/config/types' +import { Options } from '../../../settings/options/types' +import { Tools } from '../../../tools' -export type ParserType = ( - config: ScrapeConfig -) => () => (value: string) => string[] - -export default (config: ScrapeConfig) => { - const parsers = { - html: htmlParser, - json: jsonParser +const parsers = { + html: HtmlParser, + json: JsonParser +} +export const parserClassFactory = ( + config: ScrapeConfig, + options: Options, + tools: Tools +) => { + // TODO use type guards + if (config.parse) { + return new parsers[config.parse.expect](config, options, tools) + } else { + return new IdentityParser(config, options, tools) } - - return config.parse - ? parsers[config.parse.expect](config) - : identityParser(config) } +export type ParserClass = HtmlParser | JsonParser | IdentityParser diff --git a/src/scraper/scrape-step/parser/variations/html-parser.ts b/src/scraper/scrape-step/parser/variations/html-parser.ts deleted file mode 100644 index 37d481f..0000000 --- a/src/scraper/scrape-step/parser/variations/html-parser.ts +++ /dev/null @@ -1,32 +0,0 @@ -import cheerio from 'cheerio' -import { ParserType } from '../' - -const selectTextVals = (value: string, selector: string): string[] => { - const $ = cheerio.load(value) - const values: string[] = [] - const selection = $(selector) - selection.each(function() { - values.push($(this).text()) - }) - return values -} - -const selectAttrVals = ( - value: string, - selector: string, - attribute: string -): string[] => { - const $ = cheerio.load(value) - const values: string[] = [] - const selection = $(selector) - selection.attr(attribute, (i: number, attributeVal: string) => { - if (attributeVal) values.push(attributeVal) - }) - return values -} - -export const parser: ParserType = config => { - const { selector, attribute } = config.parse - const parseVal = attribute ? selectAttrVals : selectTextVals - return () => value => parseVal(value, selector, attribute) -} diff --git a/src/scraper/scrape-step/parser/variations/identity-parser.ts b/src/scraper/scrape-step/parser/variations/identity-parser.ts deleted file mode 100644 index 86fcf46..0000000 --- a/src/scraper/scrape-step/parser/variations/identity-parser.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { ParserType } from '../' - -export const parser: ParserType = config => () => value => [value] diff --git a/src/scraper/scrape-step/parser/variations/json-parser.ts b/src/scraper/scrape-step/parser/variations/json-parser.ts deleted file mode 100644 index 8779bcc..0000000 --- a/src/scraper/scrape-step/parser/variations/json-parser.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { ParserType } from '../' - -export const parser: ParserType = config => () => value => { - // parse from config - // write values to db - return [] -} diff --git a/src/scraper/types.ts b/src/scraper/types.ts deleted file mode 100644 index e00ea52..0000000 --- a/src/scraper/types.ts +++ /dev/null @@ -1,11 +0,0 @@ -import Store from '../store' -import Emitter from '../emitter' -import Queue from '../queue' -import Logger from '../logger' - -export type Dependencies = { - store: Store - emitter: Emitter - queue: Queue - logger: Logger -} diff --git a/src/configuration/site-traversal/__tests__/make-flat-config.test.unit.ts b/src/settings/config/__tests__/make-flat-config.unit.test.ts similarity index 94% rename from src/configuration/site-traversal/__tests__/make-flat-config.test.unit.ts rename to src/settings/config/__tests__/make-flat-config.unit.test.ts index f60395c..2b3912d 100644 --- a/src/configuration/site-traversal/__tests__/make-flat-config.test.unit.ts +++ b/src/settings/config/__tests__/make-flat-config.unit.test.ts @@ -3,7 +3,7 @@ import { FlatConfig } from '../types' import { normalizeConfig, makeFlatConfig } from '../' import * as testingConfigs from '../../../../testing/resources/testing-configs' -describe('config recurser', () => { +describe('make flat config', () => { const galleryPostImgTag = testingConfigs.__GALLERY_POST_IMG_TAG__ const flatConfigGuess: FlatConfig = { @@ -11,7 +11,7 @@ describe('config recurser', () => { depth: 0, horizontalIndex: 0, name: 'gallery', - parentName: null + parentName: undefined }, post: { depth: 1, diff --git a/src/configuration/site-traversal/__tests__/normalize.unit.test.ts b/src/settings/config/__tests__/normalize.unit.test.ts similarity index 91% rename from src/configuration/site-traversal/__tests__/normalize.unit.test.ts rename to src/settings/config/__tests__/normalize.unit.test.ts index 4654ced..96cd2b5 100644 --- a/src/configuration/site-traversal/__tests__/normalize.unit.test.ts +++ b/src/settings/config/__tests__/normalize.unit.test.ts @@ -3,7 +3,7 @@ import * as testingConfigs from '../../../../testing/resources/testing-configs' import { ConfigInit } from '../types' import { expect } from 'chai' -describe('filled in defaults', () => { +describe('normalize config with', () => { describe('simple config', () => { const simpleConfig = testingConfigs.__SIMPLE_CONFIG__ const fullConfig = normalizeConfig(simpleConfig) @@ -14,12 +14,12 @@ describe('filled in defaults', () => { download: { method: 'GET', urlTemplate: (simpleConfig.scrape as any).download, - cookieTemplates: {}, headerTemplates: {} // regexCleanup: undefined }, parse: { - ...(simpleConfig.scrape as any).parse, + selector: (simpleConfig.scrape.parse as any).selector, + attribute: (simpleConfig.scrape.parse as any).attribute, expect: 'html' }, incrementUntil: 0, @@ -29,7 +29,6 @@ describe('filled in defaults', () => { download: { method: 'GET', urlTemplate: (simpleConfig.scrape as any).scrapeEach.download, - cookieTemplates: {}, headerTemplates: {} }, parse: undefined, diff --git a/src/configuration/site-traversal/assert.ts b/src/settings/config/assert.ts similarity index 62% rename from src/configuration/site-traversal/assert.ts rename to src/settings/config/assert.ts index 32b97ac..ac490a0 100644 --- a/src/configuration/site-traversal/assert.ts +++ b/src/settings/config/assert.ts @@ -1,3 +1,4 @@ import { ConfigInit } from './types' +/* eslint-disable-next-line typescript/no-unused-vars */ export const assertConfigType = (config: ConfigInit) => {} diff --git a/src/configuration/site-traversal/index.ts b/src/settings/config/index.ts similarity index 65% rename from src/configuration/site-traversal/index.ts rename to src/settings/config/index.ts index 1d8916f..3d4dcd0 100644 --- a/src/configuration/site-traversal/index.ts +++ b/src/settings/config/index.ts @@ -1,4 +1,4 @@ -export { default as normalizeConfig } from './normalize' +export { normalizeConfig } from './normalize' export { makeFlatConfig } from './make-flat-config' diff --git a/src/configuration/site-traversal/make-flat-config.ts b/src/settings/config/make-flat-config.ts similarity index 96% rename from src/configuration/site-traversal/make-flat-config.ts rename to src/settings/config/make-flat-config.ts index a71b006..7bfd2d4 100644 --- a/src/configuration/site-traversal/make-flat-config.ts +++ b/src/settings/config/make-flat-config.ts @@ -3,7 +3,7 @@ import { Config, FlatConfig } from './types' const makeFlatConfig = (fullConfig: Config): FlatConfig => { const recurse = ( config: Config['scrape'], - parentName: string = null, + parentName?: string, depth = 0, horizontalIndex = 0 ): FlatConfig => { diff --git a/src/configuration/site-traversal/normalize.ts b/src/settings/config/normalize.ts similarity index 72% rename from src/configuration/site-traversal/normalize.ts rename to src/settings/config/normalize.ts index 49a579a..89788ae 100644 --- a/src/configuration/site-traversal/normalize.ts +++ b/src/settings/config/normalize.ts @@ -7,20 +7,21 @@ import { ConfigInit, Config } from './types' -import { assertConfigType } from './runtime/assert' -const runtimeAssertConfig: any = assertConfigType +import { assertConfigType } from './' -const downloadDefaults: Partial = { +const downloadDefaults: { + headerTemplates: DownloadConfig['headerTemplates'] + method: DownloadConfig['method'] +} = { method: 'GET', - headerTemplates: {}, - cookieTemplates: {} + headerTemplates: {} } +// TODO use type guards const assignDownloadDefaults = (download: DownloadConfigInit): DownloadConfig => typeof download === 'string' ? { ...downloadDefaults, - urlTemplate: download, - method: downloadDefaults.method + urlTemplate: download } : { ...downloadDefaults, @@ -28,7 +29,10 @@ const assignDownloadDefaults = (download: DownloadConfigInit): DownloadConfig => method: download.method || downloadDefaults.method } -const parseDefaults: Partial = { +const parseDefaults: { + expect: ParseConfig['expect'] + attribute: ParseConfig['attribute'] +} = { expect: 'html', attribute: undefined } @@ -36,8 +40,7 @@ const assignParseDefaults = (parse: ParseConfigInit): ParseConfig => typeof parse === 'string' ? { ...parseDefaults, - selector: parse, - expect: parseDefaults.expect + selector: parse } : { ...parseDefaults, @@ -49,8 +52,6 @@ const fillInDefaultsRecurse = (level = 0, parentName = '') => ( scrapeConfig: ScrapeConfigInit, index = 0 ): Config['scrape'] => { - if (!scrapeConfig) return undefined - const { name, download, @@ -66,10 +67,12 @@ const fillInDefaultsRecurse = (level = 0, parentName = '') => ( return { name: name || internalName, - download: download && assignDownloadDefaults(download), - parse: parse && assignParseDefaults(parse), + download: + download === undefined ? undefined : assignDownloadDefaults(download), + parse: parse === undefined ? undefined : assignParseDefaults(parse), incrementUntil: incrementUntil || 0, - scrapeNext: fillInDefaultsRecurse(level + 1, parentName)(scrapeNext), + scrapeNext: + scrapeNext && fillInDefaultsRecurse(level + 1, parentName)(scrapeNext), scrapeEach: Array.isArray(scrapeEach) ? scrapeEach.map(fillInDefaultsRecurse(level + 1, parentName)) : [fillInDefaultsRecurse(level + 1)(scrapeEach)] @@ -82,7 +85,7 @@ const standardizeInput = (input: ConfigInit['input']): Config['input'] => { } const normalizeConfig = (config: ConfigInit): Config => { - runtimeAssertConfig(config) + assertConfigType(config) const input = standardizeInput(config.input) @@ -93,4 +96,4 @@ const normalizeConfig = (config: ConfigInit): Config => { scrape: fullConfig } } -export default normalizeConfig +export { normalizeConfig } diff --git a/src/configuration/site-traversal/runtime/assert.js b/src/settings/config/runtime/assert.js similarity index 90% rename from src/configuration/site-traversal/runtime/assert.js rename to src/settings/config/runtime/assert.js index c32f4cc..03befc4 100644 --- a/src/configuration/site-traversal/runtime/assert.js +++ b/src/settings/config/runtime/assert.js @@ -3,4 +3,5 @@ Object.defineProperty(exports, "__esModule", { value: true }); require("./tsr-declarations"); var lib_1 = require("ts-runtime/lib"); var types_1 = require("./types"); +/* eslint-disable-next-line typescript/no-unused-vars */ exports.assertConfigType = lib_1.default.annotate(function (config) { var _configType = lib_1.default.nullable(lib_1.default.ref(types_1.ConfigInit)); lib_1.default.param("config", _configType).assert(config); }, lib_1.default.function(lib_1.default.param("config", lib_1.default.nullable(lib_1.default.ref(types_1.ConfigInit))), lib_1.default.return(lib_1.default.any()))); diff --git a/src/settings/config/runtime/tsconfig.runtime.json b/src/settings/config/runtime/tsconfig.runtime.json new file mode 100644 index 0000000..f5dea14 --- /dev/null +++ b/src/settings/config/runtime/tsconfig.runtime.json @@ -0,0 +1,6 @@ +{ + "extends": "../../../../tsconfig.json", + "compilerOptions": { + "outDir": "./src/settings/config/runtime/" + } +} diff --git a/src/configuration/site-traversal/runtime/tsr-declarations.js b/src/settings/config/runtime/tsr-declarations.js similarity index 99% rename from src/configuration/site-traversal/runtime/tsr-declarations.js rename to src/settings/config/runtime/tsr-declarations.js index e71cf26..b62f725 100644 --- a/src/configuration/site-traversal/runtime/tsr-declarations.js +++ b/src/settings/config/runtime/tsr-declarations.js @@ -2,7 +2,7 @@ var _this = this; Object.defineProperty(exports, "__esModule", { value: true }); var lib_1 = require("ts-runtime/lib"); -lib_1.default.declare("Array.209944603", lib_1.default.type("Array", function (Array) { +lib_1.default.declare("Array.631742855", lib_1.default.type("Array", function (Array) { var T = Array.typeParameter("T"); return lib_1.default.object(lib_1.default.property("length", lib_1.default.nullable(lib_1.default.number())), lib_1.default.indexer("n", lib_1.default.nullable(lib_1.default.number()), lib_1.default.nullable(T)), lib_1.default.property("toString", lib_1.default.nullable(lib_1.default.function(lib_1.default.return(lib_1.default.nullable(lib_1.default.string()))))), lib_1.default.property("toLocaleString", lib_1.default.nullable(lib_1.default.function(lib_1.default.return(lib_1.default.nullable(lib_1.default.string()))))), lib_1.default.property("pop", lib_1.default.nullable(lib_1.default.function(lib_1.default.return(lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(T), lib_1.default.nullable(lib_1.default.undef()))))))), lib_1.default.property("push", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("items", lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(T))), true), lib_1.default.return(lib_1.default.nullable(lib_1.default.number()))))), lib_1.default.property("concat", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("items", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(lib_1.default.ref(ConcatArray, lib_1.default.nullable(T))))), lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(T), lib_1.default.nullable(lib_1.default.ref(ConcatArray, lib_1.default.nullable(T))))))))), true), lib_1.default.return(lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(T))))))), lib_1.default.property("join", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("separator", lib_1.default.nullable(lib_1.default.string()), true), lib_1.default.return(lib_1.default.nullable(lib_1.default.string()))))), lib_1.default.property("reverse", lib_1.default.nullable(lib_1.default.function(lib_1.default.return(lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(T))))))), lib_1.default.property("shift", lib_1.default.nullable(lib_1.default.function(lib_1.default.return(lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(T), lib_1.default.nullable(lib_1.default.undef()))))))), lib_1.default.property("slice", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("start", lib_1.default.nullable(lib_1.default.number()), true), lib_1.default.param("end", lib_1.default.nullable(lib_1.default.number()), true), lib_1.default.return(lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(T))))))), lib_1.default.property("sort", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("compareFn", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("a", lib_1.default.nullable(T)), lib_1.default.param("b", lib_1.default.nullable(T)), lib_1.default.return(lib_1.default.nullable(lib_1.default.number())))), true), lib_1.default.return(lib_1.default.nullable(lib_1.default.this(_this)))))), lib_1.default.property("splice", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("start", lib_1.default.nullable(lib_1.default.number())), lib_1.default.param("deleteCount", lib_1.default.nullable(lib_1.default.number()), true), lib_1.default.param("items", lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(T))), true), lib_1.default.return(lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(T))))))), lib_1.default.property("unshift", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("items", lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(T))), true), lib_1.default.return(lib_1.default.nullable(lib_1.default.number()))))), lib_1.default.property("indexOf", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("searchElement", lib_1.default.nullable(T)), lib_1.default.param("fromIndex", lib_1.default.nullable(lib_1.default.number()), true), lib_1.default.return(lib_1.default.nullable(lib_1.default.number()))))), lib_1.default.property("lastIndexOf", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("searchElement", lib_1.default.nullable(T)), lib_1.default.param("fromIndex", lib_1.default.nullable(lib_1.default.number()), true), lib_1.default.return(lib_1.default.nullable(lib_1.default.number()))))), lib_1.default.property("every", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("callbackfn", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("value", lib_1.default.nullable(T)), lib_1.default.param("index", lib_1.default.nullable(lib_1.default.number())), lib_1.default.param("array", lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(T)))), lib_1.default.return(lib_1.default.nullable(lib_1.default.boolean()))))), lib_1.default.param("thisArg", lib_1.default.any(), true), lib_1.default.return(lib_1.default.nullable(lib_1.default.boolean()))))), lib_1.default.property("some", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("callbackfn", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("value", lib_1.default.nullable(T)), lib_1.default.param("index", lib_1.default.nullable(lib_1.default.number())), lib_1.default.param("array", lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(T)))), lib_1.default.return(lib_1.default.nullable(lib_1.default.boolean()))))), lib_1.default.param("thisArg", lib_1.default.any(), true), lib_1.default.return(lib_1.default.nullable(lib_1.default.boolean()))))), lib_1.default.property("forEach", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("callbackfn", lib_1.default.nullable(lib_1.default.function(lib_1.default.param("value", lib_1.default.nullable(T)), lib_1.default.param("index", lib_1.default.nullable(lib_1.default.number())), lib_1.default.param("array", lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(T)))), lib_1.default.return(lib_1.default.void())))), lib_1.default.param("thisArg", lib_1.default.any(), true), lib_1.default.return(lib_1.default.void())))), lib_1.default.property("map", lib_1.default.nullable(lib_1.default.function(function (fn) { var U = fn.typeParameter("U"); diff --git a/src/configuration/site-traversal/runtime/types.js b/src/settings/config/runtime/types.js similarity index 60% rename from src/configuration/site-traversal/runtime/types.js rename to src/settings/config/runtime/types.js index f893079..8b10abd 100644 --- a/src/configuration/site-traversal/runtime/types.js +++ b/src/settings/config/runtime/types.js @@ -1,28 +1,32 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); var lib_1 = require("ts-runtime/lib"); +// scraper name +exports.ScraperName = lib_1.default.type("ScraperName", lib_1.default.nullable(lib_1.default.string())); +var ScraperGroup = lib_1.default.type("ScraperGroup", lib_1.default.nullable(lib_1.default.string())); +var InputKey = lib_1.default.type("InputKey", lib_1.default.nullable(lib_1.default.string())); var RegexRemove = lib_1.default.type("RegexRemove", lib_1.default.nullable(lib_1.default.string())); var RegexReplace = lib_1.default.type("RegexReplace", lib_1.default.object(lib_1.default.property("selector", lib_1.default.nullable(lib_1.default.string())), lib_1.default.property("replacer", lib_1.default.nullable(lib_1.default.string())))); var RegexCleanup = lib_1.default.type("RegexCleanup", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.ref(RegexRemove)), lib_1.default.nullable(lib_1.default.ref(RegexReplace))))); -var InputSimple = lib_1.default.type("InputSimple", lib_1.default.nullable(lib_1.default.string())); -var InputCleaned = lib_1.default.type("InputCleaned", lib_1.default.object(lib_1.default.property("name", lib_1.default.nullable(lib_1.default.string())), lib_1.default.property("regexCleanup", lib_1.default.nullable(lib_1.default.ref(RegexCleanup))))); +var InputSimple = lib_1.default.type("InputSimple", lib_1.default.nullable(lib_1.default.ref(InputKey))); +var InputCleaned = lib_1.default.type("InputCleaned", lib_1.default.object(lib_1.default.property("name", lib_1.default.nullable(lib_1.default.ref(InputKey))), lib_1.default.property("regexCleanup", lib_1.default.nullable(lib_1.default.ref(RegexCleanup))))); var Input = lib_1.default.type("Input", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.ref(InputSimple)), lib_1.default.nullable(lib_1.default.ref(InputCleaned))))); -var UrlTemplate = lib_1.default.type("UrlTemplate", lib_1.default.nullable(lib_1.default.string())); +var Template = lib_1.default.type("Template", lib_1.default.nullable(lib_1.default.string())); var UrlMethods = lib_1.default.type("UrlMethods", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.string("GET")), lib_1.default.nullable(lib_1.default.string("POST")), lib_1.default.nullable(lib_1.default.string("PUT")), lib_1.default.nullable(lib_1.default.string("DELETE"))))); -var DownloadConfigInterface = lib_1.default.type("DownloadConfigInterface", lib_1.default.object(lib_1.default.property("method", lib_1.default.nullable(lib_1.default.ref(UrlMethods)), true), lib_1.default.property("urlTemplate", lib_1.default.nullable(lib_1.default.ref(UrlTemplate))), lib_1.default.property("cookieTemplates", lib_1.default.object(lib_1.default.indexer("CookieName", lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.string()))), true), lib_1.default.property("headerTemplates", lib_1.default.object(lib_1.default.indexer("HeaderName", lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.string()))), true), lib_1.default.property("regexCleanup", lib_1.default.nullable(lib_1.default.ref(RegexCleanup)), true))); -exports.DownloadConfigInit = lib_1.default.type("DownloadConfigInit", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.ref(UrlTemplate)), lib_1.default.nullable(lib_1.default.ref(DownloadConfigInterface))))); -exports.DownloadConfig = lib_1.default.type("DownloadConfig", lib_1.default.intersect(lib_1.default.ref(DownloadConfigInterface), lib_1.default.object(lib_1.default.property("method", lib_1.default.nullable(lib_1.default.ref(UrlMethods)))))); +var DownloadConfigInterface = lib_1.default.type("DownloadConfigInterface", lib_1.default.object(lib_1.default.property("method", lib_1.default.nullable(lib_1.default.ref(UrlMethods)), true), lib_1.default.property("urlTemplate", lib_1.default.nullable(lib_1.default.ref(Template))), lib_1.default.property("headerTemplates", lib_1.default.object(lib_1.default.indexer("headerName", lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.ref(Template)))), true), lib_1.default.property("regexCleanup", lib_1.default.nullable(lib_1.default.ref(RegexCleanup)), true))); +exports.DownloadConfigInit = lib_1.default.type("DownloadConfigInit", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.ref(Template)), lib_1.default.nullable(lib_1.default.ref(DownloadConfigInterface))))); +exports.DownloadConfig = lib_1.default.type("DownloadConfig", lib_1.default.intersect(lib_1.default.ref(DownloadConfigInterface), lib_1.default.object(lib_1.default.property("method", lib_1.default.nullable(lib_1.default.ref(UrlMethods))), lib_1.default.property("headerTemplates", lib_1.default.object(lib_1.default.indexer("headerName", lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.ref(Template)))))))); var ExpectedFormats = lib_1.default.type("ExpectedFormats", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.string("html")), lib_1.default.nullable(lib_1.default.string("json"))))); var Selector = lib_1.default.type("Selector", lib_1.default.nullable(lib_1.default.string())); var ParseConfigInterface = lib_1.default.type("ParseConfigInterface", lib_1.default.object(lib_1.default.property("expect", lib_1.default.nullable(lib_1.default.ref(ExpectedFormats)), true), lib_1.default.property("selector", lib_1.default.nullable(lib_1.default.ref(Selector))), lib_1.default.property("attribute", lib_1.default.nullable(lib_1.default.string()), true), lib_1.default.property("regexCleanup", lib_1.default.nullable(lib_1.default.ref(RegexCleanup)), true))); exports.ParseConfigInit = lib_1.default.type("ParseConfigInit", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.ref(Selector)), lib_1.default.nullable(lib_1.default.ref(ParseConfigInterface))))); exports.ParseConfig = lib_1.default.type("ParseConfig", lib_1.default.intersect(lib_1.default.ref(ParseConfigInterface), lib_1.default.object(lib_1.default.property("expect", lib_1.default.nullable(lib_1.default.ref(ExpectedFormats)))))); var Incrementers = lib_1.default.type("Incrementers", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.string("failed-download")), lib_1.default.nullable(lib_1.default.string("empty-parse")), lib_1.default.nullable(lib_1.default.number())))); -exports.ScrapeConfigInit = lib_1.default.type("ScrapeConfigInit", function (ScrapeConfigInit) { return lib_1.default.object(lib_1.default.property("name", lib_1.default.nullable(lib_1.default.string()), true), lib_1.default.property("download", lib_1.default.nullable(lib_1.default.ref(exports.DownloadConfigInit)), true), lib_1.default.property("parse", lib_1.default.nullable(lib_1.default.ref(exports.ParseConfigInit)), true), lib_1.default.property("incrementUntil", lib_1.default.nullable(lib_1.default.ref(Incrementers)), true), lib_1.default.property("scrapeNext", lib_1.default.nullable(lib_1.default.ref(ScrapeConfigInit)), true), lib_1.default.property("scrapeEach", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.ref(ScrapeConfigInit)), lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(lib_1.default.ref(ScrapeConfigInit)))))), true)); }); +exports.ScrapeConfigInit = lib_1.default.type("ScrapeConfigInit", function (ScrapeConfigInit) { return lib_1.default.object(lib_1.default.property("name", lib_1.default.nullable(lib_1.default.ref(exports.ScraperName)), true), lib_1.default.property("download", lib_1.default.nullable(lib_1.default.ref(exports.DownloadConfigInit)), true), lib_1.default.property("parse", lib_1.default.nullable(lib_1.default.ref(exports.ParseConfigInit)), true), lib_1.default.property("incrementUntil", lib_1.default.nullable(lib_1.default.ref(Incrementers)), true), lib_1.default.property("scrapeNext", lib_1.default.nullable(lib_1.default.ref(ScrapeConfigInit)), true), lib_1.default.property("scrapeEach", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.ref(ScrapeConfigInit)), lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(lib_1.default.ref(ScrapeConfigInit)))))), true)); }); exports.ConfigInit = lib_1.default.type("ConfigInit", lib_1.default.object(lib_1.default.property("input", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.ref(Input)), lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(lib_1.default.ref(Input)))))), true), lib_1.default.property("scrape", lib_1.default.nullable(lib_1.default.ref(exports.ScrapeConfigInit))))); // returned by ./normalize.ts -exports.ScrapeConfig = lib_1.default.type("ScrapeConfig", function (ScrapeConfig) { return lib_1.default.object(lib_1.default.property("name", lib_1.default.nullable(lib_1.default.string())), lib_1.default.property("download", lib_1.default.nullable(lib_1.default.ref(exports.DownloadConfig)), true), lib_1.default.property("parse", lib_1.default.nullable(lib_1.default.ref(exports.ParseConfig)), true), lib_1.default.property("incrementUntil", lib_1.default.nullable(lib_1.default.ref(Incrementers))), lib_1.default.property("scrapeNext", lib_1.default.nullable(lib_1.default.ref(ScrapeConfig)), true), lib_1.default.property("scrapeEach", lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(lib_1.default.ref(ScrapeConfig)))))); }); -exports.Config = lib_1.default.type("Config", lib_1.default.intersect(lib_1.default.ref(exports.ConfigInit), lib_1.default.object(lib_1.default.property("scrape", lib_1.default.nullable(lib_1.default.ref(exports.ScrapeConfig)))))); +exports.ScrapeConfig = lib_1.default.type("ScrapeConfig", function (ScrapeConfig) { return lib_1.default.object(lib_1.default.property("name", lib_1.default.nullable(lib_1.default.ref(exports.ScraperName))), lib_1.default.property("download", lib_1.default.nullable(lib_1.default.ref(exports.DownloadConfig)), true), lib_1.default.property("parse", lib_1.default.nullable(lib_1.default.ref(exports.ParseConfig)), true), lib_1.default.property("incrementUntil", lib_1.default.nullable(lib_1.default.ref(Incrementers))), lib_1.default.property("scrapeNext", lib_1.default.nullable(lib_1.default.ref(ScrapeConfig)), true), lib_1.default.property("scrapeEach", lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(lib_1.default.ref(ScrapeConfig)))))); }); +exports.Config = lib_1.default.type("Config", lib_1.default.intersect(lib_1.default.ref(exports.ConfigInit), lib_1.default.object(lib_1.default.property("input", lib_1.default.nullable(lib_1.default.array(lib_1.default.nullable(lib_1.default.ref(Input))))), lib_1.default.property("scrape", lib_1.default.nullable(lib_1.default.ref(exports.ScrapeConfig)))))); // returned by ./make-flat-config.ts -exports.ConfigPositionInfo = lib_1.default.type("ConfigPositionInfo", lib_1.default.object(lib_1.default.property("depth", lib_1.default.nullable(lib_1.default.number())), lib_1.default.property("horizontalIndex", lib_1.default.nullable(lib_1.default.number())), lib_1.default.property("name", lib_1.default.nullable(lib_1.default.string())), lib_1.default.property("parentName", lib_1.default.nullable(lib_1.default.string()), true))); -exports.FlatConfig = lib_1.default.type("FlatConfig", lib_1.default.object(lib_1.default.indexer("name", lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.ref(exports.ConfigPositionInfo))))); +exports.ConfigPositionInfo = lib_1.default.type("ConfigPositionInfo", lib_1.default.object(lib_1.default.property("depth", lib_1.default.nullable(lib_1.default.number())), lib_1.default.property("horizontalIndex", lib_1.default.nullable(lib_1.default.number())), lib_1.default.property("name", lib_1.default.nullable(lib_1.default.ref(exports.ScraperName))), lib_1.default.property("parentName", lib_1.default.nullable(lib_1.default.ref(exports.ScraperName)), true))); +exports.FlatConfig = lib_1.default.type("FlatConfig", lib_1.default.object(lib_1.default.indexer("scraperName", lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.ref(exports.ConfigPositionInfo))))); diff --git a/src/configuration/site-traversal/types.ts b/src/settings/config/types.ts similarity index 75% rename from src/configuration/site-traversal/types.ts rename to src/settings/config/types.ts index e556085..0e1d1fb 100644 --- a/src/configuration/site-traversal/types.ts +++ b/src/settings/config/types.ts @@ -1,3 +1,10 @@ +// scraper name +export type ScraperName = string +// scraper group +type ScraperGroup = string +// input key +type InputKey = string + // RegexCleanup {{{ type RegexRemove = string interface RegexReplace { @@ -8,27 +15,28 @@ type RegexCleanup = RegexRemove | RegexReplace // }}} // Input {{{ -type InputSimple = string +type InputSimple = InputKey type InputCleaned = { - name: string + name: InputKey regexCleanup: RegexCleanup } type Input = InputSimple | InputCleaned // }}} // DownloadConfig {{{ -type UrlTemplate = string +// handlebars template +type Template = string type UrlMethods = 'GET' | 'POST' | 'PUT' | 'DELETE' interface DownloadConfigInterface { method?: UrlMethods - urlTemplate: UrlTemplate - cookieTemplates?: { [CookieName: string]: string } - headerTemplates?: { [HeaderName: string]: string } + urlTemplate: Template + headerTemplates?: { [headerName: string]: Template } regexCleanup?: RegexCleanup } -export type DownloadConfigInit = UrlTemplate | DownloadConfigInterface +export type DownloadConfigInit = Template | DownloadConfigInterface export interface DownloadConfig extends DownloadConfigInterface { method: UrlMethods + headerTemplates: { [headerName: string]: Template } } // }}} @@ -50,7 +58,7 @@ export interface ParseConfig extends ParseConfigInterface { type Incrementers = 'failed-download' | 'empty-parse' | number export interface ScrapeConfigInit { - name?: string + name?: ScraperName download?: DownloadConfigInit parse?: ParseConfigInit incrementUntil?: Incrementers @@ -65,7 +73,7 @@ export interface ConfigInit { // returned by ./normalize.ts export interface ScrapeConfig { - name: string + name: ScraperName download?: DownloadConfig parse?: ParseConfig incrementUntil: Incrementers @@ -74,6 +82,7 @@ export interface ScrapeConfig { } export interface Config extends ConfigInit { + input: Input[] scrape: ScrapeConfig } @@ -81,9 +90,9 @@ export interface Config extends ConfigInit { export type ConfigPositionInfo = { depth: number horizontalIndex: number - name: string - parentName?: string + name: ScraperName + parentName?: ScraperName } export type FlatConfig = { - [name: string]: ConfigPositionInfo + [scraperName: string]: ConfigPositionInfo } diff --git a/src/settings/external-utils.ts b/src/settings/external-utils.ts new file mode 100644 index 0000000..3e9a143 --- /dev/null +++ b/src/settings/external-utils.ts @@ -0,0 +1,12 @@ +import { normalizeConfig, makeFlatConfig, assertConfigType } from './config' +import { normalizeOptions, assertOptionsType } from './options' + +export const config = { + normalize: normalizeConfig, + flatten: makeFlatConfig, + verify: assertConfigType +} +export const options = { + normalize: normalizeOptions, + verify: assertOptionsType +} diff --git a/src/settings/options/__tests__/normalize.unit.test.ts b/src/settings/options/__tests__/normalize.unit.test.ts new file mode 100644 index 0000000..6da39ed --- /dev/null +++ b/src/settings/options/__tests__/normalize.unit.test.ts @@ -0,0 +1,66 @@ +import { normalizeConfig } from '../../config' +import { normalizeOptions } from '../' +import * as testingConfigs from '../../../../testing/resources/testing-configs' +import { FlatOptions } from '../types' +import { expect } from 'chai' + +describe('normalize run options with', () => { + describe('simple config', () => { + const fullConfig = normalizeConfig(testingConfigs.__SIMPLE_CONFIG__) + it('should match returned run options', () => { + const runOptionsInit = { + folder: '/nonexistent' + } + const runOptions = normalizeOptions(fullConfig, runOptionsInit) + const runOptionsExpected: FlatOptions = new Map([ + [ + 'level_0_index_0', + { + cache: true, + logLevel: 'error' as 'error', + downloadPriority: 0, + folder: '/nonexistent/level_0_index_0', + input: {} + } + ], + [ + 'level_1_index_0', + { + cache: true, + logLevel: 'error' as 'error', + downloadPriority: 0, + folder: '/nonexistent/level_1_index_0', + input: {} + } + ] + ]) + expect(runOptionsExpected).to.be.deep.equal(runOptions) + }) + }) + + describe('config with input', () => { + const fullConfig = normalizeConfig(testingConfigs.__INPUT_CONFIG__) + + it('should error out when there is no input', () => { + const runOptionsInit = { folder: '/nonexistent' } + const missingInputs = fullConfig.input.join() + + expect(() => normalizeOptions(fullConfig, runOptionsInit)).to.throw( + `Invalid input! Options is missing keys(s) [${missingInputs}]` + ) + }) + + it('should not error out when there are extra run option inputs', () => { + const runOptionsInit = { + input: { username: 'johnnybravo', password: 'sunglasses' }, + folder: '/nonexistent' + } + const runOptions = normalizeOptions(fullConfig, runOptionsInit) + const normalizedInput = runOptions.get('level_0_index_0')!.input + + expect(normalizedInput).to.be.deep.equal({ + username: runOptionsInit.input.username + }) + }) + }) +}) diff --git a/src/settings/options/assert.ts b/src/settings/options/assert.ts new file mode 100644 index 0000000..ceb86ce --- /dev/null +++ b/src/settings/options/assert.ts @@ -0,0 +1,4 @@ +import { OptionsInit } from './types' + +/* eslint-disable-next-line typescript/no-unused-vars */ +export const assertOptionsType = (options: OptionsInit) => {} diff --git a/src/configuration/run-options/index.ts b/src/settings/options/index.ts similarity index 100% rename from src/configuration/run-options/index.ts rename to src/settings/options/index.ts diff --git a/src/settings/options/normalize.ts b/src/settings/options/normalize.ts new file mode 100644 index 0000000..ddd18fd --- /dev/null +++ b/src/settings/options/normalize.ts @@ -0,0 +1,66 @@ +import { resolve } from 'path' +import { makeFlatConfig } from '../config/make-flat-config' +import { assertOptionsType } from './' +// type imports +import { Input, OptionsInit, FlatOptions } from './types' +import { Config } from '../config/types' + +const getConfigInputValues = (config: Config, options: OptionsInit) => { + const configInputKeys = config.input.map(input => { + if (typeof input === 'string') return input + else return input.name + }) + + const initInputs = options.input || {} + const filteredInputs: Input = {} + for (const inputKey of configInputKeys) { + if (initInputs[inputKey] === undefined) { + const missingKeys = configInputKeys + .filter(key => initInputs[key] === undefined) + .join() + throw new Error( + `Invalid input! Options is missing keys(s) [${missingKeys}]` + ) + } + filteredInputs[inputKey] = initInputs[inputKey] + } + return filteredInputs +} + +const normalizeOptions = ( + config: Config, + optionsInit: OptionsInit +): FlatOptions => { + assertOptionsType(optionsInit) + + const flatConfig = makeFlatConfig(config) + const { optionsEach = {}, ...globalOptions } = optionsInit + + const input = getConfigInputValues(config, optionsInit) + // assertValidInput(config, options) + + const defaults = { + cache: true, + downloadPriority: 0, + logLevel: 'error' as 'error', + ...globalOptions // user preferences for all things override + } + + const options: FlatOptions = Object.values(flatConfig).reduce( + (acc: FlatOptions, scraperConfig) => { + const { name } = scraperConfig + const scraperOptions = optionsEach[name] + acc.set(name, { + ...defaults, + folder: resolve(defaults.folder, name), + ...scraperOptions, + input + }) + return acc + }, + new Map() + ) + + return options +} +export { normalizeOptions } diff --git a/src/settings/options/runtime/assert.js b/src/settings/options/runtime/assert.js new file mode 100644 index 0000000..ed87b82 --- /dev/null +++ b/src/settings/options/runtime/assert.js @@ -0,0 +1,7 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +require("./tsr-declarations"); +var lib_1 = require("ts-runtime/lib"); +var types_1 = require("./types"); +/* eslint-disable-next-line typescript/no-unused-vars */ +exports.assertOptionsType = lib_1.default.annotate(function (options) { var _optionsType = lib_1.default.nullable(lib_1.default.ref(types_1.OptionsInit)); lib_1.default.param("options", _optionsType).assert(options); }, lib_1.default.function(lib_1.default.param("options", lib_1.default.nullable(lib_1.default.ref(types_1.OptionsInit))), lib_1.default.return(lib_1.default.any()))); diff --git a/src/settings/options/runtime/tsconfig.runtime.json b/src/settings/options/runtime/tsconfig.runtime.json new file mode 100644 index 0000000..83eda9b --- /dev/null +++ b/src/settings/options/runtime/tsconfig.runtime.json @@ -0,0 +1,6 @@ +{ + "extends": "../../../../tsconfig.json", + "compilerOptions": { + "outDir": "./src/settings/options/runtime/" + } +} diff --git a/src/settings/options/runtime/tsr-declarations.js b/src/settings/options/runtime/tsr-declarations.js new file mode 100644 index 0000000..64549db --- /dev/null +++ b/src/settings/options/runtime/tsr-declarations.js @@ -0,0 +1,8 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +var lib_1 = require("ts-runtime/lib"); +lib_1.default.declare("global.Map.1935072597", lib_1.default.type("global.Map", function (Map) { + var K = Map.typeParameter("K"); + var V = Map.typeParameter("V"); + return lib_1.default.object(); +})); diff --git a/src/settings/options/runtime/types.js b/src/settings/options/runtime/types.js new file mode 100644 index 0000000..3064bec --- /dev/null +++ b/src/settings/options/runtime/types.js @@ -0,0 +1,13 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +var lib_1 = require("ts-runtime/lib"); +var ScraperName = lib_1.default.type("ScraperName", lib_1.default.nullable(lib_1.default.string())); +var LogLevel = lib_1.default.type("LogLevel", lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.string("debug")), lib_1.default.nullable(lib_1.default.string("info")), lib_1.default.nullable(lib_1.default.string("warn")), lib_1.default.nullable(lib_1.default.string("error"))))); +exports.Input = lib_1.default.type("Input", lib_1.default.object(lib_1.default.indexer("inputName", lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.union(lib_1.default.nullable(lib_1.default.number()), lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.boolean())))))); +var OptionsAny = lib_1.default.type("OptionsAny", lib_1.default.object(lib_1.default.property("cache", lib_1.default.nullable(lib_1.default.boolean()), true))); +var ScraperOptionsInit = lib_1.default.type("ScraperOptionsInit", lib_1.default.intersect(lib_1.default.ref(OptionsAny), lib_1.default.object(lib_1.default.property("downloadPriority", lib_1.default.nullable(lib_1.default.number()), true), lib_1.default.property("logLevel", lib_1.default.nullable(lib_1.default.ref(LogLevel)), true)))); +var ScraperOptions = lib_1.default.type("ScraperOptions", lib_1.default.intersect(lib_1.default.ref(OptionsAny), lib_1.default.object(lib_1.default.property("cache", lib_1.default.nullable(lib_1.default.boolean())), lib_1.default.property("downloadPriority", lib_1.default.nullable(lib_1.default.number())), lib_1.default.property("logLevel", lib_1.default.nullable(lib_1.default.ref(LogLevel)))))); +exports.Parallelism = lib_1.default.type("Parallelism", lib_1.default.object(lib_1.default.property("maxConcurrent", lib_1.default.nullable(lib_1.default.number()), true), lib_1.default.property("rateLimit", lib_1.default.object(lib_1.default.property("rate", lib_1.default.nullable(lib_1.default.number())), lib_1.default.property("limit", lib_1.default.nullable(lib_1.default.number()))), true))); +exports.OptionsInit = lib_1.default.type("OptionsInit", lib_1.default.intersect(lib_1.default.ref(OptionsAny), lib_1.default.ref(exports.Parallelism), lib_1.default.object(lib_1.default.property("input", lib_1.default.nullable(lib_1.default.ref(exports.Input)), true), lib_1.default.property("folder", lib_1.default.nullable(lib_1.default.string())), lib_1.default.property("cleanFolder", lib_1.default.nullable(lib_1.default.boolean()), true), lib_1.default.property("logLevel", lib_1.default.nullable(lib_1.default.ref(LogLevel)), true), lib_1.default.property("logToFile", lib_1.default.nullable(lib_1.default.string()), true), lib_1.default.property("optionsEach", lib_1.default.object(lib_1.default.indexer("scraperName", lib_1.default.nullable(lib_1.default.string()), lib_1.default.nullable(lib_1.default.ref(ScraperOptionsInit)))), true)))); +exports.Options = lib_1.default.type("Options", lib_1.default.intersect(lib_1.default.ref(ScraperOptions), lib_1.default.object(lib_1.default.property("input", lib_1.default.nullable(lib_1.default.ref(exports.Input))), lib_1.default.property("folder", lib_1.default.nullable(lib_1.default.string()))))); +exports.FlatOptions = lib_1.default.type("FlatOptions", lib_1.default.nullable(lib_1.default.ref("global.Map.1935072597", lib_1.default.nullable(lib_1.default.ref(ScraperName)), lib_1.default.nullable(lib_1.default.ref(exports.Options))))); diff --git a/src/settings/options/types.ts b/src/settings/options/types.ts new file mode 100644 index 0000000..593d6b2 --- /dev/null +++ b/src/settings/options/types.ts @@ -0,0 +1,46 @@ +// until webpack can load ts-runtime, this is far more convienent than importing other files +type ScraperName = string +// likewise, this should come from bunyan, but it does not work well with ts-runtime +type LogLevel = 'debug' | 'info' | 'warn' | 'error' + +export type Input = { [inputName: string]: number | string | boolean } + +interface OptionsAny { + cache?: boolean +} + +interface ScraperOptionsInit extends OptionsAny { + downloadPriority?: number + logLevel?: LogLevel +} +interface ScraperOptions extends OptionsAny { + cache: boolean + downloadPriority: number + logLevel: LogLevel +} + +export interface Parallelism { + maxConcurrent?: number + rateLimit?: { + rate: number + limit: number + } +} + +export interface OptionsInit extends OptionsAny, Parallelism { + input?: Input + folder: string + cleanFolder?: boolean + logLevel?: LogLevel + logToFile?: string + optionsEach?: { + [scraperName: string]: ScraperOptionsInit + } +} + +export interface Options extends ScraperOptions { + input: Input + folder: string +} + +export type FlatOptions = Map diff --git a/src/store/sql-generators/util/find-lowest-common-parent.ts b/src/store/sql-generators/util/find-lowest-common-parent.ts deleted file mode 100644 index 62302b9..0000000 --- a/src/store/sql-generators/util/find-lowest-common-parent.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { - FlatConfig, - ConfigPositionInfo -} from '../../../configuration/site-traversal/types' -/* - * G - * | - * P - * / \ - * T I - * - * findLowestCommonParent(T, I) => P - */ -export const findLowestCommonParent = ( - flatConfig: FlatConfig, - level1: ConfigPositionInfo, - level2: ConfigPositionInfo -): FlatConfig['name'] => { - if (level1.parentName === null) return level1 - if (level2.parentName === null) return level2 - const recurse = ( - level1?: ConfigPositionInfo, - level2?: ConfigPositionInfo - ): ConfigPositionInfo => { - const level1Parent = flatConfig[level1.parentName] - const level2Parent = flatConfig[level2.parentName] - if (level1 === level2) return level1 - if (!level1 || !level2 || !level1Parent || !level2Parent) return null - - const commonParent1 = recurse(level1Parent, level2) - if (commonParent1) return commonParent1 - - const commonParent2 = recurse(level1, level2Parent) - if (commonParent2) return commonParent2 - } - // make sure the level lower on the tree is second - return recurse(...[level1, level2].sort((a, b) => b.depth - a.depth)) -} diff --git a/src/store/types.ts b/src/store/types.ts deleted file mode 100644 index c5aaa1a..0000000 --- a/src/store/types.ts +++ /dev/null @@ -1,7 +0,0 @@ -import DB from './database' -import { FlatConfig } from '../configuration/site-traversal/types' - -export type CreateQuery = ( - flatConfig: FlatConfig, - database: DB -) => StatementFunction diff --git a/src/tools/emitter.ts b/src/tools/emitter.ts new file mode 100644 index 0000000..589cd76 --- /dev/null +++ b/src/tools/emitter.ts @@ -0,0 +1,95 @@ +import EventEmitter from 'events' +import * as Rx from 'rxjs' +import * as Fetch from 'node-fetch' +// type imports +import { makeFlatConfig } from '../settings/config/make-flat-config' +import { Config } from '../settings/config/types' + +const scraperEvents = { + QUEUED: 'queued', + PROGRESS: 'progress', + COMPLETE: 'complete' +} + +class ScraperEmitter { + public emit = { + queued: (id: number) => { + this.emitter.emit(`${this.name}:${scraperEvents.QUEUED}`, id) + }, + progress: (id: number, response: Fetch.Response) => { + const emitKey = `${this.name}:${scraperEvents.PROGRESS}` + if (this.emitter.listenerCount(emitKey)) { + const contentLength = parseInt( + response.headers.get('content-length') || '0' + ) + let bytesLength = 0 + response.body.on('data', chunk => { + bytesLength += chunk.length + const progress = bytesLength / contentLength + this.emitter.emit(emitKey, progress, id) + }) + } + }, + completed: (id: number) => { + this.emitter.emit(`${this.name}:${scraperEvents.COMPLETE}`, id) + } + } + private emitter: EventEmitter + private name: string + + public constructor(name: string, emitter: EventEmitter) { + this.emitter = emitter + this.name = name + } +} + +const events = { + // listenable + DONE: 'done', + ERROR: 'error', + // emittable + STOP: 'stop', + USE_RATE_LIMITER: 'useRateLimiter' +} +type EmitterOn = (event: string, callback: (...args: any[]) => void) => void +type EmitterEmit = ( + event: 'stop' | 'useRateLimiter', + ...emittedValues: any[] +) => void + +class Emitter { + public emitter: EventEmitter + + public emit = { + done: () => { + this.emitter.emit(events.DONE) + }, + error: (error: Error) => { + this.emitter.emit(events.ERROR, error) + } + } + public on = { + stop: (callback: () => void) => { + this.emitter.on(events.STOP, callback) + } + } + private scrapers: { [scraper: string]: ScraperEmitter } = {} + + public constructor(config: Config) { + this.emitter = new EventEmitter() + + const flatConfig = makeFlatConfig(config) + for (const name of Object.keys(flatConfig)) { + this.scrapers[name] = new ScraperEmitter(name, this.emitter) + } + } + public scraper = (name: string) => this.scrapers[name] + public getBoundOn = (): EmitterOn => this.emitter.on.bind(this.emitter) + public getBoundEmit = (): EmitterEmit => this.emitter.emit.bind(this.emitter) + public getRxEventStream = (eventName: string) => + Rx.fromEvent(this.emitter, eventName) + + private hasListenerFor = (eventName: string): boolean => + this.emitter.listenerCount(eventName) !== 0 +} +export { Emitter } diff --git a/src/tools/index.ts b/src/tools/index.ts new file mode 100644 index 0000000..0c805ae --- /dev/null +++ b/src/tools/index.ts @@ -0,0 +1,35 @@ +import * as Rx from 'rxjs' +import { Emitter } from './emitter' +import { Store } from './store' +import { Logger } from './logger' +import { Queue } from './queue' +// type imports +import { Config } from '../settings/config/types' +import { OptionsInit, FlatOptions } from '../settings/options/types' + +export type Tools = { + store: Store + emitter: Emitter + logger: Logger + queue: Queue +} +export const initTools = ( + config: Config, + optionsInit: OptionsInit, + flatOptions: FlatOptions +): Tools => { + const store = new Store(config, optionsInit) + const emitter = new Emitter(config) + const logger = new Logger(optionsInit, flatOptions) + const rateLimiterEventStream = emitter.getRxEventStream( + 'useRateLimiter' + ) as Rx.Observable // deal with incoming values on this event as truthy or falsey + const queue = new Queue(optionsInit, flatOptions, rateLimiterEventStream) + + return { + store, + emitter, + logger, + queue + } +} diff --git a/src/tools/logger.ts b/src/tools/logger.ts new file mode 100644 index 0000000..28c0cec --- /dev/null +++ b/src/tools/logger.ts @@ -0,0 +1,46 @@ +import * as path from 'path' +import * as bunyan from 'bunyan' +// import { createLogger, transports, format } from 'winston' +import { tap } from 'rxjs/operators' +// type imports +import { OptionsInit, FlatOptions } from '../settings/options/types' +import { ScraperName } from '../settings/config/types' +// import * as winston from 'winston' +import { ParsedValue } from '../scraper/scrape-step' + +const serializers = { + parsedValuesWithId: (values: ParsedValue[]) => values.map(v => v.parsedValue) +} +class Logger { + public debug: typeof bunyan.prototype.debug + public info: typeof bunyan.prototype.info + public warn: typeof bunyan.prototype.warn + public error: typeof bunyan.prototype.error + private logger: bunyan + private scrapers: { [scraperName: string]: bunyan } + + public constructor(options: OptionsInit, flatOptions: FlatOptions) { + this.logger = bunyan.createLogger({ + name: 'root', + level: options.logLevel || ('error' as 'error'), + serializers, + streams: options.logToFile + ? [{ path: path.resolve(options.folder, options.logToFile) }] + : [{ stream: process.stdout }] + }) + this.scrapers = {} + flatOptions.forEach((options, name) => { + const logger = this.logger.child({ scraper: name }) + this.scrapers[name] = logger + }) + this.debug = this.logger.debug.bind(this.logger) + this.info = this.logger.info.bind(this.logger) + this.warn = this.logger.warn.bind(this.logger) + this.error = this.logger.error.bind(this.logger) + } + public tap = (name = 'TAP') => + tap((...args: any[]) => this.logger.debug({ tap: name, ...args })) + public scraper = (name: ScraperName) => this.scrapers[name] +} +export type LogLevel = bunyan.LogLevel +export { Logger } diff --git a/src/queue/index.ts b/src/tools/queue/index.ts similarity index 69% rename from src/queue/index.ts rename to src/tools/queue/index.ts index bf53085..55abc60 100644 --- a/src/queue/index.ts +++ b/src/tools/queue/index.ts @@ -1,31 +1,26 @@ -import EventEmitter from 'events' import * as Rx from 'rxjs' import * as ops from 'rxjs/operators' -import { rateLimitToggle } from '../util/rxjs/operators' -import { - RunOptionsInit, - FlatRunOptions -} from '../configuration/run-options/types' +import { rateLimitToggle } from '../../util/rxjs/operators' +import { OptionsInit, FlatOptions } from '../../settings/options/types' import { PriorityQueue } from './priority-queue' -import { Task } from '../util/rxjs/operators/conditional-rate-limiter' +import { Task } from '../../util/rxjs/operators/conditional-rate-limiter' -type ErrorCallback = (error: Error, value: any) => void -class Queuer { +type ErrorCallback = (error?: Error, value?: any) => void +class Queue { private isOpen: boolean private enqueueSubject: Rx.Subject // private taskObservable: Rx.Observable<{}> private queuePromise: Promise private queue: PriorityQueue - constructor( - { maxConcurrent = 1, rateLimit }: RunOptionsInit, - flatRunOptions: FlatRunOptions, + public constructor( + { maxConcurrent = 1, rateLimit }: OptionsInit, + flatOptions: FlatOptions, toggler: Rx.Observable ) { - const priorities = Object.values(flatRunOptions).map( - options => options.downloadPriority - ) - this.queue = new PriorityQueue(Array.from(new Set(priorities))) + const priorities = new Set() + flatOptions.forEach(options => priorities.add(options.downloadPriority)) + this.queue = new PriorityQueue(Array.from(priorities)) // rx subject that adds tasks to the observable pipeline const enqueueSubject = new Rx.Subject() @@ -51,18 +46,8 @@ class Queuer { this.queuePromise = taskObservable.toPromise() } - private executor = (): Promise => { - const taskWithCallback = this.queue.pop() - return taskWithCallback() - } - - private wrapTask = (task: Task, callback: ErrorCallback): Task => () => - task() - .then(value => callback(null, value)) - .catch(error => callback(error, null)) - // returns a promise that resolves or rejects according to the promise passed in - add = (task: () => Promise, priority: number): Promise => { + public add = (task: () => Promise, priority: number): Promise => { return new Promise((resolve, reject) => { const callback: ErrorCallback = (error, value) => { if (error) reject(error) @@ -74,7 +59,7 @@ class Queuer { } // called add(task) anywhere after this method is called will throw an error - closeQueue() { + public closeQueue() { this.isOpen = false this.enqueueSubject.complete() } @@ -82,8 +67,21 @@ class Queuer { // queuePromise will never resolve until closeQueue() is called // if you are waiting on a promise that will never close, your program may exit unexpectedly // see https://stackoverflow.com/q/46966890/3795137 for an explaination of the nodejs event cycle - toPromise() { + public toPromise() { return this.queuePromise } + + private executor = (): Promise => { + const taskWithCallback = this.queue.pop() + if (!taskWithCallback) { + throw new TypeError('queue popped an undefined task.') + } + return taskWithCallback() + } + + private wrapTask = (task: Task, callback: ErrorCallback): Task => () => + task() + .then(value => callback(undefined, value)) + .catch(error => callback(error, undefined)) } -export default Queuer +export { Queue } diff --git a/src/queue/priority-queue.ts b/src/tools/queue/priority-queue.ts similarity index 86% rename from src/queue/priority-queue.ts rename to src/tools/queue/priority-queue.ts index f562bbd..f3e6fc1 100644 --- a/src/queue/priority-queue.ts +++ b/src/tools/queue/priority-queue.ts @@ -4,12 +4,12 @@ * e.g. priority one is popped before priority zero */ export class PriorityQueue { + public length = 0 private priorities: number[] = [] private queue: { [priority: number]: T[] } = {} - public length = 0 // all priorities used in the push function are defined in the constructor - constructor(availablePriorities: number[]) { + public constructor(availablePriorities: number[]) { for (const priority of availablePriorities) { this.priorities.push(priority) this.queue[priority] = [] @@ -17,7 +17,7 @@ export class PriorityQueue { this.priorities.sort((a: number, b: number) => b - a) } - pop = () => { + public pop = () => { for (const priority of this.priorities) { const queueSlot = this.queue[priority] if (queueSlot.length) { @@ -27,7 +27,7 @@ export class PriorityQueue { } } - push = (val: T, priority: number) => { + public push = (val: T, priority: number) => { this.queue[priority].push(val) this.length++ } diff --git a/src/store/__tests__/find-lowest-common-parent.test.unit.ts b/src/tools/store/__tests__/find-lowest-common-parent.unit.test.ts similarity index 87% rename from src/store/__tests__/find-lowest-common-parent.test.unit.ts rename to src/tools/store/__tests__/find-lowest-common-parent.unit.test.ts index 40df706..8b69207 100644 --- a/src/store/__tests__/find-lowest-common-parent.test.unit.ts +++ b/src/tools/store/__tests__/find-lowest-common-parent.unit.test.ts @@ -1,10 +1,7 @@ import { expect } from 'chai' -import { - makeFlatConfig, - normalizeConfig -} from '../../configuration/site-traversal' +import { makeFlatConfig, normalizeConfig } from '../../../settings/config' import { findLowestCommonParent } from '../sql-generators/util/find-lowest-common-parent' -import * as testingConfigs from '../../../testing/resources/testing-configs' +import * as testingConfigs from '../../../../testing/resources/testing-configs' describe('find lowest common parent', () => { const galleryPostImgTag = testingConfigs.__GALLERY_POST_IMG_TAG__ diff --git a/src/store/__tests__/make-dynamic-order-level-column.test.unit.ts b/src/tools/store/__tests__/make-dynamic-order-level-column.unit.test.ts similarity index 91% rename from src/store/__tests__/make-dynamic-order-level-column.test.unit.ts rename to src/tools/store/__tests__/make-dynamic-order-level-column.unit.test.ts index c0cdc09..0d21f7d 100644 --- a/src/store/__tests__/make-dynamic-order-level-column.test.unit.ts +++ b/src/tools/store/__tests__/make-dynamic-order-level-column.unit.test.ts @@ -1,10 +1,7 @@ import { expect } from 'chai' -import { - makeFlatConfig, - normalizeConfig -} from '../../configuration/site-traversal' +import { makeFlatConfig, normalizeConfig } from '../../../settings/config' import { makeDynamicOrderLevelColumn } from '../sql-generators' -import * as testingConfigs from '../../../testing/resources/testing-configs' +import * as testingConfigs from '../../../../testing/resources/testing-configs' // should only give the higher one when their depths are unequal describe('make dynamic order level column', () => { diff --git a/src/store/__tests__/make-waiting-conditional-joins.test.unit.ts b/src/tools/store/__tests__/make-waiting-conditional-joins.unit.test.ts similarity index 89% rename from src/store/__tests__/make-waiting-conditional-joins.test.unit.ts rename to src/tools/store/__tests__/make-waiting-conditional-joins.unit.test.ts index 0e411e1..6397fd5 100644 --- a/src/store/__tests__/make-waiting-conditional-joins.test.unit.ts +++ b/src/tools/store/__tests__/make-waiting-conditional-joins.unit.test.ts @@ -1,10 +1,7 @@ import { expect } from 'chai' -import { - makeFlatConfig, - normalizeConfig -} from '../../configuration/site-traversal' +import { makeFlatConfig, normalizeConfig } from '../../../settings/config' import { makeWaitingConditionalJoins } from '../sql-generators' -import * as testingConfigs from '../../../testing/resources/testing-configs' +import * as testingConfigs from '../../../../testing/resources/testing-configs' // should only give the higher one when their depths are unequal describe('make waiting conditional joins', () => { diff --git a/src/store/database.ts b/src/tools/store/database.ts similarity index 75% rename from src/store/database.ts rename to src/tools/store/database.ts index 4d0a6d2..58a0dae 100644 --- a/src/store/database.ts +++ b/src/tools/store/database.ts @@ -2,8 +2,8 @@ import { resolve } from 'path' import BetterSqlite3Database from 'better-sqlite3' class Database extends BetterSqlite3Database { - constructor(folder: string) { + public constructor(folder: string) { super(resolve(folder, 'store.sqlite')) } } -export default Database +export { Database } diff --git a/src/store/index.ts b/src/tools/store/index.ts similarity index 62% rename from src/store/index.ts rename to src/tools/store/index.ts index 19d67a5..7e8d8f7 100644 --- a/src/store/index.ts +++ b/src/tools/store/index.ts @@ -1,41 +1,35 @@ -import DB from './database' -import { makeFlatConfig } from '../configuration/site-traversal' -import * as queries from './queries' -import { groupBy as groupByKey } from '../util/array' -import { Config, FlatConfig } from '../configuration/site-traversal/types' +import { Database } from './database' +import { makeFlatConfig } from '../../settings/config' +import { groupBy as groupByKey } from '../../util/array' +import { Config, FlatConfig } from '../../settings/config/types' import { createTables, createStatements } from './queries' +// type imports +import { Transaction } from 'better-sqlite3' +import { OptionsInit } from '../../settings/options/types' class Store { + public qs: ReturnType private config: Config private flatConfig: FlatConfig - private database: DB - public qs: ReturnType + private database: Database - constructor(config: Config) { + public constructor(config: Config, { folder }: OptionsInit) { this.config = config this.flatConfig = makeFlatConfig(config) - } - - init = ({ folder }: { folder: string }) => { - this.database = new DB(folder) + // initialize sqlite3 database + this.database = new Database(folder) this.database.pragma('journal_mode = WAL') - + // initialize tables (if they do not exist already) createTables(this.flatConfig, this.database)() + // prepare statements this.qs = createStatements(this.flatConfig, this.database) } - asTransaction = (func: () => T) => (): T => { - this.qs.beginTransaction.run() - try { - const result = func() - this.qs.commitTransaction.run() - return result - } finally { - if (this.database.inTransaction) this.qs.rollbackTransaction.run() - } + public transaction = (func: () => T): Transaction => { + return this.database.transaction(func) } - queryFor = ({ + public query = ({ scrapers, groupBy }: { @@ -47,9 +41,9 @@ class Store { const matchingScrapers = scrapers.filter(s => this.flatConfig[s]) if (!matchingScrapers.length) return [{}] - const matchingAll = Array.from(new Set(scrapers.concat(groupBy))).filter( - s => this.flatConfig[s] - ) + const matchingAll = Array.from( + new Set(scrapers.concat(groupBy === undefined ? [] : groupBy)) + ).filter(s => this.flatConfig[s]) // const matchingAll = Array.from( // new Set(scraperNames.concat(groupBy)) // ).filter(s => this.flatConfig[s]) @@ -63,17 +57,18 @@ class Store { // }, {}) // ) - const headers = [ - 'id', - 'parentId', - 'incrementIndex', - 'levelOrder', - 'recurseDepth', - 'currentScraper', - 'filename', - 'parsedValue', - 'scraper' - ] + /** logging helper + // const headers = [ + // 'id', + // 'parentId', + // 'incrementIndex', + // 'levelOrder', + // 'recurseDepth', + // 'currentScraper', + // 'filename', + // 'parsedValue', + // 'scraper' + // ] // console.log([ // headers.join(' | '), // ...result.map(r => { @@ -88,6 +83,7 @@ class Store { // .join(' | ') // }) // ]) + */ // TODO move this into sql // const objectPicker = ( @@ -108,7 +104,7 @@ class Store { result, 'scraper', groupBy, - scrapers.includes(groupBy), + groupBy !== undefined && scrapers.includes(groupBy), selector => selector // selector => objectPicker(selector, scrapers[selector.scraper]) ) @@ -116,4 +112,4 @@ class Store { } } -export default Store +export { Store } diff --git a/src/store/queries/create-tables/index.ts b/src/tools/store/queries/create-tables/index.ts similarity index 100% rename from src/store/queries/create-tables/index.ts rename to src/tools/store/queries/create-tables/index.ts diff --git a/src/store/queries/create-tables/template.sql b/src/tools/store/queries/create-tables/template.sql similarity index 89% rename from src/store/queries/create-tables/template.sql rename to src/tools/store/queries/create-tables/template.sql index 5c6155f..bce5c15 100644 --- a/src/store/queries/create-tables/template.sql +++ b/src/tools/store/queries/create-tables/template.sql @@ -3,7 +3,6 @@ BEGIN TRANSACTION; CREATE TABLE IF NOT EXISTS downloads ( id INTEGER PRIMARY KEY NOT NULL, scraper TEXT NOT NULL, - scrapeNextIndex INT NOT NULL, -- index of a looped config step (priority high) incrementIndex INT NOT NULL, -- index of a url incrementer (priority low) parseParentId INT, -- necessary to distinguish identity steps url TEXT, @@ -29,5 +28,5 @@ CREATE TABLE IF NOT EXISTS parsedTree ( -- horizontalIndex INT, -- index used for consitent order when two parsers are next to each other (priority low and only under certain circumstances) CREATE UNIQUE INDEX IF NOT EXISTS downloadId ON downloads(id); -CREATE UNIQUE INDEX IF NOT EXISTS indexes ON downloads(scraper, scrapeNextIndex, incrementIndex, parseParentId); +CREATE UNIQUE INDEX IF NOT EXISTS indexes ON downloads(scraper, incrementIndex, parseParentId); COMMIT; diff --git a/src/store/queries/index.ts b/src/tools/store/queries/index.ts similarity index 84% rename from src/store/queries/index.ts rename to src/tools/store/queries/index.ts index c8ce4bb..1a27e61 100644 --- a/src/store/queries/index.ts +++ b/src/tools/store/queries/index.ts @@ -1,5 +1,6 @@ -import DB from '../database' -import { FlatConfig } from '../../configuration/site-traversal/types' +// type imports +import { Database } from '../database' +import { FlatConfig } from '../../../settings/config/types' export { query as createTables } from './create-tables' import { query as selectOrderedScrapers } from './select-ordered-scrapers' @@ -9,7 +10,10 @@ import { query as updateDownloadToComplete } from './update-download-to-complete import { query as insertBatchParsedValues } from './insert-batch-parsed-values' import { query as selectParsedValues } from './select-parsed-values' -export const createStatements = (flatConfig: FlatConfig, database: DB) => ({ +export const createStatements = ( + flatConfig: FlatConfig, + database: Database +) => ({ selectOrderedScrapers: selectOrderedScrapers(flatConfig, database), selectCompletedDownload: selectCompletedDownload(flatConfig, database), insertQueuedDownload: insertQueuedDownload(flatConfig, database), diff --git a/src/store/queries/insert-batch-parsed-values/index.ts b/src/tools/store/queries/insert-batch-parsed-values/index.ts similarity index 88% rename from src/store/queries/insert-batch-parsed-values/index.ts rename to src/tools/store/queries/insert-batch-parsed-values/index.ts index 3475538..7567d62 100644 --- a/src/store/queries/insert-batch-parsed-values/index.ts +++ b/src/tools/store/queries/insert-batch-parsed-values/index.ts @@ -4,9 +4,9 @@ import { CreateQuery } from '../../types' type Statement = ( params: { name: string - parentId: number + parentId?: number downloadId: number - parsedValues: string[] + parsedValues: string[] | [string | undefined] } ) => void export const query: CreateQuery = (flatConfig, database) => { diff --git a/src/store/queries/insert-batch-parsed-values/template.sql b/src/tools/store/queries/insert-batch-parsed-values/template.sql similarity index 100% rename from src/store/queries/insert-batch-parsed-values/template.sql rename to src/tools/store/queries/insert-batch-parsed-values/template.sql diff --git a/src/store/queries/insert-queued-download/index.ts b/src/tools/store/queries/insert-queued-download/index.ts similarity index 59% rename from src/store/queries/insert-queued-download/index.ts rename to src/tools/store/queries/insert-queued-download/index.ts index 3e0a3b5..62760ab 100644 --- a/src/store/queries/insert-queued-download/index.ts +++ b/src/tools/store/queries/insert-queued-download/index.ts @@ -2,25 +2,22 @@ import SQL_TEMPLATE from './template.sql' import { CreateQuery } from '../../types' type Statement = ( + scraper: string, params: { - scraper: string parentId?: number - scrapeNextIndex: number incrementIndex: number - url?: string - } + }, + downloadData: {} ) => number export const query: CreateQuery = (flatConfig, database) => { - // TODO batch this call? (less readable but doable) const statement = database.prepare(SQL_TEMPLATE) - return ({ scraper, parentId, scrapeNextIndex, incrementIndex, url }) => { + return (scraper, { parentId, incrementIndex }, downloadData) => { const info = statement.run( scraper, parentId, - scrapeNextIndex, incrementIndex, - url + JSON.stringify(downloadData) ) - return info.lastInsertROWID as number + return info.lastInsertRowid as number } } diff --git a/src/store/queries/insert-queued-download/template.sql b/src/tools/store/queries/insert-queued-download/template.sql similarity index 65% rename from src/store/queries/insert-queued-download/template.sql rename to src/tools/store/queries/insert-queued-download/template.sql index 4f736e9..bb72dfa 100644 --- a/src/store/queries/insert-queued-download/template.sql +++ b/src/tools/store/queries/insert-queued-download/template.sql @@ -1,8 +1,6 @@ INSERT OR IGNORE INTO downloads ( scraper, parseParentId, - scrapeNextIndex, incrementIndex, url - -) VALUES (?, ?, ?, ?, ?) +) VALUES (?, ?, ?, ?) diff --git a/src/store/queries/select-completed-download/index.ts b/src/tools/store/queries/select-completed-download/index.ts similarity index 64% rename from src/store/queries/select-completed-download/index.ts rename to src/tools/store/queries/select-completed-download/index.ts index 728f722..59b8649 100644 --- a/src/store/queries/select-completed-download/index.ts +++ b/src/tools/store/queries/select-completed-download/index.ts @@ -5,16 +5,13 @@ type SelectedRow = { id?: number } type Statement = ( params: { incrementIndex: number - scrapeNextIndex?: number parentId?: number scraper: string } ) => SelectedRow export const query: CreateQuery = (flatConfig, database) => { const statement = database.prepare(SQL_TEMPLATE) - return ({ incrementIndex, scrapeNextIndex = 0, parentId = -1, scraper }) => { - return ( - statement.get([scrapeNextIndex, incrementIndex, parentId, scraper]) || {} - ) + return ({ incrementIndex, parentId = -1, scraper }) => { + return statement.get([incrementIndex, parentId, scraper]) || {} } } diff --git a/src/store/queries/select-completed-download/template.sql b/src/tools/store/queries/select-completed-download/template.sql similarity index 65% rename from src/store/queries/select-completed-download/template.sql rename to src/tools/store/queries/select-completed-download/template.sql index 6e3fe55..7a6d883 100644 --- a/src/store/queries/select-completed-download/template.sql +++ b/src/tools/store/queries/select-completed-download/template.sql @@ -1,6 +1,5 @@ SELECT id FROM downloads -WHERE scrapeNextIndex = ? -AND incrementIndex = ? +WHERE incrementIndex = ? AND IFNULL(parseParentId, -1) = ? AND complete = 1 AND scraper = ? diff --git a/src/store/queries/select-ordered-scrapers/index.ts b/src/tools/store/queries/select-ordered-scrapers/index.ts similarity index 91% rename from src/store/queries/select-ordered-scrapers/index.ts rename to src/tools/store/queries/select-ordered-scrapers/index.ts index b70f35d..8f95e5b 100644 --- a/src/store/queries/select-ordered-scrapers/index.ts +++ b/src/tools/store/queries/select-ordered-scrapers/index.ts @@ -1,4 +1,4 @@ -import { compileTemplate } from '../../../util/handlebars' +import { compileTemplate } from '../../../../util/handlebars' import { makeDynamicOrderLevelColumn, makeWaitingConditionalJoins @@ -33,7 +33,6 @@ export const query: CreateQuery = ( selectedScrapers, lowestDepth }) - // console.log(selectOrderedSql) const statement = database.prepare(selectOrderedSql) return statement.all() } diff --git a/src/store/queries/select-ordered-scrapers/template.sql b/src/tools/store/queries/select-ordered-scrapers/template.sql similarity index 100% rename from src/store/queries/select-ordered-scrapers/template.sql rename to src/tools/store/queries/select-ordered-scrapers/template.sql diff --git a/src/store/queries/select-parsed-values/index.ts b/src/tools/store/queries/select-parsed-values/index.ts similarity index 100% rename from src/store/queries/select-parsed-values/index.ts rename to src/tools/store/queries/select-parsed-values/index.ts diff --git a/src/store/queries/select-parsed-values/template.sql b/src/tools/store/queries/select-parsed-values/template.sql similarity index 100% rename from src/store/queries/select-parsed-values/template.sql rename to src/tools/store/queries/select-parsed-values/template.sql diff --git a/src/store/queries/update-download-to-complete/index.ts b/src/tools/store/queries/update-download-to-complete/index.ts similarity index 79% rename from src/store/queries/update-download-to-complete/index.ts rename to src/tools/store/queries/update-download-to-complete/index.ts index bf94424..f74624b 100644 --- a/src/store/queries/update-download-to-complete/index.ts +++ b/src/tools/store/queries/update-download-to-complete/index.ts @@ -1,7 +1,7 @@ import SQL_TEMPLATE from './template.sql' import { CreateQuery } from '../../types' -type Statement = (params: { downloadId: number; filename: string }) => void +type Statement = (params: { downloadId: number; filename?: string }) => void export const query: CreateQuery = (flatConfig, database) => { const statement = database.prepare(SQL_TEMPLATE) return ({ downloadId, filename }) => { diff --git a/src/store/queries/update-download-to-complete/template.sql b/src/tools/store/queries/update-download-to-complete/template.sql similarity index 100% rename from src/store/queries/update-download-to-complete/template.sql rename to src/tools/store/queries/update-download-to-complete/template.sql diff --git a/src/store/sql-generators/index.ts b/src/tools/store/sql-generators/index.ts similarity index 100% rename from src/store/sql-generators/index.ts rename to src/tools/store/sql-generators/index.ts diff --git a/src/store/sql-generators/make-dynamic-order-level-column.ts b/src/tools/store/sql-generators/make-dynamic-order-level-column.ts similarity index 88% rename from src/store/sql-generators/make-dynamic-order-level-column.ts rename to src/tools/store/sql-generators/make-dynamic-order-level-column.ts index 145b04c..b01df7f 100644 --- a/src/store/sql-generators/make-dynamic-order-level-column.ts +++ b/src/tools/store/sql-generators/make-dynamic-order-level-column.ts @@ -1,9 +1,6 @@ import { findLowestCommonParent } from './util/find-lowest-common-parent' // type imports -import { - FlatConfig, - ConfigPositionInfo -} from '../../configuration/site-traversal/types' +import { FlatConfig, ConfigPositionInfo } from '../../../settings/config/types' /** * ensures that when multiple scrapes are selected at once, the proper order is attached at each level of the @@ -36,15 +33,14 @@ const makeDynamicOrderLevelColumn = ( } } } - ancestors[min.name] = ancestors[min.name] || [] - ancestors[min.name].push(current) + ancestors[min!.name] = ancestors[min!.name] || [] + ancestors[min!.name].push(current) } const diagonalOrderColumn = Object.keys(ancestors) .map(commonParentName => { const orderAtRecurseDepth = lowestDepth - flatConfig[commonParentName].depth - 1 - const orderAtScraper = flatConfig[commonParentName].name const scrapersToOrder = ancestors[commonParentName] return scrapersToOrder .map(({ name, depth, horizontalIndex }) => { diff --git a/src/store/sql-generators/make-waiting-conditional-joins.ts b/src/tools/store/sql-generators/make-waiting-conditional-joins.ts similarity index 89% rename from src/store/sql-generators/make-waiting-conditional-joins.ts rename to src/tools/store/sql-generators/make-waiting-conditional-joins.ts index 22f892d..e1ff8bb 100644 --- a/src/store/sql-generators/make-waiting-conditional-joins.ts +++ b/src/tools/store/sql-generators/make-waiting-conditional-joins.ts @@ -1,8 +1,5 @@ // type imports -import { - FlatConfig, - ConfigPositionInfo -} from '../../configuration/site-traversal/types' +import { FlatConfig } from '../../../settings/config/types' const makeWaitingConditionalJoins = ( flatConfig: FlatConfig, diff --git a/src/tools/store/sql-generators/util/find-lowest-common-parent.ts b/src/tools/store/sql-generators/util/find-lowest-common-parent.ts new file mode 100644 index 0000000..29aa0de --- /dev/null +++ b/src/tools/store/sql-generators/util/find-lowest-common-parent.ts @@ -0,0 +1,49 @@ +import { + FlatConfig, + ConfigPositionInfo +} from '../../../../settings/config/types' +/* + * G + * | + * P + * / \ + * T I + * + * findLowestCommonParent(T, I) => P + */ +export const findLowestCommonParent = ( + flatConfig: FlatConfig, + scraperA: ConfigPositionInfo, + scraperB: ConfigPositionInfo +): FlatConfig['name'] => { + if (scraperA.parentName === null) return scraperA + if (scraperB.parentName === null) return scraperB + const recurse = ( + scraperA?: ConfigPositionInfo, + scraperB?: ConfigPositionInfo + ): Voidable => { + if (scraperA === scraperB) return scraperA + if ( + scraperA === undefined || + scraperB === undefined || + scraperA.parentName === undefined || + scraperB.parentName === undefined + ) + return + + const scraperAParent = flatConfig[scraperA.parentName] + const scraperBParent = flatConfig[scraperB.parentName] + if (!scraperAParent || !scraperBParent) return + + const commonParent1 = recurse(scraperAParent, scraperB) + if (commonParent1) return commonParent1 + + const commonParent2 = recurse(scraperA, scraperBParent) + if (commonParent2) return commonParent2 + } + // make sure the level lower on the tree is second + return ( + recurse(...[scraperA, scraperB].sort((a, b) => b.depth - a.depth)) || + scraperA + ) +} diff --git a/src/tools/store/types.ts b/src/tools/store/types.ts new file mode 100644 index 0000000..81f14cc --- /dev/null +++ b/src/tools/store/types.ts @@ -0,0 +1,7 @@ +import { Database } from './database' +import { FlatConfig } from '../../settings/config/types' + +export type CreateQuery = ( + flatConfig: FlatConfig, + database: Database +) => StatementFunction diff --git a/src/util/error.ts b/src/util/error.ts new file mode 100644 index 0000000..7705d33 --- /dev/null +++ b/src/util/error.ts @@ -0,0 +1,5 @@ +import * as Rx from 'rxjs' +import VError from 'verror' + +export const wrapError = (message: any) => (e: Error) => + Rx.throwError(new VError({ name: e.name, cause: e }, message)) diff --git a/src/util/fs.ts b/src/util/fs.ts index 54395bf..a3467b9 100644 --- a/src/util/fs.ts +++ b/src/util/fs.ts @@ -1,6 +1,7 @@ import path from 'path' import fs from 'fs' import { promisify } from 'util' +import sanitize from 'sanitize-filename' const [exists, readFile, mkdir, readdir, stat, unlink, rmdir] = [ fs.readFile, @@ -14,18 +15,6 @@ const [exists, readFile, mkdir, readdir, stat, unlink, rmdir] = [ export { mkdir, exists, readFile, readdir, stat, unlink } -export const mkdirpSync = (folder: string) => { - try { - fs.mkdirSync(folder) - } catch (e) { - if (e.code === 'ENOENT') { - mkdirpSync(path.dirname(folder)) - mkdirpSync(folder) - } else if (e.code !== 'EEXIST') { - throw e - } - } -} export const mkdirp = async (folder: string) => { try { await mkdir(folder) @@ -40,15 +29,24 @@ export const mkdirp = async (folder: string) => { } export const rmrf = async (folder: string) => { - const files = await readdir(folder) - for (const file of files) { - const fullPath = path.resolve(folder, file) - const fileStats = await stat(fullPath) - if (fileStats.isDirectory()) { - await rmrf(fullPath) - await rmdir(fullPath) - } else { - await unlink(fullPath) + try { + const files = await readdir(folder) + for (const file of files) { + const fullPath = path.resolve(folder, file) + const fileStats = await stat(fullPath) + if (fileStats.isDirectory()) { + await rmrf(fullPath) + await rmdir(fullPath) + } else { + await unlink(fullPath) + } + } + } catch (e) { + if (e.code !== 'ENOENT') { + throw e } } } + +export const sanitizeFilename = (filename: string) => + sanitize(filename, { replacement: '_' }) diff --git a/src/util/handlebars.ts b/src/util/handlebars.ts index 9d9b406..e387045 100644 --- a/src/util/handlebars.ts +++ b/src/util/handlebars.ts @@ -1,4 +1,4 @@ -import Handlebars from 'handlebars' +import * as Handlebars from 'handlebars' Handlebars.registerHelper('+', (x: number, y: number) => x + y) Handlebars.registerHelper('-', (x: number, y: number) => x - y) diff --git a/src/util/rxjs/observables/from-async-generator.ts b/src/util/rxjs/observables/from-async-generator.ts index d990ad7..89a23f7 100644 --- a/src/util/rxjs/observables/from-async-generator.ts +++ b/src/util/rxjs/observables/from-async-generator.ts @@ -1,5 +1,4 @@ import * as Rx from 'rxjs' -import * as ops from 'rxjs/operators' const translateAsyncGeneratorToObservable = async ( generator: () => AsyncIterableIterator, diff --git a/src/util/rxjs/observables/while-loop.ts b/src/util/rxjs/observables/while-loop.ts index 89135b7..95cd55e 100644 --- a/src/util/rxjs/observables/while-loop.ts +++ b/src/util/rxjs/observables/while-loop.ts @@ -1,7 +1,4 @@ import * as Rx from 'rxjs' -import * as ops from 'rxjs/operators' -// types -import { DownloadParseFunction } from '../../../scraper/scrape-step/incrementer' export const whileLoopObservable = ( inLoopFunction: (initialVal: In, index: number) => Promise, @@ -20,7 +17,6 @@ export const whileLoopObservable = ( observer.next(nextVal) index++ } catch (e) { - // TODO in incrementUntil number, allow loop to keep going when there is a failed download observer.error(e) break } diff --git a/src/util/rxjs/operators/conditional-rate-limiter.ts b/src/util/rxjs/operators/conditional-rate-limiter.ts index 56481b6..77cb4db 100644 --- a/src/util/rxjs/operators/conditional-rate-limiter.ts +++ b/src/util/rxjs/operators/conditional-rate-limiter.ts @@ -1,6 +1,7 @@ import * as Rx from 'rxjs' import * as ops from 'rxjs/operators' +// TODO replace any with T export type Task = () => Promise export interface Queue { push: Function @@ -17,12 +18,10 @@ export interface Queue { const rateLimitToggle = ( { toggler, - executor, - timer + executor }: { toggler: Rx.Observable executor: () => Promise - timer?: typeof Rx.timer }, { limit, @@ -69,14 +68,14 @@ const rateLimitToggle = ( ) .subscribe(val => { inProgressExecutions-- - concurrentLimiter.next(null) + concurrentLimiter.next(false) subscriber.next(val) }) source.subscribe({ - next(val) { + next() { plannedExecutions++ - concurrentLimiter.next(null) + concurrentLimiter.next(false) }, complete() { closed = true diff --git a/testing/functional/increment-gallery-site/config.ts b/testing/functional/increment-gallery-site/config.ts index 88a33db..07669ed 100644 --- a/testing/functional/increment-gallery-site/config.ts +++ b/testing/functional/increment-gallery-site/config.ts @@ -1,4 +1,4 @@ -import { ConfigInit } from '../../../src/configuration/site-traversal/types' +import { ConfigInit } from '../../../src/settings/config/types' export const config: ConfigInit = { scrape: { diff --git a/testing/functional/increment-gallery-site/functional.test.ts b/testing/functional/increment-gallery-site/functional.test.ts index a705941..c76df89 100644 --- a/testing/functional/increment-gallery-site/functional.test.ts +++ b/testing/functional/increment-gallery-site/functional.test.ts @@ -1,38 +1,32 @@ import os from 'os' import path from 'path' -import { expect, use } from 'chai' -import chaiExclude from 'chai-exclude' -use(chaiExclude) +import { expect } from 'chai' import _ from 'lodash/fp' -import { NockFolderMock } from '../../nock-folder-mock' -import PageScraper from '../../../src' +import '../../use-chai-plugins' +import { nockMockFolder } from '../../nock-folder-mock' import { config } from './config' import expectedQueryResult from './resources/expected-query-result.json' +import { scrape } from '../../../src' describe('increment gallery site', () => { describe('with instant scraper', function() { let scraperQueryForFunction: any before(done => { ;(async () => { - const endpointMock = new NockFolderMock( + await nockMockFolder( `${__dirname}/resources/mock-endpoints`, 'http://increment-gallery-site.com' ) - await endpointMock.init() - const downloadDir = path.resolve(os.tmpdir(), this.fullTitle()) - const siteScraper = new PageScraper(config) - siteScraper - .run({ - folder: downloadDir, - cleanFolder: true - }) - .on('done', queryFor => { - scraperQueryForFunction = queryFor - done() - }) + const options = { + folder: path.resolve(os.tmpdir(), this.fullTitle()), + cleanFolder: true + } + const { on, query } = await scrape(config, options) + scraperQueryForFunction = query + on('done', done) })() }) @@ -60,24 +54,19 @@ describe('increment gallery site', () => { let scraperQueryForFunction: any before(done => { ;(async () => { - const endpointMock = new NockFolderMock( + await nockMockFolder( `${__dirname}/resources/mock-endpoints`, 'http://increment-gallery-site.com', { randomSeed: 1 } ) - await endpointMock.init() - const downloadDir = path.resolve(os.tmpdir(), this.fullTitle()) - const siteScraper = new PageScraper(config) - siteScraper - .run({ - folder: downloadDir, - cleanFolder: true - }) - .on('done', queryFor => { - scraperQueryForFunction = queryFor - done() - }) + const options = { + folder: path.resolve(os.tmpdir(), this.fullTitle()), + cleanFolder: true + } + const { on, query } = await scrape(config, options) + scraperQueryForFunction = query + on('done', done) })() }) diff --git a/testing/functional/increment-gallery-site/resources/expected-query-result.json b/testing/functional/increment-gallery-site/resources/expected-query-result.json index f676c20..3d4c00b 100644 --- a/testing/functional/increment-gallery-site/resources/expected-query-result.json +++ b/testing/functional/increment-gallery-site/resources/expected-query-result.json @@ -4,7 +4,7 @@ { "scraper": "image", "parsedValue": null, - "url": "http://increment-gallery-site.com/image/the.jpg", + "url": "[\"http://increment-gallery-site.com/image/the.jpg\",{\"headers\":{},\"method\":\"GET\"}]", "filename": -1 } ], @@ -28,7 +28,7 @@ { "scraper": "image", "parsedValue": null, - "url": "http://increment-gallery-site.com/image/quick.jpg", + "url": "[\"http://increment-gallery-site.com/image/quick.jpg\",{\"headers\":{},\"method\":\"GET\"}]", "filename": -1 } ], @@ -52,7 +52,7 @@ { "scraper": "image", "parsedValue": null, - "url": "http://increment-gallery-site.com/image/brown.jpg", + "url": "[\"http://increment-gallery-site.com/image/brown.jpg\",{\"headers\":{},\"method\":\"GET\"}]", "filename": -1 } ], @@ -76,7 +76,7 @@ { "scraper": "image", "parsedValue": null, - "url": "http://increment-gallery-site.com/image/fox.jpg", + "url": "[\"http://increment-gallery-site.com/image/fox.jpg\",{\"headers\":{},\"method\":\"GET\"}]", "filename": -1 } ], diff --git a/testing/functional/scrape-next-site/config.ts b/testing/functional/scrape-next-site/config.ts index 2d1b6a6..4db8c06 100644 --- a/testing/functional/scrape-next-site/config.ts +++ b/testing/functional/scrape-next-site/config.ts @@ -1,4 +1,4 @@ -import { ConfigInit } from '../../../src/configuration/site-traversal/types' +import { ConfigInit } from '../../../src/settings/config/types' export const config: ConfigInit = { scrape: { diff --git a/testing/functional/scrape-next-site/functional.test.ts b/testing/functional/scrape-next-site/functional.test.ts index 635ae56..92282f2 100644 --- a/testing/functional/scrape-next-site/functional.test.ts +++ b/testing/functional/scrape-next-site/functional.test.ts @@ -1,38 +1,32 @@ import os from 'os' import path from 'path' -import { expect, use } from 'chai' -import chaiExclude from 'chai-exclude' -use(chaiExclude) +import { expect } from 'chai' import _ from 'lodash/fp' -import { NockFolderMock } from '../../nock-folder-mock' -import PageScraper from '../../../src' +import '../../use-chai-plugins' +import { nockMockFolder } from '../../nock-folder-mock' import { config } from './config' import expectedQueryResult from './resources/expected-query-result.json' +import { scrape } from '../../../src' describe('scrape next site', () => { describe('with instant scraper', function() { let scraperQueryForFunction: any before(done => { ;(async () => { - const endpointMock = new NockFolderMock( + await nockMockFolder( `${__dirname}/resources/mock-endpoints`, 'http://scrape-next-site.com' ) - await endpointMock.init() - const downloadDir = path.resolve(os.tmpdir(), this.fullTitle()) - const siteScraper = new PageScraper(config) - siteScraper - .run({ - folder: downloadDir, - cleanFolder: true - }) - .on('done', queryFor => { - scraperQueryForFunction = queryFor - done() - }) + const options = { + folder: path.resolve(os.tmpdir(), this.fullTitle()), + cleanFolder: true + } + const { on, query } = await scrape(config, options) + scraperQueryForFunction = query + on('done', done) })() }) @@ -60,25 +54,19 @@ describe('scrape next site', () => { let scraperQueryForFunction: any before(done => { ;(async () => { - const endpointMock = new NockFolderMock( + await nockMockFolder( `${__dirname}/resources/mock-endpoints`, 'http://scrape-next-site.com', { randomSeed: 2 } ) - await endpointMock.init() - const downloadDir = path.resolve(os.tmpdir(), this.fullTitle()) - const siteScraper = new PageScraper(config) - siteScraper - .run({ - folder: downloadDir, - cleanFolder: true, - maxConcurrent: 10 - }) - .on('done', queryFor => { - scraperQueryForFunction = queryFor - done() - }) + const options = { + folder: path.resolve(os.tmpdir(), this.fullTitle()), + cleanFolder: true + } + const { on, query } = await scrape(config, options) + scraperQueryForFunction = query + on('done', done) })() }) diff --git a/testing/functional/scrape-next-site/resources/expected-query-result.json b/testing/functional/scrape-next-site/resources/expected-query-result.json index 7f07644..01b8306 100644 --- a/testing/functional/scrape-next-site/resources/expected-query-result.json +++ b/testing/functional/scrape-next-site/resources/expected-query-result.json @@ -4,7 +4,7 @@ { "scraper": "image", "parsedValue": null, - "url": "http://scrape-next-site.com/image/the.jpg", + "url": "[\"http://scrape-next-site.com/image/the.jpg\",{\"headers\":{},\"method\":\"GET\"}]", "filename": -1 } ], @@ -28,7 +28,7 @@ { "scraper": "image", "parsedValue": null, - "url": "http://scrape-next-site.com/image/quick.jpg", + "url": "[\"http://scrape-next-site.com/image/quick.jpg\",{\"headers\":{},\"method\":\"GET\"}]", "filename": -1 } ], @@ -52,7 +52,7 @@ { "scraper": "image", "parsedValue": null, - "url": "http://scrape-next-site.com/image/brown.jpg", + "url": "[\"http://scrape-next-site.com/image/brown.jpg\",{\"headers\":{},\"method\":\"GET\"}]", "filename": -1 } ], @@ -76,7 +76,7 @@ { "scraper": "image", "parsedValue": null, - "url": "http://scrape-next-site.com/image/fox.jpg", + "url": "[\"http://scrape-next-site.com/image/fox.jpg\",{\"headers\":{},\"method\":\"GET\"}]", "filename": -1 } ], diff --git a/testing/nock-folder-mock.ts b/testing/nock-folder-mock.ts index b4d5a56..659090b 100644 --- a/testing/nock-folder-mock.ts +++ b/testing/nock-folder-mock.ts @@ -18,44 +18,34 @@ const findFilesRecursive = async (folder: string): Promise => { } class PsuedoSeedRandom { - _seed: number - constructor(seed: number) { + private _seed: number + public constructor(seed: number) { this._seed = seed % 2147483647 if (this._seed <= 0) this._seed += 2147483646 } - next = () => (this._seed = (this._seed * 16807) % 2147483647) - nextFloat = () => (this.next() - 1) / 2147483646 + public next = () => (this._seed = (this._seed * 16807) % 2147483647) + public nextFloat = () => (this.next() - 1) / 2147483646 } -export class NockFolderMock { - mockEndpointsFolder: string - baseUrl: string - scope: nock.Scope - random: PsuedoSeedRandom +export const nockMockFolder = async ( + mockEndpointsFolder: string, + baseUrl: string, + { randomSeed }: { randomSeed?: number } = {} +) => { + const scope = nock(baseUrl) + const random = randomSeed && new PsuedoSeedRandom(randomSeed) - constructor( - mockEndpointsFolder: string, - baseUrl: string, - { randomSeed }: { randomSeed?: number } = {} - ) { - this.mockEndpointsFolder = mockEndpointsFolder - this.baseUrl = baseUrl - this.scope = nock(baseUrl) - if (randomSeed) this.random = new PsuedoSeedRandom(randomSeed) - } - init = async () => { - const files = await findFilesRecursive(this.mockEndpointsFolder) - for (const file of files) { - const relativePath = path.relative(this.mockEndpointsFolder, file) - const fullPath = path.resolve(this.mockEndpointsFolder, file) - if (this.random) { - this.scope - .get(`/${relativePath}`) - .delay(this.random.nextFloat() * 100) - .replyWithFile(200, fullPath) - } else { - this.scope.get(`/${relativePath}`).replyWithFile(200, fullPath) - } + const files = await findFilesRecursive(mockEndpointsFolder) + for (const file of files) { + const relativePath = path.relative(mockEndpointsFolder, file) + const fullPath = path.resolve(mockEndpointsFolder, file) + if (random) { + scope + .get(`/${relativePath}`) + .delay(random.nextFloat() * 100) + .replyWithFile(200, fullPath) + } else { + scope.get(`/${relativePath}`).replyWithFile(200, fullPath) } } } diff --git a/testing/resources/testing-configs.ts b/testing/resources/testing-configs.ts index 261d50d..8a41d1e 100644 --- a/testing/resources/testing-configs.ts +++ b/testing/resources/testing-configs.ts @@ -1,4 +1,4 @@ -import { ConfigInit } from '../../src/configuration/site-traversal/types' +import { ConfigInit } from '../../src/settings/config/types' // setup reusable variables export const __SIMPLE_CONFIG__: ConfigInit = { @@ -53,3 +53,12 @@ export const __GALLERY_POST_IMG_TAG__: ConfigInit = { } } } + +export const __EMPTY_CONFIG__: ConfigInit = { + scrape: {} +} + +export const __INPUT_CONFIG__: ConfigInit = { + input: ['username'], + scrape: {} +} diff --git a/testing/use-chai-plugins.ts b/testing/use-chai-plugins.ts new file mode 100644 index 0000000..2b8e6ce --- /dev/null +++ b/testing/use-chai-plugins.ts @@ -0,0 +1,3 @@ +import { use } from 'chai' +import chaiExclude from 'chai-exclude' +use(chaiExclude) diff --git a/tsconfig.json b/tsconfig.json index 7e074bd..bfa3055 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,6 +3,7 @@ "outDir": "./lib/", "sourceMap": true, "noImplicitAny": true, + "strictNullChecks": true, "lib": ["es2015", "es2016", "es2017"], "module": "commonjs", "moduleResolution": "node", diff --git a/webpack.config.js b/webpack.config.js index 3cc216c..2e9d3df 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -2,9 +2,14 @@ const { resolve } = require('path') const CopyWebpackPlugin = require('copy-webpack-plugin') const nodeExternals = require('webpack-node-externals') const TerserPlugin = require('terser-webpack-plugin') -const CleanTerminalPlugin = require('clean-terminal-webpack-plugin') -const devPlugins = [new CleanTerminalPlugin()] +class ClearTerminalInWatchMode { + apply(compiler) { + compiler.hooks.afterCompile.tap('ClearTerminalInWatchMode', () => { + if (compiler.watchMode) console.clear() + }) + } +} module.exports = (env, { mode = 'development' } = {}) => ({ target: 'node', @@ -15,7 +20,7 @@ module.exports = (env, { mode = 'development' } = {}) => ({ devtool: 'inline-source-map', entry: { index: './src/index.ts', - 'normalize-config': './src/configuration/site-traversal/normalize.ts' + util: './src/settings/external-utils.ts' }, output: { path: resolve(__dirname, 'lib'), @@ -46,7 +51,7 @@ module.exports = (env, { mode = 'development' } = {}) => ({ 'LICENSE', 'README.md' ]), - ...(mode === 'development' ? devPlugins : []) + new ClearTerminalInWatchMode() ], optimization: { minimizer: [new TerserPlugin()] diff --git a/website/package-lock.json b/website/package-lock.json index 020ab32..f8bc99d 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -309,9 +309,9 @@ } }, "ajv-errors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-1.0.0.tgz", - "integrity": "sha1-7PAh+hCP0X37Xms4Py3SM+Mf/Fk=", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-1.0.1.tgz", + "integrity": "sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==", "dev": true }, "ajv-keywords": { @@ -327,9 +327,9 @@ "dev": true }, "ansi-colors": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.1.0.tgz", - "integrity": "sha512-hTv1qPdi+sVEk3jYsdjox5nQI0C9HTbjKShbCdYLKb1LOfNbb7wsF4d7OEKIZoxIHx02tSp3m94jcPW2EfMjmA==", + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-3.2.3.tgz", + "integrity": "sha512-LEHHyuhlPY3TmuUYMh2oz89lTShfvgbmzaBcxve9t/9Wuy7Dwf4yoAKcND7KFT1HAQfqZ12qtc+DUrBMeKF9nw==", "dev": true }, "ansi-escapes": { @@ -403,9 +403,9 @@ "dev": true }, "array-flatten": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.1.tgz", - "integrity": "sha1-Qmu52oQJDBg42BLIFQryCoMx4pY=", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.2.tgz", + "integrity": "sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ==", "dev": true }, "array-union": { @@ -480,7 +480,7 @@ }, "async": { "version": "1.5.2", - "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", "dev": true }, @@ -2425,12 +2425,12 @@ "dev": true }, "eventsource": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-0.1.6.tgz", - "integrity": "sha1-Cs7ehJ7X3RzMMsgRuxG5RNTykjI=", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-1.0.7.tgz", + "integrity": "sha512-4Ln17+vVT0k8aWq+t/bF5arcS3EpT9gYtW66EPacdj/mAFevznsnyoHLPy2BA8gbIQeIHoPsvwmfBftfcG//BQ==", "dev": true, "requires": { - "original": ">=0.0.5" + "original": "^1.0.0" } }, "evp_bytestokey": { @@ -2752,7 +2752,7 @@ }, "finalhandler": { "version": "1.1.1", - "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz", "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==", "dev": true, "requires": { @@ -2818,9 +2818,9 @@ } }, "follow-redirects": { - "version": "1.5.9", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.5.9.tgz", - "integrity": "sha512-Bh65EZI/RU8nx0wbYF9shkFZlqLP+6WT/5FnA3cE/djNSuKNHJEinGGZgu/cQEkeeb2GdFOgenAmn8qaqYke2w==", + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.1.tgz", + "integrity": "sha512-t2JCjbzxQpWvbhts3l6SH1DKzSrx8a+SsaVf4h6bG4kOXUuPYS/kg2Lr4gQSb7eemaHqJkOThF1BGyjlUkO1GQ==", "dev": true, "requires": { "debug": "=3.1.0" @@ -2928,14 +2928,12 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, - "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2950,20 +2948,17 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "core-util-is": { "version": "1.0.2", @@ -3080,8 +3075,7 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "ini": { "version": "1.3.5", @@ -3093,7 +3087,6 @@ "version": "1.0.0", "bundled": true, "dev": true, - "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -3108,7 +3101,6 @@ "version": "3.0.4", "bundled": true, "dev": true, - "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -3116,14 +3108,12 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "minipass": { "version": "2.2.4", "bundled": true, "dev": true, - "optional": true, "requires": { "safe-buffer": "^5.1.1", "yallist": "^3.0.0" @@ -3142,7 +3132,6 @@ "version": "0.5.1", "bundled": true, "dev": true, - "optional": true, "requires": { "minimist": "0.0.8" } @@ -3223,8 +3212,7 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true, - "optional": true + "dev": true }, "object-assign": { "version": "4.1.1", @@ -3236,7 +3224,6 @@ "version": "1.4.0", "bundled": true, "dev": true, - "optional": true, "requires": { "wrappy": "1" } @@ -3358,7 +3345,6 @@ "version": "1.0.2", "bundled": true, "dev": true, - "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -3529,9 +3515,9 @@ "dev": true }, "handle-thing": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz", - "integrity": "sha1-/Xqtcmvxpf0W38KbL3pmAdJxOcQ=", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-2.0.0.tgz", + "integrity": "sha512-d4sze1JNC454Wdo2fkuyzCr6aHcbL6PGGuFAz0Li/NcOm1tCHGnWDRmJP85dh9IhQErTc2svWFEX5xHIOo//kQ==", "dev": true }, "has": { @@ -3814,7 +3800,7 @@ }, "http-errors": { "version": "1.6.3", - "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz", "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=", "dev": true, "requires": { @@ -3825,9 +3811,9 @@ } }, "http-parser-js": { - "version": "0.4.13", - "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.4.13.tgz", - "integrity": "sha1-O9bW/ebjFyyTNMOzO2wZPYD+ETc=", + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.0.tgz", + "integrity": "sha512-cZdEF7r4gfRIq7ezX9J0T+kQmJNOub71dWbgAXVHDct80TKP4MCETtZQ31xyv38UwgzkWPYF/Xc0ge55dW9Z9w==", "dev": true }, "http-proxy": { @@ -3843,7 +3829,7 @@ }, "http-proxy-middleware": { "version": "0.18.0", - "resolved": "http://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz", "integrity": "sha512-Fs25KVMPAIIcgjMZkVHJoKg9VcXcC1C8yb9JUgeDvVXY0S/zgVIhMb+qVswDIgtJe2DfckMSY2d6TuTEutlk6Q==", "dev": true, "requires": { @@ -4488,9 +4474,9 @@ "dev": true }, "map-age-cleaner": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.2.tgz", - "integrity": "sha512-UN1dNocxQq44IhJyMI4TU8phc2m9BddacHRPRjKGLYaF0jqd3xLz0jS0skpAU9WgYyoR4gHtUpzytNBS385FWQ==", + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz", + "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==", "dev": true, "requires": { "p-defer": "^1.0.0" @@ -4608,18 +4594,18 @@ "dev": true }, "mime-db": { - "version": "1.36.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.36.0.tgz", - "integrity": "sha512-L+xvyD9MkoYMXb1jAmzI/lWYAxAMCPvIBSWur0PZ5nOf5euahRLVqH//FKW9mWp2lkqUgYiXPgkzfMUFi4zVDw==", + "version": "1.37.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.37.0.tgz", + "integrity": "sha512-R3C4db6bgQhlIhPU48fUtdVmKnflq+hRdad7IyKhtFj06VPNVdk2RhiYL3UjQIlso8L+YxAtFkobT0VK+S/ybg==", "dev": true }, "mime-types": { - "version": "2.1.20", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.20.tgz", - "integrity": "sha512-HrkrPaP9vGuWbLK1B1FfgAkbqNjIuy4eHlIYnFi7kamZyLLrGlo2mpcx0bBmNpKqBtYtAfGbodDddIgddSJC2A==", + "version": "2.1.21", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.21.tgz", + "integrity": "sha512-3iL6DbwpyLzjR3xHSFNFeb9Nz/M8WDkX33t1GFQnFOllWk8pOrh/LSrB5OXlnlW5P9LH73X6loW/eogc+F5lJg==", "dev": true, "requires": { - "mime-db": "~1.36.0" + "mime-db": "~1.37.0" } }, "mimic-fn": { @@ -5120,7 +5106,7 @@ }, "p-is-promise": { "version": "1.1.0", - "resolved": "http://registry.npmjs.org/p-is-promise/-/p-is-promise-1.1.0.tgz", + "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-1.1.0.tgz", "integrity": "sha1-nJRWmJ6fZYgBewQ01WCXZ1w9oF4=", "dev": true }, @@ -5305,9 +5291,9 @@ } }, "portfinder": { - "version": "1.0.18", - "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.18.tgz", - "integrity": "sha512-KanzLOERzKoX3En5yTiV8K/arnU1ykYVokmtEn0PgCzqKZG9489tqW8ifp9+v3/VJZ5YDjvDt/PAP5WaPgk7FA==", + "version": "1.0.20", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.20.tgz", + "integrity": "sha512-Yxe4mTyDzTd59PZJY4ojZR8F+E5e97iq2ZOHPz3HDgSvYC5siNad2tLooQ5y5QHyQhc3xVqvyk/eNA3wuoa7Sw==", "dev": true, "requires": { "async": "^1.5.2", @@ -6527,26 +6513,26 @@ } }, "sockjs-client": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/sockjs-client/-/sockjs-client-1.1.5.tgz", - "integrity": "sha1-G7fA9yIsQPQq3xT0RCy9Eml3GoM=", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/sockjs-client/-/sockjs-client-1.3.0.tgz", + "integrity": "sha512-R9jxEzhnnrdxLCNln0xg5uGHqMnkhPSTzUZH2eXcR03S/On9Yvoq2wyUZILRUhZCNVu2PmwWVoyuiPz8th8zbg==", "dev": true, "requires": { - "debug": "^2.6.6", - "eventsource": "0.1.6", - "faye-websocket": "~0.11.0", - "inherits": "^2.0.1", + "debug": "^3.2.5", + "eventsource": "^1.0.7", + "faye-websocket": "~0.11.1", + "inherits": "^2.0.3", "json3": "^3.3.2", - "url-parse": "^1.1.8" + "url-parse": "^1.4.3" }, "dependencies": { "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", "dev": true, "requires": { - "ms": "2.0.0" + "ms": "^2.1.1" } }, "faye-websocket": { @@ -6557,6 +6543,12 @@ "requires": { "websocket-driver": ">=0.5.1" } + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "dev": true } } }, @@ -6608,52 +6600,73 @@ "dev": true }, "spdy": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/spdy/-/spdy-3.4.7.tgz", - "integrity": "sha1-Qv9B7OXMD5mjpsKKq7c/XDsDrLw=", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/spdy/-/spdy-4.0.0.tgz", + "integrity": "sha512-ot0oEGT/PGUpzf/6uk4AWLqkq+irlqHXkrdbk51oWONh3bxQmBuljxPNl66zlRRcIJStWq0QkLUCPOPjgjvU0Q==", "dev": true, "requires": { - "debug": "^2.6.8", - "handle-thing": "^1.2.5", + "debug": "^4.1.0", + "handle-thing": "^2.0.0", "http-deceiver": "^1.2.7", - "safe-buffer": "^5.0.1", "select-hose": "^2.0.0", - "spdy-transport": "^2.0.18" + "spdy-transport": "^3.0.0" }, "dependencies": { "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "dev": true, "requires": { - "ms": "2.0.0" + "ms": "^2.1.1" } + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "dev": true } } }, "spdy-transport": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/spdy-transport/-/spdy-transport-2.1.0.tgz", - "integrity": "sha512-bpUeGpZcmZ692rrTiqf9/2EUakI6/kXX1Rpe0ib/DyOzbiexVfXkw6GnvI9hVGvIwVaUhkaBojjCZwLNRGQg1g==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdy-transport/-/spdy-transport-3.0.0.tgz", + "integrity": "sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==", "dev": true, "requires": { - "debug": "^2.6.8", - "detect-node": "^2.0.3", + "debug": "^4.1.0", + "detect-node": "^2.0.4", "hpack.js": "^2.1.6", - "obuf": "^1.1.1", - "readable-stream": "^2.2.9", - "safe-buffer": "^5.0.1", - "wbuf": "^1.7.2" + "obuf": "^1.1.2", + "readable-stream": "^3.0.6", + "wbuf": "^1.7.3" }, "dependencies": { "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", "dev": true, "requires": { - "ms": "2.0.0" + "ms": "^2.1.1" + } + }, + "ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "dev": true + }, + "readable-stream": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.1.1.tgz", + "integrity": "sha512-DkN66hPyqDhnIQ6Jcsvx9bFjhw214O4poMBcIMgPVpQvNy9a0e0Uhg5SqySyDKAmUlwt8LonTBz1ezOnM8pUdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" } } } @@ -7217,9 +7230,9 @@ } }, "url-parse": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.3.tgz", - "integrity": "sha512-rh+KuAW36YKo0vClhQzLLveoj8FwPJNu65xLb7Mrt+eZht0IPT0IXgSv8gcMegZ6NvjJUALf6Mf25POlMwD1Fw==", + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.4.4.tgz", + "integrity": "sha512-/92DTTorg4JjktLNLe6GPS2/RvAd/RGr6LuktmWSMLEOa6rjnlrFXNgSbSmkNvCoL2T028A0a1JaJLzRMlFoHg==", "dev": true, "requires": { "querystringify": "^2.0.0", @@ -7391,9 +7404,9 @@ }, "dependencies": { "mime": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/mime/-/mime-2.3.1.tgz", - "integrity": "sha512-OEUllcVoydBHGN1z84yfQDimn58pZNNNXgZlHXSboxMlFvgI6MXSWpWKpFRra7H1HxpVhHTkrghfRW49k6yjeg==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.0.tgz", + "integrity": "sha512-ikBcWwyqXQSHKtciCcctu9YfPbFYZ4+gbHEmE0Q8jzcTYQg5dHCr3g2wwAZjPoJfQVXZq6KXAjpXOTf5/cjT7w==", "dev": true }, "webpack-log": { @@ -7409,9 +7422,9 @@ } }, "webpack-dev-server": { - "version": "3.1.9", - "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-3.1.9.tgz", - "integrity": "sha512-fqPkuNalLuc/hRC2QMkVYJkgNmRvxZQo7ykA2e1XRg/tMJm3qY7ZaD6d89/Fqjxtj9bOrn5wZzLD2n84lJdvWg==", + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/webpack-dev-server/-/webpack-dev-server-3.1.14.tgz", + "integrity": "sha512-mGXDgz5SlTxcF3hUpfC8hrQ11yhAttuUQWf1Wmb+6zo3x6rb7b9mIfuQvAPLdfDRCGRGvakBWHdHOa0I9p/EVQ==", "dev": true, "requires": { "ansi-html": "0.0.7", @@ -7433,12 +7446,14 @@ "portfinder": "^1.0.9", "schema-utils": "^1.0.0", "selfsigned": "^1.9.1", + "semver": "^5.6.0", "serve-index": "^1.7.2", "sockjs": "0.3.19", - "sockjs-client": "1.1.5", - "spdy": "^3.4.1", + "sockjs-client": "1.3.0", + "spdy": "^4.0.0", "strip-ansi": "^3.0.0", "supports-color": "^5.1.0", + "url": "^0.11.0", "webpack-dev-middleware": "3.4.0", "webpack-log": "^2.0.0", "yargs": "12.0.2" @@ -7451,13 +7466,13 @@ "dev": true }, "execa": { - "version": "0.10.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz", - "integrity": "sha512-7XOMnz8Ynx1gGo/3hyV9loYNPWM94jG3+3T3Y8tsfSstFmETmENCMU/A/zj8Lyaj1lkgEepKepvd6240tBRvlw==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", "dev": true, "requires": { "cross-spawn": "^6.0.0", - "get-stream": "^3.0.0", + "get-stream": "^4.0.0", "is-stream": "^1.1.0", "npm-run-path": "^2.0.0", "p-finally": "^1.0.0", @@ -7474,6 +7489,15 @@ "locate-path": "^3.0.0" } }, + "get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + }, "import-local": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/import-local/-/import-local-2.0.0.tgz", @@ -7521,20 +7545,20 @@ } }, "os-locale": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.0.1.tgz", - "integrity": "sha512-7g5e7dmXPtzcP4bgsZ8ixDVqA7oWYuEz4lOSujeWyliPai4gfVDiFIcwBg3aGCPnmSGfzOKTK3ccPn0CKv3DBw==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz", + "integrity": "sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q==", "dev": true, "requires": { - "execa": "^0.10.0", + "execa": "^1.0.0", "lcid": "^2.0.0", "mem": "^4.0.0" } }, "p-limit": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.0.0.tgz", - "integrity": "sha512-fl5s52lI5ahKCernzzIyAP0QAZbGIovtVHGwpcu1Jr/EpzLVDI2myISHwGqK7m8uQFugVWSrbxH7XnhGtvEc+A==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.1.0.tgz", + "integrity": "sha512-NhURkNcrVB+8hNfLuysU8enY5xn2KXphsHBaC2YmRNTZRc7RWusw6apSpdEj3jo4CMb6W9nrF6tTnsJsJeyu6g==", "dev": true, "requires": { "p-try": "^2.0.0" @@ -7564,6 +7588,16 @@ "find-up": "^3.0.0" } }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "schema-utils": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", @@ -7575,9 +7609,15 @@ "ajv-keywords": "^3.1.0" } }, + "semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==", + "dev": true + }, "strip-ansi": { "version": "3.0.1", - "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "dev": true, "requires": { diff --git a/website/package.json b/website/package.json index b132227..63731d3 100644 --- a/website/package.json +++ b/website/package.json @@ -30,7 +30,7 @@ "optimize-css-assets-webpack-plugin": "^5.0.0", "webpack": "^4.16.2", "webpack-cli": "^3.1.0", - "webpack-dev-server": "^3.1.9" + "webpack-dev-server": "^3.1.14" }, "dependencies": { "@fortawesome/fontawesome-free": "^5.2.0",