update readme

README.md

@@ -29,26 +29,26 @@ const config = {
       parse: {
         selector: 'body > b > a',
         attribute: 'href'
+        limit: 10
       },
-      limitValuesTo: 10
     },
     post: {
-      download: 'https://apod.nasa.gov/apod/{value}',
+      download: 'https://apod.nasa.gov/apod/{{ value }}',
       parse: {
-        selector: 'a[href^="image"]',
-        attribute: 'href'
+        selector: 'img[src^="image"]',
+        attribute: 'src'
       }
     },
     image: {
-      download: 'https://apod.nasa.gov/apod/{value}'
+      download: 'https://apod.nasa.gov/apod/{{ value }}'
     }
   },
   // describe how they work together
   run: {
     scraper: 'index',
-    scrapeEach: {
+    forEach: {
       scraper: 'post',
-      scrapeEach: {
+      forEach: {
         scraper: 'image'
       }
     }
@@ -129,7 +129,6 @@ While the scraper is working, you can affect its behavior by emitting these even
 | `'useRateLimiter'` | boolean   | turn on or off the rate limit defined in the run options              |
 | `'stop'`           |           | stop the crawler (note that in progress requests will still complete) |
 
-
 #### query
 
 This function is an argument in the emitter callback and is used to get data back out of the scraper whenever

src/settings/config/__tests__/normalize.unit.test.ts

@@ -86,7 +86,7 @@ describe('normalize config with', () => {
 
   describe('poorly formed config', () => {
     const configInit: any = {}
-    it.only('should throw a type assertion error', () => {
+    it('should throw a type assertion error', () => {
       expect(() => normalizeConfig(configInit))
         .to.throw(`$: expected 'scrapers' in object`)
         .with.property('name', 'RuntimeTypeError')

src/settings/config/types.ts

@@ -29,7 +29,7 @@ interface DownloadConfigInterface {
   headerTemplates?: { [headerName: string]: Template }
   regexCleanup?: RegexCleanup
 }
-export type DownloadConfigInit = Template | DownloadConfigInterface
+export type DownloadConfigInit = Template | DownloadConfigInterface | undefined
 export interface DownloadConfig extends DownloadConfigInterface {
   method: UrlMethods
   headerTemplates: { [headerName: string]: Template }
@@ -46,7 +46,7 @@ interface ParseConfigInterface {
   limit?: number
   regexCleanup?: RegexCleanup
 }
-export type ParseConfigInit = Selector | ParseConfigInterface
+export type ParseConfigInit = Selector | ParseConfigInterface | undefined
 export interface ParseConfig extends ParseConfigInterface {
   expect: ExpectedFormats
 }

src/settings/options/__tests__/normalize.unit.test.ts

@@ -35,9 +35,7 @@ describe('normalize options with', () => {
     const fullConfig = normalizeConfig(testingConfigs.EMPTY_CONFIG)
     it('should throw a type assertion error', () => {
       expect(() => normalizeOptions(fullConfig, optionsInit))
-        .to.throw(
-          `$; cause: at $; all causes: (at $: found 'maxConcurrent' in object; at $.maxConcurrent: expected number)`
-        )
+        .to.throw('$.maxConcurrent: expected a number')
         .with.property('name', 'RuntimeTypeError')
     })
   })

src/settings/params/__tests__/normalize.unit.test.ts

@@ -62,7 +62,7 @@ describe('normalize params with', () => {
 
     it('should throw a type assertion error', () => {
       expect(() => normalizeParams(config, options, paramsInit))
-        .to.throw(`$; cause: at $; cause: at $: expected 'folder' in object`)
+        .to.throw(`$: expected 'folder' in object`)
         .with.property('name', 'RuntimeTypeError')
     })
   })