✨ Add row structure validator

Tomas2D · Oct 5, 2022 · 2e50f49 · 2e50f49
1 parent 25e5a8d
commit 2e50f49
Show file tree

Hide file tree

Showing 6 changed files with 78 additions and 16 deletions.
diff --git a/src/parseTable.ts b/src/parseTable.ts
@@ -1,7 +1,7 @@
-import { FullParserSettings } from './types';
-import { extraColsMapperFactory, diffFromSource } from './helpers';
+import { FullParserSettings, RowValidationPolicy } from './types';
+import { diffFromSource, extraColsMapperFactory } from './helpers';
 import { ElementHandle } from 'puppeteer';
-import { InvalidColumnError, MissingRequiredColumnsError } from './errors';
+import { InvalidColumnError, InvalidSettingsError, MissingRequiredColumnsError } from './errors';
 
 export function parseTableFactory(settings: FullParserSettings) {
   const extraColsMapper = extraColsMapperFactory(settings.extraCols);
@@ -31,6 +31,20 @@ export function parseTableFactory(settings: FullParserSettings) {
     return settings.rowValuesAsArray ? headerRow : headerRow.join(settings.csvSeparator);
   };
 
+  const getRowStructureValidator = (allowedIndexes: Record<string, number>) => {
+    if (settings.rowValidationPolicy === RowValidationPolicy.NONE) {
+      return () => true;
+    }
+    if (settings.rowValidationPolicy === RowValidationPolicy.NON_EMPTY) {
+      return (rows: string[]) => rows.length > 0;
+    }
+    if (settings.rowValidationPolicy === RowValidationPolicy.EXACT_MATCH) {
+      const indexesCount = Object.keys(allowedIndexes).length;
+      return (rows: string[]) => rows.length === indexesCount;
+    }
+    throw new InvalidSettingsError('Unknown mode for the "rowValidationPolicy"');
+  };
+
   const filterSortCols =
     (allowedIndexes: Record<string, number>) =>
     (row: ElementHandle): Promise<string[]> =>
@@ -126,6 +140,7 @@ export function parseTableFactory(settings: FullParserSettings) {
     };
 
     const finalRows = (await Promise.all(bodyRows.map(filterSortCols(allowedIndexes))))
+      .filter(getRowStructureValidator(allowedIndexes))
       .map((row) => extraColsMapper(row, 'data'))
       .filter((row, index, rows) => settings.rowValidator(row, getColumnIndex, index, rows))
       .map((row) => row.map((cell, index) => settings.colParser(cell, index, getColumnIndex)))

diff --git a/src/settings.ts b/src/settings.ts
@@ -1,4 +1,9 @@
-import { FullParserSettings, ParserSettings, ParserSettingsOptional } from './types';
+import {
+  FullParserSettings,
+  ParserSettings,
+  ParserSettingsOptional,
+  RowValidationPolicy,
+} from './types';
 import { InvalidSettingsError } from './errors';
 import { omitUndefined } from './helpers';
 
@@ -7,6 +12,7 @@ export const defaultSettings: ParserSettingsOptional = {
   withHeader: true,
   csvSeparator: ';',
   newLine: '\n',
+  rowValidationPolicy: RowValidationPolicy.NON_EMPTY,
   rowValidator: () => true,
   rowTransform: () => {},
   asArray: false,

diff --git a/src/types.ts b/src/types.ts
@@ -6,12 +6,19 @@ export interface ExtraCol {
 
 export type GetColumnIndexType = (colName: string) => number;
 
+export enum RowValidationPolicy {
+  NONE = 'NONE',
+  NON_EMPTY = 'NON_EMPTY',
+  EXACT_MATCH = 'EXACT_MATCH',
+}
+
 export type ParserSettingsOptional = {
   temporaryColNames: string[];
   extraCols: ExtraCol[];
   withHeader: boolean;
   csvSeparator: string;
   newLine: string;
+  rowValidationPolicy: RowValidationPolicy;
   rowValidator: (
     row: string[],
     getColumnIndex: GetColumnIndexType,

diff --git a/test/assets/1.html b/test/assets/1.html
@@ -19,6 +19,7 @@
     <td>332</td>
     <td>2015</td>
   </tr>
+  <tr class='empty-row'></tr>
   <tr>
     <td>Alfa Romeo Giulia</td>
     <td>500</td>
@@ -34,6 +35,9 @@
     <td>120</td>
     <td>2012</td>
   </tr>
+  <tr class='empty-row'>
+    <td></td>
+  </tr>
   </tbody>
 </table>
 

diff --git a/test/helpers.test.ts b/test/helpers.test.ts
@@ -1,5 +1,5 @@
 import { omitUndefined } from '../src/helpers';
-import { ParserSettingsOptional } from '../src/types';
+import type { ParserSettingsOptional } from '../src/types';
 
 describe('Helper utils', () => {
   it('Removes undefined properties', () => {

diff --git a/test/index.test.ts b/test/index.test.ts
@@ -1,8 +1,8 @@
 import { Server } from 'http';
 import { promisify } from 'util';
 import { createServer, getBaseUrl } from './createServer';
-import { launch, Browser, Page } from 'puppeteer';
-import tableParser from '../src';
+import { Browser, launch, Page } from 'puppeteer';
+import tableParser, { RowValidationPolicy } from '../src';
 
 describe('Basic parsing', () => {
   let server: Server;
@@ -128,6 +128,9 @@ describe('Basic parsing', () => {
           "120",
           "2012",
         ],
+        Array [
+          "",
+        ],
       ]
     `);
   });
@@ -278,7 +281,8 @@ describe('Basic parsing', () => {
       2021-03-15;Audi S5
       2021-03-15;Alfa Romeo Giulia
       2021-03-15;BMW X3
-      2021-03-15;Skoda Octavia"
+      2021-03-15;Skoda Octavia
+      2021-03-15;"
     `);
   });
 
@@ -329,6 +333,7 @@ describe('Basic parsing', () => {
 
     const data = await tableParser(page, {
       selector: 'table',
+      rowValidationPolicy: RowValidationPolicy.EXACT_MATCH,
       allowedColNames: {
         'Car Name': 'car',
         'Some non existing column': 'non-existing',
@@ -361,6 +366,7 @@ describe('Basic parsing', () => {
       selector: 'table',
       allowedColNames: {
         'Car Name': 'car',
+        'Horse Powers': 'hp',
       },
       extraCols: [
         {
@@ -371,20 +377,44 @@ describe('Basic parsing', () => {
           colName: 'ex2',
           data: 'ex2',
         },
+      ],
+    });
+
+    expect(data).toMatchInlineSnapshot(`
+      "car;hp;ex1;ex2
+      Audi S5;332;ex1;ex2
+      Alfa Romeo Giulia;500;ex1;ex2
+      BMW X3;215;ex1;ex2
+      Skoda Octavia;120;ex1;ex2
+      ;ex1;ex2"
+    `);
+  });
+
+  it('Handles filtering partial rows', async () => {
+    await page.goto(`${getBaseUrl()}/1.html`);
+
+    const data = await tableParser(page, {
+      selector: 'table',
+      rowValidationPolicy: RowValidationPolicy.EXACT_MATCH,
+      allowedColNames: {
+        'Car Name': 'car',
+        'Horse Powers': 'hp',
+        'Manufacture Year': 'year',
+      },
+      extraCols: [
         {
-          colName: 'ex0',
-          data: 'ex0',
-          position: 0,
+          colName: 'sellerId',
+          data: '123',
         },
       ],
     });
 
     expect(data).toMatchInlineSnapshot(`
-      "ex0;car;ex1;ex2
-      ex0;Audi S5;ex1;ex2
-      ex0;Alfa Romeo Giulia;ex1;ex2
-      ex0;BMW X3;ex1;ex2
-      ex0;Skoda Octavia;ex1;ex2"
+      "car;hp;year;sellerId
+      Audi S5;332;2015;123
+      Alfa Romeo Giulia;500;2020;123
+      BMW X3;215;2017;123
+      Skoda Octavia;120;2012;123"
     `);
   });
 });