✨ Add custom aggregations support

Author: Tomas2D

.prettierrc

@@ -1,5 +1,6 @@
 {
   "singleQuote": true,
   "trailingComma": "all",
-  "printWidth": 100
+  "printWidth": 100,
+  "quoteProps": "consistent"
 }

README.md

@@ -10,6 +10,7 @@ This library brings you abstraction between `puppeteer` and `page context`.
 - ✨ Respect the defined order of columns.
 - ✨ Appending custom columns with custom data.
 - ✨ Custom sanitization of data in cells.
+- ✨ Group and Aggregate data by your own function.
 - ✨ Merge data from two independent tables into one structure.
 - ✨ Handles invalid HTML structure
 - ✨ And much more!
@@ -39,6 +40,10 @@ interface ParserSettings {
   csvSeparator?: string; // (default: ';')
   newLine?: string; // (default: '\n')
   rowValidationPolicy?: RowValidationPolicy; // (default: 'NON_EMPTY')
+  groupBy?: {
+    cols: string[];
+    handler?: (rows: string[][], getColumnIndex: GetColumnIndexType) => string[];
+  }
   rowValidator: (
     row: string[],
     getColumnIndex: GetColumnIndexType,
@@ -63,9 +68,10 @@ interface ParserSettings {
 5. Run `rowValidator` function for every table row.
 6. Run `colParser` for every cell in a row.
 7. Run `rowTransform` function for each row.
-8. Add processed row to a temp array result.
-9. Add `header` column if `withHeader` property is `true`.
-10. Merge partial results and return them.
+8. Group results into buckets (`groupBy.cols`) property and pick the aggregated rows.
+9. Add processed row to a temp array result. 
+10. 10.Add `header` column if `withHeader` property is `true`.
+11. Merge partial results and return them.
 
 ## Examples
 
@@ -207,12 +213,27 @@ await tableParser(page, {
 });
 ```
 
-For more, look at `test` folder! 🙈
-
-## TODO
+***Grouping and Aggregating**
+```typescript
+await tableParser(page, {
+  selector: '#my-table',
+  allowedColNames: {
+    'Employee Name': 'name',
+    'Age': 'age',
+  },
+  groupBy: {
+    cols: ['name'],
+    handler: (rows: string[][], getColumnIndex) => {
+      const ageIndex = getColumnIndex('age');
+
+      // select one with the minimal age
+      return rows.reduce((previous, current) =>
+        previous[ageIndex] < current[ageIndex] ? previous : current,
+      );
+    },
+  }
+});
+```
 
-- [X] Add more examples
-- [X] Add tests
-- [X] Describe interfaces
-- [ ] Show merging table structures
+For more, look at the `test` folder! 🙈
 

src/aggregations.ts

@@ -0,0 +1,26 @@
+import { GetColumnIndexType, GroupByOptions } from './types';
+
+const takeFirstHandler = (rows: string[][]) => rows[0];
+
+export function groupBy(
+  rows: string[][],
+  { cols, handler = takeFirstHandler }: GroupByOptions,
+  getColumnIndex: GetColumnIndexType,
+): string[][] {
+  const rowsByKey = new Map<string, string[][]>();
+
+  rows.forEach((row) => {
+    const key = cols.map((col) => row[getColumnIndex(col)]).join('-');
+    if (!rowsByKey.has(key)) {
+      rowsByKey.set(key, []);
+    }
+    rowsByKey.get(key)!.push(row);
+  });
+
+  return Array.from(rowsByKey.values()).map((rows) => {
+    if (rows.length > 1) {
+      return handler(rows, getColumnIndex);
+    }
+    return rows[0];
+  });
+}

src/parseTable.ts

@@ -3,6 +3,7 @@ import { extraColsMapperFactory, getColumnsInfo } from './helpers';
 import { ElementHandle } from 'puppeteer';
 import { InvalidSettingsError } from './errors';
 import PipelineExecutor from './pipelineExecutor';
+import { groupBy } from './aggregations';
 export function parseTableFactory(settings: FullParserSettings) {
   const extraColsMapper = extraColsMapperFactory(settings.extraCols);
 
@@ -16,15 +17,11 @@ export function parseTableFactory(settings: FullParserSettings) {
     return table.$$(settings.bodyRowsSelector);
   };
 
-  const getOutputHeaderRow = (excludedKeyIndexes: number[], missingColNames: string[]) => {
+  const getOutputHeaderRow = (missingColNames: string[]) => {
     const headerRowRaw = Object.values(settings.allowedColNames);
     const sortedHeader = extraColsMapper(headerRowRaw, 'colName');
 
-    const headerRow = sortedHeader
-      .filter((_, index) => !excludedKeyIndexes.includes(index))
-      .filter((key) => !missingColNames.includes(key));
-
-    return settings.rowValuesAsArray ? headerRow : headerRow.join(settings.csvSeparator);
+    return sortedHeader.filter((key) => !missingColNames.includes(key));
   };
 
   const getRowStructureValidator = (allowedIndexes: Record<string, number>) => {
@@ -82,24 +79,29 @@ export function parseTableFactory(settings: FullParserSettings) {
       bodyRows.reverse();
     }
 
-    const finalRows = new PipelineExecutor<
-      string[][],
-      typeof settings.rowValuesAsArray extends true ? string[][] : string[]
-    >()
+    let parsedRows = new PipelineExecutor<string[][], string[][]>()
       .addFilter(getRowStructureValidator(indexes.allowed))
       .addMap((row) => extraColsMapper(row, 'data'))
       .addFilter((row, index, rows) => settings.rowValidator(row, getColumnIndex, index, rows))
       .addMap((row) => row.map((cell, index) => settings.colParser(cell, index, getColumnIndex)))
       .addTransform((row) => settings.rowTransform(row, getColumnIndex))
-      .addMap((row) => row.filter((_, index: number) => !indexes.excluded.includes(index)))
-      .addMap((row) => (settings.rowValuesAsArray ? row : row.join(settings.csvSeparator)))
       .execute(await Promise.all(bodyRows.map(getRowsData(indexes.allowed))));
 
+    if (settings.groupBy) {
+      parsedRows = groupBy(parsedRows, settings.groupBy, getColumnIndex);
+    }
+
     if (addHeader) {
-      const headerRow = getOutputHeaderRow(indexes.excluded, missingColNames);
-      finalRows.unshift(headerRow as string);
+      const headerRow = getOutputHeaderRow(missingColNames);
+      parsedRows.unshift(headerRow);
     }
 
-    return finalRows;
+    return new PipelineExecutor<
+      string[][],
+      typeof settings.rowValuesAsArray extends true ? string[][] : string[]
+    >()
+      .addMap((row) => row.filter((_, index: number) => !indexes.excluded.includes(index)))
+      .addMap((row) => (settings.rowValuesAsArray ? row : row.join(settings.csvSeparator)))
+      .execute(parsedRows);
   };
 }

src/settings.ts

@@ -13,6 +13,7 @@ export const defaultSettings: ParserSettingsOptional = {
   csvSeparator: ';',
   newLine: '\n',
   rowValidationPolicy: RowValidationPolicy.NON_EMPTY,
+  groupBy: undefined,
   rowValidator: () => true,
   rowTransform: () => {},
   asArray: false,
@@ -82,4 +83,13 @@ export function validateSettings(
       );
     }
   }
+
+  if (settings.groupBy) {
+    if (!Array.isArray(settings.groupBy.cols)) {
+      throw new InvalidSettingsError(`Columns in "groupBy" field must be typeof array`);
+    }
+    if (settings.groupBy.handler && typeof settings.groupBy.handler !== 'function') {
+      throw new InvalidSettingsError(`Passed handler to the "groupBy" is not a function`);
+    }
+  }
 }

src/types.ts

@@ -12,13 +12,19 @@ export enum RowValidationPolicy {
   EXACT_MATCH = 'EXACT_MATCH',
 }
 
+export type GroupByOptions = {
+  cols: string[];
+  handler?: (rows: string[][], getColumnIndex: GetColumnIndexType) => string[];
+};
+
 export type ParserSettingsOptional = {
   temporaryColNames: string[];
   extraCols: ExtraCol[];
   withHeader: boolean;
   csvSeparator: string;
   newLine: string;
   rowValidationPolicy: RowValidationPolicy;
+  groupBy: GroupByOptions;
   rowValidator: (
     row: string[],
     getColumnIndex: GetColumnIndexType,

test/assets/2.html

@@ -0,0 +1,44 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <title>Example #2</title>
+</head>
+<body>
+
+<table id="employee-overview">
+  <thead>
+  <tr>
+    <th>Employee Name</th>
+    <th>Age</th>
+  </tr>
+  </thead>
+  <tbody>
+  <tr>
+    <td>John M. Bolduc</td>
+    <td>32</td>
+  </tr>
+  <tr>
+    <td>Allan Meron</td>
+    <td>40</td>
+  </tr>
+  <tr>
+    <td>John M. Bolduc</td>
+    <td>29</td>
+  </tr>
+  <tr>
+    <td>Milan Lukeš</td>
+    <td>33</td>
+  </tr>
+  <tr>
+    <td>John M. Bolduc</td>
+    <td>34</td>
+  </tr>/tr>
+  <tr>
+    <td>Allan Meron</td>
+    <td>90</td>
+  </tr>
+  </tbody>
+</table>
+
+</body>
+</html>

test/index.test.ts

@@ -420,20 +420,57 @@ describe('Basic parsing', () => {
     `);
   });
 
-  it('Parses large HTML table', async () => {
-    await page.goto(`${getBaseUrl()}/large-table.html`);
+  it.only('Handles aggregation', async () => {
+    await page.goto(`${getBaseUrl()}/2.html`);
 
     const data = await tableParser(page, {
-      selector: 'table',
+      selector: '#employee-overview',
       asArray: false,
-      rowValidationPolicy: RowValidationPolicy.NON_EMPTY,
       allowedColNames: {
-        H: 'first',
-        V: 'cond',
-        N: 'last',
+        'Employee Name': 'name',
+        'Age': 'age',
+      },
+      groupBy: {
+        cols: ['name'],
       },
     });
 
-    expect(data).toBeTruthy();
+    expect(data).toMatchInlineSnapshot(`
+      "name;age
+      John M. Bolduc;32
+      Allan Meron;40
+      Milan Lukeš;33"
+    `);
+  });
+
+  it('Handles aggregation with custom handler', async () => {
+    await page.goto(`${getBaseUrl()}/2.html`);
+
+    const data = await tableParser(page, {
+      selector: '#employee-overview',
+      asArray: false,
+      allowedColNames: {
+        'Employee Name': 'name',
+        'Age': 'age',
+      },
+      groupBy: {
+        cols: ['name'],
+        handler: (rows: string[][], getColumnIndex) => {
+          const ageIndex = getColumnIndex('age');
+
+          // select one with the minimal age
+          return rows.reduce((previous, current) =>
+            previous[ageIndex] < current[ageIndex] ? previous : current,
+          );
+        },
+      },
+    });
+
+    expect(data).toMatchInlineSnapshot(`
+      "name;age
+      John M. Bolduc;29
+      Allan Meron;40
+      Milan Lukeš;33"
+    `);
   });
 });