Expose stats beneath tables property. Update README and examples

acuminous · Sep 25, 2023 · aebb3f3 · aebb3f3
1 parent 079177f
commit aebb3f3
Show file tree

Hide file tree

Showing 5 changed files with 113 additions and 31 deletions.
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # PG-Scanner
 
-PG-Scanner is a library which reports statistics about PostgreSQL databasesstatistics.
+PG-Scanner is a library which reports statistics about PostgreSQL databases.
 
 ## TL;DR
 
@@ -15,7 +15,7 @@ const config = {
   database: "mydb",
 };
 
-const scanner = new Scanner(config);
+const scanner = new Scanner({ config });
 
 (async () => {
   await scanner.init();
@@ -25,14 +25,22 @@ const scanner = new Scanner(config);
 function scheduleScan(delay) {
   setTimeout(async () => {
     const stats = await scanner.scan();
-    console.log({ stats });
+    dump(stats);
     scheduleScan(delay);
   }, delay).unref();
 }
+
+function dump(stats) {
+  const serializer = (_, value) => typeof value === "bigint" ? value.toString() : value;
+  const text = JSON.stringify(stats, serializer);
+  console.log(text);
+}
 ```
 
 In this example, we create a new instance of the Scanner class with the database configuration. We intialise the scanner to establish a baseline, then re-scan once per hour, logging the statistics each time.
 
+The custom serializer is required because JavaScripts maximum safe integer is less than the maximum value of a PostgreSQL integer, and the numerical stats have a type of BigInt.
+
 We call `unref` to ensure the scheduled scans to not prevent your application from shutting down if the event loop is otherwise inactive, but if you are running the above script in a standalone process you may wish to remove this call.
 
 ## Index
@@ -41,8 +49,10 @@ We call `unref` to ensure the scheduled scans to not prevent your application fr
 
 - [Installiation](#installation)
 - [API](#pg-scanner-api)
+  - [Constructor](#Constructor)
   - [init](#init)
   - [scan](#scan)
+  - [Stats](#stats)
 
 ### Installation
 
@@ -66,6 +76,13 @@ import { Scanner } from 'pg-scanner';
 
 ## API
 
+### Scanner(options?)
+
+| Name    | Required | Notes                                                                                                                                                                                                                                  |
+| ------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| config  | No       | A configuration object which is passed directly to [node-pg](https://www.npmjs.com/package/pg). Alternatively, you can use [environment variables](https://node-postgres.com/features/connecting#environment-variables) if you prefer. |
+| filter  | No       | A function for filtering out unwanted tables. It will be called with an object with a table and schema property and should return truthy if the table is to be included in the statistics                                              |
+
 ### init()
 
 ```js
@@ -74,14 +91,27 @@ await scanner.init();
 
 The init method is responsible for initialising the scanner wise baseline statistics. It will error if called repeatedly.
 
-### scan()
+### scan() : Promise<Stats>
 
 ```js
 await scanner.scan();
 ```
 
 The scan method is responsible for retrieving and augmenting database statistics.
 
+### Stats
+
+The stats returned by the [scan](#scan) is an array of objects with the following properties
+
+| Name                 | Notes                                                                 |
+| -------------------- | --------------------------------------------------------------------- |
+| schema               | The schema to which the stats relate                                  |
+| table                | The table to which the stats relate                                   |
+| sequentialScans      | The total number of sequential scans performed on the table           |
+| rowsScanned          | The total number of rows returned by the sequential scans             |
+| sequentialScansDelta | The change in sequential scans since the last check                   |
+| rowsScannedDelta     | The change in rows scanned since the last check                       |
+
 ### Contributing
 
 Contributions to the Scanner module are welcome. If you find a bug, have a feature request, or want to improve the code, please open an issue or submit a pull request on GitHub.

diff --git a/examples/filter.js b/examples/filter.js
@@ -0,0 +1,41 @@
+/* eslint no-console: 0 */
+const { Scanner } = require('..');
+
+const config = {
+  host: 'localhost',
+  port: 5432,
+  database: 'postgres',
+  user: 'postgres',
+  password: 'postgres',
+};
+
+const excluded = [
+  'ignore_me',
+  'ignore_me_too',
+];
+
+const filter = ({ schema, table }) => {
+  if (excluded.includes(`${schema}.${table}`)) return false;
+  return true;
+};
+
+const scanner = new Scanner({ config, filter });
+
+(async () => {
+  await scanner.init();
+  scheduleScan(1000);
+})();
+
+function scheduleScan(delay) {
+  setTimeout(async () => {
+    const stats = await scanner.scan();
+    dump(stats);
+    scheduleScan(delay);
+  }, delay);
+}
+
+function dump(stats) {
+  const serializer = (_, value) => (typeof value === 'bigint' ? value.toString() : value);
+  const text = JSON.stringify(stats, serializer);
+  console.log(text);
+}
diff --git a/example/index.js → examples/simple.js b/example/index.js → examples/simple.js
@@ -1,3 +1,4 @@
+/* eslint no-console: 0 */
 const { Scanner } = require('..');
 
 const config = {
@@ -8,7 +9,7 @@ const config = {
   password: 'postgres',
 };
 
-const scanner = new Scanner(config);
+const scanner = new Scanner({ config });
 
 (async () => {
   await scanner.init();
@@ -18,7 +19,13 @@ const scanner = new Scanner(config);
 function scheduleScan(delay) {
   setTimeout(async () => {
     const stats = await scanner.scan();
-    console.log({ stats });
+    dump(stats);
     scheduleScan(delay);
   }, delay);
 }
+
+function dump(stats) {
+  const serializer = (_, value) => (typeof value === 'bigint' ? value.toString() : value);
+  const text = JSON.stringify(stats, serializer);
+  console.log(text);
+}
diff --git a/lib/Scanner.js b/lib/Scanner.js
@@ -1,16 +1,18 @@
 const { Client } = require('pg');
 const { ConnectionError, InitialisationError, } = require('./Errors');
 
+const INCLUDE_ALL = () => true;
+
 module.exports = class Scanner {
   #config;
   #client;
   #initialised = false;
-  #previousStats = [];
+  #previousTableStats = [];
   #filter;
 
-  constructor(config, filter = () => true) {
-    this.#config = config;
-    this.#filter = filter;
+  constructor(options = {}) {
+    this.#config = options.config;
+    this.#filter = options.filter || INCLUDE_ALL;
   }
 
   async init() {
@@ -35,10 +37,10 @@ module.exports = class Scanner {
   async #scan() {
     try {
       await this.#connect();
-      const rawStats = await this.#readDatabaseTableStats();
-      const augmentedStats = this.#augmentStats(rawStats);
-      this.#previousStats = augmentedStats;
-      return augmentedStats;
+      const rawTableStats = await this.#readDatabaseTableStats();
+      const augmentedTableStats = this.#augmentTableStats(rawTableStats);
+      this.#previousTableStats = augmentedTableStats;
+      return { tables: augmentedTableStats };
     } finally {
       this.#disconnect();
     }
@@ -49,7 +51,8 @@ module.exports = class Scanner {
     try {
       await this.#client.connect();
     } catch (cause) {
-      throw new ConnectionError(`Error connecting to ${this.#config.host}:${this.#config.port} as ${this.#config.user}: ${cause.message}`, { cause });
+      const { host, port, database, user } = this.#client.connectionParameters;
+      throw new ConnectionError(`Error connecting to ${host}:${port}/${database} as ${user}: ${cause.message}`, { cause });
     }
     return this;
   }
@@ -63,8 +66,8 @@ module.exports = class Scanner {
     return rows.map(fromColumnNames).filter(this.#filter);
   }
 
-  #augmentStats(rawStats) {
-    return rawStats.map((tableStats) => {
+  #augmentTableStats(rawTableStats) {
+    return rawTableStats.map((tableStats) => {
       const previousTableStats = this.#findPreviousTableStats(tableStats.schema, tableStats.table);
       const rowsScannedDelta = tableStats.rowsScanned - previousTableStats.rowsScanned;
       const sequentialScansDelta = tableStats.sequentialScans - previousTableStats.sequentialScans;
@@ -73,7 +76,7 @@ module.exports = class Scanner {
   }
 
   #findPreviousTableStats(schema, table) {
-    return this.#previousStats.find((entry) => entry.schema === schema && entry.table === table) || this.#getNewTableStats();
+    return this.#previousTableStats.find((entry) => entry.schema === schema && entry.table === table) || this.#getNewTableStats();
   }
 
   #getNewTableStats() {

diff --git a/test/index.test.js b/test/index.test.js
@@ -34,10 +34,11 @@ describe('PG Scanner', () => {
     });
 
     it('should report connection errors', async () => {
-      scanner = new Scanner({ host: 'doesnotexist-wibble-panda-totem.com', port: 1111, user: 'bob' });
+      const badConfig = { host: 'doesnotexist-wibble-panda-totem.com', port: 1111, database: 'db', user: 'bob' };
+      scanner = new Scanner({ config: badConfig });
 
       await rejects(() => scanner.init(), (err) => {
-        eq(err.message, 'Error connecting to doesnotexist-wibble-panda-totem.com:1111 as bob: getaddrinfo ENOTFOUND doesnotexist-wibble-panda-totem.com');
+        eq(err.message, 'Error connecting to doesnotexist-wibble-panda-totem.com:1111/db as bob: getaddrinfo ENOTFOUND doesnotexist-wibble-panda-totem.com');
         eq(err.code, 'ERR_PG_SCANNER_CONNECTION_ERROR');
         eq(err.cause.code, 'ENOTFOUND');
         return true;
@@ -58,7 +59,7 @@ describe('PG Scanner', () => {
     it('should ignore standard tables', async () => {
       await initialiseScanner();
       const stats = await scanner.scan();
-      eq(stats.length, 0);
+      eq(stats.tables.length, 0);
     });
 
     it('should filter by specified schemas/tables', async () => {
@@ -69,19 +70,19 @@ describe('PG Scanner', () => {
         if (schema === 'public' && table === 'exclude_table') return false;
         return true;
       };
-      scanner = new Scanner(config, filter);
+      scanner = new Scanner({ config, filter });
 
       await scanner.init();
       const stats = await scanner.scan();
 
-      eq(stats.length, 1);
+      eq(stats.tables.length, 1);
     });
 
     it('should return stats for custom tables', async () => {
       await database.createTable('test_table');
       await initialiseScanner();
 
-      const [stats] = await scanner.scan();
+      const { tables: [stats] } = await scanner.scan();
       ok(stats, 'No custom tables');
       eq(stats.schema, 'public');
       eq(stats.table, 'test_table');
@@ -93,12 +94,12 @@ describe('PG Scanner', () => {
       await database.createTable('test_table');
       await initialiseScanner();
 
-      const [stats1] = await scanner.scan();
+      const { tables: [stats1] } = await scanner.scan();
       eq(stats1.sequentialScans, BigInt(1));
 
       await database.readTable('test_table');
 
-      const [stats2] = await scanner.scan();
+      const { tables: [stats2] } = await scanner.scan();
       eq(stats2.sequentialScans, BigInt(2));
     });
 
@@ -109,7 +110,7 @@ describe('PG Scanner', () => {
       await setupTable(tableName, numberOfRows, numberOfReads);
 
       await initialiseScanner();
-      const [stats] = await scanner.scan();
+      const { tables: [stats] } = await scanner.scan();
 
       const numberOfRowsScanned = numberOfRows * numberOfReads;
       eq(stats.rowsScanned, BigInt(numberOfRowsScanned));
@@ -131,7 +132,7 @@ describe('PG Scanner', () => {
 
       const delta = additionalNumberOfReads;
 
-      const [stats] = await scanner.scan();
+      const { tables: [stats] } = await scanner.scan();
       eq(stats.sequentialScansDelta, BigInt(delta));
     });
 
@@ -154,7 +155,7 @@ describe('PG Scanner', () => {
       const totalNumberOfRowsScanned = startingNumberOfRowsScanned + additionalNumberOfRowsScanned;
       const delta = totalNumberOfRowsScanned - startingNumberOfRowsScanned;
 
-      const [stats] = await scanner.scan();
+      const { tables: [stats] } = await scanner.scan();
       eq(stats.rowsScannedDelta, BigInt(delta));
     });
 
@@ -173,14 +174,14 @@ describe('PG Scanner', () => {
 
       await scanner.scan();
       await database.readTable(tableOne);
-      const [stats] = await scanner.scan();
+      const { tables: [stats] } = await scanner.scan();
 
       eq(stats.rowsScannedDelta, BigInt(2));
     });
   });
 
   async function initialiseScanner() {
-    scanner = new Scanner(config);
+    scanner = new Scanner({ config });
     await scanner.init();
   }