From 7d1792c828ba6bf13a561c666ff9be10394acab9 Mon Sep 17 00:00:00 2001 From: Trygve Lie <173003+trygve-lie@users.noreply.github.com> Date: Tue, 7 Jul 2020 23:17:08 +0200 Subject: [PATCH] feat: Gather metrics from sinks (#148) * fix: Gather metrics from sinks * fix: make a difference in error and accessing the storage in the sink metrics Co-authored-by: Trygve Lie --- lib/sinks/fs.js | 81 +++++++++++++- lib/sinks/mem.js | 59 +++++++++++ lib/sinks/test.js | 63 ++++++++++- tap-snapshots/test-sinks-fs.js-TAP.test.js | 105 +++++++++++++++++++ tap-snapshots/test-sinks-mem.js-TAP.test.js | 105 +++++++++++++++++++ tap-snapshots/test-sinks-test.js-TAP.test.js | 105 +++++++++++++++++++ test/sinks/fs.js | 93 ++++++++++++---- test/sinks/mem.js | 91 ++++++++++++---- test/sinks/test.js | 90 ++++++++++++---- 9 files changed, 726 insertions(+), 66 deletions(-) create mode 100644 tap-snapshots/test-sinks-fs.js-TAP.test.js create mode 100644 tap-snapshots/test-sinks-mem.js-TAP.test.js create mode 100644 tap-snapshots/test-sinks-test.js-TAP.test.js diff --git a/lib/sinks/fs.js b/lib/sinks/fs.js index 153f6d5b..37be1092 100644 --- a/lib/sinks/fs.js +++ b/lib/sinks/fs.js @@ -1,6 +1,7 @@ 'use strict'; const { ReadFile } = require('@eik/common'); +const Metrics = require('@metrics/client'); const rimraf = require('rimraf'); const Sink = require('@eik/sink'); const mime = require('mime/lite'); @@ -20,14 +21,31 @@ const SinkFS = class SinkFS extends Sink { constructor(config = {}) { super(); this._config = { ...conf, ...config}; + this._metrics = new Metrics(); + this._counter = this._metrics.counter({ + name: 'eik_core_sink_fs', + description: 'Counter measuring access to the file system storage sink', + labels: { + operation: 'n/a', + success: false, + access: false, + }, + }); + } + + get metrics() { + return this._metrics; } write(filePath, contentType) { return new Promise((resolve, reject) => { + const operation = 'write'; + try { super.constructor.validateFilePath(filePath); super.constructor.validateContentType(contentType); } catch (error) { + this._counter.inc({ labels: { operation } }); reject(error); return; } @@ -35,6 +53,7 @@ const SinkFS = class SinkFS extends Sink { const pathname = path.join(this._config.sinkFsRootPath, filePath); if (pathname.indexOf(this._config.sinkFsRootPath) !== 0) { + this._counter.inc({ labels: { operation } }); reject(new Error(`Directory traversal - ${filePath}`)); return; } @@ -48,6 +67,7 @@ const SinkFS = class SinkFS extends Sink { }, error => { if (error) { + this._counter.inc({ labels: { access: true, operation } }); reject( new Error(`Could not create directory - ${dir}`), ); @@ -59,6 +79,12 @@ const SinkFS = class SinkFS extends Sink { emitClose: true, }); + this._counter.inc({ labels: { + success: true, + access: true, + operation + } }); + resolve(stream); }, ); @@ -67,9 +93,12 @@ const SinkFS = class SinkFS extends Sink { read(filePath) { return new Promise((resolve, reject) => { + const operation = 'read'; + try { super.constructor.validateFilePath(filePath); } catch (error) { + this._counter.inc({ labels: { operation } }); reject(error); return; } @@ -77,18 +106,34 @@ const SinkFS = class SinkFS extends Sink { const pathname = path.join(this._config.sinkFsRootPath, filePath); if (pathname.indexOf(this._config.sinkFsRootPath) !== 0) { + this._counter.inc({ labels: { operation } }); reject(new Error(`Directory traversal - ${filePath}`)); return; } const closeFd = fd => { - fs.close(fd, () => { - // TODO: Log errors + fs.close(fd, (error) => { + if (error) { + this._counter.inc({ labels: { + access: true, + operation + } }); + return; + } + this._counter.inc({ labels: { + success: true, + access: true, + operation + } }); }); } fs.open(pathname, 'r', (error, fd) => { if (error) { + this._counter.inc({ labels: { + access: true, + operation + } }); reject(error); return; }; @@ -119,6 +164,21 @@ const SinkFS = class SinkFS extends Sink { fd, }); + obj.stream.on('error', () => { + this._counter.inc({ labels: { + access: true, + operation + } }); + }); + + obj.stream.on('end', () => { + this._counter.inc({ labels: { + success: true, + access: true, + operation + } }); + }); + resolve(obj); }); @@ -128,9 +188,12 @@ const SinkFS = class SinkFS extends Sink { delete(filePath) { return new Promise((resolve, reject) => { + const operation = 'delete'; + try { super.constructor.validateFilePath(filePath); } catch (error) { + this._counter.inc({ labels: { operation } }); reject(error); return; } @@ -138,15 +201,22 @@ const SinkFS = class SinkFS extends Sink { const pathname = path.join(this._config.sinkFsRootPath, filePath); if (pathname.indexOf(this._config.sinkFsRootPath) !== 0) { + this._counter.inc({ labels: { operation } }); reject(new Error(`Directory traversal - ${filePath}`)); return; } rimraf(pathname, error => { if (error) { + this._counter.inc({ labels: { access: true, operation } }); reject(error); return; } + this._counter.inc({ labels: { + success: true, + access: true, + operation + } }); resolve(); }); }); @@ -154,9 +224,12 @@ const SinkFS = class SinkFS extends Sink { exist(filePath) { return new Promise((resolve, reject) => { + const operation = 'exist'; + try { super.constructor.validateFilePath(filePath); } catch (error) { + this._counter.inc({ labels: { operation } }); reject(error); return; } @@ -164,15 +237,19 @@ const SinkFS = class SinkFS extends Sink { const pathname = path.join(this._config.sinkFsRootPath, filePath); if (pathname.indexOf(this._config.sinkFsRootPath) !== 0) { + this._counter.inc({ labels: { operation } }); reject(new Error(`Directory traversal - ${filePath}`)); return; } fs.stat(pathname, (error, stat) => { + this._counter.inc({ labels: { success: true, access: true, operation } }); + if (stat && stat.isFile()) { resolve(); return; } + if (error) { reject(error); return; diff --git a/lib/sinks/mem.js b/lib/sinks/mem.js index b45f6f48..c4de8e52 100644 --- a/lib/sinks/mem.js +++ b/lib/sinks/mem.js @@ -3,6 +3,7 @@ const { Writable, Readable } = require('stream'); const { ReadFile } = require('@eik/common'); const { join } = require('path'); +const Metrics = require('@metrics/client'); const Sink = require('@eik/sink'); const Entry = require('./mem-entry'); @@ -19,15 +20,32 @@ const SinkMem = class SinkMem extends Sink { constructor({ rootPath = DEFAULT_ROOT_PATH } = {}) { super(); this._rootPath = rootPath; + this._metrics = new Metrics(); + this._counter = this._metrics.counter({ + name: 'eik_core_sink_mem', + description: 'Counter measuring access to the in memory storage sink', + labels: { + operation: 'n/a', + success: false, + access: false, + }, + }); this._state = new Map(); } + get metrics() { + return this._metrics; + } + write(filePath, contentType) { return new Promise((resolve, reject) => { + const operation = 'write'; + try { super.constructor.validateFilePath(filePath); super.constructor.validateContentType(contentType); } catch (error) { + this._counter.inc({ labels: { operation } }); reject(error); return; } @@ -36,6 +54,7 @@ const SinkMem = class SinkMem extends Sink { if (pathname.indexOf(this._rootPath) !== 0) { reject(new Error(`Directory traversal - ${filePath}`)); + this._counter.inc({ labels: { operation } }); return; } @@ -52,7 +71,14 @@ const SinkMem = class SinkMem extends Sink { mimeType: contentType, payload }); + this._state.set(pathname, entry); + + this._counter.inc({ labels: { + success: true, + access: true, + operation + } }); }); resolve(stream); @@ -61,9 +87,12 @@ const SinkMem = class SinkMem extends Sink { read(filePath) { return new Promise((resolve, reject) => { + const operation = 'read'; + try { super.constructor.validateFilePath(filePath); } catch (error) { + this._counter.inc({ labels: { operation } }); reject(error); return; } @@ -71,6 +100,7 @@ const SinkMem = class SinkMem extends Sink { const pathname = join(this._rootPath, filePath); if (pathname.indexOf(this._rootPath) !== 0) { + this._counter.inc({ labels: { operation } }); reject(new Error(`Directory traversal - ${filePath}`)); return; } @@ -91,15 +121,26 @@ const SinkMem = class SinkMem extends Sink { }, }); + obj.stream.on('end', () => { + this._counter.inc({ labels: { + success: true, + access: true, + operation + } }); + }); + resolve(obj); }); } delete(filePath) { return new Promise((resolve, reject) => { + const operation = 'delete'; + try { super.constructor.validateFilePath(filePath); } catch (error) { + this._counter.inc({ labels: { operation } }); reject(error); return; } @@ -107,6 +148,7 @@ const SinkMem = class SinkMem extends Sink { const pathname = join(this._rootPath, filePath); if (pathname.indexOf(this._rootPath) !== 0) { + this._counter.inc({ labels: { operation } }); reject(new Error(`Directory traversal - ${filePath}`)); return; } @@ -118,15 +160,24 @@ const SinkMem = class SinkMem extends Sink { } }); + this._counter.inc({ labels: { + success: true, + access: true, + operation + } }); + resolve(); }); } exist(filePath) { return new Promise((resolve, reject) => { + const operation = 'exist'; + try { super.constructor.validateFilePath(filePath); } catch (error) { + this._counter.inc({ labels: { operation } }); reject(error); return; } @@ -134,14 +185,22 @@ const SinkMem = class SinkMem extends Sink { const pathname = join(this._rootPath, filePath); if (pathname.indexOf(this._rootPath) !== 0) { + this._counter.inc({ labels: { operation } }); reject(new Error(`Directory traversal - ${filePath}`)); return; } + this._counter.inc({ labels: { + success: true, + access: true, + operation + } }); + if (this._state.has(pathname)) { resolve(); return; } + reject(new Error('File does not exist')); }); } diff --git a/lib/sinks/test.js b/lib/sinks/test.js index b610d2bb..60641c17 100644 --- a/lib/sinks/test.js +++ b/lib/sinks/test.js @@ -2,6 +2,7 @@ const { Writable, Readable } = require('stream'); const { ReadFile } = require('@eik/common'); +const Metrics = require('@metrics/client'); const Sink = require('@eik/sink'); const mime = require('mime/lite'); const path = require('path'); @@ -14,12 +15,27 @@ const SinkTest = class SinkTest extends Sink { constructor({ rootPath = DEFAULT_ROOT_PATH } = {}) { super(); this._rootPath = rootPath; + this._metrics = new Metrics(); this._state = new Map(); + this._counter = this._metrics.counter({ + name: 'eik_core_sink_test', + description: 'Counter measuring access to the in memory test storage sink', + labels: { + operation: 'n/a', + success: false, + access: false, + }, + }); + this._writeDelayResolve = () => -1; this._writeDelayChunks = () => -1; } + get metrics() { + return this._metrics; + } + set(filePath, payload) { const pathname = path.join(this._rootPath, filePath); const mimeType = mime.getType(pathname) || 'application/octet-stream'; @@ -73,10 +89,13 @@ const SinkTest = class SinkTest extends Sink { write(filePath, contentType) { return new Promise((resolve, reject) => { + const operation = 'write'; + try { super.constructor.validateFilePath(filePath); super.constructor.validateContentType(contentType); } catch (error) { + this._counter.inc({ labels: { operation } }); reject(error); return; } @@ -84,6 +103,7 @@ const SinkTest = class SinkTest extends Sink { const pathname = path.join(this._rootPath, filePath); if (pathname.indexOf(this._rootPath) !== 0) { + this._counter.inc({ labels: { operation } }); reject(new Error(`Directory traversal - ${filePath}`)); return; } @@ -113,7 +133,14 @@ const SinkTest = class SinkTest extends Sink { mimeType: contentType, payload }); + this._state.set(pathname, entry); + + this._counter.inc({ labels: { + success: true, + access: true, + operation + } }); }); const resolveDelay = this._writeDelayResolve(); @@ -129,9 +156,12 @@ const SinkTest = class SinkTest extends Sink { read(filePath) { return new Promise((resolve, reject) => { + const operation = 'read'; + try { super.constructor.validateFilePath(filePath); } catch (error) { + this._counter.inc({ labels: { operation } }); reject(error); return; } @@ -139,6 +169,7 @@ const SinkTest = class SinkTest extends Sink { const pathname = path.join(this._rootPath, filePath); if (pathname.indexOf(this._rootPath) !== 0) { + this._counter.inc({ labels: { operation } }); reject(new Error(`Directory traversal - ${filePath}`)); return; } @@ -157,19 +188,28 @@ const SinkTest = class SinkTest extends Sink { }); this.push(null); }, - });; + }); - resolve(file); + file.stream.on('end', () => { + this._counter.inc({ labels: { + success: true, + access: true, + operation + } }); + }); - // TODO: Handle if stream never opens or errors, set a timeout which will reject with an error + resolve(file); }); } delete(filePath) { return new Promise((resolve, reject) => { + const operation = 'delete'; + try { super.constructor.validateFilePath(filePath); } catch (error) { + this._counter.inc({ labels: { operation } }); reject(error); return; } @@ -177,6 +217,7 @@ const SinkTest = class SinkTest extends Sink { const pathname = path.join(this._rootPath, filePath); if (pathname.indexOf(this._rootPath) !== 0) { + this._counter.inc({ labels: { operation } }); reject(new Error(`Directory traversal - ${filePath}`)); return; } @@ -188,15 +229,24 @@ const SinkTest = class SinkTest extends Sink { } }); + this._counter.inc({ labels: { + success: true, + access: true, + operation + } }); + resolve(); }); } exist(filePath) { return new Promise((resolve, reject) => { + const operation = 'exist'; + try { super.constructor.validateFilePath(filePath); } catch (error) { + this._counter.inc({ labels: { operation } }); reject(error); return; } @@ -204,10 +254,17 @@ const SinkTest = class SinkTest extends Sink { const pathname = path.join(this._rootPath, filePath); if (pathname.indexOf(this._rootPath) !== 0) { + this._counter.inc({ labels: { operation } }); reject(new Error(`Directory traversal - ${filePath}`)); return; } + this._counter.inc({ labels: { + success: true, + access: true, + operation + } }); + if (this._state.has(pathname)) { resolve(); return; diff --git a/tap-snapshots/test-sinks-fs.js-TAP.test.js b/tap-snapshots/test-sinks-fs.js-TAP.test.js new file mode 100644 index 00000000..7a9639d5 --- /dev/null +++ b/tap-snapshots/test-sinks-fs.js-TAP.test.js @@ -0,0 +1,105 @@ +/* IMPORTANT + * This snapshot file is auto-generated, but designed for humans. + * It should be checked into source control and tracked carefully. + * Re-generate by setting TAP_SNAPSHOT=1 and running tests. + * Make sure to inspect the output below. Do not ignore changes! + */ + +'use strict' + +exports[`test/sinks/fs.js TAP Sink() - .metrics - all successfull operations > metrics should match snapshot 1`] = ` +[ + { + "name": "eik_core_sink_fs", + "description": "Counter measuring access to the file system storage sink", + "timestamp": -1, + "type": 2, + "value": 1, + "labels": [ + { + "name": "operation", + "value": "write" + }, + { + "name": "success", + "value": true + }, + { + "name": "access", + "value": true + } + ], + "time": null, + "meta": {} + }, + { + "name": "eik_core_sink_fs", + "description": "Counter measuring access to the file system storage sink", + "timestamp": -1, + "type": 2, + "value": 1, + "labels": [ + { + "name": "operation", + "value": "exist" + }, + { + "name": "success", + "value": true + }, + { + "name": "access", + "value": true + } + ], + "time": null, + "meta": {} + }, + { + "name": "eik_core_sink_fs", + "description": "Counter measuring access to the file system storage sink", + "timestamp": -1, + "type": 2, + "value": 1, + "labels": [ + { + "name": "operation", + "value": "read" + }, + { + "name": "success", + "value": true + }, + { + "name": "access", + "value": true + } + ], + "time": null, + "meta": {} + }, + { + "name": "eik_core_sink_fs", + "description": "Counter measuring access to the file system storage sink", + "timestamp": -1, + "type": 2, + "value": 1, + "labels": [ + { + "name": "operation", + "value": "delete" + }, + { + "name": "success", + "value": true + }, + { + "name": "access", + "value": true + } + ], + "time": null, + "meta": {} + } +] +` diff --git a/tap-snapshots/test-sinks-mem.js-TAP.test.js b/tap-snapshots/test-sinks-mem.js-TAP.test.js new file mode 100644 index 00000000..2dc370da --- /dev/null +++ b/tap-snapshots/test-sinks-mem.js-TAP.test.js @@ -0,0 +1,105 @@ +/* IMPORTANT + * This snapshot file is auto-generated, but designed for humans. + * It should be checked into source control and tracked carefully. + * Re-generate by setting TAP_SNAPSHOT=1 and running tests. + * Make sure to inspect the output below. Do not ignore changes! + */ + +'use strict' + +exports[`test/sinks/mem.js TAP Sink() - .metrics - all successfull operations > metrics should match snapshot 1`] = ` +[ + { + "name": "eik_core_sink_mem", + "description": "Counter measuring access to the in memory storage sink", + "timestamp": -1, + "type": 2, + "value": 1, + "labels": [ + { + "name": "operation", + "value": "write" + }, + { + "name": "success", + "value": true + }, + { + "name": "access", + "value": true + } + ], + "time": null, + "meta": {} + }, + { + "name": "eik_core_sink_mem", + "description": "Counter measuring access to the in memory storage sink", + "timestamp": -1, + "type": 2, + "value": 1, + "labels": [ + { + "name": "operation", + "value": "exist" + }, + { + "name": "success", + "value": true + }, + { + "name": "access", + "value": true + } + ], + "time": null, + "meta": {} + }, + { + "name": "eik_core_sink_mem", + "description": "Counter measuring access to the in memory storage sink", + "timestamp": -1, + "type": 2, + "value": 1, + "labels": [ + { + "name": "operation", + "value": "read" + }, + { + "name": "success", + "value": true + }, + { + "name": "access", + "value": true + } + ], + "time": null, + "meta": {} + }, + { + "name": "eik_core_sink_mem", + "description": "Counter measuring access to the in memory storage sink", + "timestamp": -1, + "type": 2, + "value": 1, + "labels": [ + { + "name": "operation", + "value": "delete" + }, + { + "name": "success", + "value": true + }, + { + "name": "access", + "value": true + } + ], + "time": null, + "meta": {} + } +] +` diff --git a/tap-snapshots/test-sinks-test.js-TAP.test.js b/tap-snapshots/test-sinks-test.js-TAP.test.js new file mode 100644 index 00000000..668d798a --- /dev/null +++ b/tap-snapshots/test-sinks-test.js-TAP.test.js @@ -0,0 +1,105 @@ +/* IMPORTANT + * This snapshot file is auto-generated, but designed for humans. + * It should be checked into source control and tracked carefully. + * Re-generate by setting TAP_SNAPSHOT=1 and running tests. + * Make sure to inspect the output below. Do not ignore changes! + */ + +'use strict' + +exports[`test/sinks/test.js TAP Sink() - .metrics - all successfull operations > metrics should match snapshot 1`] = ` +[ + { + "name": "eik_core_sink_test", + "description": "Counter measuring access to the in memory test storage sink", + "timestamp": -1, + "type": 2, + "value": 1, + "labels": [ + { + "name": "operation", + "value": "write" + }, + { + "name": "success", + "value": true + }, + { + "name": "access", + "value": true + } + ], + "time": null, + "meta": {} + }, + { + "name": "eik_core_sink_test", + "description": "Counter measuring access to the in memory test storage sink", + "timestamp": -1, + "type": 2, + "value": 1, + "labels": [ + { + "name": "operation", + "value": "exist" + }, + { + "name": "success", + "value": true + }, + { + "name": "access", + "value": true + } + ], + "time": null, + "meta": {} + }, + { + "name": "eik_core_sink_test", + "description": "Counter measuring access to the in memory test storage sink", + "timestamp": -1, + "type": 2, + "value": 1, + "labels": [ + { + "name": "operation", + "value": "read" + }, + { + "name": "success", + "value": true + }, + { + "name": "access", + "value": true + } + ], + "time": null, + "meta": {} + }, + { + "name": "eik_core_sink_test", + "description": "Counter measuring access to the in memory test storage sink", + "timestamp": -1, + "type": 2, + "value": 1, + "labels": [ + { + "name": "operation", + "value": "delete" + }, + { + "name": "success", + "value": true + }, + { + "name": "access", + "value": true + } + ], + "time": null, + "meta": {} + } +] +` diff --git a/test/sinks/fs.js b/test/sinks/fs.js index 1c59741e..c407bbf5 100644 --- a/test/sinks/fs.js +++ b/test/sinks/fs.js @@ -2,19 +2,46 @@ const { Writable, pipeline } = require('stream'); const { stream } = require('@eik/common'); -const { test } = require('tap'); const slug = require('unique-slug'); const path = require('path'); +const tap = require('tap'); const fs = require('fs'); const os = require('os'); const Sink = require('../../lib/sinks/fs'); +// Ignore the value for "timestamp" field in the snapshots +tap.cleanSnapshot = (s) => { + const regex = /"timestamp": [0-9.]+,/gi; + return s.replace(regex, '"timestamp": -1,'); +}; + const DEFAULT_CONFIG = { sinkFsRootPath: path.join(os.tmpdir(), '/eik-test-files') }; const FIXTURE = fs.readFileSync(path.join(__dirname, '../../fixtures/import-map.json')).toString(); +const MetricsInto = class MetricsInto extends Writable { + constructor() { + super({ objectMode : true }); + this._metrics = []; + } + + _write(chunk, encoding, callback) { + this._metrics.push(chunk); + callback(); + } + + done () { + return new Promise((resolve) => { + this.once('finish', () => { + resolve(JSON.stringify(this._metrics, null, 2)); + }); + this.end(); + }) + } +} + const readFileStream = (file = '../README.md') => { const pathname = path.join(__dirname, file); return fs.createReadStream(pathname); @@ -49,14 +76,14 @@ const pipe = (...streams) => { }); } -test('Sink() - Object type', (t) => { +tap.test('Sink() - Object type', (t) => { const sink = new Sink(DEFAULT_CONFIG); const name = Object.prototype.toString.call(sink); t.true(name.startsWith('[object Sink'), 'should begin with Sink'); t.end(); }); -test('Sink() - .write()', async (t) => { +tap.test('Sink() - .write()', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/bar/map.json`; @@ -71,7 +98,7 @@ test('Sink() - .write()', async (t) => { t.end(); }); -test('Sink() - .write() - arguments is illegal', async (t) => { +tap.test('Sink() - .write() - arguments is illegal', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); @@ -80,7 +107,7 @@ test('Sink() - .write() - arguments is illegal', async (t) => { t.end(); }); -test('Sink() - .write() - directory traversal prevention', async (t) => { +tap.test('Sink() - .write() - directory traversal prevention', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); @@ -96,7 +123,7 @@ test('Sink() - .write() - directory traversal prevention', async (t) => { t.end(); }); -test('Sink() - .read() - File exists', async (t) => { +tap.test('Sink() - .read() - File exists', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/bar/map.json`; @@ -122,20 +149,20 @@ test('Sink() - .read() - File exists', async (t) => { t.end(); }); -test('Sink() - .read() - File does NOT exist', (t) => { +tap.test('Sink() - .read() - File does NOT exist', (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); t.rejects(sink.read(`/${dir}/foo/not-exist.json`), 'should reject'); t.end(); }); -test('Sink() - .read() - arguments is illegal', async (t) => { +tap.test('Sink() - .read() - arguments is illegal', async (t) => { const sink = new Sink(DEFAULT_CONFIG); t.rejects(sink.read(300), new TypeError('Argument must be a String'), 'should reject on illegal filepath'); t.end(); }); -test('Sink() - .read() - directory traversal prevention', async (t) => { +tap.test('Sink() - .read() - directory traversal prevention', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/map.json`; @@ -157,7 +184,7 @@ test('Sink() - .read() - directory traversal prevention', async (t) => { t.end(); }); -test('Sink() - .read() - value of .mimeType of known file type', async (t) => { +tap.test('Sink() - .read() - value of .mimeType of known file type', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/bar/map.json`; @@ -177,7 +204,7 @@ test('Sink() - .read() - value of .mimeType of known file type', async (t) => { t.end(); }); -test('Sink() - .read() - value of .mimeType of unknown file type', async (t) => { +tap.test('Sink() - .read() - value of .mimeType of unknown file type', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/bar/map.foo`; @@ -197,7 +224,7 @@ test('Sink() - .read() - value of .mimeType of unknown file type', async (t) => t.end(); }); -test('Sink() - .delete() - Delete existing file', async (t) => { +tap.test('Sink() - .delete() - Delete existing file', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); @@ -219,13 +246,13 @@ test('Sink() - .delete() - Delete existing file', async (t) => { t.end(); }); -test('Sink() - .delete() - Delete non existing file', (t) => { +tap.test('Sink() - .delete() - Delete non existing file', (t) => { const sink = new Sink(DEFAULT_CONFIG); t.resolves(sink.delete('/bar/foo/not-exist.json'), 'should resolve'); t.end(); }); -test('Sink() - .delete() - Delete file in tree structure', async (t) => { +tap.test('Sink() - .delete() - Delete file in tree structure', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const fileA = `${dir}/a/map.json`; @@ -249,7 +276,7 @@ test('Sink() - .delete() - Delete file in tree structure', async (t) => { t.end(); }); -test('Sink() - .delete() - Delete files recursively', async (t) => { +tap.test('Sink() - .delete() - Delete files recursively', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const fileA = `${dir}/a/map.json`; @@ -271,13 +298,13 @@ test('Sink() - .delete() - Delete files recursively', async (t) => { t.end(); }); -test('Sink() - .delete() - arguments is illegal', async (t) => { +tap.test('Sink() - .delete() - arguments is illegal', async (t) => { const sink = new Sink(DEFAULT_CONFIG); t.rejects(sink.delete(300), new TypeError('Argument must be a String'), 'should reject on illegal filepath'); t.end(); }); -test('Sink() - .delete() - directory traversal prevention', async (t) => { +tap.test('Sink() - .delete() - directory traversal prevention', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/map.json`; @@ -294,7 +321,7 @@ test('Sink() - .delete() - directory traversal prevention', async (t) => { t.end(); }); -test('Sink() - .exist() - Check existing file', async (t) => { +tap.test('Sink() - .exist() - Check existing file', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/map.json`; @@ -311,19 +338,19 @@ test('Sink() - .exist() - Check existing file', async (t) => { t.end(); }); -test('Sink() - .exist() - Check non existing file', (t) => { +tap.test('Sink() - .exist() - Check non existing file', (t) => { const sink = new Sink(DEFAULT_CONFIG); t.rejects(sink.exist('/bar/foo/not-exist.json'), 'should reject - file does not exist'); t.end(); }); -test('Sink() - .exist() - arguments is illegal', async (t) => { +tap.test('Sink() - .exist() - arguments is illegal', async (t) => { const sink = new Sink(DEFAULT_CONFIG); t.rejects(sink.exist(300), new TypeError('Argument must be a String'), 'should reject on illegal filepath'); t.end(); }); -test('Sink() - .exist() - directory traversal prevention', async (t) => { +tap.test('Sink() - .exist() - directory traversal prevention', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/map.json`; @@ -344,3 +371,27 @@ test('Sink() - .exist() - directory traversal prevention', async (t) => { await sink.delete(dir); t.end(); }); + +tap.test('Sink() - .metrics - all successfull operations', async (t) => { + const sink = new Sink(DEFAULT_CONFIG); + const dir = slug(); + const file = `${dir}/bar/map.json`; + + const metricsInto = new MetricsInto(); + sink.metrics.pipe(metricsInto); + + // write, check, read and delete file + const writeFrom = readFileStream('../../fixtures/import-map.json'); + const writeTo = await sink.write(file, 'application/json'); + await pipe(writeFrom, writeTo); + + await sink.exist(file) + + const readFrom = await sink.read(file); + await pipeInto(readFrom.stream); + + await sink.delete(dir); + + const metrics = await metricsInto.done(); + t.matchSnapshot(metrics, 'metrics should match snapshot'); +}); diff --git a/test/sinks/mem.js b/test/sinks/mem.js index 6aa328f8..56490076 100644 --- a/test/sinks/mem.js +++ b/test/sinks/mem.js @@ -2,16 +2,43 @@ const { Writable, pipeline } = require('stream'); const { stream } = require('@eik/common'); -const { test } = require('tap'); const slug = require('unique-slug'); const path = require('path'); +const tap = require('tap'); const fs = require('fs'); const Sink = require('../../lib/sinks/mem'); +// Ignore the value for "timestamp" field in the snapshots +tap.cleanSnapshot = (s) => { + const regex = /"timestamp": [0-9.]+,/gi; + return s.replace(regex, '"timestamp": -1,'); +}; + const DEFAULT_CONFIG = {}; const FIXTURE = fs.readFileSync(path.join(__dirname, '../../fixtures/import-map.json')).toString(); +const MetricsInto = class MetricsInto extends Writable { + constructor() { + super({ objectMode : true }); + this._metrics = []; + } + + _write(chunk, encoding, callback) { + this._metrics.push(chunk); + callback(); + } + + done () { + return new Promise((resolve) => { + this.once('finish', () => { + resolve(JSON.stringify(this._metrics, null, 2)); + }); + this.end(); + }) + } +} + const readFileStream = (file = '../README.md') => { const pathname = path.join(__dirname, file); return fs.createReadStream(pathname); @@ -46,14 +73,14 @@ const pipe = (...streams) => { }); } -test('Sink() - Object type', (t) => { +tap.test('Sink() - Object type', (t) => { const sink = new Sink(DEFAULT_CONFIG); const name = Object.prototype.toString.call(sink); t.true(name.startsWith('[object Sink'), 'should begin with Sink'); t.end(); }); -test('Sink() - .write()', async (t) => { +tap.test('Sink() - .write()', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/bar/map.json`; @@ -68,7 +95,7 @@ test('Sink() - .write()', async (t) => { t.end(); }); -test('Sink() - .write() - arguments is illegal', async (t) => { +tap.test('Sink() - .write() - arguments is illegal', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); @@ -77,7 +104,7 @@ test('Sink() - .write() - arguments is illegal', async (t) => { t.end(); }); -test('Sink() - .write() - directory traversal prevention', async (t) => { +tap.test('Sink() - .write() - directory traversal prevention', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); @@ -93,7 +120,7 @@ test('Sink() - .write() - directory traversal prevention', async (t) => { t.end(); }); -test('Sink() - .read() - File exists', async (t) => { +tap.test('Sink() - .read() - File exists', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/bar/map.json`; @@ -119,20 +146,20 @@ test('Sink() - .read() - File exists', async (t) => { t.end(); }); -test('Sink() - .read() - File does NOT exist', (t) => { +tap.test('Sink() - .read() - File does NOT exist', (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); t.rejects(sink.read(`/${dir}/foo/not-exist.json`), 'should reject'); t.end(); }); -test('Sink() - .read() - arguments is illegal', async (t) => { +tap.test('Sink() - .read() - arguments is illegal', async (t) => { const sink = new Sink(DEFAULT_CONFIG); t.rejects(sink.read(300), new TypeError('Argument must be a String'), 'should reject on illegal filepath'); t.end(); }); -test('Sink() - .read() - directory traversal prevention', async (t) => { +tap.test('Sink() - .read() - directory traversal prevention', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/map.json`; @@ -154,7 +181,7 @@ test('Sink() - .read() - directory traversal prevention', async (t) => { t.end(); }); -test('Sink() - .read() - value of .mimeType of known file type', async (t) => { +tap.test('Sink() - .read() - value of .mimeType of known file type', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/bar/map.json`; @@ -174,7 +201,7 @@ test('Sink() - .read() - value of .mimeType of known file type', async (t) => { t.end(); }); -test('Sink() - .delete() - Delete existing file', async (t) => { +tap.test('Sink() - .delete() - Delete existing file', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); @@ -196,13 +223,13 @@ test('Sink() - .delete() - Delete existing file', async (t) => { t.end(); }); -test('Sink() - .delete() - Delete non existing file', (t) => { +tap.test('Sink() - .delete() - Delete non existing file', (t) => { const sink = new Sink(DEFAULT_CONFIG); t.resolves(sink.delete('/bar/foo/not-exist.json'), 'should resolve'); t.end(); }); -test('Sink() - .delete() - Delete file in tree structure', async (t) => { +tap.test('Sink() - .delete() - Delete file in tree structure', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const fileA = `${dir}/a/map.json`; @@ -226,7 +253,7 @@ test('Sink() - .delete() - Delete file in tree structure', async (t) => { t.end(); }); -test('Sink() - .delete() - Delete files recursively', async (t) => { +tap.test('Sink() - .delete() - Delete files recursively', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const fileA = `${dir}/a/map.json`; @@ -248,13 +275,13 @@ test('Sink() - .delete() - Delete files recursively', async (t) => { t.end(); }); -test('Sink() - .delete() - arguments is illegal', async (t) => { +tap.test('Sink() - .delete() - arguments is illegal', async (t) => { const sink = new Sink(DEFAULT_CONFIG); t.rejects(sink.delete(300), new TypeError('Argument must be a String'), 'should reject on illegal filepath'); t.end(); }); -test('Sink() - .delete() - directory traversal prevention', async (t) => { +tap.test('Sink() - .delete() - directory traversal prevention', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/map.json`; @@ -271,7 +298,7 @@ test('Sink() - .delete() - directory traversal prevention', async (t) => { t.end(); }); -test('Sink() - .exist() - Check existing file', async (t) => { +tap.test('Sink() - .exist() - Check existing file', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/map.json`; @@ -288,19 +315,19 @@ test('Sink() - .exist() - Check existing file', async (t) => { t.end(); }); -test('Sink() - .exist() - Check non existing file', (t) => { +tap.test('Sink() - .exist() - Check non existing file', (t) => { const sink = new Sink(DEFAULT_CONFIG); t.rejects(sink.exist('/bar/foo/not-exist.json'), 'should reject - file does not exist'); t.end(); }); -test('Sink() - .exist() - arguments is illegal', async (t) => { +tap.test('Sink() - .exist() - arguments is illegal', async (t) => { const sink = new Sink(DEFAULT_CONFIG); t.rejects(sink.exist(300), new TypeError('Argument must be a String'), 'should reject on illegal filepath'); t.end(); }); -test('Sink() - .exist() - directory traversal prevention', async (t) => { +tap.test('Sink() - .exist() - directory traversal prevention', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/map.json`; @@ -321,3 +348,27 @@ test('Sink() - .exist() - directory traversal prevention', async (t) => { await sink.delete(dir); t.end(); }); + +tap.test('Sink() - .metrics - all successfull operations', async (t) => { + const sink = new Sink(DEFAULT_CONFIG); + const dir = slug(); + const file = `${dir}/bar/map.json`; + + const metricsInto = new MetricsInto(); + sink.metrics.pipe(metricsInto); + + // write, check, read and delete file + const writeFrom = readFileStream('../../fixtures/import-map.json'); + const writeTo = await sink.write(file, 'application/json'); + await pipe(writeFrom, writeTo); + + await sink.exist(file) + + const readFrom = await sink.read(file); + await pipeInto(readFrom.stream); + + await sink.delete(dir); + + const metrics = await metricsInto.done(); + t.matchSnapshot(metrics, 'metrics should match snapshot'); +}); diff --git a/test/sinks/test.js b/test/sinks/test.js index f18b9271..b3f0f408 100644 --- a/test/sinks/test.js +++ b/test/sinks/test.js @@ -2,16 +2,43 @@ const { Writable, pipeline } = require('stream'); const { stream } = require('@eik/common'); -const { test } = require('tap'); const slug = require('unique-slug'); const path = require('path'); +const tap = require('tap'); const fs = require('fs'); const Sink = require('../../lib/sinks/test'); +// Ignore the value for "timestamp" field in the snapshots +tap.cleanSnapshot = (s) => { + const regex = /"timestamp": [0-9.]+,/gi; + return s.replace(regex, '"timestamp": -1,'); +}; + const DEFAULT_CONFIG = {}; const FIXTURE = fs.readFileSync(path.join(__dirname, '../../fixtures/import-map.json')).toString(); +const MetricsInto = class MetricsInto extends Writable { + constructor() { + super({ objectMode : true }); + this._metrics = []; + } + + _write(chunk, encoding, callback) { + this._metrics.push(chunk); + callback(); + } + + done () { + return new Promise((resolve) => { + this.once('finish', () => { + resolve(JSON.stringify(this._metrics, null, 2)); + }); + this.end(); + }) + } +} + const readFileStream = (file = '../README.md') => { const pathname = path.join(__dirname, file); return fs.createReadStream(pathname); @@ -46,14 +73,14 @@ const pipe = (...streams) => { }); } -test('Sink() - Object type', (t) => { +tap.test('Sink() - Object type', (t) => { const sink = new Sink(DEFAULT_CONFIG); const name = Object.prototype.toString.call(sink); t.true(name.startsWith('[object Sink'), 'should begin with Sink'); t.end(); }); -test('Sink() - .write()', async (t) => { +tap.test('Sink() - .write()', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/bar/map.json`; @@ -68,7 +95,7 @@ test('Sink() - .write()', async (t) => { t.end(); }); -test('Sink() - .write() - arguments is illegal', async (t) => { +tap.test('Sink() - .write() - arguments is illegal', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); @@ -77,7 +104,7 @@ test('Sink() - .write() - arguments is illegal', async (t) => { t.end(); }); -test('Sink() - .write() - directory traversal prevention', async (t) => { +tap.test('Sink() - .write() - directory traversal prevention', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); @@ -93,7 +120,7 @@ test('Sink() - .write() - directory traversal prevention', async (t) => { t.end(); }); -test('Sink() - .read() - File exists', async (t) => { +tap.test('Sink() - .read() - File exists', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/bar/map.json`; @@ -119,20 +146,20 @@ test('Sink() - .read() - File exists', async (t) => { t.end(); }); -test('Sink() - .read() - File does NOT exist', (t) => { +tap.test('Sink() - .read() - File does NOT exist', (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); t.rejects(sink.read(`/${dir}/foo/not-exist.json`), 'should reject'); t.end(); }); -test('Sink() - .read() - arguments is illegal', async (t) => { +tap.test('Sink() - .read() - arguments is illegal', async (t) => { const sink = new Sink(DEFAULT_CONFIG); t.rejects(sink.read(300), new TypeError('Argument must be a String'), 'should reject on illegal filepath'); t.end(); }); -test('Sink() - .read() - directory traversal prevention', async (t) => { +tap.test('Sink() - .read() - directory traversal prevention', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/map.json`; @@ -154,7 +181,7 @@ test('Sink() - .read() - directory traversal prevention', async (t) => { t.end(); }); -test('Sink() - .read() - value of .mimeType of known file type', async (t) => { +tap.test('Sink() - .read() - value of .mimeType of known file type', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/bar/map.json`; @@ -174,7 +201,7 @@ test('Sink() - .read() - value of .mimeType of known file type', async (t) => { t.end(); }); -test('Sink() - .delete() - Delete existing file', async (t) => { +tap.test('Sink() - .delete() - Delete existing file', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); @@ -196,13 +223,13 @@ test('Sink() - .delete() - Delete existing file', async (t) => { t.end(); }); -test('Sink() - .delete() - Delete non existing file', (t) => { +tap.test('Sink() - .delete() - Delete non existing file', (t) => { const sink = new Sink(DEFAULT_CONFIG); t.resolves(sink.delete('/bar/foo/not-exist.json'), 'should resolve'); t.end(); }); -test('Sink() - .delete() - Delete file in tree structure', async (t) => { +tap.test('Sink() - .delete() - Delete file in tree structure', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const fileA = `${dir}/a/map.json`; @@ -226,7 +253,7 @@ test('Sink() - .delete() - Delete file in tree structure', async (t) => { t.end(); }); -test('Sink() - .delete() - Delete files recursively', async (t) => { +tap.test('Sink() - .delete() - Delete files recursively', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const fileA = `${dir}/a/map.json`; @@ -248,13 +275,13 @@ test('Sink() - .delete() - Delete files recursively', async (t) => { t.end(); }); -test('Sink() - .delete() - arguments is illegal', async (t) => { +tap.test('Sink() - .delete() - arguments is illegal', async (t) => { const sink = new Sink(DEFAULT_CONFIG); t.rejects(sink.delete(300), new TypeError('Argument must be a String'), 'should reject on illegal filepath'); t.end(); }); -test('Sink() - .delete() - directory traversal prevention', async (t) => { +tap.test('Sink() - .delete() - directory traversal prevention', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/map.json`; @@ -271,7 +298,7 @@ test('Sink() - .delete() - directory traversal prevention', async (t) => { t.end(); }); -test('Sink() - .exist() - Check existing file', async (t) => { +tap.test('Sink() - .exist() - Check existing file', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/map.json`; @@ -288,19 +315,19 @@ test('Sink() - .exist() - Check existing file', async (t) => { t.end(); }); -test('Sink() - .exist() - Check non existing file', (t) => { +tap.test('Sink() - .exist() - Check non existing file', (t) => { const sink = new Sink(DEFAULT_CONFIG); t.rejects(sink.exist('/bar/foo/not-exist.json'), 'should reject - file does not exist'); t.end(); }); -test('Sink() - .exist() - arguments is illegal', async (t) => { +tap.test('Sink() - .exist() - arguments is illegal', async (t) => { const sink = new Sink(DEFAULT_CONFIG); t.rejects(sink.exist(300), new TypeError('Argument must be a String'), 'should reject on illegal filepath'); t.end(); }); -test('Sink() - .exist() - directory traversal prevention', async (t) => { +tap.test('Sink() - .exist() - directory traversal prevention', async (t) => { const sink = new Sink(DEFAULT_CONFIG); const dir = slug(); const file = `${dir}/map.json`; @@ -322,3 +349,26 @@ test('Sink() - .exist() - directory traversal prevention', async (t) => { t.end(); }); +tap.test('Sink() - .metrics - all successfull operations', async (t) => { + const sink = new Sink(DEFAULT_CONFIG); + const dir = slug(); + const file = `${dir}/bar/map.json`; + + const metricsInto = new MetricsInto(); + sink.metrics.pipe(metricsInto); + + // write, check, read and delete file + const writeFrom = readFileStream('../../fixtures/import-map.json'); + const writeTo = await sink.write(file, 'application/json'); + await pipe(writeFrom, writeTo); + + await sink.exist(file) + + const readFrom = await sink.read(file); + await pipeInto(readFrom.stream); + + await sink.delete(dir); + + const metrics = await metricsInto.done(); + t.matchSnapshot(metrics, 'metrics should match snapshot'); +});