Skip to content

Commit

Permalink
feat: Gather metrics from sinks (#148)
Browse files Browse the repository at this point in the history 
* fix: Gather metrics from sinks

* fix: make a difference in error and accessing the storage in the sink metrics

Co-authored-by: Trygve Lie <[email protected]>
  • Loading branch information
@trygve-lie
trygve-lie and Trygve Lie authored Jul 7, 2020
1 parent 8bb655d commit 7d1792c
Show file tree
Hide file tree
Showing 9 changed files with 726 additions and 66 deletions.
81 changes: 79 additions & 2 deletions lib/sinks/fs.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
'use strict';

const { ReadFile } = require('@eik/common');
const Metrics = require('@metrics/client');
const rimraf = require('rimraf');
const Sink = require('@eik/sink');
const mime = require('mime/lite');
Expand All @@ -20,21 +21,39 @@ const SinkFS = class SinkFS extends Sink {
constructor(config = {}) {
super();
this._config = { ...conf, ...config};
this._metrics = new Metrics();
this._counter = this._metrics.counter({
name: 'eik_core_sink_fs',
description: 'Counter measuring access to the file system storage sink',
labels: {
operation: 'n/a',
success: false,
access: false,
},
});
}

get metrics() {
return this._metrics;
}

write(filePath, contentType) {
return new Promise((resolve, reject) => {
const operation = 'write';

try {
super.constructor.validateFilePath(filePath);
super.constructor.validateContentType(contentType);
} catch (error) {
this._counter.inc({ labels: { operation } });
reject(error);
return;
}

const pathname = path.join(this._config.sinkFsRootPath, filePath);

if (pathname.indexOf(this._config.sinkFsRootPath) !== 0) {
this._counter.inc({ labels: { operation } });
reject(new Error(`Directory traversal - ${filePath}`));
return;
}
Expand All @@ -48,6 +67,7 @@ const SinkFS = class SinkFS extends Sink {
},
error => {
if (error) {
this._counter.inc({ labels: { access: true, operation } });
reject(
new Error(`Could not create directory - ${dir}`),
);
Expand All @@ -59,6 +79,12 @@ const SinkFS = class SinkFS extends Sink {
emitClose: true,
});

this._counter.inc({ labels: {
success: true,
access: true,
operation
} });

resolve(stream);
},
);
Expand All @@ -67,28 +93,47 @@ const SinkFS = class SinkFS extends Sink {

read(filePath) {
return new Promise((resolve, reject) => {
const operation = 'read';

try {
super.constructor.validateFilePath(filePath);
} catch (error) {
this._counter.inc({ labels: { operation } });
reject(error);
return;
}

const pathname = path.join(this._config.sinkFsRootPath, filePath);

if (pathname.indexOf(this._config.sinkFsRootPath) !== 0) {
this._counter.inc({ labels: { operation } });
reject(new Error(`Directory traversal - ${filePath}`));
return;
}

const closeFd = fd => {
fs.close(fd, () => {
// TODO: Log errors
fs.close(fd, (error) => {
if (error) {
this._counter.inc({ labels: {
access: true,
operation
} });
return;
}
this._counter.inc({ labels: {
success: true,
access: true,
operation
} });
});
}

fs.open(pathname, 'r', (error, fd) => {
if (error) {
this._counter.inc({ labels: {
access: true,
operation
} });
reject(error);
return;
};
Expand Down Expand Up @@ -119,6 +164,21 @@ const SinkFS = class SinkFS extends Sink {
fd,
});

obj.stream.on('error', () => {
this._counter.inc({ labels: {
access: true,
operation
} });
});

obj.stream.on('end', () => {
this._counter.inc({ labels: {
success: true,
access: true,
operation
} });
});

resolve(obj);
});

Expand All @@ -128,51 +188,68 @@ const SinkFS = class SinkFS extends Sink {

delete(filePath) {
return new Promise((resolve, reject) => {
const operation = 'delete';

try {
super.constructor.validateFilePath(filePath);
} catch (error) {
this._counter.inc({ labels: { operation } });
reject(error);
return;
}

const pathname = path.join(this._config.sinkFsRootPath, filePath);

if (pathname.indexOf(this._config.sinkFsRootPath) !== 0) {
this._counter.inc({ labels: { operation } });
reject(new Error(`Directory traversal - ${filePath}`));
return;
}

rimraf(pathname, error => {
if (error) {
this._counter.inc({ labels: { access: true, operation } });
reject(error);
return;
}
this._counter.inc({ labels: {
success: true,
access: true,
operation
} });
resolve();
});
});
}

exist(filePath) {
return new Promise((resolve, reject) => {
const operation = 'exist';

try {
super.constructor.validateFilePath(filePath);
} catch (error) {
this._counter.inc({ labels: { operation } });
reject(error);
return;
}

const pathname = path.join(this._config.sinkFsRootPath, filePath);

if (pathname.indexOf(this._config.sinkFsRootPath) !== 0) {
this._counter.inc({ labels: { operation } });
reject(new Error(`Directory traversal - ${filePath}`));
return;
}

fs.stat(pathname, (error, stat) => {
this._counter.inc({ labels: { success: true, access: true, operation } });

if (stat && stat.isFile()) {
resolve();
return;
}

if (error) {
reject(error);
return;
Expand Down
59 changes: 59 additions & 0 deletions lib/sinks/mem.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
const { Writable, Readable } = require('stream');
const { ReadFile } = require('@eik/common');
const { join } = require('path');
const Metrics = require('@metrics/client');
const Sink = require('@eik/sink');

const Entry = require('./mem-entry');
Expand All @@ -19,15 +20,32 @@ const SinkMem = class SinkMem extends Sink {
constructor({ rootPath = DEFAULT_ROOT_PATH } = {}) {
super();
this._rootPath = rootPath;
this._metrics = new Metrics();
this._counter = this._metrics.counter({
name: 'eik_core_sink_mem',
description: 'Counter measuring access to the in memory storage sink',
labels: {
operation: 'n/a',
success: false,
access: false,
},
});
this._state = new Map();
}

get metrics() {
return this._metrics;
}

write(filePath, contentType) {
return new Promise((resolve, reject) => {
const operation = 'write';

try {
super.constructor.validateFilePath(filePath);
super.constructor.validateContentType(contentType);
} catch (error) {
this._counter.inc({ labels: { operation } });
reject(error);
return;
}
Expand All @@ -36,6 +54,7 @@ const SinkMem = class SinkMem extends Sink {

if (pathname.indexOf(this._rootPath) !== 0) {
reject(new Error(`Directory traversal - ${filePath}`));
this._counter.inc({ labels: { operation } });
return;
}

Expand All @@ -52,7 +71,14 @@ const SinkMem = class SinkMem extends Sink {
mimeType: contentType,
payload
});

this._state.set(pathname, entry);

this._counter.inc({ labels: {
success: true,
access: true,
operation
} });
});

resolve(stream);
Expand All @@ -61,16 +87,20 @@ const SinkMem = class SinkMem extends Sink {

read(filePath) {
return new Promise((resolve, reject) => {
const operation = 'read';

try {
super.constructor.validateFilePath(filePath);
} catch (error) {
this._counter.inc({ labels: { operation } });
reject(error);
return;
}

const pathname = join(this._rootPath, filePath);

if (pathname.indexOf(this._rootPath) !== 0) {
this._counter.inc({ labels: { operation } });
reject(new Error(`Directory traversal - ${filePath}`));
return;
}
Expand All @@ -91,22 +121,34 @@ const SinkMem = class SinkMem extends Sink {
},
});

obj.stream.on('end', () => {
this._counter.inc({ labels: {
success: true,
access: true,
operation
} });
});

resolve(obj);
});
}

delete(filePath) {
return new Promise((resolve, reject) => {
const operation = 'delete';

try {
super.constructor.validateFilePath(filePath);
} catch (error) {
this._counter.inc({ labels: { operation } });
reject(error);
return;
}

const pathname = join(this._rootPath, filePath);

if (pathname.indexOf(this._rootPath) !== 0) {
this._counter.inc({ labels: { operation } });
reject(new Error(`Directory traversal - ${filePath}`));
return;
}
Expand All @@ -118,30 +160,47 @@ const SinkMem = class SinkMem extends Sink {
}
});

this._counter.inc({ labels: {
success: true,
access: true,
operation
} });

resolve();
});
}

exist(filePath) {
return new Promise((resolve, reject) => {
const operation = 'exist';

try {
super.constructor.validateFilePath(filePath);
} catch (error) {
this._counter.inc({ labels: { operation } });
reject(error);
return;
}

const pathname = join(this._rootPath, filePath);

if (pathname.indexOf(this._rootPath) !== 0) {
this._counter.inc({ labels: { operation } });
reject(new Error(`Directory traversal - ${filePath}`));
return;
}

this._counter.inc({ labels: {
success: true,
access: true,
operation
} });

if (this._state.has(pathname)) {
resolve();
return;
}

reject(new Error('File does not exist'));
});
}
Expand Down
Loading

0 comments on commit 7d1792c

Please sign in to comment.