Add benchmark comparison based on an TSV file

.gitignore

@@ -10,3 +10,5 @@ npm-debug.log
 
 # WebStorm
 .idea
+
+benchmark.tsv

README.md

@@ -161,12 +161,43 @@ See the [ECMAScript spec](http://www.ecma-international.org/ecma-262/6.0/index.h
 
 Go to `/test`
 
-In Node run:
+#### In Node
+
+Run:
 
 ```
 npm test
 ```
 
+#### Benchmarking
+
+When you run `npm run bench`, few things happen:
+
+1. Five benchmark specs defined in `proxyBenchmark.js` are executed. This might take a minute.
+2. The results are appended to the `benchmark.tsv` file, including the following information for each spec:
+   - current Git commit SHA
+   - number of operations per second
+   - name of the spec
+3. At the end, in your console you will see the detailed information about the current benchmark, and the comparison of all data found in `benchmark.tsv` relatively to the first results for each given spec name, e.g.:
+
+```julia
+Observe and generate, small object (JSONPatcherProxy)
+ ec7b9bf: 136720 Ops/sec
+ 92da649: 136351 Ops/sec (0.3% worse)
+Observe and generate (JSONPatcherProxy)
+ ec7b9bf: 2762 Ops/sec
+ 92da649: 2793 Ops/sec (1.1% better)
+Primitive mutation (JSONPatcherProxy)
+ ec7b9bf: 781852 Ops/sec
+ 92da649: 781270 Ops/sec (0.1% worse)
+Complex mutation (JSONPatcherProxy)
+ ec7b9bf: 11808 Ops/sec
+ 92da649: 10692 Ops/sec (9.5% worse)
+Serialization (JSONPatcherProxy)
+ ec7b9bf: 1719 Ops/sec
+ 92da649: 1719 Ops/sec (no difference)
+```
+
 ## Contributing
 
 * Fork it.

test/helpers/benchmarkComparisonAdder.js

@@ -0,0 +1,54 @@
+const fs = require('fs');
+const { execSync } = require('child_process');
+
+const filePath = './benchmark.tsv';
+
+function getGitCommitHash() {
+    return execSync('git rev-parse HEAD').toString().trim();
+}
+
+const commitHash = getGitCommitHash().substr(0, 7);
+let headerRow = 'Version\tOps/s\tCategory';
+
+function benchmarkComparisonToFile(suite){
+    let bench;
+
+    let tsv = [];
+    if (fs.existsSync(filePath)) {
+        tsv = fs.readFileSync(filePath).toString();
+        tsv = tsv.replace(/\r\n/g, '\n');
+        tsv = tsv.split('\n');
+        if (tsv.length > 0) {
+            if (tsv[0] !== headerRow) {
+                throw new Error(`File ${filePath} exists but it does not have expected columns in the header. Expected: \n${headerRow}\n Found: \n${tsv[0]}\n`);
+            }
+        }
+    }
+
+    if (tsv.length === 0) {
+        tsv.push(headerRow);
+    }
+
+    for(let testNo = 0; testNo < suite.length; testNo++){
+        bench = suite[testNo];
+
+        if (bench.name.indexOf(' * 1000') > -1) {
+            bench.hz = bench.hz/1000;
+        }
+        else if (bench.name.indexOf(' * 100') > -1) {
+            bench.hz = bench.hz/100;
+        }
+        else if (bench.name.indexOf(' * 10') > -1) {
+            bench.hz = bench.hz/10;
+        }
+
+        let resultAsFormattedString = bench.hz.toFixed(bench.hz < 100 ? 2 : 0);
+        tsv.push(`${commitHash}\t${resultAsFormattedString}\t${bench.name}`);
+
+    }
+
+    fs.writeFileSync(filePath, tsv.join('\n'));
+}
+if (typeof exports !== "undefined") {
+    exports.benchmarkComparisonToFile = benchmarkComparisonToFile;
+}

test/helpers/benchmarkComparisonReporter.js

@@ -0,0 +1,70 @@
+const chalk = require('chalk');
+const fs = require('fs');
+
+function benchmarkComparisonToConsole() {
+    console.log("\n");
+    console.log("==================");
+    console.log("Benchmark comparison:");
+    console.log("------------------");
+
+    const filePath = './benchmark.tsv';
+    let tsv = [];
+    if (fs.existsSync(filePath)) {
+        tsv = fs.readFileSync(filePath).toString();
+        tsv = tsv.replace(/\r\n/g, '\n');
+        tsv = tsv.split('\n');
+    }
+
+    if (tsv.length < 2) {
+        throw new Error(`File ${filePath} does not contain TSV data`);
+    }
+
+    tsv.shift(); // remove header
+
+    const groups = new Map();
+
+    tsv.forEach(row => {
+        const [hash, ops, suiteName] = row.split('\t');
+        if (!groups.has(suiteName)) {
+            groups.set(suiteName, []);
+        }
+
+        const group = groups.get(suiteName);
+        group.push({
+            hash,
+            ops
+        });
+    });
+
+    groups.forEach((group, suiteName) => {
+        console.log(chalk.yellow.underline(suiteName));
+        let first;
+        group.forEach(row => {
+            if (first === undefined) {
+                first = row;
+                console.log(` ${row.hash}: ${chalk.magenta(row.ops)} Ops/sec`);
+            }
+            else {
+                const relative = (((row.ops / first.ops) * 100) - 100).toFixed(1);
+                let relativeString;
+                if (relative > 0) {
+                    relativeString = chalk.bold.green(`${relative}% better`);
+                }
+                else if (relative < 0) {
+                    relativeString = chalk.bold.red(`${-relative}% worse`);
+                }
+                else {
+                    relativeString = chalk.gray(`no difference`);
+                }
+                console.log(` ${row.hash}: ${chalk.magenta(row.ops)} Ops/sec (${relativeString})`);
+            }
+        })
+    });
+
+    console.log("===================");
+    console.log(chalk.gray("Run `npm run bench` at another Git commit to create a comparison"));
+
+}
+if (typeof exports !== "undefined") {
+    exports.benchmarkComparisonToConsole = benchmarkComparisonToConsole;
+}

test/spec/proxyBenchmark.js

@@ -31,8 +31,9 @@ if (typeof JSONPatcherProxy === 'undefined') {
 
 if (typeof Benchmark === 'undefined') {
   global.Benchmark = require('benchmark');
-  global.benchmarkResultsToConsole = require('./../helpers/benchmarkReporter.js')
-    .benchmarkResultsToConsole;
+  global.benchmarkResultsToConsole = require('./../helpers/benchmarkReporter.js').benchmarkResultsToConsole;
+  global.benchmarkComparisonToFile = require('./../helpers/benchmarkComparisonAdder.js').benchmarkComparisonToFile;
+  global.benchmarkComparisonToConsole = require('./../helpers/benchmarkComparisonReporter.js').benchmarkComparisonToConsole;
 }
 
 const suite = new Benchmark.Suite();
@@ -122,7 +123,7 @@ function reverseString(str) {
       jsonPatcherProxy.generate();
     });
   }
-  
+
   if (includeComparisons) {
     suite.add(`${suiteName} (fast-json-patch)`, function() {
       const obj = generateBigObjectFixture(100);
@@ -247,6 +248,8 @@ if (typeof benchmarkReporter !== 'undefined') {
 } else {
   suite.on('complete', function() {
     benchmarkResultsToConsole(suite);
+    benchmarkComparisonToFile(suite);
+    benchmarkComparisonToConsole();
   });
   suite.run();
 }