From b811a8c0b851480928a0d0eaeea0edc6d1738b4e Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Sat, 28 Sep 2024 17:11:28 -0300 Subject: [PATCH 1/5] benchmark: add --runs support to run.js --- benchmark/run.js | 85 +++++++++++-------- .../writing-and-running-benchmarks.md | 10 +++ 2 files changed, 59 insertions(+), 36 deletions(-) diff --git a/benchmark/run.js b/benchmark/run.js index 6a61df71221710..6f6a78ddbd2df9 100644 --- a/benchmark/run.js +++ b/benchmark/run.js @@ -12,6 +12,8 @@ const cli = new CLI(`usage: ./node run.js [options] [--] ... (can be repeated) --exclude pattern excludes scripts matching (can be repeated) + --runs variable=value set the amount of benchmark suite execution. + Default: 1 --set variable=value set benchmark variable (can be repeated) --format [simple|csv] optional value that specifies the output format test only run a single configuration from the options @@ -45,8 +47,7 @@ if (format === 'csv') { console.log('"filename", "configuration", "rate", "time"'); } -(function recursive(i) { - const filename = benchmarks[i]; +async function runBenchmark(filename) { const scriptPath = path.resolve(__dirname, filename); const args = cli.test ? ['--test'] : cli.optional.set; @@ -63,42 +64,54 @@ if (format === 'csv') { ); } - if (format !== 'csv') { - console.log(); - console.log(filename); - } - - child.on('message', (data) => { - if (data.type !== 'report') { - return; - } - // Construct configuration string, " A=a, B=b, ..." - let conf = ''; - for (const key of Object.keys(data.conf)) { - if (conf !== '') - conf += ' '; - conf += `${key}=${JSON.stringify(data.conf[key])}`; - } - if (format === 'csv') { - // Escape quotes (") for correct csv formatting - conf = conf.replace(/"/g, '""'); - console.log(`"${data.name}", "${conf}", ${data.rate}, ${data.time}`); - } else { - let rate = data.rate.toString().split('.'); - rate[0] = rate[0].replace(/(\d)(?=(?:\d\d\d)+(?!\d))/g, '$1,'); - rate = (rate[1] ? rate.join('.') : rate[0]); - console.log(`${data.name} ${conf}: ${rate}`); - } + return new Promise((resolve, reject) => { + child.on('message', (data) => { + if (data.type !== 'report') { + return; + } + // Construct configuration string, " A=a, B=b, ..." + let conf = ''; + for (const key of Object.keys(data.conf)) { + if (conf !== '') + conf += ' '; + conf += `${key}=${JSON.stringify(data.conf[key])}`; + } + if (format === 'csv') { + // Escape quotes (") for correct csv formatting + conf = conf.replace(/"/g, '""'); + console.log(`"${data.name}", "${conf}", ${data.rate}, ${data.time}`); + } else { + let rate = data.rate.toString().split('.'); + rate[0] = rate[0].replace(/(\d)(?=(?:\d\d\d)+(?!\d))/g, '$1,'); + rate = (rate[1] ? rate.join('.') : rate[0]); + console.log(`${data.name} ${conf}: ${rate}`); + } + }); + child.once('error', (err) => reject(err)); + child.once('close', (code) => { + if (code) { + reject(code); + } else { + resolve(code); + } + }); }); +} + +let runs = cli.optional.runs ?? 1; - child.once('close', (code) => { - if (code) { - process.exit(code); +async function run() { + for (let i = 0; i < benchmarks.length; ++i) { + const filename = benchmarks[i]; + if (format !== 'csv') { + console.log(); + console.log(filename); } - // If there are more benchmarks execute the next - if (i + 1 < benchmarks.length) { - recursive(i + 1); + while (runs--) { + await runBenchmark(filename); } - }); -})(0); + } +} + +run(); diff --git a/doc/contributing/writing-and-running-benchmarks.md b/doc/contributing/writing-and-running-benchmarks.md index c258280ce9572b..9a314b5542e04b 100644 --- a/doc/contributing/writing-and-running-benchmarks.md +++ b/doc/contributing/writing-and-running-benchmarks.md @@ -174,6 +174,16 @@ It is possible to execute more groups by adding extra process arguments. node benchmark/run.js assert async_hooks ``` +It's also possible to execute the benchmark more than once using the +`--runs` flag. + +```bash +node benchmark/run.js --runs 10 assert async_hooks +``` + +This command will run the benchmark suite 10 times for both assert and +async\_hooks group + #### Specifying CPU Cores for Benchmarks with run.js When using `run.js` to execute a group of benchmarks, From 8bd9e7fe9c3c7fc0789aaa168e52d6b6c5f02d7a Mon Sep 17 00:00:00 2001 From: Rafael Gonzaga Date: Sun, 29 Sep 2024 14:51:32 -0300 Subject: [PATCH 2/5] Update doc/contributing/writing-and-running-benchmarks.md Co-authored-by: Antoine du Hamel --- doc/contributing/writing-and-running-benchmarks.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/contributing/writing-and-running-benchmarks.md b/doc/contributing/writing-and-running-benchmarks.md index 9a314b5542e04b..3d16c7d14fc033 100644 --- a/doc/contributing/writing-and-running-benchmarks.md +++ b/doc/contributing/writing-and-running-benchmarks.md @@ -181,8 +181,8 @@ It's also possible to execute the benchmark more than once using the node benchmark/run.js --runs 10 assert async_hooks ``` -This command will run the benchmark suite 10 times for both assert and -async\_hooks group +This command will run the benchmark files in `benchmark/assert` and `benchmark/async_hooks` +10 times each. #### Specifying CPU Cores for Benchmarks with run.js From 5ede3211e48a33abf64de9eb8f0f7eb36a757cf9 Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Thu, 17 Oct 2024 18:19:59 -0300 Subject: [PATCH 3/5] fixup! benchmark: add --runs support to run.js --- benchmark/run.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/benchmark/run.js b/benchmark/run.js index 6f6a78ddbd2df9..57359eb58eaa71 100644 --- a/benchmark/run.js +++ b/benchmark/run.js @@ -87,28 +87,26 @@ async function runBenchmark(filename) { console.log(`${data.name} ${conf}: ${rate}`); } }); - child.once('error', (err) => reject(err)); child.once('close', (code) => { if (code) { - reject(code); - } else { resolve(code); + } else { + reject(code); } }); }); } -let runs = cli.optional.runs ?? 1; - async function run() { for (let i = 0; i < benchmarks.length; ++i) { + let runs = cli.optional.runs ?? 1; const filename = benchmarks[i]; if (format !== 'csv') { console.log(); console.log(filename); } - while (runs--) { + while (runs-- > 0) { await runBenchmark(filename); } } From cff977ebdaa70a1e79f3cacf7f03b66f4e1eb5ba Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Thu, 17 Oct 2024 22:48:10 -0300 Subject: [PATCH 4/5] fixup! fixup! benchmark: add --runs support to run.js --- benchmark/run.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmark/run.js b/benchmark/run.js index 57359eb58eaa71..74ee5f1ed23ed4 100644 --- a/benchmark/run.js +++ b/benchmark/run.js @@ -89,9 +89,9 @@ async function runBenchmark(filename) { }); child.once('close', (code) => { if (code) { - resolve(code); - } else { reject(code); + } else { + resolve(code); } }); }); From f9ed2906ef47d6a5c83e55236d16949051d51a0d Mon Sep 17 00:00:00 2001 From: RafaelGSS Date: Fri, 18 Oct 2024 10:01:54 -0300 Subject: [PATCH 5/5] fixup! fixup! fixup! benchmark: add --runs support to run.js --- benchmark/run.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/run.js b/benchmark/run.js index 74ee5f1ed23ed4..ea0dc415e91ec6 100644 --- a/benchmark/run.js +++ b/benchmark/run.js @@ -47,7 +47,7 @@ if (format === 'csv') { console.log('"filename", "configuration", "rate", "time"'); } -async function runBenchmark(filename) { +function runBenchmark(filename) { const scriptPath = path.resolve(__dirname, filename); const args = cli.test ? ['--test'] : cli.optional.set;