From 8af9889f561fcf97ac7c1446ceb72f9324348dd4 Mon Sep 17 00:00:00 2001 From: Alexandru Gheorghe <49718502+alexggh@users.noreply.github.com> Date: Wed, 7 Aug 2024 17:22:27 +0300 Subject: [PATCH] Bring reference_hardware.json inline with machine used for weights (#5196) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since `May 2023` after https://github.com/paritytech/substrate/pull/13548 optimization, `Blake2256` is faster with about 30%, that means that there is a difference of ~30% between the benchmark values we ask validators to run against and the machine we use for generating the weights.So if all validators, just barely pass the benchmarks our weights are potentially underestimated with about ~20%, so let's bring this two in sync. Same thing happened when we merged https://github.com/paritytech/polkadot-sdk/pull/2524 in `Nov 2023` SR25519-Verify became faster with about 10-15% ## Results Generated on machine from here: https://github.com/paritytech/devops/pull/3210 ``` +----------+----------------+--------------+-------------+-------------------+ | Category | Function | Score | Minimum | Result | +============================================================================+ | CPU | BLAKE2-256 | 1.00 GiBs | 783.27 MiBs | ✅ Pass (130.7 %) | |----------+----------------+--------------+-------------+-------------------| | CPU | SR25519-Verify | 637.62 KiBs | 560.67 KiBs | ✅ Pass (113.7 %) | |----------+----------------+--------------+-------------+-------------------| | Memory | Copy | 12.19 GiBs | 11.49 GiBs | ✅ Pass (106.1 %) | ``` Discovered and discussed here: https://github.com/paritytech/polkadot-sdk/pull/5127#issuecomment-2258423469 ## Downsides Machines that barely passed the benchmark will suddenly find themselves bellow the benchmark, but since that is just an warning and everything else continues as before it shouldn't be too impactful and should give the validators the necessary information that they need to become compliant, since they actually aren't when compared with the used weights. --------- Signed-off-by: Alexandru Gheorghe --- prdoc/pr_5196.prdoc | 23 +++++++++++++++++++ .../benchmarking-cli/src/machine/hardware.rs | 4 ++-- .../src/machine/reference_hardware.json | 4 ++-- 3 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 prdoc/pr_5196.prdoc diff --git a/prdoc/pr_5196.prdoc b/prdoc/pr_5196.prdoc new file mode 100644 index 000000000000..3ed4fbdff3f3 --- /dev/null +++ b/prdoc/pr_5196.prdoc @@ -0,0 +1,23 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Bring benchmark inline with reference machine used for weights + +doc: + - audience: Node Operator + description: | + - BLAKE2-256 reference values were too low(~30%) when compared with the machine used for generating + the weights, so it was brought in sync with results on the reference hardware recommended here: + https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#reference-hardware + - SR25519-Verify reference values were too low(~10%) when compared with the machine used for generating + the weights, so it was brought in sync with results on the reference hardware recommended here: + https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#reference-hardware + - Validators where the `BLAKE2-256` and `SR25519-Verify` were barely passing, might received the + warning that they are not compliant anymore, this should not be treated as critical, but they + should take the necessary steps to become compliant in the near/mid-term future. + - Note!: The reference hardware requirements have not been increased we just fixed the benchmark which + was wrongly reporting lower spec HW as being compliant. + +crates: + - name: frame-benchmarking-cli + bump: minor diff --git a/substrate/utils/frame/benchmarking-cli/src/machine/hardware.rs b/substrate/utils/frame/benchmarking-cli/src/machine/hardware.rs index 5a4b7c797b6f..555e848f8ccc 100644 --- a/substrate/utils/frame/benchmarking-cli/src/machine/hardware.rs +++ b/substrate/utils/frame/benchmarking-cli/src/machine/hardware.rs @@ -51,10 +51,10 @@ mod tests { assert_eq!( *SUBSTRATE_REFERENCE_HARDWARE, Requirements(vec![ - Requirement { metric: Metric::Blake2256, minimum: Throughput::from_mibs(783.27) }, + Requirement { metric: Metric::Blake2256, minimum: Throughput::from_mibs(1000.00) }, Requirement { metric: Metric::Sr25519Verify, - minimum: Throughput::from_kibs(560.670000128), + minimum: Throughput::from_kibs(637.619999744), }, Requirement { metric: Metric::MemCopy, diff --git a/substrate/utils/frame/benchmarking-cli/src/machine/reference_hardware.json b/substrate/utils/frame/benchmarking-cli/src/machine/reference_hardware.json index c2fb4c7d4a28..cec42b8f245c 100644 --- a/substrate/utils/frame/benchmarking-cli/src/machine/reference_hardware.json +++ b/substrate/utils/frame/benchmarking-cli/src/machine/reference_hardware.json @@ -1,11 +1,11 @@ [ { "metric": "Blake2256", - "minimum": 783.27 + "minimum": 1000.00 }, { "metric": "Sr25519Verify", - "minimum": 0.547529297 + "minimum": 0.622675781 }, { "metric": "MemCopy",