From f835f3f97c3423d856a37e09bfb159cce29086ac Mon Sep 17 00:00:00 2001 From: Lidi Zheng Date: Thu, 1 Jul 2021 09:48:37 -0700 Subject: [PATCH] Tighten the error tolerance requirement by 100x (#26588) * Tighten the error tolerance requirement by 10x * Make it 5 sigma instead of 4.5 * Rewrap comments --- test/cpp/end2end/xds_end2end_test.cc | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/test/cpp/end2end/xds_end2end_test.cc b/test/cpp/end2end/xds_end2end_test.cc index 46fcb7c316196..5c82a36d1b7f0 100644 --- a/test/cpp/end2end/xds_end2end_test.cc +++ b/test/cpp/end2end/xds_end2end_test.cc @@ -1615,28 +1615,25 @@ grpc_millis NowFromCycleCounter() { return grpc_cycle_counter_to_millis_round_up(now); } -// Returns the number of RPCs needed to pass error_tolerance at 99.995% chance. -// Rolling dices in drop/fault-injection generates a binomial distribution (if -// our code is not horribly wrong). Let's make "n" the number of samples, "p" -// the probabilty. If we have np>5 & n(1-p)>5, we can approximately treat the -// binomial distribution as a normal distribution. +// Returns the number of RPCs needed to pass error_tolerance at 99.99994% +// chance. Rolling dices in drop/fault-injection generates a binomial +// distribution (if our code is not horribly wrong). Let's make "n" the number +// of samples, "p" the probabilty. If we have np>5 & n(1-p)>5, we can +// approximately treat the binomial distribution as a normal distribution. // // For normal distribution, we can easily look up how many standard deviation we // need to reach 99.995%. Based on Wiki's table -// https://en.wikipedia.org/wiki/Standard_normal_table, we need 4.00 sigma -// (standard deviation) to cover the probability area of 99.995%. In another -// word, for a sample with size "n" probability "p" error-tolerance "k", we want -// the error always land within 4.00 sigma. The sigma of binominal distribution -// and be computed as sqrt(np(1-p)). Hence, we have the equation: +// https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule, we need 5.00 +// sigma (standard deviation) to cover the probability area of 99.99994%. In +// another word, for a sample with size "n" probability "p" error-tolerance "k", +// we want the error always land within 5.00 sigma. The sigma of binominal +// distribution and be computed as sqrt(np(1-p)). Hence, we have the equation: // -// kn <= 4.00 * sqrt(np(1-p)) -// -// E.g., with p=0.5 k=0.1, n >= 400; with p=0.5 k=0.05, n >= 1600; with p=0.5 -// k=0.01, n >= 40000. +// kn <= 5.00 * sqrt(np(1-p)) size_t ComputeIdealNumRpcs(double p, double error_tolerance) { GPR_ASSERT(p >= 0 && p <= 1); size_t num_rpcs = - ceil(p * (1 - p) * 4.00 * 4.00 / error_tolerance / error_tolerance); + ceil(p * (1 - p) * 5.00 * 5.00 / error_tolerance / error_tolerance); gpr_log(GPR_INFO, "Sending %" PRIuPTR " RPCs for percentage=%.3f error_tolerance=%.3f", num_rpcs, p, error_tolerance);