From d02b7d572b8527c3a8541501cfa724971b5f5d53 Mon Sep 17 00:00:00 2001 From: Ying Zhu Date: Mon, 22 Aug 2022 11:10:07 -0700 Subject: [PATCH] istio: add metric for debouncing (#40523) * istio: add metric for debouncing This CL adds metric for the delay between a first config change enters deboucing until the final merged push request is pushed into the push queue. This time plus the proxy convergence time give us an upper bound on the total delay between a config change and the change is pushed to proxies. Also increased the buckets since logging shows that the debounce time is pretty long (more than 1 minute). Change-Id: I3220f9c3188824ea6925151ff6837f91aac5a15a Reviewed-on: https://gerrit.musta.ch/c/public/istio/+/3512 Reviewed-by: Weibo He Reviewed-by: Ryan Smick Reviewed-by: Jungho Ahn * istio: fix debounceTime typo Change-Id: I9977c597768360cc3dd485dbf21bd9afdb2f5151 Reviewed-on: https://gerrit.musta.ch/c/public/istio/+/3517 Reviewed-by: Weibo He * istio: handle debounce time entirely in the debounce() function To address comments in https://github.com/istio/istio/pull/40523, we will record after push finishes inside the debounce function. We can actually remove the DebounceStart field in the push context as a result. Also change the buckets as discussed in the PR. Change-Id: I3b7a7860590e7e5ed4f13282b4398527de089c81 Reviewed-on: https://gerrit.musta.ch/c/public/istio/+/3519 Reviewed-by: Jungho Ahn Reviewed-by: Weibo He --- pilot/pkg/xds/discovery.go | 5 +++-- pilot/pkg/xds/monitoring.go | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pilot/pkg/xds/discovery.go b/pilot/pkg/xds/discovery.go index 14678389d2a9..8aa246071909 100644 --- a/pilot/pkg/xds/discovery.go +++ b/pilot/pkg/xds/discovery.go @@ -414,9 +414,10 @@ func debounce(ch chan *model.PushRequest, stopCh <-chan struct{}, opts debounceO free := true freeCh := make(chan struct{}, 1) - push := func(req *model.PushRequest, debouncedEvents int) { + push := func(req *model.PushRequest, debouncedEvents int, startDebounce time.Time) { pushFn(req) updateSent.Add(int64(debouncedEvents)) + debounceTime.Record(time.Since(startDebounce).Seconds()) freeCh <- struct{}{} } @@ -437,7 +438,7 @@ func debounce(ch chan *model.PushRequest, stopCh <-chan struct{}, opts debounceO quietTime, eventDelay, req.Full) } free = false - go push(req, debouncedEvents) + go push(req, debouncedEvents, startDebounce) req = nil debouncedEvents = 0 } diff --git a/pilot/pkg/xds/monitoring.go b/pilot/pkg/xds/monitoring.go index da0760c24efe..624931ed6100 100644 --- a/pilot/pkg/xds/monitoring.go +++ b/pilot/pkg/xds/monitoring.go @@ -118,6 +118,12 @@ var ( ldsSendErrPushes = pushes.With(typeTag.Value("lds_senderr")) rdsSendErrPushes = pushes.With(typeTag.Value("rds_senderr")) + debounceTime = monitoring.NewDistribution( + "pilot_debounce_time", + "Delay in seconds between the first config enters debouncing and the merged push request is pushed into the push queue.", + []float64{.01, .1, 1, 3, 5, 10, 20, 30}, + ) + pushContextInitTime = monitoring.NewDistribution( "pilot_pushcontext_init_seconds", "Total time in seconds Pilot takes to init pushContext.", @@ -292,6 +298,8 @@ func init() { xdsClients, xdsResponseWriteTimeouts, pushes, + debounceTime, + pushContextInitTime, pushTime, proxiesConvergeDelay, proxiesQueueTime,