istio: add metric for debouncing (istio#40523)

This CL patches commit c9557ae from
upstream istio into air-release-1.14.3 to add a metric for pilot
debounce time. Original PR can be found here:
istio#40523.
The description from that PR is reproduced below.

This CL adds metric for the delay between a first config change
enters deboucing until the final merged push request is pushed into
the push queue. This time plus the proxy convergence time give us
an upper bound on the total delay between a config change and the
change is pushed to proxies.

Also increased the buckets since logging shows that the debounce time
is pretty long (more than 1 minute).

Change-Id: If6cbdd7ee9fd5aed94001a3857989bada1919c87
Reviewed-on: https://gerrit.musta.ch/c/public/istio/+/3559
Reviewed-by: Ying Zhu <[email protected]>
Reviewed-by: Douglas Jordan <[email protected]>

pilot/pkg/xds/discovery.go

@@ -394,9 +394,10 @@ func debounce(ch chan *model.PushRequest, stopCh <-chan struct{}, opts debounceO
 	free := true
 	freeCh := make(chan struct{}, 1)
 
-	push := func(req *model.PushRequest, debouncedEvents int) {
+	push := func(req *model.PushRequest, debouncedEvents int, startDebounce time.Time) {
 		pushFn(req)
 		updateSent.Add(int64(debouncedEvents))
+		debounceTime.Record(time.Since(startDebounce).Seconds())
 		freeCh <- struct{}{}
 	}
 
@@ -417,7 +418,7 @@ func debounce(ch chan *model.PushRequest, stopCh <-chan struct{}, opts debounceO
 						quietTime, eventDelay, req.Full)
 				}
 				free = false
-				go push(req, debouncedEvents)
+				go push(req, debouncedEvents, startDebounce)
 				req = nil
 				debouncedEvents = 0
 			}

pilot/pkg/xds/monitoring.go

@@ -118,6 +118,12 @@ var (
 	ldsSendErrPushes = pushes.With(typeTag.Value("lds_senderr"))
 	rdsSendErrPushes = pushes.With(typeTag.Value("rds_senderr"))
 
+	debounceTime = monitoring.NewDistribution(
+		"pilot_debounce_time",
+		"Delay in seconds between the first config enters debouncing and the merged push request is pushed into the push queue.",
+		[]float64{.01, .1, 1, 3, 5, 10, 20, 30},
+	)
+
 	pushContextInitTime = monitoring.NewDistribution(
 		"pilot_pushcontext_init_seconds",
 		"Total time in seconds Pilot takes to init pushContext.",
@@ -292,6 +298,7 @@ func init() {
 		xdsClients,
 		xdsResponseWriteTimeouts,
 		pushes,
+		debounceTime,
 		pushContextInitTime,
 		pushTime,
 		proxiesConvergeDelay,