From 783b0e99469ee2542fb6761452008a9b6a4e2e81 Mon Sep 17 00:00:00 2001
From: mstmdev <mstmdev@gmail.com>
Date: Wed, 30 Oct 2024 19:08:36 +0800
Subject: [PATCH] Reduce the frequency of retries and logging when an error
 occurs

---
 monitor/remote_client_monitor.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/monitor/remote_client_monitor.go b/monitor/remote_client_monitor.go
index 177e6f4..26b79de 100644
--- a/monitor/remote_client_monitor.go
+++ b/monitor/remote_client_monitor.go
@@ -133,6 +133,8 @@ func (m *remoteClientMonitor) readMessage(st *atomic.Bool, wd wait.Done) {
 	if err != nil {
 		return
 	}
+	errCount := 0
+	errThreshold := 20
 	for {
 		if m.closed.Load() {
 			wd.DoneWithError(errors.New("remote monitor is closed"))
@@ -143,6 +145,7 @@ func (m *remoteClientMonitor) readMessage(st *atomic.Bool, wd wait.Done) {
 			if st.Load() {
 				break
 			}
+			errCount++
 			m.logger.Error(err, "receive monitor message error")
 			if m.client.IsClosed(err) {
 				m.retry.Do(func() error {
@@ -161,8 +164,13 @@ func (m *remoteClientMonitor) readMessage(st *atomic.Bool, wd wait.Done) {
 						m.logger.Info("monitor the remote server success")
 					}
 				}
+			} else if errCount > errThreshold {
+				w := min(errCount/errThreshold, errThreshold)
+				m.logger.Info("receive monitor message error threshold exceeded, will retry after %d seconds", w)
+				time.Sleep(time.Second * time.Duration(w))
 			}
 		} else {
+			errCount = 0
 			m.messages.PushBack(msg)
 		}
 	}