add a throttling function that by default is disable, and can be cust…

panics.go

@@ -19,6 +19,8 @@ import (
 
 	"strings"
 
+	"github.com/tokopedia/panics/svc/throttle"
+
 	"github.com/eapache/go-resiliency/breaker"
 	"github.com/julienschmidt/httprouter"
 	"github.com/nsqio/go-nsq"
@@ -34,6 +36,7 @@ var (
 	capturedBadDeployment bool
 	customMessage         string
 
+	checkThrottle bool
 	// circuitbreaker
 	cb *breaker.Breaker
 )
@@ -52,8 +55,13 @@ type Options struct {
 	Tags            Tags
 	CustomMessage   string
 	DontLetMeDie    bool
+
+	MaxSendMessage        int // maximum message being send to channel
+	RetrySendMessageAfter int // time in second
+	FlagThrottle          bool
 }
 
+//SetOptions setting up general function
 func SetOptions(o *Options) {
 	filepath = o.Filepath
 	slackWebhookURL = o.SlackWebhookURL
@@ -73,13 +81,21 @@ func SetOptions(o *Options) {
 	if o.DontLetMeDie {
 		cb = nil
 	}
+
+	//add new flag function
+	if o.FlagThrottle {
+		checkThrottle = true
+		throttle.Setup(o.MaxSendMessage, o.RetrySendMessageAfter)
+	}
+
 	CaptureBadDeployment()
 }
 
 func init() {
 	env = os.Getenv("TKPENV")
 	// circuitbreaker to let apps died when got too many panics
 	cb = breaker.New(3, 2, time.Minute*1)
+
 }
 
 // CaptureHandler handle panic on http handler.
@@ -232,6 +248,13 @@ func recovery(r interface{}) error {
 }
 
 func publishError(errs error, reqBody []byte, withStackTrace bool) {
+	//do nothing if system exeding threshold that was being put on in
+	if checkThrottle {
+		if !throttle.AllowedSend() {
+
+		}
+	}
+
 	var text string
 	var snip string
 	var buffer bytes.Buffer
@@ -274,6 +297,12 @@ func publishError(errs error, reqBody []byte, withStackTrace bool) {
 }
 
 func postToSlack(text, snip string) {
+	//do nothing if system exeding threshold that was being put on in
+	if checkThrottle {
+		if !throttle.AllowedSend() {
+			return
+		}
+	}
 	payload := map[string]interface{}{
 		"text": text,
 		//Enable slack to parse mention @<someone>

svc/throttle/function.go

@@ -0,0 +1,11 @@
+package throttle
+
+//AllowedSend verifying send message
+func AllowedSend() bool {
+	if counter > maxSendMessage {
+		return false
+	}
+
+	counter++
+	return true
+}

svc/throttle/throttle.go

@@ -0,0 +1,24 @@
+package throttle
+
+import (
+	"time"
+)
+
+//Setup do initial setup
+func Setup(max, retryAfter int) {
+	maxSendMessage = defaultMaxSendMessage
+	if max != 0 {
+		maxSendMessage = max
+	}
+
+	go func() {
+		waitTime := defaultRetrySendMessage
+		if retryAfter != 0 {
+			waitTime = retryAfter
+		}
+		for {
+			time.Sleep(time.Duration(waitTime) * time.Second)
+			sendMessageCounter = 0
+		}
+	}()
+}

svc/throttle/types.go

@@ -0,0 +1,14 @@
+package throttle
+
+var (
+	sendMessageCounter int
+	maxSendMessage     int
+	counter            int
+)
+
+type throttle struct{}
+
+const (
+	defaultRetrySendMessage = 600 // 10 minutes
+	defaultMaxSendMessage   = 10
+)