diff --git a/panics.go b/panics.go index 718910b..af9d394 100644 --- a/panics.go +++ b/panics.go @@ -19,6 +19,8 @@ import ( "strings" + "github.com/tokopedia/panics/svc/throttle" + "github.com/eapache/go-resiliency/breaker" "github.com/julienschmidt/httprouter" "github.com/nsqio/go-nsq" @@ -34,6 +36,7 @@ var ( capturedBadDeployment bool customMessage string + checkThrottle bool // circuitbreaker cb *breaker.Breaker ) @@ -52,8 +55,13 @@ type Options struct { Tags Tags CustomMessage string DontLetMeDie bool + + MaxSendMessage int // maximum message being send to channel + RetrySendMessageAfter int // time in second + FlagThrottle bool } +//SetOptions setting up general function func SetOptions(o *Options) { filepath = o.Filepath slackWebhookURL = o.SlackWebhookURL @@ -73,6 +81,13 @@ func SetOptions(o *Options) { if o.DontLetMeDie { cb = nil } + + //add new flag function + if o.FlagThrottle { + checkThrottle = true + throttle.Setup(o.MaxSendMessage, o.RetrySendMessageAfter) + } + CaptureBadDeployment() } @@ -80,6 +95,7 @@ func init() { env = os.Getenv("TKPENV") // circuitbreaker to let apps died when got too many panics cb = breaker.New(3, 2, time.Minute*1) + } // CaptureHandler handle panic on http handler. @@ -232,6 +248,13 @@ func recovery(r interface{}) error { } func publishError(errs error, reqBody []byte, withStackTrace bool) { + //do nothing if system exeding threshold that was being put on in + if checkThrottle { + if !throttle.AllowedSend() { + + } + } + var text string var snip string var buffer bytes.Buffer @@ -274,6 +297,12 @@ func publishError(errs error, reqBody []byte, withStackTrace bool) { } func postToSlack(text, snip string) { + //do nothing if system exeding threshold that was being put on in + if checkThrottle { + if !throttle.AllowedSend() { + return + } + } payload := map[string]interface{}{ "text": text, //Enable slack to parse mention @ diff --git a/svc/throttle/function.go b/svc/throttle/function.go new file mode 100644 index 0000000..2652135 --- /dev/null +++ b/svc/throttle/function.go @@ -0,0 +1,11 @@ +package throttle + +//AllowedSend verifying send message +func AllowedSend() bool { + if counter > maxSendMessage { + return false + } + + counter++ + return true +} diff --git a/svc/throttle/throttle.go b/svc/throttle/throttle.go new file mode 100644 index 0000000..4792993 --- /dev/null +++ b/svc/throttle/throttle.go @@ -0,0 +1,24 @@ +package throttle + +import ( + "time" +) + +//Setup do initial setup +func Setup(max, retryAfter int) { + maxSendMessage = defaultMaxSendMessage + if max != 0 { + maxSendMessage = max + } + + go func() { + waitTime := defaultRetrySendMessage + if retryAfter != 0 { + waitTime = retryAfter + } + for { + time.Sleep(time.Duration(waitTime) * time.Second) + sendMessageCounter = 0 + } + }() +} diff --git a/svc/throttle/types.go b/svc/throttle/types.go new file mode 100644 index 0000000..9246711 --- /dev/null +++ b/svc/throttle/types.go @@ -0,0 +1,14 @@ +package throttle + +var ( + sendMessageCounter int + maxSendMessage int + counter int +) + +type throttle struct{} + +const ( + defaultRetrySendMessage = 600 // 10 minutes + defaultMaxSendMessage = 10 +)