From 9b8c17e43b86a50bda4cc6050948ed884a208031 Mon Sep 17 00:00:00 2001 From: Nguyen Thuy Linh Date: Mon, 4 Nov 2024 11:50:41 +0700 Subject: [PATCH 1/2] add error metrics --- go.mod | 4 ++-- go.sum | 8 ++++---- internal/worker/worker.go | 23 +++++++++++++++++++++-- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index f1a77a0..250c3a2 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/IBM/sarama v1.43.3 github.com/KyberNetwork/cclog v1.1.0 github.com/KyberNetwork/go-binance/v2 v2.0.3 - github.com/KyberNetwork/tradinglib v0.5.10 + github.com/KyberNetwork/tradinglib v0.6.1 github.com/TheZeroSlave/zapsentry v1.20.2 github.com/ethereum/go-ethereum v1.13.14 github.com/getsentry/sentry-go v0.26.0 @@ -67,7 +67,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect diff --git a/go.sum b/go.sum index 5367634..49a743a 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/KyberNetwork/go-binance/v2 v2.0.3 h1:VsBBHXAJTxryBsdO6U9MyK+oHEJ9qZtQ github.com/KyberNetwork/go-binance/v2 v2.0.3/go.mod h1:ihNzjOXDgxZ4cSENV2sQfXnO30wobXQPpqTuKxxUVjw= github.com/KyberNetwork/kyber-trace-go v0.1.1 h1:YgzZb7jSg0Rgj7zFv9zUnSlB13Iksv1eVO5MSWAvjmM= github.com/KyberNetwork/kyber-trace-go v0.1.1/go.mod h1:X6hVacmKMeOEOlFh4TyfEHaEVRerFQ5YLuQ4keRV3hw= -github.com/KyberNetwork/tradinglib v0.5.10 h1:4R9XWkfCDeisuUyxq8px1JMXNGl73+c3tIuqBX9jWl8= -github.com/KyberNetwork/tradinglib v0.5.10/go.mod h1:3Ciie58Qmd0mFSnUWAg6phbalao6f2GGcAZ6nxw/lcQ= +github.com/KyberNetwork/tradinglib v0.6.1 h1:6BIeci/MawEo+vV/3LFQBzBbulimK4gCNKZSwz6h6fg= +github.com/KyberNetwork/tradinglib v0.6.1/go.mod h1:fRH5mz+qKrwreW2eSppCYRVYGN5RYHlF5xfJBBDzPW8= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= @@ -219,8 +219,8 @@ github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= diff --git a/internal/worker/worker.go b/internal/worker/worker.go index e6fc75a..ee93d18 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -11,13 +11,15 @@ import ( "github.com/KyberNetwork/tradelogs/pkg/parser" "github.com/KyberNetwork/tradelogs/pkg/pricefiller" "github.com/KyberNetwork/tradelogs/pkg/storage" + "github.com/KyberNetwork/tradinglib/pkg/metrics" ethTypes "github.com/ethereum/go-ethereum/core/types" "go.uber.org/zap" ) const ( - RetryInterval = 4 * time.Second - RemoveInterval = time.Hour + RetryInterval = 4 * time.Second + RemoveInterval = time.Hour + ParsingErrorMetricName = "tradelogs_parse_error" ) type Worker struct { @@ -79,6 +81,13 @@ func (w *Worker) Run(ctx context.Context) error { } } func (w *Worker) processMessages(m []evmlistenerclient.Message) error { + var errCount int64 + defer func() { + if err := metrics.RecordCounter(context.Background(), ParsingErrorMetricName, errCount); err != nil { + w.l.Errorw("error when record parsing error", "err", err) + } + }() + for _, message := range m { var ( insertOrders []storage.TradeLog @@ -99,6 +108,7 @@ func (w *Worker) processMessages(m []evmlistenerclient.Message) error { } order, err := ps.Parse(ethLog, block.Timestamp) if err != nil { + errCount++ w.l.Errorw("error when parse log", "log", log, "order", order, "err", err) if err := w.s.InsertErrorLog(storage.EVMLog{ Address: log.Address, @@ -141,6 +151,14 @@ func (w *Worker) retryParseLog() error { return err } w.l.Infow("start retry logs", "len", len(logs)) + + var errCount int64 + defer func() { + if err := metrics.RecordCounter(context.Background(), ParsingErrorMetricName, errCount); err != nil { + w.l.Errorw("error when record parsing error", "err", err) + } + }() + for _, l := range logs { topics := strings.Split(l.Topics, ",") if len(topics) == 0 { @@ -162,6 +180,7 @@ func (w *Worker) retryParseLog() error { } order, err := ps.Parse(ethLog, l.Time) if err != nil { + errCount++ w.l.Errorw("error when retry log", "log", l, "err", err) continue } From e3608a84708f6519be01cfe8567393586c59d04f Mon Sep 17 00:00:00 2001 From: Nguyen Thuy Linh Date: Mon, 4 Nov 2024 13:56:27 +0700 Subject: [PATCH 2/2] update --- internal/worker/worker.go | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/internal/worker/worker.go b/internal/worker/worker.go index ee93d18..99b5256 100644 --- a/internal/worker/worker.go +++ b/internal/worker/worker.go @@ -19,7 +19,7 @@ import ( const ( RetryInterval = 4 * time.Second RemoveInterval = time.Hour - ParsingErrorMetricName = "tradelogs_parse_error" + ParsingErrorMetricName = "trading_tradelogs_parse_error" ) type Worker struct { @@ -81,13 +81,6 @@ func (w *Worker) Run(ctx context.Context) error { } } func (w *Worker) processMessages(m []evmlistenerclient.Message) error { - var errCount int64 - defer func() { - if err := metrics.RecordCounter(context.Background(), ParsingErrorMetricName, errCount); err != nil { - w.l.Errorw("error when record parsing error", "err", err) - } - }() - for _, message := range m { var ( insertOrders []storage.TradeLog @@ -108,7 +101,9 @@ func (w *Worker) processMessages(m []evmlistenerclient.Message) error { } order, err := ps.Parse(ethLog, block.Timestamp) if err != nil { - errCount++ + if err = metrics.RecordCounter(context.Background(), ParsingErrorMetricName, 1); err != nil { + w.l.Errorw("error when record parsing error", "err", err) + } w.l.Errorw("error when parse log", "log", log, "order", order, "err", err) if err := w.s.InsertErrorLog(storage.EVMLog{ Address: log.Address, @@ -152,13 +147,6 @@ func (w *Worker) retryParseLog() error { } w.l.Infow("start retry logs", "len", len(logs)) - var errCount int64 - defer func() { - if err := metrics.RecordCounter(context.Background(), ParsingErrorMetricName, errCount); err != nil { - w.l.Errorw("error when record parsing error", "err", err) - } - }() - for _, l := range logs { topics := strings.Split(l.Topics, ",") if len(topics) == 0 { @@ -180,7 +168,9 @@ func (w *Worker) retryParseLog() error { } order, err := ps.Parse(ethLog, l.Time) if err != nil { - errCount++ + if err := metrics.RecordCounter(context.Background(), ParsingErrorMetricName, 1); err != nil { + w.l.Errorw("error when record parsing error", "err", err) + } w.l.Errorw("error when retry log", "log", l, "err", err) continue }