From 3ba5176012202985799158fe511f94f6ac413f15 Mon Sep 17 00:00:00 2001 From: Krzysztof Tomecki <152964795+chris-4chain@users.noreply.github.com> Date: Mon, 12 Feb 2024 07:03:31 +0100 Subject: [PATCH 1/6] feat(567): metrics collectors & trackers --- client.go | 7 ++++++ client_internal.go | 1 + client_options.go | 14 +++++++++++ cron_job_declarations.go | 12 ++++++++- cron_job_definitions.go | 17 +++++++++++++ interface.go | 2 ++ metrics/interface.go | 22 ++++++++++++++++ metrics/metrics.go | 50 +++++++++++++++++++++++++++++++++++++ metrics/naming.go | 10 ++++++++ metrics/stats.go | 12 +++++++++ paymail_service_provider.go | 18 ++++++++++--- record_tx.go | 14 +++++++++-- 12 files changed, 172 insertions(+), 7 deletions(-) create mode 100644 metrics/interface.go create mode 100644 metrics/metrics.go create mode 100644 metrics/naming.go create mode 100644 metrics/stats.go diff --git a/client.go b/client.go index 89458213..1d932e10 100644 --- a/client.go +++ b/client.go @@ -7,6 +7,7 @@ import ( "github.com/BuxOrg/bux/chainstate" "github.com/BuxOrg/bux/cluster" "github.com/BuxOrg/bux/logging" + "github.com/BuxOrg/bux/metrics" "github.com/BuxOrg/bux/notifications" "github.com/BuxOrg/bux/taskmanager" "github.com/bitcoin-sv/go-paymail" @@ -35,6 +36,7 @@ type ( httpClient HTTPInterface // HTTP interface to use iuc bool // (Input UTXO Check) True will check input utxos when saving transactions logger *zerolog.Logger // Internal logging + metrics *metrics.Metrics // Metrics with a collector interface models *modelOptions // Configuration options for the loaded models newRelic *newRelicOptions // Configuration options for NewRelic notifications *notificationsOptions // Configuration options for Notifications @@ -416,3 +418,8 @@ func (c *Client) UserAgent() string { func (c *Client) Version() string { return version } + +// Metrics will return the metrics client (if it's enabled) +func (c *Client) Metrics() (metrics *metrics.Metrics, enabled bool) { + return c.options.metrics, c.options.metrics != nil +} diff --git a/client_internal.go b/client_internal.go index 8b746878..b642250c 100644 --- a/client_internal.go +++ b/client_internal.go @@ -2,6 +2,7 @@ package bux import ( "context" + "github.com/BuxOrg/bux/chainstate" "github.com/BuxOrg/bux/cluster" "github.com/BuxOrg/bux/notifications" diff --git a/client_options.go b/client_options.go index 776a5bfa..78355c61 100644 --- a/client_options.go +++ b/client_options.go @@ -10,6 +10,7 @@ import ( "github.com/BuxOrg/bux/chainstate" "github.com/BuxOrg/bux/cluster" "github.com/BuxOrg/bux/logging" + "github.com/BuxOrg/bux/metrics" "github.com/BuxOrg/bux/notifications" "github.com/BuxOrg/bux/taskmanager" "github.com/BuxOrg/bux/utils" @@ -283,6 +284,19 @@ func WithLogger(customLogger *zerolog.Logger) ClientOps { } } +// ----------------------------------------------------------------- +// METRICS +// ----------------------------------------------------------------- + +// WithMetrics will set the metrics with a collector interface +func WithMetrics(collector metrics.Collector) ClientOps { + return func(c *clientOptions) { + if collector != nil { + c.metrics = metrics.NewMetrics(collector) + } + } +} + // ----------------------------------------------------------------- // CACHESTORE // ----------------------------------------------------------------- diff --git a/cron_job_declarations.go b/cron_job_declarations.go index 1d039ecc..0672818d 100644 --- a/cron_job_declarations.go +++ b/cron_job_declarations.go @@ -12,6 +12,7 @@ const ( CronJobNameDraftTransactionCleanUp = "draft_transaction_clean_up" CronJobNameSyncTransactionBroadcast = "sync_transaction_broadcast" CronJobNameSyncTransactionSync = "sync_transaction_sync" + CronJobNameCalculateMetrics = "calculate_metrics" ) type cronJobHandler func(ctx context.Context, client *Client) error @@ -25,7 +26,7 @@ func (c *Client) cronJobs() taskmanager.CronJobs { } } - return taskmanager.CronJobs{ + jobs := taskmanager.CronJobs{ CronJobNameDraftTransactionCleanUp: { Period: 60 * time.Second, Handler: handler(taskCleanupDraftTransactions), @@ -39,4 +40,13 @@ func (c *Client) cronJobs() taskmanager.CronJobs { Handler: handler(taskSyncTransactions), }, } + + if _, enabled := c.Metrics(); enabled { + jobs[CronJobNameCalculateMetrics] = taskmanager.CronJob{ + Period: 15 * time.Second, + Handler: handler(taskCalculateMetrics), + } + } + + return jobs } diff --git a/cron_job_definitions.go b/cron_job_definitions.go index 6ccf3d95..cce97437 100644 --- a/cron_job_definitions.go +++ b/cron_job_definitions.go @@ -85,3 +85,20 @@ func taskSyncTransactions(ctx context.Context, client *Client) error { } return err } + +func taskCalculateMetrics(ctx context.Context, client *Client) error { + metrics, enabled := client.Metrics() + if !enabled { + return errors.New("metrics are not enabled") + } + + modelOpts := client.DefaultModelOptions() + + if xpubsCount, err := getXPubsCount(ctx, nil, nil, modelOpts...); err != nil { + client.options.logger.Error().Err(err).Msg("error getting xpubs count") + } else { + metrics.Stats.XPub.Set(float64(xpubsCount)) + } + + return nil +} diff --git a/interface.go b/interface.go index 51838b6e..f93509ea 100644 --- a/interface.go +++ b/interface.go @@ -6,6 +6,7 @@ import ( "github.com/BuxOrg/bux/chainstate" "github.com/BuxOrg/bux/cluster" + "github.com/BuxOrg/bux/metrics" "github.com/BuxOrg/bux/notifications" "github.com/BuxOrg/bux/taskmanager" "github.com/bitcoin-sv/go-broadcast-client/broadcast" @@ -186,4 +187,5 @@ type ClientInterface interface { SetNotificationsClient(notifications.ClientInterface) UserAgent() string Version() string + Metrics() (metrics *metrics.Metrics, enabled bool) } diff --git a/metrics/interface.go b/metrics/interface.go new file mode 100644 index 00000000..641d0997 --- /dev/null +++ b/metrics/interface.go @@ -0,0 +1,22 @@ +package metrics + +// Collector is an interface that is used to register metrics +type Collector interface { + RegisterGauge(name string) GaugeInterface + RegisterHistogramVec(name string, labels ...string) HistogramVecInterface +} + +// GaugeInterface is an interface that is used to track gauges of values +type GaugeInterface interface { + Set(value float64) +} + +// HistogramVecInterface is an interface that is used to register histograms with labels +type HistogramVecInterface interface { + WithLabelValues(lvs ...string) HistogramInterface +} + +// HistogramInterface is an interface that is used to track histograms of values +type HistogramInterface interface { + Observe(value float64) +} diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 00000000..1bc57e6c --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,50 @@ +/* +Package metrics provides a way to track metrics in the application. Functionality is strictly tailored to the needs of the package and is not meant to be a general purpose metrics library. +*/ +package metrics + +import "time" + +// Metrics is a struct that contains all the metrics that are used to track in the package +type Metrics struct { + collector Collector + Stats Stats + verifyMerkleRoots HistogramVecInterface + recordTransaction HistogramVecInterface +} + +// NewMetrics is a constructor for the Metrics struct +func NewMetrics(collector Collector) *Metrics { + return &Metrics{ + collector: collector, + Stats: registerStats(collector), + verifyMerkleRoots: collector.RegisterHistogramVec(verifyMerkleRootsHistogramName, "classification"), + recordTransaction: collector.RegisterHistogramVec(recordTransactionHistogramName, "classification", "strategy"), + } +} + +// EndWithClassification is a function returned by Track* methods that should be called when the tracked operation is finished +type EndWithClassification func(success bool) + +// TrackVerifyMerkleRoots is used to track the time it takes to verify merkle roots +func (m *Metrics) TrackVerifyMerkleRoots() EndWithClassification { + start := time.Now() + return func(success bool) { + m.verifyMerkleRoots.WithLabelValues(classify(success)).Observe(time.Since(start).Seconds()) + } +} + +// TrackRecordTransaction is used to track the time it takes to record a transaction +func (m *Metrics) TrackRecordTransaction(strategyName string) EndWithClassification { + start := time.Now() + return func(success bool) { + m.verifyMerkleRoots.WithLabelValues(classify(success), strategyName).Observe(time.Since(start).Seconds()) + } +} + +func classify(success bool) string { + if success { + return "success" + } + return "failure" +} diff --git a/metrics/naming.go b/metrics/naming.go new file mode 100644 index 00000000..9cea9c48 --- /dev/null +++ b/metrics/naming.go @@ -0,0 +1,10 @@ +package metrics + +const domainPrefix = "bux_" + +const ( + verifyMerkleRootsHistogramName = domainPrefix + "verify_merkle_roots_histogram" + recordTransactionHistogramName = domainPrefix + "record_transaction_histogram" +) + +const xpubGaugeName = domainPrefix + "xpub_gauge" diff --git a/metrics/stats.go b/metrics/stats.go new file mode 100644 index 00000000..0dd11780 --- /dev/null +++ b/metrics/stats.go @@ -0,0 +1,12 @@ +package metrics + +// Stats is a struct that contains all the gauges that are used to track the calculated stats of the application +type Stats struct { + XPub GaugeInterface +} + +func registerStats(collector Collector) Stats { + return Stats{ + XPub: collector.RegisterGauge(xpubGaugeName), + } +} diff --git a/paymail_service_provider.go b/paymail_service_provider.go index a7632dc0..1fc3f73e 100644 --- a/paymail_service_provider.go +++ b/paymail_service_provider.go @@ -4,6 +4,8 @@ import ( "context" "encoding/hex" "fmt" + "reflect" + "github.com/BuxOrg/bux/chainstate" "github.com/BuxOrg/bux/utils" "github.com/bitcoin-sv/go-paymail" @@ -12,7 +14,6 @@ import ( "github.com/bitcoin-sv/go-paymail/spv" "github.com/bitcoinschema/go-bitcoin/v2" "github.com/libsv/go-bk/bec" - "reflect" ) // PaymailDefaultServiceProvider is an interface for overriding the paymail actions in go-paymail/server @@ -187,15 +188,24 @@ func (p *PaymailDefaultServiceProvider) RecordTransaction(ctx context.Context, func (p *PaymailDefaultServiceProvider) VerifyMerkleRoots( ctx context.Context, merkleRoots []*spv.MerkleRootConfirmationRequestItem, -) error { - request := make([]chainstate.MerkleRootConfirmationRequestItem, 0) +) (err error) { + if metrics, enabled := p.client.Metrics(); enabled { + end := metrics.TrackVerifyMerkleRoots() + defer func() { + success := err == nil + end(success) + }() + } + + request := make([]chainstate.MerkleRootConfirmationRequestItem, 0, len(merkleRoots)) for _, m := range merkleRoots { request = append(request, chainstate.MerkleRootConfirmationRequestItem{ MerkleRoot: m.MerkleRoot, BlockHeight: m.BlockHeight, }) } - return p.client.Chainstate().VerifyMerkleRoots(ctx, request) + err = p.client.Chainstate().VerifyMerkleRoots(ctx, request) + return } func (p *PaymailDefaultServiceProvider) createPaymailInformation(ctx context.Context, alias, domain string, opts ...ModelOps) (paymailAddress *PaymailAddress, pubKey *derivedPubKey, err error) { diff --git a/record_tx.go b/record_tx.go index a338f9b5..c6d629ec 100644 --- a/record_tx.go +++ b/record_tx.go @@ -19,11 +19,21 @@ type recordIncomingTxStrategy interface { FailOnBroadcastError(forceFail bool) } -func recordTransaction(ctx context.Context, c ClientInterface, strategy recordTxStrategy, opts ...ModelOps) (*Transaction, error) { +func recordTransaction(ctx context.Context, c ClientInterface, strategy recordTxStrategy, opts ...ModelOps) (transaction *Transaction, err error) { + if metrics, enabled := c.Metrics(); enabled { + strategyType := fmt.Sprintf("%T", strategy) + end := metrics.TrackRecordTransaction(strategyType) + defer func() { + success := err == nil + end(success) + }() + } + unlock := waitForRecordTxWriteLock(ctx, c, strategy.LockKey()) defer unlock() - return strategy.Execute(ctx, c, opts) + transaction, err = strategy.Execute(ctx, c, opts) + return } func getRecordTxStrategy(ctx context.Context, c ClientInterface, xPubKey, txHex, draftID string) (recordTxStrategy, error) { From 25fd0c486c10093a2a2cd151be1b5717c87b5cc7 Mon Sep 17 00:00:00 2001 From: Krzysztof Tomecki <152964795+chris-4chain@users.noreply.github.com> Date: Mon, 12 Feb 2024 07:03:31 +0100 Subject: [PATCH 2/6] feat(BUX-567): track query transaction --- chainstate/chainstate.go | 10 +++++++++- chainstate/client.go | 14 +++++++------- chainstate/client_options.go | 9 +++++++++ client_internal.go | 1 + metrics/metrics.go | 10 ++++++++++ metrics/naming.go | 1 + 6 files changed, 37 insertions(+), 8 deletions(-) diff --git a/chainstate/chainstate.go b/chainstate/chainstate.go index 6ba7243d..99452acc 100644 --- a/chainstate/chainstate.go +++ b/chainstate/chainstate.go @@ -44,7 +44,15 @@ func (c *Client) Broadcast(ctx context.Context, id, txHex string, timeout time.D // Note: this is slow, but follows a specific order: mAPI -> WhatsOnChain func (c *Client) QueryTransaction( ctx context.Context, id string, requiredIn RequiredIn, timeout time.Duration, -) (*TransactionInfo, error) { +) (transaction *TransactionInfo, err error) { + if c.options.metrics != nil { + end := c.options.metrics.TrackQueryTransaction() + defer func() { + success := err == nil + end(success) + }() + } + // Basic validation if len(id) < 50 { return nil, ErrInvalidTransactionID diff --git a/chainstate/client.go b/chainstate/client.go index 9cfe320a..9bd5c1d0 100644 --- a/chainstate/client.go +++ b/chainstate/client.go @@ -7,6 +7,7 @@ import ( "time" "github.com/BuxOrg/bux/logging" + "github.com/BuxOrg/bux/metrics" "github.com/BuxOrg/bux/utils" "github.com/bitcoin-sv/go-broadcast-client/broadcast" "github.com/newrelic/go-agent/v3/newrelic" @@ -23,11 +24,12 @@ type ( // clientOptions holds all the configuration for the client clientOptions struct { - config *syncConfig // Configuration for broadcasting and other chain-state actions - debug bool // For extra logs and additional debug information - logger *zerolog.Logger // Logger interface - newRelicEnabled bool // If NewRelic is enabled (parent application) - userAgent string // Custom user agent for outgoing HTTP Requests + config *syncConfig // Configuration for broadcasting and other chain-state actions + debug bool // For extra logs and additional debug information + logger *zerolog.Logger // Logger interface + metrics *metrics.Metrics // For collecting metrics (if enabled) + newRelicEnabled bool // If NewRelic is enabled (parent application) + userAgent string // Custom user agent for outgoing HTTP Requests } // syncConfig holds all the configuration about the different sync processes @@ -95,12 +97,10 @@ func (c *Client) Close(ctx context.Context) { defer txn.StartSegment("close_chainstate").End() } if c != nil && c.options.config != nil { - // Close minercraft if c.options.config.minercraft != nil { c.options.config.minercraft = nil } - } } diff --git a/chainstate/client_options.go b/chainstate/client_options.go index fd4ecfb0..290287e1 100644 --- a/chainstate/client_options.go +++ b/chainstate/client_options.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/BuxOrg/bux/metrics" "github.com/BuxOrg/bux/utils" "github.com/bitcoin-sv/go-broadcast-client/broadcast" "github.com/newrelic/go-agent/v3/newrelic" @@ -33,6 +34,7 @@ func defaultClientOptions() *clientOptions { }, debug: false, newRelicEnabled: false, + metrics: nil, } } @@ -173,3 +175,10 @@ func WithCallback(callbackURL, callbackAuthToken string) ClientOps { c.config.callbackToken = callbackAuthToken } } + +// WithMetrics will set metrics +func WithMetrics(metrics *metrics.Metrics) ClientOps { + return func(c *clientOptions) { + c.metrics = metrics + } +} diff --git a/client_internal.go b/client_internal.go index b642250c..44889077 100644 --- a/client_internal.go +++ b/client_internal.go @@ -38,6 +38,7 @@ func (c *Client) loadChainstate(ctx context.Context) (err error) { if c.options.chainstate.ClientInterface == nil { c.options.chainstate.options = append(c.options.chainstate.options, chainstate.WithUserAgent(c.UserAgent())) c.options.chainstate.options = append(c.options.chainstate.options, chainstate.WithHTTPClient(c.HTTPClient())) + c.options.chainstate.options = append(c.options.chainstate.options, chainstate.WithMetrics(c.options.metrics)) c.options.chainstate.ClientInterface, err = chainstate.NewClient(ctx, c.options.chainstate.options...) } diff --git a/metrics/metrics.go b/metrics/metrics.go index 1bc57e6c..55f8c38c 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -11,6 +11,7 @@ type Metrics struct { Stats Stats verifyMerkleRoots HistogramVecInterface recordTransaction HistogramVecInterface + queryTransaction HistogramVecInterface } // NewMetrics is a constructor for the Metrics struct @@ -20,6 +21,7 @@ func NewMetrics(collector Collector) *Metrics { Stats: registerStats(collector), verifyMerkleRoots: collector.RegisterHistogramVec(verifyMerkleRootsHistogramName, "classification"), recordTransaction: collector.RegisterHistogramVec(recordTransactionHistogramName, "classification", "strategy"), + queryTransaction: collector.RegisterHistogramVec(queryTransactionHistogramName, "classification"), } } @@ -42,6 +44,14 @@ func (m *Metrics) TrackRecordTransaction(strategyName string) EndWithClassificat } } +// TrackQueryTransaction is used to track the time it takes to query a transaction +func (m *Metrics) TrackQueryTransaction() EndWithClassification { + start := time.Now() + return func(success bool) { + m.verifyMerkleRoots.WithLabelValues(classify(success)).Observe(time.Since(start).Seconds()) + } +} + func classify(success bool) string { if success { return "success" diff --git a/metrics/naming.go b/metrics/naming.go index 9cea9c48..ca351bfd 100644 --- a/metrics/naming.go +++ b/metrics/naming.go @@ -5,6 +5,7 @@ const domainPrefix = "bux_" const ( verifyMerkleRootsHistogramName = domainPrefix + "verify_merkle_roots_histogram" recordTransactionHistogramName = domainPrefix + "record_transaction_histogram" + queryTransactionHistogramName = domainPrefix + "query_transaction_histogram" ) const xpubGaugeName = domainPrefix + "xpub_gauge" From 1b286c1b0018d8a7f69940a0f63b2687c1a7454f Mon Sep 17 00:00:00 2001 From: Krzysztof Tomecki <152964795+chris-4chain@users.noreply.github.com> Date: Mon, 12 Feb 2024 07:03:31 +0100 Subject: [PATCH 3/6] feat(BUX-567): track metrics of cronJobs --- cron_job_declarations.go | 61 +++++++++++++++++++++++++--------------- metrics/interface.go | 5 ++++ metrics/metrics.go | 13 +++++++++ metrics/naming.go | 5 ++++ 4 files changed, 62 insertions(+), 22 deletions(-) diff --git a/cron_job_declarations.go b/cron_job_declarations.go index 0672818d..acea93b7 100644 --- a/cron_job_declarations.go +++ b/cron_job_declarations.go @@ -19,33 +19,50 @@ type cronJobHandler func(ctx context.Context, client *Client) error // here is where we define all the cron jobs for the client func (c *Client) cronJobs() taskmanager.CronJobs { - // handler adds the client pointer to the cronJobTask by using a closure - handler := func(cronJobTask cronJobHandler) taskmanager.CronJobHandler { - return func(ctx context.Context) error { - return cronJobTask(ctx, c) + jobs := taskmanager.CronJobs{} + + addJob := func(name string, period time.Duration, task cronJobHandler) { + // handler adds the client pointer to the cronJobTask by using a closure + handler := func(ctx context.Context) (err error) { + if metrics, enabled := c.Metrics(); enabled { + end := metrics.TrackCron(name) + defer func() { + success := err == nil + end(success) + }() + } + err = task(ctx, c) + return } - } - jobs := taskmanager.CronJobs{ - CronJobNameDraftTransactionCleanUp: { - Period: 60 * time.Second, - Handler: handler(taskCleanupDraftTransactions), - }, - CronJobNameSyncTransactionBroadcast: { - Period: 2 * time.Minute, - Handler: handler(taskBroadcastTransactions), - }, - CronJobNameSyncTransactionSync: { - Period: 5 * time.Minute, - Handler: handler(taskSyncTransactions), - }, + jobs[name] = taskmanager.CronJob{ + Handler: handler, + Period: period, + } } + addJob( + CronJobNameDraftTransactionCleanUp, + 60*time.Second, + taskCleanupDraftTransactions, + ) + addJob( + CronJobNameSyncTransactionBroadcast, + 2*time.Minute, + taskBroadcastTransactions, + ) + addJob( + CronJobNameSyncTransactionSync, + 5*time.Minute, + taskSyncTransactions, + ) + if _, enabled := c.Metrics(); enabled { - jobs[CronJobNameCalculateMetrics] = taskmanager.CronJob{ - Period: 15 * time.Second, - Handler: handler(taskCalculateMetrics), - } + addJob( + CronJobNameCalculateMetrics, + 15*time.Second, + taskCalculateMetrics, + ) } return jobs diff --git a/metrics/interface.go b/metrics/interface.go index 641d0997..4983a039 100644 --- a/metrics/interface.go +++ b/metrics/interface.go @@ -3,9 +3,14 @@ package metrics // Collector is an interface that is used to register metrics type Collector interface { RegisterGauge(name string) GaugeInterface + RegisterGaugeVec(name string, labels ...string) GaugeVecInterface RegisterHistogramVec(name string, labels ...string) HistogramVecInterface } +type GaugeVecInterface interface { + WithLabelValues(lvs ...string) GaugeInterface +} + // GaugeInterface is an interface that is used to track gauges of values type GaugeInterface interface { Set(value float64) diff --git a/metrics/metrics.go b/metrics/metrics.go index 55f8c38c..e87b71b0 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -12,6 +12,8 @@ type Metrics struct { verifyMerkleRoots HistogramVecInterface recordTransaction HistogramVecInterface queryTransaction HistogramVecInterface + cronHistogram HistogramVecInterface + cronLastExecution GaugeVecInterface } // NewMetrics is a constructor for the Metrics struct @@ -22,6 +24,8 @@ func NewMetrics(collector Collector) *Metrics { verifyMerkleRoots: collector.RegisterHistogramVec(verifyMerkleRootsHistogramName, "classification"), recordTransaction: collector.RegisterHistogramVec(recordTransactionHistogramName, "classification", "strategy"), queryTransaction: collector.RegisterHistogramVec(queryTransactionHistogramName, "classification"), + cronHistogram: collector.RegisterHistogramVec(cronHistogramName, "name"), + cronLastExecution: collector.RegisterGaugeVec(cronLastExecutionGaugeName, "name"), } } @@ -52,6 +56,15 @@ func (m *Metrics) TrackQueryTransaction() EndWithClassification { } } +// TrackCron is used to track the time it takes to execute a cron job +func (m *Metrics) TrackCron(name string) EndWithClassification { + start := time.Now() + m.cronLastExecution.WithLabelValues(name).Set(float64(start.Unix())) + return func(success bool) { + m.cronHistogram.WithLabelValues(name).Observe(time.Since(start).Seconds()) + } +} + func classify(success bool) string { if success { return "success" diff --git a/metrics/naming.go b/metrics/naming.go index ca351bfd..3d1f74a9 100644 --- a/metrics/naming.go +++ b/metrics/naming.go @@ -8,4 +8,9 @@ const ( queryTransactionHistogramName = domainPrefix + "query_transaction_histogram" ) +const ( + cronHistogramName = domainPrefix + "cron_histogram" + cronLastExecutionGaugeName = domainPrefix + "cron_last_execution_gauge" +) + const xpubGaugeName = domainPrefix + "xpub_gauge" From 0c9e6b709ee040ad1f313b250de7fc0e13542374 Mon Sep 17 00:00:00 2001 From: Krzysztof Tomecki <152964795+chris-4chain@users.noreply.github.com> Date: Mon, 12 Feb 2024 07:03:31 +0100 Subject: [PATCH 4/6] feat(BUX-567): collecting stats in a cronJob --- cron_job_definitions.go | 46 +++++++++++++++++++++++++++++++++++++++-- metrics/naming.go | 10 ++++++++- metrics/stats.go | 16 ++++++++++++-- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/cron_job_definitions.go b/cron_job_definitions.go index cce97437..84a52572 100644 --- a/cron_job_definitions.go +++ b/cron_job_definitions.go @@ -87,7 +87,7 @@ func taskSyncTransactions(ctx context.Context, client *Client) error { } func taskCalculateMetrics(ctx context.Context, client *Client) error { - metrics, enabled := client.Metrics() + m, enabled := client.Metrics() if !enabled { return errors.New("metrics are not enabled") } @@ -97,7 +97,49 @@ func taskCalculateMetrics(ctx context.Context, client *Client) error { if xpubsCount, err := getXPubsCount(ctx, nil, nil, modelOpts...); err != nil { client.options.logger.Error().Err(err).Msg("error getting xpubs count") } else { - metrics.Stats.XPub.Set(float64(xpubsCount)) + m.Stats.XPub.Set(float64(xpubsCount)) + } + + if utxosCount, err := getUtxosCount(ctx, nil, nil, modelOpts...); err != nil { + client.options.logger.Error().Err(err).Msg("error getting utxos count") + } else { + m.Stats.Utxo.Set(float64(utxosCount)) + } + + if paymailsCount, err := getPaymailAddressesCount(ctx, nil, nil, modelOpts...); err != nil { + client.options.logger.Error().Err(err).Msg("error getting paymails count") + } else { + m.Stats.Paymail.Set(float64(paymailsCount)) + } + + if destinationsCount, err := getDestinationsCount(ctx, nil, nil, modelOpts...); err != nil { + client.options.logger.Error().Err(err).Msg("error getting destinations count") + } else { + m.Stats.Destination.Set(float64(destinationsCount)) + } + + if accessKeysCount, err := getAccessKeysCount(ctx, nil, nil, modelOpts...); err != nil { + client.options.logger.Error().Err(err).Msg("error getting access keys count") + } else { + m.Stats.AccessKey.Set(float64(accessKeysCount)) + } + + inTransactionsFilter := map[string]interface{}{ + "direction": TransactionDirectionIn, + } + if transactionsCount, err := getTransactionsCount(ctx, nil, &inTransactionsFilter, modelOpts...); err != nil { + client.options.logger.Error().Err(err).Msg("error getting transactions count") + } else { + m.Stats.TransactionIn.Set(float64(transactionsCount)) + } + + outTransactionsFilter := map[string]interface{}{ + "direction": TransactionDirectionOut, + } + if transactionsCount, err := getTransactionsCount(ctx, nil, &outTransactionsFilter, modelOpts...); err != nil { + client.options.logger.Error().Err(err).Msg("error getting transactions count") + } else { + m.Stats.TransactionOut.Set(float64(transactionsCount)) } return nil diff --git a/metrics/naming.go b/metrics/naming.go index 3d1f74a9..32b60fbd 100644 --- a/metrics/naming.go +++ b/metrics/naming.go @@ -13,4 +13,12 @@ const ( cronLastExecutionGaugeName = domainPrefix + "cron_last_execution_gauge" ) -const xpubGaugeName = domainPrefix + "xpub_gauge" +const ( + xpubGaugeName = domainPrefix + "xpub_gauge" + utxoGaugeName = domainPrefix + "utxo_gauge" + transactionInGaugeName = domainPrefix + "transaction_in_gauge" + transactionOutGaugeName = domainPrefix + "transaction_out_gauge" + paymailGaugeName = domainPrefix + "paymail_gauge" + destinationGaugeName = domainPrefix + "destination_gauge" + accessKeyGaugeName = domainPrefix + "access_key_gauge" +) diff --git a/metrics/stats.go b/metrics/stats.go index 0dd11780..7d09432f 100644 --- a/metrics/stats.go +++ b/metrics/stats.go @@ -2,11 +2,23 @@ package metrics // Stats is a struct that contains all the gauges that are used to track the calculated stats of the application type Stats struct { - XPub GaugeInterface + XPub GaugeInterface + Utxo GaugeInterface + TransactionIn GaugeInterface + TransactionOut GaugeInterface + Paymail GaugeInterface + Destination GaugeInterface + AccessKey GaugeInterface } func registerStats(collector Collector) Stats { return Stats{ - XPub: collector.RegisterGauge(xpubGaugeName), + XPub: collector.RegisterGauge(xpubGaugeName), + Utxo: collector.RegisterGauge(utxoGaugeName), + TransactionIn: collector.RegisterGauge(transactionInGaugeName), + TransactionOut: collector.RegisterGauge(transactionOutGaugeName), + Paymail: collector.RegisterGauge(paymailGaugeName), + Destination: collector.RegisterGauge(destinationGaugeName), + AccessKey: collector.RegisterGauge(accessKeyGaugeName), } } From 263fc956d530f9823b016e4af93fdaf6c16856f1 Mon Sep 17 00:00:00 2001 From: Krzysztof Tomecki <152964795+chris-4chain@users.noreply.github.com> Date: Mon, 12 Feb 2024 07:03:31 +0100 Subject: [PATCH 5/6] feat(BUX-567): prometheus instead of plain interfaces --- cron_job_definitions.go | 18 ------------------ go.mod | 9 +++++++++ go.sum | 17 +++++++++++++++++ metrics/interface.go | 27 +++++---------------------- metrics/metrics.go | 16 ++++++++++------ metrics/stats.go | 26 ++++++++++++-------------- 6 files changed, 53 insertions(+), 60 deletions(-) diff --git a/cron_job_definitions.go b/cron_job_definitions.go index 84a52572..50543044 100644 --- a/cron_job_definitions.go +++ b/cron_job_definitions.go @@ -124,23 +124,5 @@ func taskCalculateMetrics(ctx context.Context, client *Client) error { m.Stats.AccessKey.Set(float64(accessKeysCount)) } - inTransactionsFilter := map[string]interface{}{ - "direction": TransactionDirectionIn, - } - if transactionsCount, err := getTransactionsCount(ctx, nil, &inTransactionsFilter, modelOpts...); err != nil { - client.options.logger.Error().Err(err).Msg("error getting transactions count") - } else { - m.Stats.TransactionIn.Set(float64(transactionsCount)) - } - - outTransactionsFilter := map[string]interface{}{ - "direction": TransactionDirectionOut, - } - if transactionsCount, err := getTransactionsCount(ctx, nil, &outTransactionsFilter, modelOpts...); err != nil { - client.options.logger.Error().Err(err).Msg("error getting transactions count") - } else { - m.Stats.TransactionOut.Set(float64(transactionsCount)) - } - return nil } diff --git a/go.mod b/go.mod index a3be03a9..ce65aed8 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,14 @@ require ( gorm.io/gorm v1.25.5 ) +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect +) + require ( github.com/99designs/gqlgen v0.17.42 // indirect github.com/acobaugh/osrelease v0.1.0 // indirect @@ -96,6 +104,7 @@ require ( github.com/newrelic/go-agent/v3/integrations/nrmongo v1.1.2 // indirect github.com/onsi/gomega v1.27.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.18.0 github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/encoding v0.4.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect diff --git a/go.sum b/go.sum index af50269c..9221c018 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/aws/aws-sdk-go v1.43.45 h1:2708Bj4uV+ym62MOtBnErm/CDX61C4mFe9V2gXy1caE= github.com/aws/aws-sdk-go v1.43.45/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bitcoin-sv/go-broadcast-client v0.16.0 h1:KadOLv+i9Y6xAOkHsSl2PIECQ59SpUyYurY6Ysvpz5A= github.com/bitcoin-sv/go-broadcast-client v0.16.0/go.mod h1:GRAliwumNBjEbLRIEkXqIKJpsgmMfjvlIDqgyw/NoJE= github.com/bitcoin-sv/go-broadcast-client v0.16.1 h1:VG4QZwJEVQY/QQupTDeLMw+PEeqh9mn4id+XzYpAmHs= @@ -92,6 +94,7 @@ github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gocraft/dbr/v2 v2.7.6 h1:ASHKFgCbTLODbb9f756Cl8VAlnvQLKqIzx9E1Cfb7eo= github.com/gocraft/dbr/v2 v2.7.6/go.mod h1:8IH98S8M8J0JSEiYk0MPH26ZDUKemiQ/GvmXL5jo+Uw= @@ -210,12 +213,17 @@ github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/goveralls v0.0.6/go.mod h1:h8b4ow6FxSPMQHF6o2ve3qsclnffZjYTNEKmLesRwqw= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/maxatome/go-testdeep v1.12.0 h1:Ql7Go8Tg0C1D/uMMX59LAoYK7LffeJQ6X2T04nTH68g= github.com/maxatome/go-testdeep v1.12.0/go.mod h1:lPZc/HAcJMP92l7yI6TRz1aZN5URwUBUAfUNvrclaNM= github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM= github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk= github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0= github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= @@ -227,6 +235,7 @@ github.com/mrz1836/go-datastore v0.5.9 h1:wNNUNCBCSddOieE5aM06GI5dN8JElbIQrzqEAx github.com/mrz1836/go-datastore v0.5.9/go.mod h1:tkc466oJtAPNxENZpfjlcerTrCy7kyCzJiVVgbIIguE= github.com/mrz1836/go-logger v0.3.2 h1:bjd23NwVaLWncXgXAyxAwWLQ02of0Ci3iJIZZEakkFU= github.com/mrz1836/go-logger v0.3.2/go.mod h1:8gWPdqxOAFNJOHDXS2ducgsokUOf0wWtAAM3LXPrhYo= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/newrelic/go-agent/v3 v3.29.0 h1:Bc1D3DoOkpJs6aIzhOjUp+yIKJ2RfZ+LMQemZOs9t9k= github.com/newrelic/go-agent/v3 v3.29.0/go.mod h1:9utrgxlSryNqRrTvII2XBL+0lpofXbqXApvVWPpbzUg= github.com/newrelic/go-agent/v3/integrations/nrhttprouter v1.0.2 h1:Y+bKuryqCg+TAvBkBaEKwak+Boy5OdosQNdDGFwyDLo= @@ -251,6 +260,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rafaeljusto/redigomock v2.4.0+incompatible h1:d7uo5MVINMxnRr20MxbgDkmZ8QRfevjOVgEa4n0OZyY= github.com/rafaeljusto/redigomock v2.4.0+incompatible/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= diff --git a/metrics/interface.go b/metrics/interface.go index 4983a039..c5e3aa9b 100644 --- a/metrics/interface.go +++ b/metrics/interface.go @@ -1,27 +1,10 @@ package metrics +import "github.com/prometheus/client_golang/prometheus" + // Collector is an interface that is used to register metrics type Collector interface { - RegisterGauge(name string) GaugeInterface - RegisterGaugeVec(name string, labels ...string) GaugeVecInterface - RegisterHistogramVec(name string, labels ...string) HistogramVecInterface -} - -type GaugeVecInterface interface { - WithLabelValues(lvs ...string) GaugeInterface -} - -// GaugeInterface is an interface that is used to track gauges of values -type GaugeInterface interface { - Set(value float64) -} - -// HistogramVecInterface is an interface that is used to register histograms with labels -type HistogramVecInterface interface { - WithLabelValues(lvs ...string) HistogramInterface -} - -// HistogramInterface is an interface that is used to track histograms of values -type HistogramInterface interface { - Observe(value float64) + RegisterGauge(name string) prometheus.Gauge + RegisterGaugeVec(name string, labels ...string) *prometheus.GaugeVec + RegisterHistogramVec(name string, labels ...string) *prometheus.HistogramVec } diff --git a/metrics/metrics.go b/metrics/metrics.go index e87b71b0..827a745a 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -3,17 +3,21 @@ Package metrics provides a way to track metrics in the application. Functionalit */ package metrics -import "time" +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) // Metrics is a struct that contains all the metrics that are used to track in the package type Metrics struct { collector Collector Stats Stats - verifyMerkleRoots HistogramVecInterface - recordTransaction HistogramVecInterface - queryTransaction HistogramVecInterface - cronHistogram HistogramVecInterface - cronLastExecution GaugeVecInterface + verifyMerkleRoots *prometheus.HistogramVec + recordTransaction *prometheus.HistogramVec + queryTransaction *prometheus.HistogramVec + cronHistogram *prometheus.HistogramVec + cronLastExecution *prometheus.GaugeVec } // NewMetrics is a constructor for the Metrics struct diff --git a/metrics/stats.go b/metrics/stats.go index 7d09432f..3068ffb8 100644 --- a/metrics/stats.go +++ b/metrics/stats.go @@ -1,24 +1,22 @@ package metrics +import "github.com/prometheus/client_golang/prometheus" + // Stats is a struct that contains all the gauges that are used to track the calculated stats of the application type Stats struct { - XPub GaugeInterface - Utxo GaugeInterface - TransactionIn GaugeInterface - TransactionOut GaugeInterface - Paymail GaugeInterface - Destination GaugeInterface - AccessKey GaugeInterface + XPub prometheus.Gauge + Utxo prometheus.Gauge + Paymail prometheus.Gauge + Destination prometheus.Gauge + AccessKey prometheus.Gauge } func registerStats(collector Collector) Stats { return Stats{ - XPub: collector.RegisterGauge(xpubGaugeName), - Utxo: collector.RegisterGauge(utxoGaugeName), - TransactionIn: collector.RegisterGauge(transactionInGaugeName), - TransactionOut: collector.RegisterGauge(transactionOutGaugeName), - Paymail: collector.RegisterGauge(paymailGaugeName), - Destination: collector.RegisterGauge(destinationGaugeName), - AccessKey: collector.RegisterGauge(accessKeyGaugeName), + XPub: collector.RegisterGauge(xpubGaugeName), + Utxo: collector.RegisterGauge(utxoGaugeName), + Paymail: collector.RegisterGauge(paymailGaugeName), + Destination: collector.RegisterGauge(destinationGaugeName), + AccessKey: collector.RegisterGauge(accessKeyGaugeName), } } From 36a09b80059fa0955910896ffd13276e234f4fb5 Mon Sep 17 00:00:00 2001 From: Krzysztof Tomecki <152964795+chris-4chain@users.noreply.github.com> Date: Mon, 12 Feb 2024 07:03:32 +0100 Subject: [PATCH 6/6] feat(BUX-467): adjustments after with-grafana tests --- cron_job_definitions.go | 10 +++--- metrics/metrics.go | 17 +++++++--- metrics/naming.go | 8 +---- metrics/stats.go | 36 ++++++++++++---------- record_tx.go | 4 +-- record_tx_strategy_external_incoming_tx.go | 4 +++ record_tx_strategy_internal_incoming_tx.go | 4 +++ record_tx_strategy_outgoing_tx.go | 6 ++-- 8 files changed, 52 insertions(+), 37 deletions(-) diff --git a/cron_job_definitions.go b/cron_job_definitions.go index 50543044..6674adbd 100644 --- a/cron_job_definitions.go +++ b/cron_job_definitions.go @@ -97,31 +97,31 @@ func taskCalculateMetrics(ctx context.Context, client *Client) error { if xpubsCount, err := getXPubsCount(ctx, nil, nil, modelOpts...); err != nil { client.options.logger.Error().Err(err).Msg("error getting xpubs count") } else { - m.Stats.XPub.Set(float64(xpubsCount)) + m.SetXPubCount(xpubsCount) } if utxosCount, err := getUtxosCount(ctx, nil, nil, modelOpts...); err != nil { client.options.logger.Error().Err(err).Msg("error getting utxos count") } else { - m.Stats.Utxo.Set(float64(utxosCount)) + m.SetUtxoCount(utxosCount) } if paymailsCount, err := getPaymailAddressesCount(ctx, nil, nil, modelOpts...); err != nil { client.options.logger.Error().Err(err).Msg("error getting paymails count") } else { - m.Stats.Paymail.Set(float64(paymailsCount)) + m.SetPaymailCount(paymailsCount) } if destinationsCount, err := getDestinationsCount(ctx, nil, nil, modelOpts...); err != nil { client.options.logger.Error().Err(err).Msg("error getting destinations count") } else { - m.Stats.Destination.Set(float64(destinationsCount)) + m.SetDestinationCount(destinationsCount) } if accessKeysCount, err := getAccessKeysCount(ctx, nil, nil, modelOpts...); err != nil { client.options.logger.Error().Err(err).Msg("error getting access keys count") } else { - m.Stats.AccessKey.Set(float64(accessKeysCount)) + m.SetAccessKeyCount(accessKeysCount) } return nil diff --git a/metrics/metrics.go b/metrics/metrics.go index 827a745a..8f8a4831 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -11,11 +11,18 @@ import ( // Metrics is a struct that contains all the metrics that are used to track in the package type Metrics struct { - collector Collector - Stats Stats + // collector is used to register the metrics + collector Collector + + // Stats contains all the gauges that track the db-calculated stats + stats *prometheus.GaugeVec + + // the histograms that track the time it takes to perform certain operations verifyMerkleRoots *prometheus.HistogramVec recordTransaction *prometheus.HistogramVec queryTransaction *prometheus.HistogramVec + + // each cronJob is observed by the duration it takes to execute and the last time it was executed cronHistogram *prometheus.HistogramVec cronLastExecution *prometheus.GaugeVec } @@ -24,7 +31,7 @@ type Metrics struct { func NewMetrics(collector Collector) *Metrics { return &Metrics{ collector: collector, - Stats: registerStats(collector), + stats: collector.RegisterGaugeVec(statsGaugeName, "name"), verifyMerkleRoots: collector.RegisterHistogramVec(verifyMerkleRootsHistogramName, "classification"), recordTransaction: collector.RegisterHistogramVec(recordTransactionHistogramName, "classification", "strategy"), queryTransaction: collector.RegisterHistogramVec(queryTransactionHistogramName, "classification"), @@ -48,7 +55,7 @@ func (m *Metrics) TrackVerifyMerkleRoots() EndWithClassification { func (m *Metrics) TrackRecordTransaction(strategyName string) EndWithClassification { start := time.Now() return func(success bool) { - m.verifyMerkleRoots.WithLabelValues(classify(success), strategyName).Observe(time.Since(start).Seconds()) + m.recordTransaction.WithLabelValues(classify(success), strategyName).Observe(time.Since(start).Seconds()) } } @@ -56,7 +63,7 @@ func (m *Metrics) TrackRecordTransaction(strategyName string) EndWithClassificat func (m *Metrics) TrackQueryTransaction() EndWithClassification { start := time.Now() return func(success bool) { - m.verifyMerkleRoots.WithLabelValues(classify(success)).Observe(time.Since(start).Seconds()) + m.queryTransaction.WithLabelValues(classify(success)).Observe(time.Since(start).Seconds()) } } diff --git a/metrics/naming.go b/metrics/naming.go index 32b60fbd..aa40f37d 100644 --- a/metrics/naming.go +++ b/metrics/naming.go @@ -14,11 +14,5 @@ const ( ) const ( - xpubGaugeName = domainPrefix + "xpub_gauge" - utxoGaugeName = domainPrefix + "utxo_gauge" - transactionInGaugeName = domainPrefix + "transaction_in_gauge" - transactionOutGaugeName = domainPrefix + "transaction_out_gauge" - paymailGaugeName = domainPrefix + "paymail_gauge" - destinationGaugeName = domainPrefix + "destination_gauge" - accessKeyGaugeName = domainPrefix + "access_key_gauge" + statsGaugeName = domainPrefix + "stats_total" ) diff --git a/metrics/stats.go b/metrics/stats.go index 3068ffb8..7ed7b89e 100644 --- a/metrics/stats.go +++ b/metrics/stats.go @@ -1,22 +1,26 @@ package metrics -import "github.com/prometheus/client_golang/prometheus" +// SetXPubCount adds a value to the stats gauge with the label "xpub" +func (m *Metrics) SetXPubCount(value int64) { + m.stats.WithLabelValues("xpub").Set(float64(value)) +} + +// SetUtxoCount adds a value to the stats gauge with the label "utxo" +func (m *Metrics) SetUtxoCount(value int64) { + m.stats.WithLabelValues("utxo").Set(float64(value)) +} + +// SetPaymailCount adds a value to the stats gauge with the label "paymail" +func (m *Metrics) SetPaymailCount(value int64) { + m.stats.WithLabelValues("paymail").Set(float64(value)) +} -// Stats is a struct that contains all the gauges that are used to track the calculated stats of the application -type Stats struct { - XPub prometheus.Gauge - Utxo prometheus.Gauge - Paymail prometheus.Gauge - Destination prometheus.Gauge - AccessKey prometheus.Gauge +// SetDestinationCount adds a value to the stats gauge with the label "destination" +func (m *Metrics) SetDestinationCount(value int64) { + m.stats.WithLabelValues("destination").Set(float64(value)) } -func registerStats(collector Collector) Stats { - return Stats{ - XPub: collector.RegisterGauge(xpubGaugeName), - Utxo: collector.RegisterGauge(utxoGaugeName), - Paymail: collector.RegisterGauge(paymailGaugeName), - Destination: collector.RegisterGauge(destinationGaugeName), - AccessKey: collector.RegisterGauge(accessKeyGaugeName), - } +// SetAccessKeyCount adds a value to the stats gauge with the label "access_key +func (m *Metrics) SetAccessKeyCount(value int64) { + m.stats.WithLabelValues("access_key").Set(float64(value)) } diff --git a/record_tx.go b/record_tx.go index c6d629ec..9712b4ff 100644 --- a/record_tx.go +++ b/record_tx.go @@ -7,6 +7,7 @@ import ( ) type recordTxStrategy interface { + Name() string TxID() string LockKey() string Validate() error @@ -21,8 +22,7 @@ type recordIncomingTxStrategy interface { func recordTransaction(ctx context.Context, c ClientInterface, strategy recordTxStrategy, opts ...ModelOps) (transaction *Transaction, err error) { if metrics, enabled := c.Metrics(); enabled { - strategyType := fmt.Sprintf("%T", strategy) - end := metrics.TrackRecordTransaction(strategyType) + end := metrics.TrackRecordTransaction(strategy.Name()) defer func() { success := err == nil end(success) diff --git a/record_tx_strategy_external_incoming_tx.go b/record_tx_strategy_external_incoming_tx.go index 758f69bf..36fa1be1 100644 --- a/record_tx_strategy_external_incoming_tx.go +++ b/record_tx_strategy_external_incoming_tx.go @@ -14,6 +14,10 @@ type externalIncomingTx struct { allowBroadcastErrors bool // only BEEF cannot allow for broadcast errors } +func (strategy *externalIncomingTx) Name() string { + return "external_incoming_tx" +} + func (strategy *externalIncomingTx) Execute(ctx context.Context, c ClientInterface, opts []ModelOps) (*Transaction, error) { logger := c.Logger() transaction, err := _createExternalTxToRecord(ctx, strategy, c, opts) diff --git a/record_tx_strategy_internal_incoming_tx.go b/record_tx_strategy_internal_incoming_tx.go index 20ecf12d..7e8d69fe 100644 --- a/record_tx_strategy_internal_incoming_tx.go +++ b/record_tx_strategy_internal_incoming_tx.go @@ -14,6 +14,10 @@ type internalIncomingTx struct { allowBroadcastErrors bool // only BEEF cannot allow for broadcast errors } +func (strategy *internalIncomingTx) Name() string { + return "internal_incoming_tx" +} + func (strategy *internalIncomingTx) Execute(ctx context.Context, c ClientInterface, _ []ModelOps) (*Transaction, error) { logger := c.Logger() logger.Info(). diff --git a/record_tx_strategy_outgoing_tx.go b/record_tx_strategy_outgoing_tx.go index edc7465a..67049c8d 100644 --- a/record_tx_strategy_outgoing_tx.go +++ b/record_tx_strategy_outgoing_tx.go @@ -15,6 +15,10 @@ type outgoingTx struct { XPubKey string } +func (strategy *outgoingTx) Name() string { + return "outgoing_tx" +} + func (strategy *outgoingTx) Execute(ctx context.Context, c ClientInterface, opts []ModelOps) (*Transaction, error) { logger := c.Logger() logger.Info(). @@ -96,7 +100,6 @@ func _createOutgoingTxToRecord(ctx context.Context, oTx *outgoingTx, c ClientInt tx, err := newTransactionWithDraftID( oTx.Hex, oTx.RelatedDraftID, newOpts..., ) - if err != nil { return nil, err } @@ -117,7 +120,6 @@ func _createOutgoingTxToRecord(ctx context.Context, oTx *outgoingTx, c ClientInt func _hydrateOutgoingWithDraft(ctx context.Context, tx *Transaction) error { draft, err := getDraftTransactionID(ctx, tx.XPubID, tx.DraftID, tx.GetOptions(false)...) - if err != nil { return err }