From 74e1161e0cc3d5d69ef9dfa049a285cc39696edd Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 8 Jan 2025 15:06:13 -0500 Subject: [PATCH 1/9] Add NewHeadsPollInterval --- multinode/config/config.go | 81 ++++---------------------------------- 1 file changed, 8 insertions(+), 73 deletions(-) diff --git a/multinode/config/config.go b/multinode/config/config.go index b56eb57..73555e0 100644 --- a/multinode/config/config.go +++ b/multinode/config/config.go @@ -4,8 +4,6 @@ import ( "time" "github.com/smartcontractkit/chainlink-common/pkg/config" - - "github.com/smartcontractkit/chainlink-framework/multinode" ) // MultiNodeConfig is a wrapper to provide required functions while keeping configs Public @@ -24,6 +22,7 @@ type MultiNode struct { SyncThreshold *uint32 NodeIsSyncingEnabled *bool LeaseDuration *config.Duration + NewHeadsPollInterval *config.Duration FinalizedBlockPollInterval *config.Duration EnforceRepeatableRead *bool DeathDeclarationDelay *config.Duration @@ -62,6 +61,10 @@ func (c *MultiNodeConfig) NodeIsSyncingEnabled() bool { func (c *MultiNodeConfig) LeaseDuration() time.Duration { return c.MultiNode.LeaseDuration.Duration() } +func (c *MultiNodeConfig) NewHeadsPollInterval() time.Duration { + return c.MultiNode.NewHeadsPollInterval.Duration() +} + func (c *MultiNodeConfig) FinalizedBlockPollInterval() time.Duration { return c.MultiNode.FinalizedBlockPollInterval.Duration() } @@ -86,73 +89,6 @@ func (c *MultiNodeConfig) FinalityTagEnabled() bool { return *c.MultiNode.Finali func (c *MultiNodeConfig) FinalizedBlockOffset() uint32 { return *c.MultiNode.FinalizedBlockOffset } -func (c *MultiNodeConfig) SetDefaults() { - // MultiNode is disabled as it's not fully implemented yet: BCFR-122 - if c.MultiNode.Enabled == nil { - c.MultiNode.Enabled = ptr(false) - } - - /* Node Configs */ - // Failure threshold for polling set to 5 to tolerate some polling failures before taking action. - if c.MultiNode.PollFailureThreshold == nil { - c.MultiNode.PollFailureThreshold = ptr(uint32(5)) - } - // Poll interval is set to 15 seconds to ensure timely updates while minimizing resource usage. - if c.MultiNode.PollInterval == nil { - c.MultiNode.PollInterval = config.MustNewDuration(15 * time.Second) - } - // Selection mode defaults to priority level to enable using node priorities - if c.MultiNode.SelectionMode == nil { - c.MultiNode.SelectionMode = ptr(multinode.NodeSelectionModePriorityLevel) - } - // The sync threshold is set to 10 to allow for some flexibility in node synchronization before considering it out of sync. - if c.MultiNode.SyncThreshold == nil { - c.MultiNode.SyncThreshold = ptr(uint32(10)) - } - // Lease duration is set to 1 minute by default to allow node locks for a reasonable amount of time. - if c.MultiNode.LeaseDuration == nil { - c.MultiNode.LeaseDuration = config.MustNewDuration(time.Minute) - } - // Node syncing is not relevant for Solana and is disabled by default. - if c.MultiNode.NodeIsSyncingEnabled == nil { - c.MultiNode.NodeIsSyncingEnabled = ptr(false) - } - // The finalized block polling interval is set to 5 seconds to ensure timely updates while minimizing resource usage. - if c.MultiNode.FinalizedBlockPollInterval == nil { - c.MultiNode.FinalizedBlockPollInterval = config.MustNewDuration(5 * time.Second) - } - // Repeatable read guarantee should be enforced by default. - if c.MultiNode.EnforceRepeatableRead == nil { - c.MultiNode.EnforceRepeatableRead = ptr(true) - } - // The delay before declaring a node dead is set to 20 seconds to give nodes time to recover from temporary issues. - if c.MultiNode.DeathDeclarationDelay == nil { - c.MultiNode.DeathDeclarationDelay = config.MustNewDuration(20 * time.Second) - } - - /* Chain Configs */ - // Threshold for no new heads is set to 20 seconds, assuming that heads should update at a reasonable pace. - if c.MultiNode.NodeNoNewHeadsThreshold == nil { - c.MultiNode.NodeNoNewHeadsThreshold = config.MustNewDuration(20 * time.Second) - } - // Similar to heads, finalized heads should be updated within 20 seconds. - if c.MultiNode.NoNewFinalizedHeadsThreshold == nil { - c.MultiNode.NoNewFinalizedHeadsThreshold = config.MustNewDuration(20 * time.Second) - } - // Finality tags are used in Solana and enabled by default. - if c.MultiNode.FinalityTagEnabled == nil { - c.MultiNode.FinalityTagEnabled = ptr(true) - } - // Finality depth will not be used since finality tags are enabled. - if c.MultiNode.FinalityDepth == nil { - c.MultiNode.FinalityDepth = ptr(uint32(0)) - } - // Finalized block offset allows for RPCs to be slightly behind the finalized block. - if c.MultiNode.FinalizedBlockOffset == nil { - c.MultiNode.FinalizedBlockOffset = ptr(uint32(50)) - } -} - func (c *MultiNodeConfig) SetFrom(f *MultiNodeConfig) { if f.MultiNode.Enabled != nil { c.MultiNode.Enabled = f.MultiNode.Enabled @@ -177,6 +113,9 @@ func (c *MultiNodeConfig) SetFrom(f *MultiNodeConfig) { if f.MultiNode.LeaseDuration != nil { c.MultiNode.LeaseDuration = f.MultiNode.LeaseDuration } + if f.MultiNode.NewHeadsPollInterval != nil { + c.MultiNode.NewHeadsPollInterval = f.MultiNode.NewHeadsPollInterval + } if f.MultiNode.FinalizedBlockPollInterval != nil { c.MultiNode.FinalizedBlockPollInterval = f.MultiNode.FinalizedBlockPollInterval } @@ -204,7 +143,3 @@ func (c *MultiNodeConfig) SetFrom(f *MultiNodeConfig) { c.MultiNode.FinalizedBlockOffset = f.MultiNode.FinalizedBlockOffset } } - -func ptr[T any](t T) *T { - return &t -} From 504635357ecdfe83932cdc4beb35e483a952a8c8 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 8 Jan 2025 15:45:03 -0500 Subject: [PATCH 2/9] Initialize metrics once --- multinode/multi_node.go | 8 ++++++-- multinode/node.go | 8 +++++++- multinode/node_fsm.go | 12 +++++++++++- multinode/node_lifecycle.go | 11 ++++++++++- multinode/transaction_sender.go | 6 +++++- 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/multinode/multi_node.go b/multinode/multi_node.go index 019c678..135a955 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -16,12 +16,16 @@ import ( var ( // PromMultiNodeRPCNodeStates reports current RPC node state + PromMultiNodeRPCNodeStates *prometheus.GaugeVec + ErrNodeError = fmt.Errorf("no live nodes available") +) + +func init() { PromMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "multi_node_states", Help: "The number of RPC nodes currently in the given state for the given chain", }, []string{"network", "chainId", "state"}) - ErrNodeError = fmt.Errorf("no live nodes available") -) +} // MultiNode is a generalized multi node client interface that includes methods to interact with different chains. // It also handles multiple node RPC connections simultaneously. diff --git a/multinode/node.go b/multinode/node.go index f1b0aa2..415c04c 100644 --- a/multinode/node.go +++ b/multinode/node.go @@ -20,6 +20,12 @@ const QueryTimeout = 10 * time.Second var errInvalidChainID = errors.New("invalid chain id") var ( + promPoolRPCNodeVerifies *prometheus.CounterVec + promPoolRPCNodeVerifiesFailed *prometheus.CounterVec + promPoolRPCNodeVerifiesSuccess *prometheus.CounterVec +) + +func init() { promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pool_rpc_node_verifies", Help: "The total number of chain ID verifications for the given RPC node", @@ -32,7 +38,7 @@ var ( Name: "pool_rpc_node_verifies_success", Help: "The total number of successful chain ID verifications for the given RPC node", }, []string{"network", "chainID", "nodeName"}) -) +} type NodeConfig interface { PollFailureThreshold() uint32 diff --git a/multinode/node_fsm.go b/multinode/node_fsm.go index c10a5f8..ea4a7e0 100644 --- a/multinode/node_fsm.go +++ b/multinode/node_fsm.go @@ -8,6 +8,16 @@ import ( ) var ( + promPoolRPCNodeTransitionsToAlive *prometheus.CounterVec + promPoolRPCNodeTransitionsToInSync *prometheus.CounterVec + promPoolRPCNodeTransitionsToOutOfSync *prometheus.CounterVec + promPoolRPCNodeTransitionsToUnreachable *prometheus.CounterVec + promPoolRPCNodeTransitionsToInvalidChainID *prometheus.CounterVec + promPoolRPCNodeTransitionsToUnusable *prometheus.CounterVec + promPoolRPCNodeTransitionsToSyncing *prometheus.CounterVec +) + +func init() { promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "pool_rpc_node_num_transitions_to_alive", Help: transitionString(nodeStateAlive), @@ -36,7 +46,7 @@ var ( Name: "pool_rpc_node_num_transitions_to_syncing", Help: transitionString(nodeStateSyncing), }, []string{"chainID", "nodeName"}) -) +} // nodeState represents the current state of the node // Node is a FSM (finite state machine) diff --git a/multinode/node_lifecycle.go b/multinode/node_lifecycle.go index ae64851..55b7e90 100644 --- a/multinode/node_lifecycle.go +++ b/multinode/node_lifecycle.go @@ -16,6 +16,15 @@ import ( ) var ( + promPoolRPCNodeHighestSeenBlock *prometheus.GaugeVec + promPoolRPCNodeHighestFinalizedBlock *prometheus.GaugeVec + promPoolRPCNodeNumSeenBlocks *prometheus.CounterVec + promPoolRPCNodePolls *prometheus.CounterVec + promPoolRPCNodePollsFailed *prometheus.CounterVec + promPoolRPCNodePollsSuccess *prometheus.CounterVec +) + +func init() { promPoolRPCNodeHighestSeenBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "pool_rpc_node_highest_seen_block", Help: "The highest seen block for the given RPC node", @@ -40,7 +49,7 @@ var ( Name: "pool_rpc_node_polls_success", Help: "The total number of successful poll checks for the given RPC node", }, []string{"chainID", "nodeName"}) -) +} // zombieNodeCheckInterval controls how often to re-check to see if we need to // state change in case we have to force a state transition due to no available diff --git a/multinode/transaction_sender.go b/multinode/transaction_sender.go index f04c6df..dc6df43 100644 --- a/multinode/transaction_sender.go +++ b/multinode/transaction_sender.go @@ -17,11 +17,15 @@ import ( var ( // PromMultiNodeInvariantViolations reports violation of our assumptions + PromMultiNodeInvariantViolations *prometheus.CounterVec +) + +func init() { PromMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "multi_node_invariant_violations", Help: "The number of invariant violations", }, []string{"network", "chainId", "invariant"}) -) +} type SendTxResult interface { Code() SendTxReturnCode From 8e2d56c9cdfaf10f9819f8987c8e0c922cd69349 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 8 Jan 2025 16:24:20 -0500 Subject: [PATCH 3/9] Create metrics file --- multinode/metrics.go | 98 +++++++++++++++++++++++++++++++++ multinode/multi_node.go | 9 --- multinode/node.go | 17 ------ multinode/node_fsm.go | 32 ----------- multinode/node_lifecycle.go | 29 ---------- multinode/transaction_sender.go | 9 --- 6 files changed, 98 insertions(+), 96 deletions(-) create mode 100644 multinode/metrics.go diff --git a/multinode/metrics.go b/multinode/metrics.go new file mode 100644 index 0000000..7cbebb1 --- /dev/null +++ b/multinode/metrics.go @@ -0,0 +1,98 @@ +package multinode + +import ( + "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "sync" +) + +var ( + metricsInit sync.Once +) + +func init() { + metricsInit.Do(func() { + // Node Metrics + promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies", + Help: "The total number of chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeVerifiesFailed = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies_failed", + Help: "The total number of failed chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeVerifiesSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies_success", + Help: "The total number of successful chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + + // Node FSM Metrics + promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_alive", + Help: transitionString(nodeStateAlive), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_in_sync", + Help: fmt.Sprintf("%s to %s", transitionString(nodeStateOutOfSync), nodeStateAlive), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_out_of_sync", + Help: transitionString(nodeStateOutOfSync), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_unreachable", + Help: transitionString(nodeStateUnreachable), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", + Help: transitionString(nodeStateInvalidChainID), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_unusable", + Help: transitionString(nodeStateUnusable), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_syncing", + Help: transitionString(nodeStateSyncing), + }, []string{"chainID", "nodeName"}) + + // Node Lifecycle Metrics + promPoolRPCNodeHighestSeenBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "pool_rpc_node_highest_seen_block", + Help: "The highest seen block for the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeHighestFinalizedBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "pool_rpc_node_highest_finalized_block", + Help: "The highest seen finalized block for the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeNumSeenBlocks = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_seen_blocks", + Help: "The total number of new blocks seen by the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodePolls = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_polls_total", + Help: "The total number of poll checks for the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodePollsFailed = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_polls_failed", + Help: "The total number of failed poll checks for the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodePollsSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_polls_success", + Help: "The total number of successful poll checks for the given RPC node", + }, []string{"chainID", "nodeName"}) + + // MultiNode Metrics + PromMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "multi_node_states", + Help: "The number of RPC nodes currently in the given state for the given chain", + }, []string{"network", "chainId", "state"}) + + // Transaction Sender Metrics + PromMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multi_node_invariant_violations", + Help: "The number of invariant violations", + }, []string{"network", "chainId", "invariant"}) + }) +} diff --git a/multinode/multi_node.go b/multinode/multi_node.go index 135a955..7696e2d 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -8,8 +8,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) @@ -20,13 +18,6 @@ var ( ErrNodeError = fmt.Errorf("no live nodes available") ) -func init() { - PromMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "multi_node_states", - Help: "The number of RPC nodes currently in the given state for the given chain", - }, []string{"network", "chainId", "state"}) -} - // MultiNode is a generalized multi node client interface that includes methods to interact with different chains. // It also handles multiple node RPC connections simultaneously. type MultiNode[ diff --git a/multinode/node.go b/multinode/node.go index 415c04c..ac221ce 100644 --- a/multinode/node.go +++ b/multinode/node.go @@ -9,8 +9,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) @@ -25,21 +23,6 @@ var ( promPoolRPCNodeVerifiesSuccess *prometheus.CounterVec ) -func init() { - promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies", - Help: "The total number of chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) - promPoolRPCNodeVerifiesFailed = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies_failed", - Help: "The total number of failed chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) - promPoolRPCNodeVerifiesSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies_success", - Help: "The total number of successful chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) -} - type NodeConfig interface { PollFailureThreshold() uint32 PollInterval() time.Duration diff --git a/multinode/node_fsm.go b/multinode/node_fsm.go index ea4a7e0..bc98e96 100644 --- a/multinode/node_fsm.go +++ b/multinode/node_fsm.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" ) var ( @@ -17,37 +16,6 @@ var ( promPoolRPCNodeTransitionsToSyncing *prometheus.CounterVec ) -func init() { - promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_alive", - Help: transitionString(nodeStateAlive), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_in_sync", - Help: fmt.Sprintf("%s to %s", transitionString(nodeStateOutOfSync), nodeStateAlive), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_out_of_sync", - Help: transitionString(nodeStateOutOfSync), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_unreachable", - Help: transitionString(nodeStateUnreachable), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", - Help: transitionString(nodeStateInvalidChainID), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_unusable", - Help: transitionString(nodeStateUnusable), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_syncing", - Help: transitionString(nodeStateSyncing), - }, []string{"chainID", "nodeName"}) -} - // nodeState represents the current state of the node // Node is a FSM (finite state machine) type nodeState int diff --git a/multinode/node_lifecycle.go b/multinode/node_lifecycle.go index 55b7e90..8cb31ea 100644 --- a/multinode/node_lifecycle.go +++ b/multinode/node_lifecycle.go @@ -8,8 +8,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/utils" bigmath "github.com/smartcontractkit/chainlink-common/pkg/utils/big_math" @@ -24,33 +22,6 @@ var ( promPoolRPCNodePollsSuccess *prometheus.CounterVec ) -func init() { - promPoolRPCNodeHighestSeenBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "pool_rpc_node_highest_seen_block", - Help: "The highest seen block for the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeHighestFinalizedBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "pool_rpc_node_highest_finalized_block", - Help: "The highest seen finalized block for the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeNumSeenBlocks = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_seen_blocks", - Help: "The total number of new blocks seen by the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodePolls = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_polls_total", - Help: "The total number of poll checks for the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodePollsFailed = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_polls_failed", - Help: "The total number of failed poll checks for the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodePollsSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_polls_success", - Help: "The total number of successful poll checks for the given RPC node", - }, []string{"chainID", "nodeName"}) -} - // zombieNodeCheckInterval controls how often to re-check to see if we need to // state change in case we have to force a state transition due to no available // nodes. diff --git a/multinode/transaction_sender.go b/multinode/transaction_sender.go index dc6df43..1189cf9 100644 --- a/multinode/transaction_sender.go +++ b/multinode/transaction_sender.go @@ -9,8 +9,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) @@ -20,13 +18,6 @@ var ( PromMultiNodeInvariantViolations *prometheus.CounterVec ) -func init() { - PromMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multi_node_invariant_violations", - Help: "The number of invariant violations", - }, []string{"network", "chainId", "invariant"}) -} - type SendTxResult interface { Code() SendTxReturnCode Error() error From 1ede19aceefe0eaf445149e9c02784606a790487 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 8 Jan 2025 17:13:33 -0500 Subject: [PATCH 4/9] Create metrics registry --- multinode/metrics.go | 98 --------------------------------- multinode/multi_node.go | 10 +++- multinode/node.go | 16 +++++- multinode/node_fsm.go | 36 +++++++++--- multinode/node_lifecycle.go | 31 +++++++++-- multinode/transaction_sender.go | 6 +- 6 files changed, 79 insertions(+), 118 deletions(-) delete mode 100644 multinode/metrics.go diff --git a/multinode/metrics.go b/multinode/metrics.go deleted file mode 100644 index 7cbebb1..0000000 --- a/multinode/metrics.go +++ /dev/null @@ -1,98 +0,0 @@ -package multinode - -import ( - "fmt" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "sync" -) - -var ( - metricsInit sync.Once -) - -func init() { - metricsInit.Do(func() { - // Node Metrics - promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies", - Help: "The total number of chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) - promPoolRPCNodeVerifiesFailed = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies_failed", - Help: "The total number of failed chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) - promPoolRPCNodeVerifiesSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies_success", - Help: "The total number of successful chain ID verifications for the given RPC node", - }, []string{"network", "chainID", "nodeName"}) - - // Node FSM Metrics - promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_alive", - Help: transitionString(nodeStateAlive), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_in_sync", - Help: fmt.Sprintf("%s to %s", transitionString(nodeStateOutOfSync), nodeStateAlive), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_out_of_sync", - Help: transitionString(nodeStateOutOfSync), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_unreachable", - Help: transitionString(nodeStateUnreachable), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", - Help: transitionString(nodeStateInvalidChainID), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_unusable", - Help: transitionString(nodeStateUnusable), - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_syncing", - Help: transitionString(nodeStateSyncing), - }, []string{"chainID", "nodeName"}) - - // Node Lifecycle Metrics - promPoolRPCNodeHighestSeenBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "pool_rpc_node_highest_seen_block", - Help: "The highest seen block for the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeHighestFinalizedBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "pool_rpc_node_highest_finalized_block", - Help: "The highest seen finalized block for the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodeNumSeenBlocks = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_seen_blocks", - Help: "The total number of new blocks seen by the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodePolls = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_polls_total", - Help: "The total number of poll checks for the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodePollsFailed = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_polls_failed", - Help: "The total number of failed poll checks for the given RPC node", - }, []string{"chainID", "nodeName"}) - promPoolRPCNodePollsSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_polls_success", - Help: "The total number of successful poll checks for the given RPC node", - }, []string{"chainID", "nodeName"}) - - // MultiNode Metrics - PromMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "multi_node_states", - Help: "The number of RPC nodes currently in the given state for the given chain", - }, []string{"network", "chainId", "state"}) - - // Transaction Sender Metrics - PromMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multi_node_invariant_violations", - Help: "The number of invariant violations", - }, []string{"network", "chainId", "invariant"}) - }) -} diff --git a/multinode/multi_node.go b/multinode/multi_node.go index 7696e2d..864a636 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -8,14 +8,18 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) var ( - // PromMultiNodeRPCNodeStates reports current RPC node state - PromMultiNodeRPCNodeStates *prometheus.GaugeVec - ErrNodeError = fmt.Errorf("no live nodes available") + metricsRegistry = prometheus.NewRegistry() + PromMultiNodeRPCNodeStates = promauto.With(metricsRegistry).NewGaugeVec(prometheus.GaugeOpts{ + Name: "multi_node_states", + Help: "The number of RPC nodes currently in the given state for the given chain", + }, []string{"network", "chainId", "state"}) + ErrNodeError = fmt.Errorf("no live nodes available") ) // MultiNode is a generalized multi node client interface that includes methods to interact with different chains. diff --git a/multinode/node.go b/multinode/node.go index ac221ce..9b48bf3 100644 --- a/multinode/node.go +++ b/multinode/node.go @@ -9,6 +9,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) @@ -18,9 +19,18 @@ const QueryTimeout = 10 * time.Second var errInvalidChainID = errors.New("invalid chain id") var ( - promPoolRPCNodeVerifies *prometheus.CounterVec - promPoolRPCNodeVerifiesFailed *prometheus.CounterVec - promPoolRPCNodeVerifiesSuccess *prometheus.CounterVec + promPoolRPCNodeVerifies = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies", + Help: "The total number of chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeVerifiesFailed = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies_failed", + Help: "The total number of failed chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) + promPoolRPCNodeVerifiesSuccess = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_verifies_success", + Help: "The total number of successful chain ID verifications for the given RPC node", + }, []string{"network", "chainID", "nodeName"}) ) type NodeConfig interface { diff --git a/multinode/node_fsm.go b/multinode/node_fsm.go index bc98e96..c1ec1d7 100644 --- a/multinode/node_fsm.go +++ b/multinode/node_fsm.go @@ -4,16 +4,38 @@ import ( "fmt" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" ) var ( - promPoolRPCNodeTransitionsToAlive *prometheus.CounterVec - promPoolRPCNodeTransitionsToInSync *prometheus.CounterVec - promPoolRPCNodeTransitionsToOutOfSync *prometheus.CounterVec - promPoolRPCNodeTransitionsToUnreachable *prometheus.CounterVec - promPoolRPCNodeTransitionsToInvalidChainID *prometheus.CounterVec - promPoolRPCNodeTransitionsToUnusable *prometheus.CounterVec - promPoolRPCNodeTransitionsToSyncing *prometheus.CounterVec + promPoolRPCNodeTransitionsToAlive = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_alive", + Help: transitionString(nodeStateAlive), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToInSync = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_in_sync", + Help: fmt.Sprintf("%s to %s", transitionString(nodeStateOutOfSync), nodeStateAlive), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToOutOfSync = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_out_of_sync", + Help: transitionString(nodeStateOutOfSync), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToUnreachable = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_unreachable", + Help: transitionString(nodeStateUnreachable), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToInvalidChainID = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", + Help: transitionString(nodeStateInvalidChainID), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToUnusable = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_unusable", + Help: transitionString(nodeStateUnusable), + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeTransitionsToSyncing = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_transitions_to_syncing", + Help: transitionString(nodeStateSyncing), + }, []string{"chainID", "nodeName"}) ) // nodeState represents the current state of the node diff --git a/multinode/node_lifecycle.go b/multinode/node_lifecycle.go index 8cb31ea..21cadae 100644 --- a/multinode/node_lifecycle.go +++ b/multinode/node_lifecycle.go @@ -8,18 +8,37 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/utils" bigmath "github.com/smartcontractkit/chainlink-common/pkg/utils/big_math" ) var ( - promPoolRPCNodeHighestSeenBlock *prometheus.GaugeVec - promPoolRPCNodeHighestFinalizedBlock *prometheus.GaugeVec - promPoolRPCNodeNumSeenBlocks *prometheus.CounterVec - promPoolRPCNodePolls *prometheus.CounterVec - promPoolRPCNodePollsFailed *prometheus.CounterVec - promPoolRPCNodePollsSuccess *prometheus.CounterVec + promPoolRPCNodeHighestSeenBlock = promauto.With(metricsRegistry).NewGaugeVec(prometheus.GaugeOpts{ + Name: "pool_rpc_node_highest_seen_block", + Help: "The highest seen block for the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeHighestFinalizedBlock = promauto.With(metricsRegistry).NewGaugeVec(prometheus.GaugeOpts{ + Name: "pool_rpc_node_highest_finalized_block", + Help: "The highest seen finalized block for the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodeNumSeenBlocks = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_num_seen_blocks", + Help: "The total number of new blocks seen by the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodePolls = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_polls_total", + Help: "The total number of poll checks for the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodePollsFailed = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_polls_failed", + Help: "The total number of failed poll checks for the given RPC node", + }, []string{"chainID", "nodeName"}) + promPoolRPCNodePollsSuccess = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ + Name: "pool_rpc_node_polls_success", + Help: "The total number of successful poll checks for the given RPC node", + }, []string{"chainID", "nodeName"}) ) // zombieNodeCheckInterval controls how often to re-check to see if we need to diff --git a/multinode/transaction_sender.go b/multinode/transaction_sender.go index 1189cf9..ed50701 100644 --- a/multinode/transaction_sender.go +++ b/multinode/transaction_sender.go @@ -9,13 +9,17 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) var ( // PromMultiNodeInvariantViolations reports violation of our assumptions - PromMultiNodeInvariantViolations *prometheus.CounterVec + PromMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multi_node_invariant_violations", + Help: "The number of invariant violations", + }, []string{"network", "chainId", "invariant"}) ) type SendTxResult interface { From 42df8bd43bbc4c7125facc248c5ff9eb1bcc7dca Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Wed, 8 Jan 2025 17:17:15 -0500 Subject: [PATCH 5/9] fix imports --- multinode/multi_node.go | 1 + multinode/node.go | 1 + multinode/node_lifecycle.go | 1 + 3 files changed, 3 insertions(+) diff --git a/multinode/multi_node.go b/multinode/multi_node.go index 864a636..d460518 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -9,6 +9,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) diff --git a/multinode/node.go b/multinode/node.go index 9b48bf3..56bb00a 100644 --- a/multinode/node.go +++ b/multinode/node.go @@ -10,6 +10,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) diff --git a/multinode/node_lifecycle.go b/multinode/node_lifecycle.go index 21cadae..4e43127 100644 --- a/multinode/node_lifecycle.go +++ b/multinode/node_lifecycle.go @@ -9,6 +9,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/utils" bigmath "github.com/smartcontractkit/chainlink-common/pkg/utils/big_math" From 6cff9bc49452bde57113ff0805ff2695b188d586 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 10 Jan 2025 14:52:36 -0500 Subject: [PATCH 6/9] Prefix metrics with multinode --- multinode/multi_node.go | 5 ++--- multinode/node.go | 12 ++++++------ multinode/node_fsm.go | 28 ++++++++++++++-------------- multinode/node_lifecycle.go | 24 ++++++++++++------------ multinode/transaction_sender.go | 2 +- 5 files changed, 35 insertions(+), 36 deletions(-) diff --git a/multinode/multi_node.go b/multinode/multi_node.go index d460518..39a6e29 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -15,9 +15,8 @@ import ( ) var ( - metricsRegistry = prometheus.NewRegistry() - PromMultiNodeRPCNodeStates = promauto.With(metricsRegistry).NewGaugeVec(prometheus.GaugeOpts{ - Name: "multi_node_states", + PromMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "multinode_states", Help: "The number of RPC nodes currently in the given state for the given chain", }, []string{"network", "chainId", "state"}) ErrNodeError = fmt.Errorf("no live nodes available") diff --git a/multinode/node.go b/multinode/node.go index 56bb00a..1ad551f 100644 --- a/multinode/node.go +++ b/multinode/node.go @@ -20,16 +20,16 @@ const QueryTimeout = 10 * time.Second var errInvalidChainID = errors.New("invalid chain id") var ( - promPoolRPCNodeVerifies = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies", + promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_verifies", Help: "The total number of chain ID verifications for the given RPC node", }, []string{"network", "chainID", "nodeName"}) - promPoolRPCNodeVerifiesFailed = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies_failed", + promPoolRPCNodeVerifiesFailed = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_verifies_failed", Help: "The total number of failed chain ID verifications for the given RPC node", }, []string{"network", "chainID", "nodeName"}) - promPoolRPCNodeVerifiesSuccess = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_verifies_success", + promPoolRPCNodeVerifiesSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_verifies_success", Help: "The total number of successful chain ID verifications for the given RPC node", }, []string{"network", "chainID", "nodeName"}) ) diff --git a/multinode/node_fsm.go b/multinode/node_fsm.go index c1ec1d7..83f0665 100644 --- a/multinode/node_fsm.go +++ b/multinode/node_fsm.go @@ -8,32 +8,32 @@ import ( ) var ( - promPoolRPCNodeTransitionsToAlive = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_alive", + promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_num_transitions_to_alive", Help: transitionString(nodeStateAlive), }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToInSync = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_in_sync", + promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_num_transitions_to_in_sync", Help: fmt.Sprintf("%s to %s", transitionString(nodeStateOutOfSync), nodeStateAlive), }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToOutOfSync = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_out_of_sync", + promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_num_transitions_to_out_of_sync", Help: transitionString(nodeStateOutOfSync), }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToUnreachable = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_unreachable", + promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_num_transitions_to_unreachable", Help: transitionString(nodeStateUnreachable), }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToInvalidChainID = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", + promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_num_transitions_to_invalid_chain_id", Help: transitionString(nodeStateInvalidChainID), }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToUnusable = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_unusable", + promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_num_transitions_to_unusable", Help: transitionString(nodeStateUnusable), }, []string{"chainID", "nodeName"}) - promPoolRPCNodeTransitionsToSyncing = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_transitions_to_syncing", + promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_num_transitions_to_syncing", Help: transitionString(nodeStateSyncing), }, []string{"chainID", "nodeName"}) ) diff --git a/multinode/node_lifecycle.go b/multinode/node_lifecycle.go index 4e43127..7dffe89 100644 --- a/multinode/node_lifecycle.go +++ b/multinode/node_lifecycle.go @@ -16,28 +16,28 @@ import ( ) var ( - promPoolRPCNodeHighestSeenBlock = promauto.With(metricsRegistry).NewGaugeVec(prometheus.GaugeOpts{ - Name: "pool_rpc_node_highest_seen_block", + promPoolRPCNodeHighestSeenBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "multinode_pool_rpc_node_highest_seen_block", Help: "The highest seen block for the given RPC node", }, []string{"chainID", "nodeName"}) - promPoolRPCNodeHighestFinalizedBlock = promauto.With(metricsRegistry).NewGaugeVec(prometheus.GaugeOpts{ - Name: "pool_rpc_node_highest_finalized_block", + promPoolRPCNodeHighestFinalizedBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "multinode_pool_rpc_node_highest_finalized_block", Help: "The highest seen finalized block for the given RPC node", }, []string{"chainID", "nodeName"}) - promPoolRPCNodeNumSeenBlocks = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_num_seen_blocks", + promPoolRPCNodeNumSeenBlocks = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_num_seen_blocks", Help: "The total number of new blocks seen by the given RPC node", }, []string{"chainID", "nodeName"}) - promPoolRPCNodePolls = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_polls_total", + promPoolRPCNodePolls = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_polls_total", Help: "The total number of poll checks for the given RPC node", }, []string{"chainID", "nodeName"}) - promPoolRPCNodePollsFailed = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_polls_failed", + promPoolRPCNodePollsFailed = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_polls_failed", Help: "The total number of failed poll checks for the given RPC node", }, []string{"chainID", "nodeName"}) - promPoolRPCNodePollsSuccess = promauto.With(metricsRegistry).NewCounterVec(prometheus.CounterOpts{ - Name: "pool_rpc_node_polls_success", + promPoolRPCNodePollsSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "multinode_pool_rpc_node_polls_success", Help: "The total number of successful poll checks for the given RPC node", }, []string{"chainID", "nodeName"}) ) diff --git a/multinode/transaction_sender.go b/multinode/transaction_sender.go index ed50701..79da3bf 100644 --- a/multinode/transaction_sender.go +++ b/multinode/transaction_sender.go @@ -17,7 +17,7 @@ import ( var ( // PromMultiNodeInvariantViolations reports violation of our assumptions PromMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multi_node_invariant_violations", + Name: "multinode_transaction_sender_invariant_violations", Help: "The number of invariant violations", }, []string{"network", "chainId", "invariant"}) ) From 4b8f1e21ba8f3477570f18ca2bae64ee7f4a7b8a Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Fri, 10 Jan 2025 14:57:31 -0500 Subject: [PATCH 7/9] lint --- multinode/multi_node.go | 1 + multinode/transaction_sender.go | 1 + 2 files changed, 2 insertions(+) diff --git a/multinode/multi_node.go b/multinode/multi_node.go index 39a6e29..f31a9aa 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -15,6 +15,7 @@ import ( ) var ( + // PromMultiNodeRPCNodeStates reports current RPC node state PromMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "multinode_states", Help: "The number of RPC nodes currently in the given state for the given chain", diff --git a/multinode/transaction_sender.go b/multinode/transaction_sender.go index 79da3bf..8b0b98f 100644 --- a/multinode/transaction_sender.go +++ b/multinode/transaction_sender.go @@ -10,6 +10,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" ) From 42d03f14573db522dc5ae1277a22b292eac190a6 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 14 Jan 2025 09:51:57 -0500 Subject: [PATCH 8/9] Revert metrics names --- multinode/multi_node.go | 2 +- multinode/node.go | 6 +++--- multinode/node_fsm.go | 14 +++++++------- multinode/transaction_sender.go | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/multinode/multi_node.go b/multinode/multi_node.go index f31a9aa..019c678 100644 --- a/multinode/multi_node.go +++ b/multinode/multi_node.go @@ -17,7 +17,7 @@ import ( var ( // PromMultiNodeRPCNodeStates reports current RPC node state PromMultiNodeRPCNodeStates = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "multinode_states", + Name: "multi_node_states", Help: "The number of RPC nodes currently in the given state for the given chain", }, []string{"network", "chainId", "state"}) ErrNodeError = fmt.Errorf("no live nodes available") diff --git a/multinode/node.go b/multinode/node.go index 1ad551f..f1b0aa2 100644 --- a/multinode/node.go +++ b/multinode/node.go @@ -21,15 +21,15 @@ var errInvalidChainID = errors.New("invalid chain id") var ( promPoolRPCNodeVerifies = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_verifies", + Name: "pool_rpc_node_verifies", Help: "The total number of chain ID verifications for the given RPC node", }, []string{"network", "chainID", "nodeName"}) promPoolRPCNodeVerifiesFailed = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_verifies_failed", + Name: "pool_rpc_node_verifies_failed", Help: "The total number of failed chain ID verifications for the given RPC node", }, []string{"network", "chainID", "nodeName"}) promPoolRPCNodeVerifiesSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_verifies_success", + Name: "pool_rpc_node_verifies_success", Help: "The total number of successful chain ID verifications for the given RPC node", }, []string{"network", "chainID", "nodeName"}) ) diff --git a/multinode/node_fsm.go b/multinode/node_fsm.go index 83f0665..c10a5f8 100644 --- a/multinode/node_fsm.go +++ b/multinode/node_fsm.go @@ -9,31 +9,31 @@ import ( var ( promPoolRPCNodeTransitionsToAlive = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_num_transitions_to_alive", + Name: "pool_rpc_node_num_transitions_to_alive", Help: transitionString(nodeStateAlive), }, []string{"chainID", "nodeName"}) promPoolRPCNodeTransitionsToInSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_num_transitions_to_in_sync", + Name: "pool_rpc_node_num_transitions_to_in_sync", Help: fmt.Sprintf("%s to %s", transitionString(nodeStateOutOfSync), nodeStateAlive), }, []string{"chainID", "nodeName"}) promPoolRPCNodeTransitionsToOutOfSync = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_num_transitions_to_out_of_sync", + Name: "pool_rpc_node_num_transitions_to_out_of_sync", Help: transitionString(nodeStateOutOfSync), }, []string{"chainID", "nodeName"}) promPoolRPCNodeTransitionsToUnreachable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_num_transitions_to_unreachable", + Name: "pool_rpc_node_num_transitions_to_unreachable", Help: transitionString(nodeStateUnreachable), }, []string{"chainID", "nodeName"}) promPoolRPCNodeTransitionsToInvalidChainID = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_num_transitions_to_invalid_chain_id", + Name: "pool_rpc_node_num_transitions_to_invalid_chain_id", Help: transitionString(nodeStateInvalidChainID), }, []string{"chainID", "nodeName"}) promPoolRPCNodeTransitionsToUnusable = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_num_transitions_to_unusable", + Name: "pool_rpc_node_num_transitions_to_unusable", Help: transitionString(nodeStateUnusable), }, []string{"chainID", "nodeName"}) promPoolRPCNodeTransitionsToSyncing = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_num_transitions_to_syncing", + Name: "pool_rpc_node_num_transitions_to_syncing", Help: transitionString(nodeStateSyncing), }, []string{"chainID", "nodeName"}) ) diff --git a/multinode/transaction_sender.go b/multinode/transaction_sender.go index 8b0b98f..f04c6df 100644 --- a/multinode/transaction_sender.go +++ b/multinode/transaction_sender.go @@ -18,7 +18,7 @@ import ( var ( // PromMultiNodeInvariantViolations reports violation of our assumptions PromMultiNodeInvariantViolations = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_transaction_sender_invariant_violations", + Name: "multi_node_invariant_violations", Help: "The number of invariant violations", }, []string{"network", "chainId", "invariant"}) ) From c816a400dcb3cbe2afc30751ed1ba3962c321e44 Mon Sep 17 00:00:00 2001 From: Dylan Tinianov Date: Tue, 14 Jan 2025 09:53:18 -0500 Subject: [PATCH 9/9] revert metrics names --- multinode/node_lifecycle.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/multinode/node_lifecycle.go b/multinode/node_lifecycle.go index 7dffe89..ae64851 100644 --- a/multinode/node_lifecycle.go +++ b/multinode/node_lifecycle.go @@ -17,27 +17,27 @@ import ( var ( promPoolRPCNodeHighestSeenBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "multinode_pool_rpc_node_highest_seen_block", + Name: "pool_rpc_node_highest_seen_block", Help: "The highest seen block for the given RPC node", }, []string{"chainID", "nodeName"}) promPoolRPCNodeHighestFinalizedBlock = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "multinode_pool_rpc_node_highest_finalized_block", + Name: "pool_rpc_node_highest_finalized_block", Help: "The highest seen finalized block for the given RPC node", }, []string{"chainID", "nodeName"}) promPoolRPCNodeNumSeenBlocks = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_num_seen_blocks", + Name: "pool_rpc_node_num_seen_blocks", Help: "The total number of new blocks seen by the given RPC node", }, []string{"chainID", "nodeName"}) promPoolRPCNodePolls = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_polls_total", + Name: "pool_rpc_node_polls_total", Help: "The total number of poll checks for the given RPC node", }, []string{"chainID", "nodeName"}) promPoolRPCNodePollsFailed = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_polls_failed", + Name: "pool_rpc_node_polls_failed", Help: "The total number of failed poll checks for the given RPC node", }, []string{"chainID", "nodeName"}) promPoolRPCNodePollsSuccess = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "multinode_pool_rpc_node_polls_success", + Name: "pool_rpc_node_polls_success", Help: "The total number of successful poll checks for the given RPC node", }, []string{"chainID", "nodeName"}) )