diff --git a/node/metrics.go b/node/metrics.go index 5b7fc53ce7..65bf149fbb 100644 --- a/node/metrics.go +++ b/node/metrics.go @@ -43,6 +43,8 @@ type Metrics struct { AccuSocketUpdates prometheus.Counter // avs node spec eigen_ metrics: https://eigen.nethermind.io/docs/spec/metrics/metrics-prom-spec EigenMetrics eigenmetrics.Metrics + // Reachability gauge to monitoring the reachability of the node's retrieval/dispersal sockets + ReachabilityGauge *prometheus.GaugeVec registry *prometheus.Registry // socketAddr is the address at which the metrics server will be listening. @@ -129,6 +131,14 @@ func NewMetrics(eigenMetrics eigenmetrics.Metrics, reg *prometheus.Registry, log Help: "the total number of node's socket address updates", }, ), + ReachabilityGauge: promauto.With(reg).NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: Namespace, + Name: "reachability_status", + Help: "the reachability status of the nodes retrievel/dispersal sockets", + }, + []string{"service"}, + ), EigenMetrics: eigenMetrics, logger: logger.With("component", "NodeMetrics"), registry: reg, diff --git a/node/node.go b/node/node.go index 95e0cb4e22..bde3732466 100644 --- a/node/node.go +++ b/node/node.go @@ -474,6 +474,17 @@ func (n *Node) checkNodeReachability() { return } + if n.Config.DataApiUrl == "" { + n.Logger.Error("Unable to perform reachability check - NODE_DATAAPI_URL is not defined in .env") + return + } + + checkUrl, err := url.Parse(fmt.Sprintf("%s/api/v1/operators-info/port-check?operator_id=%s", n.Config.DataApiUrl, n.Config.ID.Hex())) + if err != nil { + n.Logger.Error("Reachability check failed - invalid check url", err, "checkUrl", checkUrl.String()) + return + } + n.Logger.Info("Start nodeReachabilityCheck goroutine in background to check the reachability of the operator node") ticker := time.NewTicker(time.Duration(n.Config.ReachabilityPollIntervalSec) * time.Second) defer ticker.Stop() @@ -481,18 +492,7 @@ func (n *Node) checkNodeReachability() { for { <-ticker.C - if n.Config.DataApiUrl == "" { - n.Logger.Error("Unable to perform reachability check - NODE_DATAAPI_URL is not defined in .env") - continue - } - - checkUrl, err := url.Parse(fmt.Sprintf("%s/api/v1/operators-info/port-check?operator_id=%s", n.Config.DataApiUrl, n.Config.ID.Hex())) - if err != nil { - n.Logger.Error("Reachability check failed - invalid check url", err, "checkUrl", checkUrl.String()) - return - } - - n.Logger.Info("Calling reachability check", "url", checkUrl.String()) + n.Logger.Debug("Calling reachability check", "url", checkUrl.String()) resp, err := http.Get(checkUrl.String()) if err != nil { @@ -521,13 +521,17 @@ func (n *Node) checkNodeReachability() { if responseObject.DispersalOnline { n.Logger.Info("Reachability check - dispersal socket is ONLINE", "socket", responseObject.DispersalSocket) + n.Metrics.ReachabilityGauge.WithLabelValues("dispersal").Set(1.0) } else { n.Logger.Error("Reachability check - dispersal socket is UNREACHABLE", "socket", responseObject.DispersalSocket) + n.Metrics.ReachabilityGauge.WithLabelValues("dispersal").Set(0.0) } if responseObject.RetrievalOnline { n.Logger.Info("Reachability check - retrieval socket is ONLINE", "socket", responseObject.RetrievalSocket) + n.Metrics.ReachabilityGauge.WithLabelValues("retrieval").Set(1.0) } else { n.Logger.Error("Reachability check - retrieval socket is UNREACHABLE", "socket", responseObject.RetrievalSocket) + n.Metrics.ReachabilityGauge.WithLabelValues("retrieval").Set(0.0) } } }