diff --git a/fly/cmd/historical_uptime/main.go b/fly/cmd/historical_uptime/main.go index 5d0be54f..87e999e6 100644 --- a/fly/cmd/historical_uptime/main.go +++ b/fly/cmd/historical_uptime/main.go @@ -68,6 +68,22 @@ var ( []string{"guardian", "chain"}, ) + guardianChainSafeHeight = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "guardian_chain_safe_height", + Help: "Safe height of each guardian on each chain over time", + }, + []string{"guardian", "chain"}, + ) + + guardianChainFinalizedHeight = promauto.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "guardian_chain_finalized_height", + Help: "Finalized height of each guardian on each chain over time", + }, + []string{"guardian", "chain"}, + ) + guardianChainHeightDifferences = promauto.NewGaugeVec( prometheus.GaugeOpts{ Name: "guardian_chain_height_differences", @@ -143,7 +159,7 @@ func recordGuardianHeightDifferences() { chainName := vaa.ChainID(chainId).String() for guardian, heightDifference := range guardianHeightDifferences { - guardianChainHeightDifferences.WithLabelValues(guardian, chainName).Set(float64(heightDifference)) + guardianChainHeightDifferences.WithLabelValues(guardian, chainName).Set(float64(heightDifference.Latest)) } } } @@ -166,9 +182,11 @@ func initPromScraper(promRemoteURL string, logger *zap.Logger, errC chan error) for { select { case <-ctx.Done(): + logger.Info("Prometheus scraper context done") return nil case <-t.C: recordGuardianHeightDifferences() + for i := 1; i < 36; i++ { if i == PYTHNET_CHAIN_ID { continue @@ -186,6 +204,7 @@ func initPromScraper(promRemoteURL string, logger *zap.Logger, errC chan error) guardianMissedObservations.WithLabelValues(guardianName, chainName).Add(0) } } + err := promremotew.ScrapeAndSendLocalMetrics(ctx, info, promLogger) if err != nil { promLogger.Error("ScrapeAndSendLocalMetrics error", zap.Error(err)) @@ -194,6 +213,8 @@ func initPromScraper(promRemoteURL string, logger *zap.Logger, errC chan error) } } }) + } else { + logger.Warn("Prometheus remote write URL not provided, metrics will not be sent") } } @@ -387,13 +408,34 @@ func main() { guardianChainHeights[network.Id] = make(common.GuardianHeight) } - guardianChainHeights[network.Id][guardianName] = uint64(network.Height) + guardianChainHeights[network.Id][guardianName] = common.HeightInfo{ + Latest: uint64(network.Height), + Safe: uint64(network.SafeHeight), + Finalized: uint64(network.FinalizedHeight), + } + + chain := vaa.ChainID(network.Id).String() + guardianChainHeight.With( prometheus.Labels{ "guardian": guardianName, - "chain": vaa.ChainID(network.Id).String(), + "chain": chain, }, ).Set(float64(network.Height)) + + guardianChainSafeHeight.With( + prometheus.Labels{ + "guardian": guardianName, + "chain": chain, + }, + ).Set(float64(network.SafeHeight)) + + guardianChainFinalizedHeight.With( + prometheus.Labels{ + "guardian": guardianName, + "chain": chain, + }, + ).Set(float64(network.FinalizedHeight)) } guardianHeartbeats.With( diff --git a/fly/common/types.go b/fly/common/types.go index 12dd777f..f6c8c556 100644 --- a/fly/common/types.go +++ b/fly/common/types.go @@ -1,6 +1,12 @@ package common -type GuardianHeight map[string]uint64 // map of guardian addr to chain height +type HeightInfo struct { + Latest uint64 + Safe uint64 + Finalized uint64 +} + +type GuardianHeight map[string]HeightInfo // map of guardian addr to height info type GuardianChainHeights map[uint32]GuardianHeight // map of chainIds to guardian heights diff --git a/fly/pkg/historical_uptime/helpers.go b/fly/pkg/historical_uptime/helpers.go index fb8801fc..579ded3c 100644 --- a/fly/pkg/historical_uptime/helpers.go +++ b/fly/pkg/historical_uptime/helpers.go @@ -108,9 +108,9 @@ func computeMaxChainHeights(guardianChainHeights common.GuardianChainHeights) co for chainId, guardianHeights := range guardianChainHeights { highest := uint64(0) - for _, guardianHeight := range guardianHeights { - if highest < guardianHeight { - highest = guardianHeight + for _, heightInfo := range guardianHeights { + if highest < heightInfo.Latest { + highest = heightInfo.Latest } } @@ -124,13 +124,16 @@ func computeGuardianChainHeightDifferences(guardianChainHeights common.GuardianC heightDifferences := make(common.GuardianChainHeights) for chainId, guardianHeights := range guardianChainHeights { - for guardian, height := range guardianHeights { + for guardian, heightInfo := range guardianHeights { if heightDifferences[chainId] == nil { heightDifferences[chainId] = make(common.GuardianHeight) } - // maxChainHeights[chain] always guaranteed to be at least height since it's computed in `computeMaxChainHeights` - heightDifferences[chainId][guardian] = maxChainHeights[chainId] - height + heightDifferences[chainId][guardian] = common.HeightInfo{ + Latest: maxChainHeights[chainId] - heightInfo.Latest, + Safe: 0, // We're not calculating differences for Safe + Finalized: 0, // We're not calculating differences for Finalized + } } }