From 479bfb824a955a3bbdd2db53580cef10e6a97c72 Mon Sep 17 00:00:00 2001
From: Itay <itaylevy134@gmail.com>
Date: Mon, 24 Jul 2023 16:20:17 +0300
Subject: [PATCH] feat: Add Prometheus HTTP Metric Server for Enhanced
 Monitoring (#419)

---
 config/config.go   | 20 ++++++++++++++++----
 config/defaults.go |  4 ++++
 config/toml.go     | 15 +++++++++++++++
 node/node.go       | 27 +++++++++++++++++++++++++++
 4 files changed, 62 insertions(+), 4 deletions(-)

diff --git a/config/config.go b/config/config.go
index a6d0155f2..a03ce15ca 100644
--- a/config/config.go
+++ b/config/config.go
@@ -27,10 +27,11 @@ type NodeConfig struct {
 	// parameters below are dymint specific and read from config
 	Aggregator         bool `mapstructure:"aggregator"`
 	BlockManagerConfig `mapstructure:",squash"`
-	DALayer            string            `mapstructure:"da_layer"`
-	DAConfig           string            `mapstructure:"da_config"`
-	SettlementLayer    string            `mapstructure:"settlement_layer"`
-	SettlementConfig   settlement.Config `mapstructure:",squash"`
+	DALayer            string                 `mapstructure:"da_layer"`
+	DAConfig           string                 `mapstructure:"da_config"`
+	SettlementLayer    string                 `mapstructure:"settlement_layer"`
+	SettlementConfig   settlement.Config      `mapstructure:",squash"`
+	Instrumentation    *InstrumentationConfig `mapstructure:"instrumentation"`
 }
 
 // BlockManagerConfig consists of all parameters required by BlockManagerConfig
@@ -131,3 +132,14 @@ func (c NodeConfig) Validate() error {
 
 	return nil
 }
+
+// InstrumentationConfig defines the configuration for metrics reporting.
+type InstrumentationConfig struct {
+	// When true, Prometheus metrics are served under /metrics on
+	// PrometheusListenAddr.
+	// Check out the documentation for the list of available metrics.
+	Prometheus bool `mapstructure:"prometheus"`
+
+	// Address to listen for Prometheus collector(s) connections.
+	PrometheusListenAddr string `mapstructure:"prometheus_listen_addr"`
+}
diff --git a/config/defaults.go b/config/defaults.go
index d29d2b8d0..918f615f2 100644
--- a/config/defaults.go
+++ b/config/defaults.go
@@ -36,6 +36,10 @@ func DefaultConfig(home, chainId string) *NodeConfig {
 			BlockBatchMaxSizeBytes: 1500000},
 		DALayer:         "mock",
 		SettlementLayer: "mock",
+		Instrumentation: &InstrumentationConfig{
+			Prometheus:           false,
+			PrometheusListenAddr: ":2112",
+		},
 	}
 
 	if home == "" {
diff --git a/config/toml.go b/config/toml.go
index 565c717ce..be68a865f 100644
--- a/config/toml.go
+++ b/config/toml.go
@@ -103,4 +103,19 @@ gas_fees = "{{ .SettlementConfig.GasFees }}"
 keyring_backend = "{{ .SettlementConfig.KeyringBackend }}"
 keyring_home_dir = "{{ .SettlementConfig.KeyringHomeDir }}"
 dym_account_name = "{{ .SettlementConfig.DymAccountName }}"
+
+#######################################################
+###       Instrumentation Configuration Options     ###
+#######################################################
+[instrumentation]
+
+# When true, Prometheus metrics are served under /metrics on
+# PrometheusListenAddr.
+# Check out the documentation for the list of available metrics.
+prometheus = {{ .Instrumentation.Prometheus }}
+
+# Address to listen for Prometheus collector(s) connections
+prometheus_listen_addr = "{{ .Instrumentation.PrometheusListenAddr }}"
+
+
 `
diff --git a/node/node.go b/node/node.go
index d225c047e..3168465ca 100644
--- a/node/node.go
+++ b/node/node.go
@@ -5,7 +5,11 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"github.com/libp2p/go-libp2p/core/crypto"
 
@@ -263,6 +267,12 @@ func (n *Node) OnStart() error {
 	if err != nil {
 		return fmt.Errorf("error while starting settlement layer client: %w", err)
 	}
+	go func() {
+		if err := n.startPrometheusServer(); err != nil {
+			panic(err)
+		}
+	}()
+
 	n.baseLayersHealthStatus = BaseLayersHealthStatus{
 		settlementHealthy: true,
 		daHealthy:         true,
@@ -400,5 +410,22 @@ func (n *Node) healthStatusHandler(err error) {
 		if err = n.pubsubServer.PublishWithEvents(n.ctx, healthStatusEvent, map[string][]string{events.EventNodeTypeKey: {events.EventHealthStatus}}); err != nil {
 			panic(err)
 		}
+
 	}
 }
+
+func (n *Node) startPrometheusServer() error {
+	if n.conf.Instrumentation != nil && n.conf.Instrumentation.Prometheus {
+		http.Handle("/metrics", promhttp.Handler())
+		srv := &http.Server{
+			Addr:         n.conf.Instrumentation.PrometheusListenAddr,
+			ReadTimeout:  5 * time.Second,
+			WriteTimeout: 10 * time.Second,
+			Handler:      http.DefaultServeMux,
+		}
+		if err := srv.ListenAndServe(); err != nil {
+			return err
+		}
+	}
+	return nil
+}