Sergey/simplified metrics

forester-utils/Cargo.toml

@@ -21,6 +21,7 @@ light-system-program = { workspace = true, features = ["cpi"] }
 light-utils = { workspace = true }
 light-batched-merkle-tree = { workspace = true }
 light-verifier = { workspace = true }
+light-sdk = { workspace = true }
 photon-api = { workspace = true }
 light-client = { workspace = true }
 
@@ -30,6 +31,7 @@ anchor-spl = { workspace = true }
 
 # Solana
 spl-token = { workspace = true, features = ["no-entrypoint"] }
+solana-program = { workspace = true }
 solana-program-test = { workspace = true }
 solana-sdk = { workspace = true }
 solana-client = { workspace = true }
@@ -51,4 +53,12 @@ num-traits = { workspace = true }
 
 # HTTP client
 reqwest = "0.11.26"
+bb8 = { workspace = true }
 
+# Metrics
+prometheus = "0.13"
+warp = "0.3"
+lazy_static = "1.4"
+anyhow = "1.0.94"
+borsh = "0.10.3"
+tracing = "0.1.40"

forester-utils/src/lib.rs

@@ -18,6 +18,10 @@ pub mod address_merkle_tree_config;
 pub mod forester_epoch;
 pub mod instructions;
 pub mod registry;
+pub mod metrics;
+pub mod rpc_pool;
+pub mod solana_rpc;
+pub use solana_rpc::{RetryConfig, SolanaRpcConnection};
 
 pub fn create_account_instruction(
     payer: &Pubkey,

forester/src/metrics.rs → forester-utils/src/metrics/helpers.rs

@@ -1,15 +1,14 @@
 use std::{
     sync::Once,
-    time::{SystemTime, UNIX_EPOCH},
+    time::{Instant, SystemTime, UNIX_EPOCH},
 };
 
 use lazy_static::lazy_static;
-use prometheus::{Encoder, GaugeVec, IntCounterVec, IntGauge, IntGaugeVec, Registry, TextEncoder};
+use prometheus::{Encoder, GaugeVec, HistogramOpts, HistogramVec, IntCounterVec, IntGauge, IntGaugeVec, Registry, TextEncoder};
 use reqwest::Client;
 use tokio::sync::Mutex;
 use tracing::{debug, error};
-
-use crate::Result;
+use anyhow::Result;
 
 lazy_static! {
     pub static ref REGISTRY: Registry = Registry::new();
@@ -55,11 +54,71 @@ lazy_static! {
         &["pubkey"]
     )
     .expect("metric can be created");
+
     pub static ref REGISTERED_FORESTERS: GaugeVec = GaugeVec::new(
         prometheus::opts!("registered_foresters", "Foresters registered per epoch"),
         &["epoch", "authority"]
     )
     .expect("metric can be created");
+
+    pub static ref RPC_REQUESTS_TOTAL: IntCounterVec = IntCounterVec::new(
+        prometheus::opts!(
+            "solana_rpc_requests_total",
+            "Total number of RPC requests made"
+        ),
+        &["method", "status"]
+    ).expect("metric can be created");
+
+    pub static ref RPC_REQUEST_DURATION: HistogramVec = {
+        let opts = HistogramOpts::new(
+            "solana_rpc_request_duration_seconds",
+            "RPC request latency by method"
+        ).buckets(vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0]);
+
+        HistogramVec::new(opts, &["method"]).expect("metric can be created")
+    };
+
+    pub static ref RPC_POOL_CONNECTIONS: IntGaugeVec = IntGaugeVec::new(
+        prometheus::opts!(
+            "solana_rpc_pool_connections",
+            "Number of connections in the RPC pool"
+        ),
+        &["state"]  // "active", "idle"
+    ).expect("metric can be created");
+
+    pub static ref RPC_REQUEST_ERRORS: IntCounterVec = IntCounterVec::new(
+        prometheus::opts!(
+            "solana_rpc_request_errors_total",
+            "Total number of RPC request errors"
+        ),
+        &["method", "error_type"]
+    ).expect("metric can be created");
+
+    pub static ref RPC_POOL_WAIT_DURATION: HistogramVec = {
+        let opts = HistogramOpts::new(
+            "solana_rpc_pool_wait_duration_seconds",
+            "Time spent waiting for an RPC connection from the pool"
+        ).buckets(vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]);
+
+        HistogramVec::new(opts, &["pool_id"]).expect("metric can be created")
+    };
+
+    pub static ref RPC_POOL_ACQUISITION_TOTAL: IntCounterVec = IntCounterVec::new(
+        prometheus::opts!(
+            "solana_rpc_pool_acquisitions_total",
+            "Total number of RPC connection acquisitions"
+        ),
+        &["pool_id", "status"]  // status: success, timeout, error
+    ).expect("metric can be created");
+
+    pub static ref RPC_POOL_TIMEOUTS: IntCounterVec = IntCounterVec::new(
+        prometheus::opts!(
+            "solana_rpc_pool_timeouts_total",
+            "Total number of RPC pool timeouts"
+        ),
+        &["pool_id"]
+    ).expect("metric can be created");
+
     static ref METRIC_UPDATES: Mutex<Vec<(u64, usize, std::time::Duration)>> =
         Mutex::new(Vec::new());
 }
@@ -88,6 +147,20 @@ pub fn register_metrics() {
         REGISTRY
             .register(Box::new(REGISTERED_FORESTERS.clone()))
             .expect("collector can be registered");
+        REGISTRY.register(Box::new(RPC_REQUESTS_TOTAL.clone())).expect("collector can be registered");
+        REGISTRY.register(Box::new(RPC_REQUEST_DURATION.clone())).expect("collector can be registered");
+        REGISTRY.register(Box::new(RPC_POOL_CONNECTIONS.clone())).expect("collector can be registered");
+        REGISTRY.register(Box::new(RPC_REQUEST_ERRORS.clone())).expect("collector can be registered");  
+
+         REGISTRY
+            .register(Box::new(RPC_POOL_WAIT_DURATION.clone()))
+            .expect("collector can be registered");
+        REGISTRY
+            .register(Box::new(RPC_POOL_ACQUISITION_TOTAL.clone()))
+            .expect("collector can be registered");
+        REGISTRY
+            .register(Box::new(RPC_POOL_TIMEOUTS.clone()))
+            .expect("collector can be registered");  
     });
 }
 
@@ -214,3 +287,26 @@ pub async fn metrics_handler() -> Result<impl warp::Reply> {
     res.push_str(&res_prometheus);
     Ok(res)
 }
+
+pub fn track_rpc_request(method: &str, start_time: Instant, result: &Result<()>) {
+    let duration = start_time.elapsed().as_secs_f64();
+
+    match result {
+        Ok(_) => {
+            RPC_REQUESTS_TOTAL.with_label_values(&[method, "success"]).inc();
+        }
+        Err(e) => {
+            RPC_REQUESTS_TOTAL.with_label_values(&[method, "error"]).inc();
+            RPC_REQUEST_ERRORS
+                .with_label_values(&[method, &e.to_string()])
+                .inc();
+        }
+    }
+
+    RPC_REQUEST_DURATION.with_label_values(&[method]).observe(duration);
+}
+
+pub fn update_rpc_pool_connections(active: i64, idle: i64) {
+    RPC_POOL_CONNECTIONS.with_label_values(&["active"]).set(active);
+    RPC_POOL_CONNECTIONS.with_label_values(&["idle"]).set(idle);
+}

forester-utils/src/metrics/mod.rs

@@ -0,0 +1,12 @@
+pub mod helpers;
+
+pub use helpers::{
+    RPC_REQUESTS_TOTAL,
+    RPC_REQUEST_DURATION,
+    RPC_REQUEST_ERRORS,
+
+    RPC_POOL_CONNECTIONS,
+    RPC_POOL_WAIT_DURATION,
+    RPC_POOL_ACQUISITION_TOTAL,
+    RPC_POOL_TIMEOUTS,
+};