Skip to content

Commit

Permalink
Runloop maintenance stage metrics (#2988)
Browse files Browse the repository at this point in the history
# Description
Since the syncronized runloop mode was enabled, the maintenance task is
on the critical auction preparation path now, which takes some time. New
metrics should help finding the most expensive operations in order to
find a way to optimize them.
<img width="476" alt="image"
src="https://github.com/user-attachments/assets/921c7059-f5a9-493e-a71b-7af63cb479fd">

# Changes
A new HistogramVec metric with the `stage` label.

## How to test
Grafana.
  • Loading branch information
squadgazzz authored Sep 16, 2024
1 parent 649e32e commit 6a7748a
Showing 1 changed file with 33 additions and 10 deletions.
43 changes: 33 additions & 10 deletions crates/autopilot/src/maintenance.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,11 @@ use {
futures::StreamExt,
prometheus::{
core::{AtomicU64, GenericGauge},
HistogramVec,
IntCounterVec,
},
shared::maintenance::Maintaining,
std::{sync::Arc, time::Duration},
std::{future::Future, sync::Arc, time::Duration},
tokio::{sync::Mutex, time::timeout},
};

Expand Down Expand Up @@ -88,15 +89,21 @@ impl Maintenance {
async fn update_inner(&self) -> Result<()> {
// All these can run independently of each other.
tokio::try_join!(
self.settlement_indexer.run_maintenance(),
self.db_cleanup.run_maintenance(),
self.index_ethflow_orders(),
futures::future::try_join_all(
self.cow_amm_indexer
.iter()
.cloned()
.map(|indexer| async move { indexer.run_maintenance().await })
)
Self::timed_future(
"settlement_indexer",
self.settlement_indexer.run_maintenance()
),
Self::timed_future("db_cleanup", self.db_cleanup.run_maintenance()),
Self::timed_future("ethflow_indexer", self.index_ethflow_orders()),
Self::timed_future(
"cow_amm_indexer",
futures::future::try_join_all(
self.cow_amm_indexer
.iter()
.cloned()
.map(|indexer| async move { indexer.run_maintenance().await }),
)
),
)?;

Ok(())
Expand Down Expand Up @@ -170,6 +177,15 @@ impl Maintenance {
}
});
}

/// Runs the future and collects runtime metrics.
async fn timed_future<T>(label: &str, fut: impl Future<Output = T>) -> T {
let _timer = metrics()
.maintenance_stage_time
.with_label_values(&[label])
.start_timer();
fut.await
}
}

type EthflowIndexer =
Expand All @@ -187,6 +203,13 @@ struct Metrics {
/// Autopilot maintenance error counter
#[metric(labels("result"))]
updates: IntCounterVec,

/// Autopilot maintenance stage time
#[metric(
labels("stage"),
buckets(0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 1, 1.5, 2.0, 2.5, 3, 3.5, 4)
)]
maintenance_stage_time: HistogramVec,
}

fn metrics() -> &'static Metrics {
Expand Down

0 comments on commit 6a7748a

Please sign in to comment.