Add benchmark counters to prometheus

codegenerator/cli/templates/static/codegen/src/Benchmark.res

@@ -1,8 +1,8 @@
 module MillisAccum = {
-  type millis = int
+  type millis = float
   type t = {counters: dict<millis>, startTime: Js.Date.t, mutable endTime: Js.Date.t}
   let schema: S.t<t> = S.schema(s => {
-    counters: s.matches(S.dict(S.int)),
+    counters: s.matches(S.dict(S.float)),
     startTime: s.matches(S.string->S.datetime),
     endTime: s.matches(S.string->S.datetime),
   })
@@ -13,11 +13,17 @@ module MillisAccum = {
   }
 
   let increment = (self: t, label, amount) => {
+    self.endTime = Js.Date.make()
+    let amount = amount->Belt.Float.fromInt
     switch self.counters->Utils.Dict.dangerouslyGetNonOption(label) {
-    | None => self.counters->Js.Dict.set(label, amount)
-    | Some(current) => self.counters->Js.Dict.set(label, current + amount)
+    | None =>
+      self.counters->Js.Dict.set(label, amount)
+      amount
+    | Some(current) =>
+      let newAmount = current +. amount
+      self.counters->Js.Dict.set(label, newAmount)
+      newAmount
     }
-    self.endTime = Js.Date.make()
   }
 }
 
@@ -72,30 +78,30 @@ module SummaryData = {
   module Group = {
     type t = dict<DataSet.t>
     let schema: S.t<t> = S.dict(DataSet.schema)
-    let make = () => Js.Dict.empty()
+    let make = (): t => Js.Dict.empty()
 
     /**
     Adds a value to the data set for the given key. If the key does not exist, it will be created.
 
     Returns the updated data set.
     */
-    let add = (self: t, key: string, value: float, ~decimalPlaces=2) => {
-      switch self->Utils.Dict.dangerouslyGetNonOption(key) {
+    let add = (self: t, label, value: float, ~decimalPlaces=2) => {
+      switch self->Utils.Dict.dangerouslyGetNonOption(label) {
       | None =>
         let new = DataSet.make(value, ~decimalPlaces)
-        self->Js.Dict.set(key, new)
+        self->Js.Dict.set(label, new)
         new
       | Some(dataSet) =>
         let updated = dataSet->DataSet.add(value)
-        self->Js.Dict.set(key, updated)
+        self->Js.Dict.set(label, updated)
         updated
       }
     }
   }
 
   type t = dict<Group.t>
   let schema = S.dict(Group.schema)
-  let make = () => Js.Dict.empty()
+  let make = (): t => Js.Dict.empty()
 
   let add = (self: t, ~group, ~label, ~value, ~decimalPlaces=2) => {
     let group = switch self->Utils.Dict.dangerouslyGetNonOption(group) {
@@ -169,26 +175,38 @@ module Data = {
     summaryData: SummaryData.make(),
   }
 
-  let incrementMillis = (self: t, ~label, ~amount) => {
-    self.millisAccum->MillisAccum.increment(label, amount)
-  }
-
   module LiveMetrics = {
-    let saveLiveMetrics = if (
+    let addDataSet = if (
       Env.Benchmark.saveDataStrategy->Env.Benchmark.SaveDataStrategy.shouldSavePrometheus
     ) {
       (dataSet: SummaryData.DataSet.t, ~group, ~label) => {
         let {n, mean, stdDev, min, max, sum} = dataSet->Stats.makeFromDataSet
-        Prometheus.setBenchmarkSummaryData(~group, ~label, ~n, ~mean, ~stdDev, ~min, ~max, ~sum)
+        Prometheus.BenchmarkSummaryData.set(~group, ~label, ~n, ~mean, ~stdDev, ~min, ~max, ~sum)
       }
     } else {
       (_dataSet, ~group as _, ~label as _) => ()
     }
+    let setCounterMillis = if (
+      Env.Benchmark.saveDataStrategy->Env.Benchmark.SaveDataStrategy.shouldSavePrometheus
+    ) {
+      (millisAccum: MillisAccum.t, ~label, ~millis) => {
+        let totalRuntimeMillis =
+          millisAccum.endTime->Js.Date.getTime -. millisAccum.startTime->Js.Date.getTime
+        Prometheus.BenchmarkCounters.set(~label, ~millis, ~totalRuntimeMillis)
+      }
+    } else {
+      (_, ~label as _, ~millis as _) => ()
+    }
+  }
+
+  let incrementMillis = (self: t, ~label, ~amount) => {
+    let nextMillis = self.millisAccum->MillisAccum.increment(label, amount)
+    self.millisAccum->LiveMetrics.setCounterMillis(~label, ~millis=nextMillis)
   }
 
   let addSummaryData = (self: t, ~group, ~label, ~value, ~decimalPlaces=2) => {
     let updatedDataSet = self.summaryData->SummaryData.add(~group, ~label, ~value, ~decimalPlaces)
-    updatedDataSet->LiveMetrics.saveLiveMetrics(~group, ~label)
+    updatedDataSet->LiveMetrics.addDataSet(~group, ~label)
   }
 }
 
@@ -236,7 +254,7 @@ let addSummaryData = (~group, ~label, ~value, ~decimalPlaces=2) => {
 }
 
 let incrementMillis = (~label, ~amount) => {
-  data->Data.incrementMillis(~label, ~amount)
+  let _ = data->Data.incrementMillis(~label, ~amount)
   data->saveToCacheFile
 }
 
@@ -333,7 +351,9 @@ module Summary = {
       millisAccum.counters
       ->Js.Dict.entries
       ->Array.forEach(((label, millis)) =>
-        timeBreakdown->Js.Array2.push((label, DateFns.durationFromMillis(millis)))->ignore
+        timeBreakdown
+        ->Js.Array2.push((label, DateFns.durationFromMillis(millis->Belt.Int.fromFloat)))
+        ->ignore
       )
 
       timeBreakdown

codegenerator/cli/templates/static/codegen/src/Prometheus.res

@@ -40,49 +40,131 @@ let sourceChainHeight = PromClient.Gauge.makeGauge({
   "labelNames": ["chainId"],
 })
 
-let benchmarkSummaryData = PromClient.Gauge.makeGauge({
-  "name": "benchmark_summary_data",
-  "help": "All data points collected during indexer benchmark",
-  "labelNames": ["group", "label", "stat"],
-})
+module SafeGauge: {
+  type t<'a>
+  let makeOrThrow: (~name: string, ~help: string, ~labelSchema: S.t<'a>) => t<'a>
+  let setInt: (t<'a>, ~labels: 'a, ~value: int) => unit
+  let setFloat: (t<'a>, ~labels: 'a, ~value: float) => unit
+} = {
+  type t<'a> = PromClient.Gauge.gauge
+
+  let rec schemaIsString = (schema: S.t<'a>) =>
+    switch schema->S.classify {
+    | String => true
+    | Null(s)
+    | Option(s) =>
+      s->schemaIsString
+    | _ => false
+    }
+
+  let getLabelNames = (schema: S.t<'a>) =>
+    switch schema->S.classify {
+    | Object({items}) =>
+      let nonStringFields = items->Belt.Array.reduce([], (nonStringFields, item) => {
+        if item.schema->schemaIsString {
+          nonStringFields
+        } else {
+          nonStringFields->Belt.Array.concat([item.location])
+        }
+      })
+
+      switch nonStringFields {
+      | [] => items->Belt.Array.map(item => item.location)->Ok
+      | nonStringItems =>
+        let nonStringItems = nonStringItems->Js.Array2.joinWith(", ")
+        Error(
+          `Label schema must be an object with string (or optional string) values. Non string values: ${nonStringItems}`,
+        )
+      }
+    | _ => Error("Label schema must be an object")
+    }
+
+  let makeOrThrow = (~name, ~help, ~labelSchema: S.t<'a>): t<'a> =>
+    switch labelSchema->getLabelNames {
+    | Ok(labelNames) =>
+      PromClient.Gauge.makeGauge({
+        "name": name,
+        "help": help,
+        "labelNames": labelNames,
+      })
+
+    | Error(error) => Js.Exn.raiseError(error)
+    }
+
+  let makeLabels = (self: t<'a>, ~labels: 'a): t<'a> => self->PromClient.Gauge.labels(labels)
+
+  let setFloat = (self: t<'a>, ~labels: 'a, ~value) =>
+    self
+    ->makeLabels(~labels)
+    ->PromClient.Gauge.setFloat(value)
+
+  let setInt = (self: t<'a>, ~labels: 'a, ~value) =>
+    self
+    ->makeLabels(~labels)
+    ->PromClient.Gauge.set(value)
+}
 
-let setBenchmarkSummaryData = (
-  ~group: string,
-  ~label: string,
-  ~n: float,
-  ~mean: float,
-  ~stdDev: option<float>,
-  ~min: float,
-  ~max: float,
-  ~sum: float,
-) => {
-  benchmarkSummaryData
-  ->PromClient.Gauge.labels({"group": group, "label": label, "stat": "n"})
-  ->PromClient.Gauge.setFloat(n)
-
-  benchmarkSummaryData
-  ->PromClient.Gauge.labels({"group": group, "label": label, "stat": "mean"})
-  ->PromClient.Gauge.setFloat(mean)
-
-  switch stdDev {
-  | Some(stdDev) =>
-    benchmarkSummaryData
-    ->PromClient.Gauge.labels({"group": group, "label": label, "stat": "stdDev"})
-    ->PromClient.Gauge.setFloat(stdDev)
-  | None => ()
+module BenchmarkSummaryData = {
+  type labels = {
+    group: string,
+    stat: string,
+    label: string,
   }
+  let labelSchema = S.schema(s => {
+    group: s.matches(S.string),
+    stat: s.matches(S.string),
+    label: s.matches(S.string),
+  })
+
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="benchmark_summary_data",
+    ~help="All data points collected during indexer benchmark",
+    ~labelSchema,
+  )
+
+  let set = (
+    ~group: string,
+    ~label: string,
+    ~n: float,
+    ~mean: float,
+    ~stdDev: option<float>,
+    ~min: float,
+    ~max: float,
+    ~sum: float,
+  ) => {
+    let mk = stat => {
+      group,
+      stat,
+      label,
+    }
+    gauge->SafeGauge.setFloat(~labels=mk("n"), ~value=n)
+    gauge->SafeGauge.setFloat(~labels=mk("mean"), ~value=mean)
+    gauge->SafeGauge.setFloat(~labels=mk("min"), ~value=min)
+    gauge->SafeGauge.setFloat(~labels=mk("max"), ~value=max)
+    gauge->SafeGauge.setFloat(~labels=mk("sum"), ~value=sum)
+    switch stdDev {
+    | Some(stdDev) => gauge->SafeGauge.setFloat(~labels=mk("stdDev"), ~value=stdDev)
+    | None => ()
+    }
+  }
+}
 
-  benchmarkSummaryData
-  ->PromClient.Gauge.labels({"group": group, "label": label, "stat": "min"})
-  ->PromClient.Gauge.setFloat(min)
-
-  benchmarkSummaryData
-  ->PromClient.Gauge.labels({"group": group, "label": label, "stat": "max"})
-  ->PromClient.Gauge.setFloat(max)
-
-  benchmarkSummaryData
-  ->PromClient.Gauge.labels({"group": group, "label": label, "stat": "sum"})
-  ->PromClient.Gauge.setFloat(sum)
+module BenchmarkCounters = {
+  type labels = {label: string}
+  let labelSchema = S.schema(s => {
+    label: s.matches(S.string),
+  })
+
+  let gauge = SafeGauge.makeOrThrow(
+    ~name="benchmark_counters",
+    ~help="All counters collected during indexer benchmark",
+    ~labelSchema,
+  )
+
+  let set = (~label, ~millis, ~totalRuntimeMillis) => {
+    gauge->SafeGauge.setFloat(~labels={label: label}, ~value=millis)
+    gauge->SafeGauge.setFloat(~labels={label: "Total Run Time (ms)"}, ~value=totalRuntimeMillis)
+  }
 }
 
 // TODO: implement this metric that updates in batches, currently unused