diff --git a/codegenerator/cli/templates/static/codegen/src/Benchmark.res b/codegenerator/cli/templates/static/codegen/src/Benchmark.res index b96ffeaa9..14ecadbd7 100644 --- a/codegenerator/cli/templates/static/codegen/src/Benchmark.res +++ b/codegenerator/cli/templates/static/codegen/src/Benchmark.res @@ -1,8 +1,8 @@ module MillisAccum = { - type millis = int + type millis = float type t = {counters: dict, startTime: Js.Date.t, mutable endTime: Js.Date.t} let schema: S.t = S.schema(s => { - counters: s.matches(S.dict(S.int)), + counters: s.matches(S.dict(S.float)), startTime: s.matches(S.string->S.datetime), endTime: s.matches(S.string->S.datetime), }) @@ -13,11 +13,17 @@ module MillisAccum = { } let increment = (self: t, label, amount) => { + self.endTime = Js.Date.make() + let amount = amount->Belt.Float.fromInt switch self.counters->Utils.Dict.dangerouslyGetNonOption(label) { - | None => self.counters->Js.Dict.set(label, amount) - | Some(current) => self.counters->Js.Dict.set(label, current + amount) + | None => + self.counters->Js.Dict.set(label, amount) + amount + | Some(current) => + let newAmount = current +. amount + self.counters->Js.Dict.set(label, newAmount) + newAmount } - self.endTime = Js.Date.make() } } @@ -72,22 +78,22 @@ module SummaryData = { module Group = { type t = dict let schema: S.t = S.dict(DataSet.schema) - let make = () => Js.Dict.empty() + let make = (): t => Js.Dict.empty() /** Adds a value to the data set for the given key. If the key does not exist, it will be created. Returns the updated data set. */ - let add = (self: t, key: string, value: float, ~decimalPlaces=2) => { - switch self->Utils.Dict.dangerouslyGetNonOption(key) { + let add = (self: t, label, value: float, ~decimalPlaces=2) => { + switch self->Utils.Dict.dangerouslyGetNonOption(label) { | None => let new = DataSet.make(value, ~decimalPlaces) - self->Js.Dict.set(key, new) + self->Js.Dict.set(label, new) new | Some(dataSet) => let updated = dataSet->DataSet.add(value) - self->Js.Dict.set(key, updated) + self->Js.Dict.set(label, updated) updated } } @@ -95,7 +101,7 @@ module SummaryData = { type t = dict let schema = S.dict(Group.schema) - let make = () => Js.Dict.empty() + let make = (): t => Js.Dict.empty() let add = (self: t, ~group, ~label, ~value, ~decimalPlaces=2) => { let group = switch self->Utils.Dict.dangerouslyGetNonOption(group) { @@ -169,26 +175,38 @@ module Data = { summaryData: SummaryData.make(), } - let incrementMillis = (self: t, ~label, ~amount) => { - self.millisAccum->MillisAccum.increment(label, amount) - } - module LiveMetrics = { - let saveLiveMetrics = if ( + let addDataSet = if ( Env.Benchmark.saveDataStrategy->Env.Benchmark.SaveDataStrategy.shouldSavePrometheus ) { (dataSet: SummaryData.DataSet.t, ~group, ~label) => { let {n, mean, stdDev, min, max, sum} = dataSet->Stats.makeFromDataSet - Prometheus.setBenchmarkSummaryData(~group, ~label, ~n, ~mean, ~stdDev, ~min, ~max, ~sum) + Prometheus.BenchmarkSummaryData.set(~group, ~label, ~n, ~mean, ~stdDev, ~min, ~max, ~sum) } } else { (_dataSet, ~group as _, ~label as _) => () } + let setCounterMillis = if ( + Env.Benchmark.saveDataStrategy->Env.Benchmark.SaveDataStrategy.shouldSavePrometheus + ) { + (millisAccum: MillisAccum.t, ~label, ~millis) => { + let totalRuntimeMillis = + millisAccum.endTime->Js.Date.getTime -. millisAccum.startTime->Js.Date.getTime + Prometheus.BenchmarkCounters.set(~label, ~millis, ~totalRuntimeMillis) + } + } else { + (_, ~label as _, ~millis as _) => () + } + } + + let incrementMillis = (self: t, ~label, ~amount) => { + let nextMillis = self.millisAccum->MillisAccum.increment(label, amount) + self.millisAccum->LiveMetrics.setCounterMillis(~label, ~millis=nextMillis) } let addSummaryData = (self: t, ~group, ~label, ~value, ~decimalPlaces=2) => { let updatedDataSet = self.summaryData->SummaryData.add(~group, ~label, ~value, ~decimalPlaces) - updatedDataSet->LiveMetrics.saveLiveMetrics(~group, ~label) + updatedDataSet->LiveMetrics.addDataSet(~group, ~label) } } @@ -236,7 +254,7 @@ let addSummaryData = (~group, ~label, ~value, ~decimalPlaces=2) => { } let incrementMillis = (~label, ~amount) => { - data->Data.incrementMillis(~label, ~amount) + let _ = data->Data.incrementMillis(~label, ~amount) data->saveToCacheFile } @@ -333,7 +351,9 @@ module Summary = { millisAccum.counters ->Js.Dict.entries ->Array.forEach(((label, millis)) => - timeBreakdown->Js.Array2.push((label, DateFns.durationFromMillis(millis)))->ignore + timeBreakdown + ->Js.Array2.push((label, DateFns.durationFromMillis(millis->Belt.Int.fromFloat))) + ->ignore ) timeBreakdown diff --git a/codegenerator/cli/templates/static/codegen/src/Prometheus.res b/codegenerator/cli/templates/static/codegen/src/Prometheus.res index ab7f8e2fc..13d74f77f 100644 --- a/codegenerator/cli/templates/static/codegen/src/Prometheus.res +++ b/codegenerator/cli/templates/static/codegen/src/Prometheus.res @@ -40,49 +40,131 @@ let sourceChainHeight = PromClient.Gauge.makeGauge({ "labelNames": ["chainId"], }) -let benchmarkSummaryData = PromClient.Gauge.makeGauge({ - "name": "benchmark_summary_data", - "help": "All data points collected during indexer benchmark", - "labelNames": ["group", "label", "stat"], -}) +module SafeGauge: { + type t<'a> + let makeOrThrow: (~name: string, ~help: string, ~labelSchema: S.t<'a>) => t<'a> + let setInt: (t<'a>, ~labels: 'a, ~value: int) => unit + let setFloat: (t<'a>, ~labels: 'a, ~value: float) => unit +} = { + type t<'a> = PromClient.Gauge.gauge + + let rec schemaIsString = (schema: S.t<'a>) => + switch schema->S.classify { + | String => true + | Null(s) + | Option(s) => + s->schemaIsString + | _ => false + } + + let getLabelNames = (schema: S.t<'a>) => + switch schema->S.classify { + | Object({items}) => + let nonStringFields = items->Belt.Array.reduce([], (nonStringFields, item) => { + if item.schema->schemaIsString { + nonStringFields + } else { + nonStringFields->Belt.Array.concat([item.location]) + } + }) + + switch nonStringFields { + | [] => items->Belt.Array.map(item => item.location)->Ok + | nonStringItems => + let nonStringItems = nonStringItems->Js.Array2.joinWith(", ") + Error( + `Label schema must be an object with string (or optional string) values. Non string values: ${nonStringItems}`, + ) + } + | _ => Error("Label schema must be an object") + } + + let makeOrThrow = (~name, ~help, ~labelSchema: S.t<'a>): t<'a> => + switch labelSchema->getLabelNames { + | Ok(labelNames) => + PromClient.Gauge.makeGauge({ + "name": name, + "help": help, + "labelNames": labelNames, + }) + + | Error(error) => Js.Exn.raiseError(error) + } + + let makeLabels = (self: t<'a>, ~labels: 'a): t<'a> => self->PromClient.Gauge.labels(labels) + + let setFloat = (self: t<'a>, ~labels: 'a, ~value) => + self + ->makeLabels(~labels) + ->PromClient.Gauge.setFloat(value) + + let setInt = (self: t<'a>, ~labels: 'a, ~value) => + self + ->makeLabels(~labels) + ->PromClient.Gauge.set(value) +} -let setBenchmarkSummaryData = ( - ~group: string, - ~label: string, - ~n: float, - ~mean: float, - ~stdDev: option, - ~min: float, - ~max: float, - ~sum: float, -) => { - benchmarkSummaryData - ->PromClient.Gauge.labels({"group": group, "label": label, "stat": "n"}) - ->PromClient.Gauge.setFloat(n) - - benchmarkSummaryData - ->PromClient.Gauge.labels({"group": group, "label": label, "stat": "mean"}) - ->PromClient.Gauge.setFloat(mean) - - switch stdDev { - | Some(stdDev) => - benchmarkSummaryData - ->PromClient.Gauge.labels({"group": group, "label": label, "stat": "stdDev"}) - ->PromClient.Gauge.setFloat(stdDev) - | None => () +module BenchmarkSummaryData = { + type labels = { + group: string, + stat: string, + label: string, } + let labelSchema = S.schema(s => { + group: s.matches(S.string), + stat: s.matches(S.string), + label: s.matches(S.string), + }) + + let gauge = SafeGauge.makeOrThrow( + ~name="benchmark_summary_data", + ~help="All data points collected during indexer benchmark", + ~labelSchema, + ) + + let set = ( + ~group: string, + ~label: string, + ~n: float, + ~mean: float, + ~stdDev: option, + ~min: float, + ~max: float, + ~sum: float, + ) => { + let mk = stat => { + group, + stat, + label, + } + gauge->SafeGauge.setFloat(~labels=mk("n"), ~value=n) + gauge->SafeGauge.setFloat(~labels=mk("mean"), ~value=mean) + gauge->SafeGauge.setFloat(~labels=mk("min"), ~value=min) + gauge->SafeGauge.setFloat(~labels=mk("max"), ~value=max) + gauge->SafeGauge.setFloat(~labels=mk("sum"), ~value=sum) + switch stdDev { + | Some(stdDev) => gauge->SafeGauge.setFloat(~labels=mk("stdDev"), ~value=stdDev) + | None => () + } + } +} - benchmarkSummaryData - ->PromClient.Gauge.labels({"group": group, "label": label, "stat": "min"}) - ->PromClient.Gauge.setFloat(min) - - benchmarkSummaryData - ->PromClient.Gauge.labels({"group": group, "label": label, "stat": "max"}) - ->PromClient.Gauge.setFloat(max) - - benchmarkSummaryData - ->PromClient.Gauge.labels({"group": group, "label": label, "stat": "sum"}) - ->PromClient.Gauge.setFloat(sum) +module BenchmarkCounters = { + type labels = {label: string} + let labelSchema = S.schema(s => { + label: s.matches(S.string), + }) + + let gauge = SafeGauge.makeOrThrow( + ~name="benchmark_counters", + ~help="All counters collected during indexer benchmark", + ~labelSchema, + ) + + let set = (~label, ~millis, ~totalRuntimeMillis) => { + gauge->SafeGauge.setFloat(~labels={label: label}, ~value=millis) + gauge->SafeGauge.setFloat(~labels={label: "Total Run Time (ms)"}, ~value=totalRuntimeMillis) + } } // TODO: implement this metric that updates in batches, currently unused