From b5a944da4083989f01e82460e7e34c0c2baf7c9f Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 15 Oct 2024 11:20:31 -0500 Subject: [PATCH 1/3] Add feature that can disable traffic generator metrics. Signed-off-by: Cody Littley --- tools/traffic/Makefile | 2 + tools/traffic/config/config.go | 3 ++ tools/traffic/config/flags.go | 16 ++++++++ tools/traffic/config/worker_config.go | 10 ++++- tools/traffic/generator_v2.go | 6 ++- tools/traffic/metrics/count_metric.go | 7 +++- tools/traffic/metrics/gauge_metric.go | 7 +++- tools/traffic/metrics/latency_metric.go | 7 +++- tools/traffic/metrics/metrics.go | 51 +++++++++++++++++++++---- 9 files changed, 97 insertions(+), 12 deletions(-) diff --git a/tools/traffic/Makefile b/tools/traffic/Makefile index 1a1e0efab3..696d8bb293 100644 --- a/tools/traffic/Makefile +++ b/tools/traffic/Makefile @@ -48,4 +48,6 @@ run2: build2 TRAFFIC_GENERATOR_NUM_WORKERS=4 \ TRAFFIC_GENERATOR_CHAIN_RPC=http://localhost:8545 \ TRAFFIC_GENERATOR_NUM_RETRIES=2 \ + TRAFFIC_GENERATOR_METRICS_FUZZY_BLACKLIST=_returned_chunk,recombination_success \ + TRAFFIC_GENERATOR_METRICS_BLACKLIST=get_status_CONFIRMED \ ./bin/server2 diff --git a/tools/traffic/config/config.go b/tools/traffic/config/config.go index 3f7f7d41ed..2b147f1a49 100644 --- a/tools/traffic/config/config.go +++ b/tools/traffic/config/config.go @@ -106,6 +106,9 @@ func NewConfig(ctx *cli.Context) (*Config, error) { EigenDAServiceManager: retrieverConfig.EigenDAServiceManagerAddr, SignerPrivateKey: ctx.String(SignerPrivateKeyFlag.Name), CustomQuorums: customQuorumsUint8, + + MetricsBlacklist: ctx.StringSlice(MetricsBlacklistFlag.Name), + MetricsFuzzyBlacklist: ctx.StringSlice(MetricsFuzzyBlacklistFlag.Name), }, } diff --git a/tools/traffic/config/flags.go b/tools/traffic/config/flags.go index 20105de626..c4218e52cf 100644 --- a/tools/traffic/config/flags.go +++ b/tools/traffic/config/flags.go @@ -80,6 +80,20 @@ var ( EnvVar: common.PrefixEnvVar(envPrefix, "INSTANCE_LAUNCH_INTERVAL"), } + MetricsBlacklistFlag = cli.StringSliceFlag{ + Name: common.PrefixFlag(FlagPrefix, "metrics-blacklist"), + Usage: "Any metric with a label exactly matching this string will not be sent to the metrics server.", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "METRICS_BLACKLIST"), + } + + MetricsFuzzyBlacklistFlag = cli.StringSliceFlag{ + Name: common.PrefixFlag(FlagPrefix, "metrics-fuzzy-blacklist"), + Usage: "Any metric that contains any string in this list will not be sent to the metrics server.", + Required: false, + EnvVar: common.PrefixEnvVar(envPrefix, "METRICS_FUZZY_BLACKLIST"), + } + /* Configuration for the blob writer. */ NumWriteInstancesFlag = cli.UintFlag{ @@ -238,6 +252,8 @@ var optionalFlags = []cli.Flag{ GetBlobStatusTimeoutFlag, WriteTimeoutFlag, VerificationChannelCapacityFlag, + MetricsBlacklistFlag, + MetricsFuzzyBlacklistFlag, } // Flags contains the list of configuration options available to the binary. diff --git a/tools/traffic/config/worker_config.go b/tools/traffic/config/worker_config.go index 08dead2958..78eb7a6fc2 100644 --- a/tools/traffic/config/worker_config.go +++ b/tools/traffic/config/worker_config.go @@ -2,7 +2,7 @@ package config import "time" -// Config configures the traffic generator workers. +// WorkerConfig configures the traffic generator workers. type WorkerConfig struct { // The number of worker threads that generate write traffic. NumWriteInstances uint @@ -42,4 +42,12 @@ type WorkerConfig struct { SignerPrivateKey string // Custom quorum numbers to use for the traffic generator. CustomQuorums []uint8 + + // Any metric with a label exactly matching one of the strings in this list will not be sent to the metrics server. + MetricsBlacklist []string + + // Any metric that contains any string in this list will not be sent to the metrics server. For example, + // including the string "_returned_chunk" will cause all metrics in the form of + // "operator_fb390a64122db3957fb220c3c42d5f71e97ab0c995da4e1e5cc3261602dac527_returned_chunk" to be omitted. + MetricsFuzzyBlacklist []string } diff --git a/tools/traffic/generator_v2.go b/tools/traffic/generator_v2.go index cb9165d9ca..d5ce70ae67 100644 --- a/tools/traffic/generator_v2.go +++ b/tools/traffic/generator_v2.go @@ -77,7 +77,11 @@ func NewTrafficGeneratorV2(config *config.Config) (*Generator, error) { ctx, cancel := context.WithCancel(context.Background()) waitGroup := sync.WaitGroup{} - generatorMetrics := metrics.NewMetrics(config.MetricsHTTPPort, logger) + generatorMetrics := metrics.NewMetrics( + config.MetricsHTTPPort, + logger, + config.WorkerConfig.MetricsBlacklist, + config.WorkerConfig.MetricsFuzzyBlacklist) blobTable := table.NewBlobStore() diff --git a/tools/traffic/metrics/count_metric.go b/tools/traffic/metrics/count_metric.go index 0e9ba6e7d2..b9c51bcc88 100644 --- a/tools/traffic/metrics/count_metric.go +++ b/tools/traffic/metrics/count_metric.go @@ -14,14 +14,19 @@ type CountMetric interface { type countMetric struct { metrics *metrics description string + // If true, the metric is disabled and should behave as a no-op. + disabled bool } // Increment increments the count of a type of event. func (metric *countMetric) Increment() { + if metric.disabled { + return + } metric.metrics.count.WithLabelValues(metric.description).Inc() } -// NewCountMetric creates a new prometheus collector for counting metrics. +// buildCounterCollector creates a new prometheus collector for counting metrics. func buildCounterCollector(namespace string, registry *prometheus.Registry) *prometheus.CounterVec { return promauto.With(registry).NewCounterVec( prometheus.CounterOpts{ diff --git a/tools/traffic/metrics/gauge_metric.go b/tools/traffic/metrics/gauge_metric.go index 5cace773d5..e9dfb4d8c9 100644 --- a/tools/traffic/metrics/gauge_metric.go +++ b/tools/traffic/metrics/gauge_metric.go @@ -15,14 +15,19 @@ type GaugeMetric interface { type gaugeMetric struct { metrics *metrics description string + // If true, the metric is disabled and should behave as a no-op. + disabled bool } // Set sets the value of a gauge metric. func (metric *gaugeMetric) Set(value float64) { + if metric.disabled { + return + } metric.metrics.gauge.WithLabelValues(metric.description).Set(value) } -// NewGaugeMetric creates a collector for gauge metrics. +// buildGaugeCollector creates a collector for gauge metrics. func buildGaugeCollector(namespace string, registry *prometheus.Registry) *prometheus.GaugeVec { return promauto.With(registry).NewGaugeVec( prometheus.GaugeOpts{ diff --git a/tools/traffic/metrics/latency_metric.go b/tools/traffic/metrics/latency_metric.go index 8929940790..4f82bae69e 100644 --- a/tools/traffic/metrics/latency_metric.go +++ b/tools/traffic/metrics/latency_metric.go @@ -15,14 +15,19 @@ type LatencyMetric interface { type latencyMetric struct { metrics *metrics description string + // If true, the metric is disabled and should behave as a no-op. + disabled bool } // ReportLatency reports the latency of an operation. func (metric *latencyMetric) ReportLatency(latency time.Duration) { + if metric.disabled { + return + } metric.metrics.latency.WithLabelValues(metric.description).Observe(latency.Seconds()) } -// NewLatencyMetric creates a new prometheus collector for latency metrics. +// buildLatencyCollector creates a new prometheus collector for latency metrics. func buildLatencyCollector(namespace string, registry *prometheus.Registry) *prometheus.SummaryVec { return promauto.With(registry).NewSummaryVec( prometheus.SummaryOpts{ diff --git a/tools/traffic/metrics/metrics.go b/tools/traffic/metrics/metrics.go index 971dad2483..3e68777ce6 100644 --- a/tools/traffic/metrics/metrics.go +++ b/tools/traffic/metrics/metrics.go @@ -7,6 +7,7 @@ import ( "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" "net/http" + "strings" ) // Metrics allows the creation of metrics for the traffic generator. @@ -31,22 +32,40 @@ type metrics struct { httpPort string logger logging.Logger + + metricsBlacklist []string + metricsFuzzyBlacklist []string } // NewMetrics creates a new Metrics instance. -func NewMetrics(httpPort string, logger logging.Logger) Metrics { +func NewMetrics( + httpPort string, + logger logging.Logger, + metricsBlacklist []string, + metricsFuzzyBlacklist []string) Metrics { + namespace := "eigenda_generator" reg := prometheus.NewRegistry() reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) reg.MustRegister(collectors.NewGoCollector()) + if metricsBlacklist == nil { + metricsBlacklist = []string{} + } + + fmt.Printf("-------------------------------------------\n") + fmt.Printf("metricsBlacklist: %v\n", metricsBlacklist) + fmt.Printf("metricsFuzzyBlacklist: %v\n", metricsFuzzyBlacklist) + metrics := &metrics{ - count: buildCounterCollector(namespace, reg), - latency: buildLatencyCollector(namespace, reg), - gauge: buildGaugeCollector(namespace, reg), - registry: reg, - httpPort: httpPort, - logger: logger.With("component", "GeneratorMetrics"), + count: buildCounterCollector(namespace, reg), + latency: buildLatencyCollector(namespace, reg), + gauge: buildGaugeCollector(namespace, reg), + registry: reg, + httpPort: httpPort, + logger: logger.With("component", "GeneratorMetrics"), + metricsBlacklist: metricsBlacklist, + metricsFuzzyBlacklist: metricsFuzzyBlacklist, } return metrics } @@ -71,6 +90,7 @@ func (metrics *metrics) NewLatencyMetric(description string) LatencyMetric { return &latencyMetric{ metrics: metrics, description: description, + disabled: metrics.isBlacklisted(description), } } @@ -79,6 +99,7 @@ func (metrics *metrics) NewCountMetric(description string) CountMetric { return &countMetric{ metrics: metrics, description: description, + disabled: metrics.isBlacklisted(description), } } @@ -87,5 +108,21 @@ func (metrics *metrics) NewGaugeMetric(description string) GaugeMetric { return &gaugeMetric{ metrics: metrics, description: description, + disabled: metrics.isBlacklisted(description), + } +} + +// isBlacklisted returns true if the metric name is blacklisted. +func (metrics *metrics) isBlacklisted(metricName string) bool { + for _, blacklisted := range metrics.metricsBlacklist { + if metricName == blacklisted { + return true + } + } + for _, blacklisted := range metrics.metricsFuzzyBlacklist { + if strings.Contains(metricName, blacklisted) { + return true + } } + return false } From 83f0590b1bd0aadae51caee67ae7dad9123f04ea Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Tue, 15 Oct 2024 11:23:01 -0500 Subject: [PATCH 2/3] Handle nil case. Signed-off-by: Cody Littley --- tools/traffic/metrics/metrics.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/traffic/metrics/metrics.go b/tools/traffic/metrics/metrics.go index 3e68777ce6..e24a52dfc8 100644 --- a/tools/traffic/metrics/metrics.go +++ b/tools/traffic/metrics/metrics.go @@ -52,10 +52,9 @@ func NewMetrics( if metricsBlacklist == nil { metricsBlacklist = []string{} } - - fmt.Printf("-------------------------------------------\n") - fmt.Printf("metricsBlacklist: %v\n", metricsBlacklist) - fmt.Printf("metricsFuzzyBlacklist: %v\n", metricsFuzzyBlacklist) + if metricsFuzzyBlacklist == nil { + metricsFuzzyBlacklist = []string{} + } metrics := &metrics{ count: buildCounterCollector(namespace, reg), From 80597fab2f302d95b48002e73a742bc559259d60 Mon Sep 17 00:00:00 2001 From: Cody Littley Date: Wed, 16 Oct 2024 08:52:45 -0500 Subject: [PATCH 3/3] Made suggested changes. Signed-off-by: Cody Littley --- tools/traffic/metrics/count_metric.go | 2 +- tools/traffic/metrics/gauge_metric.go | 2 +- tools/traffic/metrics/latency_metric.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/traffic/metrics/count_metric.go b/tools/traffic/metrics/count_metric.go index b9c51bcc88..daa508bb80 100644 --- a/tools/traffic/metrics/count_metric.go +++ b/tools/traffic/metrics/count_metric.go @@ -14,7 +14,7 @@ type CountMetric interface { type countMetric struct { metrics *metrics description string - // If true, the metric is disabled and should behave as a no-op. + // disabled specifies whether the metrics should behave as a no-op disabled bool } diff --git a/tools/traffic/metrics/gauge_metric.go b/tools/traffic/metrics/gauge_metric.go index e9dfb4d8c9..aa2d5d641f 100644 --- a/tools/traffic/metrics/gauge_metric.go +++ b/tools/traffic/metrics/gauge_metric.go @@ -15,7 +15,7 @@ type GaugeMetric interface { type gaugeMetric struct { metrics *metrics description string - // If true, the metric is disabled and should behave as a no-op. + // disabled specifies whether the metrics should behave as a no-op disabled bool } diff --git a/tools/traffic/metrics/latency_metric.go b/tools/traffic/metrics/latency_metric.go index 4f82bae69e..79e1d12da7 100644 --- a/tools/traffic/metrics/latency_metric.go +++ b/tools/traffic/metrics/latency_metric.go @@ -15,7 +15,7 @@ type LatencyMetric interface { type latencyMetric struct { metrics *metrics description string - // If true, the metric is disabled and should behave as a no-op. + // disabled specifies whether the metrics should behave as a no-op disabled bool }