From e9d2ed70dc3ce09a9402e86336ea58c96f7570a9 Mon Sep 17 00:00:00 2001 From: Anna Tran Date: Thu, 28 Nov 2024 05:02:24 -0800 Subject: [PATCH] Add label size bytes native histogram (#6380) * Add label size bytes native histogram Signed-off-by: Anna Tran * Update DeletePerUserValidationMetrics with labelSizeBytes Signed-off-by: Anna Tran --------- Signed-off-by: Anna Tran --- CHANGELOG.md | 1 + pkg/util/validation/validate.go | 15 +++++++++++++++ pkg/util/validation/validate_test.go | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad1e276bae..c70c1b14af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ * [ENHANCEMENT] Querier/Ruler: Expose `store_gateway_consistency_check_max_attempts` for max retries when querying store gateway in consistency check. #6276 * [ENHANCEMENT] StoreGateway: Add new `cortex_bucket_store_chunk_pool_inuse_bytes` metric to track the usage in chunk pool. #6310 * [ENHANCEMENT] Distributor: Add new `cortex_distributor_inflight_client_requests` metric to track number of ingester client inflight requests. #6358 +* [ENHANCEMENT] Distributor: Expose `cortex_label_size_bytes` native histogram metric. #6372 * [BUGFIX] Runtime-config: Handle absolute file paths when working directory is not / #6224 * [BUGFIX] Ruler: Allow rule evaluation to complete during shutdown. #6326 * [BUGFIX] Ring: update ring with new ip address when instance is lost, rejoins, but heartbeat is disabled #6271 diff --git a/pkg/util/validation/validate.go b/pkg/util/validation/validate.go index be94cfa2f1..3e36ddf529 100644 --- a/pkg/util/validation/validate.go +++ b/pkg/util/validation/validate.go @@ -78,6 +78,7 @@ type ValidateMetrics struct { DiscardedExemplars *prometheus.CounterVec DiscardedMetadata *prometheus.CounterVec HistogramSamplesReducedResolution *prometheus.CounterVec + LabelSizeBytes *prometheus.HistogramVec } func registerCollector(r prometheus.Registerer, c prometheus.Collector) { @@ -120,11 +121,21 @@ func NewValidateMetrics(r prometheus.Registerer) *ValidateMetrics { []string{"user"}, ) registerCollector(r, histogramSamplesReducedResolution) + labelSizeBytes := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "cortex_label_size_bytes", + Help: "The combined size in bytes of all labels and label values for a time series.", + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }, []string{"user"}) + registerCollector(r, labelSizeBytes) + m := &ValidateMetrics{ DiscardedSamples: discardedSamples, DiscardedExemplars: discardedExemplars, DiscardedMetadata: discardedMetadata, HistogramSamplesReducedResolution: histogramSamplesReducedResolution, + LabelSizeBytes: labelSizeBytes, } return m @@ -236,6 +247,7 @@ func ValidateLabels(validateMetrics *ValidateMetrics, limits *Limits, userID str lastLabelName = l.Name labelsSizeBytes += l.Size() } + validateMetrics.LabelSizeBytes.WithLabelValues(userID).Observe(float64(labelsSizeBytes)) if maxLabelsSizeBytes > 0 && labelsSizeBytes > maxLabelsSizeBytes { validateMetrics.DiscardedSamples.WithLabelValues(labelsSizeBytesExceeded, userID).Inc() return labelSizeBytesExceededError(ls, labelsSizeBytes, maxLabelsSizeBytes) @@ -352,4 +364,7 @@ func DeletePerUserValidationMetrics(validateMetrics *ValidateMetrics, userID str if err := util.DeleteMatchingLabels(validateMetrics.HistogramSamplesReducedResolution, filter); err != nil { level.Warn(log).Log("msg", "failed to remove cortex_reduced_resolution_histogram_samples_total metric for user", "user", userID, "err", err) } + if err := util.DeleteMatchingLabels(validateMetrics.LabelSizeBytes, filter); err != nil { + level.Warn(log).Log("msg", "failed to remove cortex_label_size_bytes metric for user", "user", userID, "err", err) + } } diff --git a/pkg/util/validation/validate_test.go b/pkg/util/validation/validate_test.go index 7d7cf45930..6b6fb00bbf 100644 --- a/pkg/util/validation/validate_test.go +++ b/pkg/util/validation/validate_test.go @@ -119,6 +119,14 @@ func TestValidateLabels(t *testing.T) { cortex_discarded_samples_total{reason="random reason",user="different user"} 1 `), "cortex_discarded_samples_total")) + require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_label_size_bytes The combined size in bytes of all labels and label values for a time series. + # TYPE cortex_label_size_bytes histogram + cortex_label_size_bytes_bucket{user="testUser",le="+Inf"} 3 + cortex_label_size_bytes_sum{user="testUser"} 148 + cortex_label_size_bytes_count{user="testUser"} 3 + `), "cortex_label_size_bytes")) + DeletePerUserValidationMetrics(validateMetrics, userID, util_log.Logger) require.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(`