diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b97cd3b3f..22299f8eff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ * [ENHANCEMENT] Ruler: Add support for filtering by `state` and `health` field on Rules API. #6040 * [ENHANCEMENT] Compactor: Split cleaner cycle for active and deleted tenants. #6112 * [ENHANCEMENT] Compactor: Introduce cleaner visit marker. #6113 +* [ENHANCEMENT] Query Frontend: Add cortex_query_samples_total metric. #6142 * [BUGFIX] Configsdb: Fix endline issue in db password. #5920 * [BUGFIX] Ingester: Fix `user` and `type` labels for the `cortex_ingester_tsdb_head_samples_appended_total` TSDB metric. #5952 * [BUGFIX] Querier: Enforce max query length check for `/api/v1/series` API even though `ignoreMaxQueryLength` is set to true. #6018 diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index 61bb2590cf..64c7b55f4b 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -91,6 +91,7 @@ type Handler struct { // Metrics. querySeconds *prometheus.CounterVec querySeries *prometheus.CounterVec + querySamples *prometheus.CounterVec queryChunkBytes *prometheus.CounterVec queryDataBytes *prometheus.CounterVec rejectedQueries *prometheus.CounterVec @@ -116,6 +117,11 @@ func NewHandler(cfg HandlerConfig, roundTripper http.RoundTripper, log log.Logge Help: "Number of series fetched to execute a query.", }, []string{"user"}) + h.querySamples = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ + Name: "cortex_query_samples_total", + Help: "Number of samples fetched to execute a query.", + }, []string{"user"}) + h.queryChunkBytes = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Name: "cortex_query_fetched_chunks_bytes_total", Help: "Size of all chunks fetched to execute a query in bytes.", @@ -137,6 +143,7 @@ func NewHandler(cfg HandlerConfig, roundTripper http.RoundTripper, log log.Logge h.activeUsers = util.NewActiveUsersCleanupWithDefaultValues(func(user string) { h.querySeconds.DeleteLabelValues(user) h.querySeries.DeleteLabelValues(user) + h.querySamples.DeleteLabelValues(user) h.queryChunkBytes.DeleteLabelValues(user) h.queryDataBytes.DeleteLabelValues(user) if err := util.DeleteMatchingLabels(h.rejectedQueries, map[string]string{"user": user}); err != nil { @@ -305,6 +312,7 @@ func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString u // Track stats. f.querySeconds.WithLabelValues(userID).Add(wallTime.Seconds()) f.querySeries.WithLabelValues(userID).Add(float64(numSeries)) + f.querySamples.WithLabelValues(userID).Add(float64(numSamples)) f.queryChunkBytes.WithLabelValues(userID).Add(float64(numChunkBytes)) f.queryDataBytes.WithLabelValues(userID).Add(float64(numDataBytes)) f.activeUsers.UpdateUserTimestamp(userID, time.Now()) diff --git a/pkg/frontend/transport/handler_test.go b/pkg/frontend/transport/handler_test.go index 7967185845..b1933ffc86 100644 --- a/pkg/frontend/transport/handler_test.go +++ b/pkg/frontend/transport/handler_test.go @@ -188,7 +188,7 @@ func TestHandler_ServeHTTP(t *testing.T) { { name: "test handler with stats enabled", cfg: HandlerConfig{QueryStatsEnabled: true}, - expectedMetrics: 3, + expectedMetrics: 4, roundTripperFunc: roundTripper, expectedStatusCode: http.StatusOK, }, @@ -202,7 +202,7 @@ func TestHandler_ServeHTTP(t *testing.T) { { name: "test handler with reasonResponseTooLarge", cfg: HandlerConfig{QueryStatsEnabled: true}, - expectedMetrics: 3, + expectedMetrics: 4, roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusRequestEntityTooLarge, @@ -218,7 +218,7 @@ func TestHandler_ServeHTTP(t *testing.T) { { name: "test handler with reasonTooManyRequests", cfg: HandlerConfig{QueryStatsEnabled: true}, - expectedMetrics: 3, + expectedMetrics: 4, roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusTooManyRequests, @@ -234,7 +234,7 @@ func TestHandler_ServeHTTP(t *testing.T) { { name: "test handler with reasonTooManySamples", cfg: HandlerConfig{QueryStatsEnabled: true}, - expectedMetrics: 3, + expectedMetrics: 4, roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, @@ -250,7 +250,7 @@ func TestHandler_ServeHTTP(t *testing.T) { { name: "test handler with reasonTooLongRange", cfg: HandlerConfig{QueryStatsEnabled: true}, - expectedMetrics: 3, + expectedMetrics: 4, roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, @@ -266,7 +266,7 @@ func TestHandler_ServeHTTP(t *testing.T) { { name: "test handler with reasonSeriesFetched", cfg: HandlerConfig{QueryStatsEnabled: true}, - expectedMetrics: 3, + expectedMetrics: 4, roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, @@ -282,7 +282,7 @@ func TestHandler_ServeHTTP(t *testing.T) { { name: "test handler with reasonChunksFetched", cfg: HandlerConfig{QueryStatsEnabled: true}, - expectedMetrics: 3, + expectedMetrics: 4, roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, @@ -298,7 +298,7 @@ func TestHandler_ServeHTTP(t *testing.T) { { name: "test handler with reasonChunkBytesFetched", cfg: HandlerConfig{QueryStatsEnabled: true}, - expectedMetrics: 3, + expectedMetrics: 4, roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, @@ -314,7 +314,7 @@ func TestHandler_ServeHTTP(t *testing.T) { { name: "test handler with reasonDataBytesFetched", cfg: HandlerConfig{QueryStatsEnabled: true}, - expectedMetrics: 3, + expectedMetrics: 4, roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, @@ -330,7 +330,7 @@ func TestHandler_ServeHTTP(t *testing.T) { { name: "test handler with reasonSeriesLimitStoreGateway", cfg: HandlerConfig{QueryStatsEnabled: true}, - expectedMetrics: 3, + expectedMetrics: 4, roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, @@ -346,7 +346,7 @@ func TestHandler_ServeHTTP(t *testing.T) { { name: "test handler with reasonChunksLimitStoreGateway", cfg: HandlerConfig{QueryStatsEnabled: true}, - expectedMetrics: 3, + expectedMetrics: 4, roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, @@ -362,7 +362,7 @@ func TestHandler_ServeHTTP(t *testing.T) { { name: "test handler with reasonBytesLimitStoreGateway", cfg: HandlerConfig{QueryStatsEnabled: true}, - expectedMetrics: 3, + expectedMetrics: 4, roundTripperFunc: roundTripperFunc(func(req *http.Request) (*http.Response, error) { return &http.Response{ StatusCode: http.StatusUnprocessableEntity, @@ -393,6 +393,7 @@ func TestHandler_ServeHTTP(t *testing.T) { reg, "cortex_query_seconds_total", "cortex_query_fetched_series_total", + "cortex_query_samples_total", "cortex_query_fetched_chunks_bytes_total", )