From e74c6040d4acad413b434aa5bf1b75901e447ac5 Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Fri, 26 Apr 2024 12:06:16 -0700 Subject: [PATCH] Collect store gateway postings touched count and bytes in querier stats (#5892) * update stats Signed-off-by: Ben Ye * update stats Signed-off-by: Ben Ye * collect store gateway postings touched count and bytes in querier stats Signed-off-by: Ben Ye * changelog Signed-off-by: Ben Ye --------- Signed-off-by: Ben Ye --- CHANGELOG.md | 2 + pkg/frontend/transport/handler.go | 7 ++ pkg/frontend/transport/handler_test.go | 17 +++ pkg/querier/blocks_store_queryable.go | 2 + pkg/querier/stats/stats.go | 34 ++++++ pkg/querier/stats/stats.pb.go | 153 +++++++++++++++++++------ pkg/querier/stats/stats.proto | 6 + pkg/querier/stats/stats_test.go | 42 +++++++ 8 files changed, 230 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e66b25fb9..71b7094542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,8 @@ ## master / unreleased * [CHANGE] Ruler: Remove `experimental.ruler.api-enable-rules-backup` flag and use `ruler.ring.replication-factor` to check if rules backup is enabled +* [ENHANCEMENT] Query Frontend/Querier: Added store gateway postings touched count and touched size in Querier stats and log in Query Frontend. #5892 + ## 1.17.0 in progress * [CHANGE] Azure Storage: Upgraded objstore dependency and support Azure Workload Identity Authentication. Added `connection_string` to support authenticating via SAS token. Marked `msi_resource` config as deprecating. #5645 diff --git a/pkg/frontend/transport/handler.go b/pkg/frontend/transport/handler.go index 6da1686562..61bb2590cf 100644 --- a/pkg/frontend/transport/handler.go +++ b/pkg/frontend/transport/handler.go @@ -296,6 +296,8 @@ func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString u numSamples := stats.LoadFetchedSamples() numChunkBytes := stats.LoadFetchedChunkBytes() numDataBytes := stats.LoadFetchedDataBytes() + numStoreGatewayTouchedPostings := stats.LoadStoreGatewayTouchedPostings() + numStoreGatewayTouchedPostingBytes := stats.LoadStoreGatewayTouchedPostingBytes() splitQueries := stats.LoadSplitQueries() dataSelectMaxTime := stats.LoadDataSelectMaxTime() dataSelectMinTime := stats.LoadDataSelectMinTime() @@ -334,6 +336,11 @@ func (f *Handler) reportQueryStats(r *http.Request, userID string, queryString u "response_size", contentLength, }, stats.LoadExtraFields()...) + if numStoreGatewayTouchedPostings > 0 { + logMessage = append(logMessage, "store_gateway_touched_postings_count", numStoreGatewayTouchedPostings) + logMessage = append(logMessage, "store_gateway_touched_posting_bytes", numStoreGatewayTouchedPostingBytes) + } + grafanaFields := formatGrafanaStatsFields(r) if len(grafanaFields) > 0 { logMessage = append(logMessage, grafanaFields...) diff --git a/pkg/frontend/transport/handler_test.go b/pkg/frontend/transport/handler_test.go index d3e97d367b..7967185845 100644 --- a/pkg/frontend/transport/handler_test.go +++ b/pkg/frontend/transport/handler_test.go @@ -471,6 +471,23 @@ func TestReportQueryStatsFormat(t *testing.T) { }, expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=0 fetched_series_count=0 fetched_chunks_count=0 fetched_samples_count=0 fetched_chunks_bytes=0 fetched_data_bytes=0 split_queries=0 status_code=200 response_size=1000 data_select_max_time=1704153600 data_select_min_time=1704067200 query_length=2 param_query=up`, }, + "should include query stats with store gateway stats": { + queryStats: &querier_stats.QueryStats{ + Stats: querier_stats.Stats{ + WallTime: 3 * time.Second, + QueryStorageWallTime: 100 * time.Minute, + FetchedSeriesCount: 100, + FetchedChunksCount: 200, + FetchedSamplesCount: 300, + FetchedChunkBytes: 1024, + FetchedDataBytes: 2048, + SplitQueries: 10, + StoreGatewayTouchedPostingsCount: 20, + StoreGatewayTouchedPostingBytes: 200, + }, + }, + expectedLog: `level=info msg="query stats" component=query-frontend method=GET path=/prometheus/api/v1/query response_time=1s query_wall_time_seconds=3 fetched_series_count=100 fetched_chunks_count=200 fetched_samples_count=300 fetched_chunks_bytes=1024 fetched_data_bytes=2048 split_queries=10 status_code=200 response_size=1000 store_gateway_touched_postings_count=20 store_gateway_touched_posting_bytes=200 query_storage_wall_time_seconds=6000`, + }, } for testName, testData := range tests { diff --git a/pkg/querier/blocks_store_queryable.go b/pkg/querier/blocks_store_queryable.go index 11289bc103..7c56649a05 100644 --- a/pkg/querier/blocks_store_queryable.go +++ b/pkg/querier/blocks_store_queryable.go @@ -756,6 +756,8 @@ func (q *blocksStoreQuerier) fetchSeriesFromStores( reqStats.AddFetchedSamples(numSamples) reqStats.AddFetchedChunkBytes(uint64(chunkBytes)) reqStats.AddFetchedDataBytes(uint64(dataBytes)) + reqStats.AddStoreGatewayTouchedPostings(uint64(seriesQueryStats.PostingsTouched)) + reqStats.AddStoreGatewayTouchedPostingBytes(uint64(seriesQueryStats.PostingsTouchedSizeSum)) level.Debug(spanLog).Log("msg", "received series from store-gateway", "instance", c.RemoteAddress(), diff --git a/pkg/querier/stats/stats.go b/pkg/querier/stats/stats.go index 5fd3fb3122..f0d4a35f5d 100644 --- a/pkg/querier/stats/stats.go +++ b/pkg/querier/stats/stats.go @@ -270,6 +270,38 @@ func (s *QueryStats) LoadDataSelectMinTime() int64 { return atomic.LoadInt64(&s.DataSelectMinTime) } +func (s *QueryStats) AddStoreGatewayTouchedPostings(count uint64) { + if s == nil { + return + } + + atomic.AddUint64(&s.StoreGatewayTouchedPostingsCount, count) +} + +func (s *QueryStats) LoadStoreGatewayTouchedPostings() uint64 { + if s == nil { + return 0 + } + + return atomic.LoadUint64(&s.StoreGatewayTouchedPostingsCount) +} + +func (s *QueryStats) AddStoreGatewayTouchedPostingBytes(bytes uint64) { + if s == nil { + return + } + + atomic.AddUint64(&s.StoreGatewayTouchedPostingBytes, bytes) +} + +func (s *QueryStats) LoadStoreGatewayTouchedPostingBytes() uint64 { + if s == nil { + return 0 + } + + return atomic.LoadUint64(&s.StoreGatewayTouchedPostingBytes) +} + // Merge the provided Stats into this one. func (s *QueryStats) Merge(other *QueryStats) { if s == nil || other == nil { @@ -283,6 +315,8 @@ func (s *QueryStats) Merge(other *QueryStats) { s.AddFetchedDataBytes(other.LoadFetchedDataBytes()) s.AddFetchedSamples(other.LoadFetchedSamples()) s.AddFetchedChunks(other.LoadFetchedChunks()) + s.AddStoreGatewayTouchedPostings(other.LoadStoreGatewayTouchedPostings()) + s.AddStoreGatewayTouchedPostingBytes(other.LoadStoreGatewayTouchedPostingBytes()) s.AddExtraFields(other.LoadExtraFields()...) } diff --git a/pkg/querier/stats/stats.pb.go b/pkg/querier/stats/stats.pb.go index ebbcf9b237..eda53692cb 100644 --- a/pkg/querier/stats/stats.pb.go +++ b/pkg/querier/stats/stats.pb.go @@ -53,6 +53,12 @@ type Stats struct { SplitQueries uint64 `protobuf:"varint,9,opt,name=split_queries,json=splitQueries,proto3" json:"split_queries,omitempty"` // The sum of wall time spent in the querier to fetch and merge data from storage. QueryStorageWallTime time.Duration `protobuf:"bytes,10,opt,name=query_storage_wall_time,json=queryStorageWallTime,proto3,stdduration" json:"query_storage_wall_time"` + // The total number of postings touched in store gateway for a specific query. + // Only successful requests from querier to store gateway are included. + StoreGatewayTouchedPostingsCount uint64 `protobuf:"varint,11,opt,name=store_gateway_touched_postings_count,json=storeGatewayTouchedPostingsCount,proto3" json:"store_gateway_touched_postings_count,omitempty"` + // The total size of postings touched in store gateway for a specific query, in bytes. + // Only successful requests from querier to store gateway are included. + StoreGatewayTouchedPostingBytes uint64 `protobuf:"varint,12,opt,name=store_gateway_touched_posting_bytes,json=storeGatewayTouchedPostingBytes,proto3" json:"store_gateway_touched_posting_bytes,omitempty"` } func (m *Stats) Reset() { *m = Stats{} } @@ -157,6 +163,20 @@ func (m *Stats) GetQueryStorageWallTime() time.Duration { return 0 } +func (m *Stats) GetStoreGatewayTouchedPostingsCount() uint64 { + if m != nil { + return m.StoreGatewayTouchedPostingsCount + } + return 0 +} + +func (m *Stats) GetStoreGatewayTouchedPostingBytes() uint64 { + if m != nil { + return m.StoreGatewayTouchedPostingBytes + } + return 0 +} + func init() { proto.RegisterType((*Stats)(nil), "stats.Stats") proto.RegisterMapType((map[string]string)(nil), "stats.Stats.ExtraFieldsEntry") @@ -165,38 +185,41 @@ func init() { func init() { proto.RegisterFile("stats.proto", fileDescriptor_b4756a0aec8b9d44) } var fileDescriptor_b4756a0aec8b9d44 = []byte{ - // 486 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0x41, 0x6f, 0xd3, 0x30, - 0x14, 0xc7, 0xe3, 0xb5, 0x1d, 0x8d, 0x3b, 0xa4, 0x62, 0x8a, 0xc8, 0x2a, 0xe1, 0x15, 0xb8, 0xf4, - 0x80, 0x52, 0x54, 0x2e, 0x08, 0x24, 0x34, 0x75, 0x1b, 0xe2, 0x4a, 0x8b, 0x84, 0xb4, 0x8b, 0xe5, - 0xb6, 0x6e, 0x6a, 0x2d, 0x8d, 0x4b, 0xe2, 0x00, 0xb9, 0xf1, 0x11, 0x38, 0xf2, 0x11, 0xf8, 0x28, - 0x3d, 0xf6, 0xb8, 0xd3, 0xa0, 0xa9, 0x90, 0x38, 0xee, 0x23, 0x20, 0x3f, 0x27, 0x1b, 0xec, 0xc4, - 0xcd, 0x7e, 0xbf, 0xf7, 0x7f, 0xf2, 0xff, 0xff, 0x8c, 0x1b, 0x89, 0xe6, 0x3a, 0xf1, 0x97, 0xb1, - 0xd2, 0x8a, 0xd4, 0xe0, 0xd2, 0x6e, 0x05, 0x2a, 0x50, 0x50, 0xe9, 0x99, 0x93, 0x85, 0x6d, 0x1a, - 0x28, 0x15, 0x84, 0xa2, 0x07, 0xb7, 0x71, 0x3a, 0xeb, 0x4d, 0xd3, 0x98, 0x6b, 0xa9, 0xa2, 0x82, - 0xef, 0xdf, 0xe4, 0x3c, 0xca, 0x2c, 0x7a, 0xf4, 0xab, 0x8a, 0x6b, 0x23, 0x33, 0x9a, 0x1c, 0x62, - 0xf7, 0x13, 0x0f, 0x43, 0xa6, 0xe5, 0x42, 0x78, 0xa8, 0x83, 0xba, 0x8d, 0xfe, 0xbe, 0x6f, 0x85, - 0x7e, 0x29, 0xf4, 0x8f, 0x8b, 0xc1, 0x83, 0xfa, 0xea, 0xe2, 0xc0, 0xf9, 0xf6, 0xe3, 0x00, 0x0d, - 0xeb, 0x46, 0xf5, 0x4e, 0x2e, 0x04, 0x79, 0x8a, 0x5b, 0x33, 0xa1, 0x27, 0x73, 0x31, 0x65, 0x89, - 0x88, 0xa5, 0x48, 0xd8, 0x44, 0xa5, 0x91, 0xf6, 0x76, 0x3a, 0xa8, 0x5b, 0x1d, 0x92, 0x82, 0x8d, - 0x00, 0x1d, 0x19, 0x42, 0x7c, 0x7c, 0xb7, 0x54, 0x4c, 0xe6, 0x69, 0x74, 0xc6, 0xc6, 0x99, 0x16, - 0x89, 0x57, 0x01, 0xc1, 0x9d, 0x02, 0x1d, 0x19, 0x32, 0x30, 0x80, 0x3c, 0xc1, 0xe5, 0x14, 0x36, - 0xe5, 0x9a, 0x17, 0xed, 0x55, 0x68, 0x6f, 0x16, 0xe4, 0x98, 0x6b, 0x6e, 0xbb, 0x0f, 0xf1, 0x9e, - 0xf8, 0xac, 0x63, 0xce, 0x66, 0x52, 0x84, 0xd3, 0xc4, 0xab, 0x75, 0x2a, 0xdd, 0x46, 0xff, 0x81, - 0x6f, 0x73, 0x05, 0xd7, 0xfe, 0x89, 0x69, 0x78, 0x0d, 0xfc, 0x24, 0xd2, 0x71, 0x36, 0x6c, 0x88, - 0xeb, 0xca, 0xdf, 0x8e, 0xe0, 0x7d, 0xa5, 0xa3, 0xdd, 0x7f, 0x1c, 0xc1, 0x03, 0x0b, 0x47, 0x7d, - 0x7c, 0xef, 0x2a, 0x03, 0xbe, 0x58, 0x86, 0x57, 0x21, 0xdc, 0x02, 0x49, 0x69, 0x77, 0x64, 0x99, - 0xd5, 0x3c, 0xc4, 0x6e, 0x28, 0x17, 0x52, 0xb3, 0xb9, 0xd4, 0x5e, 0xbd, 0x83, 0xba, 0xee, 0xa0, - 0xba, 0xba, 0x30, 0xd1, 0x42, 0xf9, 0x8d, 0xd4, 0xe4, 0x31, 0xbe, 0x9d, 0x2c, 0x43, 0xa9, 0xd9, - 0x87, 0x14, 0xe2, 0xf3, 0x5c, 0x18, 0xb7, 0x07, 0xc5, 0xb7, 0xb6, 0x46, 0x4e, 0xf1, 0x7d, 0x83, - 0x33, 0x96, 0x68, 0x15, 0xf3, 0x40, 0xb0, 0xeb, 0x7d, 0xe2, 0xff, 0xdf, 0x67, 0x0b, 0x66, 0x8c, - 0xec, 0x88, 0xf7, 0xc5, 0x6e, 0xdb, 0xaf, 0x70, 0xf3, 0x66, 0x54, 0xa4, 0x89, 0x2b, 0x67, 0x22, - 0x83, 0xbf, 0xe2, 0x0e, 0xcd, 0x91, 0xb4, 0x70, 0xed, 0x23, 0x0f, 0x53, 0x01, 0x2b, 0x77, 0x87, - 0xf6, 0xf2, 0x62, 0xe7, 0x39, 0x1a, 0xbc, 0x5c, 0x6f, 0xa8, 0x73, 0xbe, 0xa1, 0xce, 0xe5, 0x86, - 0xa2, 0x2f, 0x39, 0x45, 0xdf, 0x73, 0x8a, 0x56, 0x39, 0x45, 0xeb, 0x9c, 0xa2, 0x9f, 0x39, 0x45, - 0xbf, 0x73, 0xea, 0x5c, 0xe6, 0x14, 0x7d, 0xdd, 0x52, 0x67, 0xbd, 0xa5, 0xce, 0xf9, 0x96, 0x3a, - 0xa7, 0xf6, 0xd7, 0x8f, 0x77, 0xe1, 0xbd, 0xcf, 0xfe, 0x04, 0x00, 0x00, 0xff, 0xff, 0x39, 0x10, - 0xd3, 0xa4, 0x12, 0x03, 0x00, 0x00, + // 544 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x93, 0xcf, 0x6e, 0xd3, 0x40, + 0x10, 0xc6, 0xbd, 0xcd, 0x1f, 0xe2, 0x4d, 0x90, 0x82, 0x09, 0xc2, 0x8d, 0xc4, 0x26, 0x50, 0x0e, + 0x39, 0x20, 0x07, 0x85, 0x0b, 0x02, 0x09, 0x55, 0x69, 0x0b, 0x1c, 0x10, 0x82, 0xa4, 0x12, 0x52, + 0x2f, 0xab, 0x4d, 0xb2, 0x71, 0xac, 0x3a, 0xde, 0x60, 0xaf, 0x29, 0xbe, 0x21, 0xf1, 0x02, 0x1c, + 0x79, 0x04, 0x1e, 0x25, 0xc7, 0x1c, 0x7b, 0x2a, 0xc4, 0xb9, 0x70, 0xec, 0x23, 0xa0, 0x9d, 0xb5, + 0x5b, 0xa8, 0x04, 0xe2, 0xe6, 0x9d, 0xdf, 0x37, 0x9f, 0xf6, 0x9b, 0x59, 0xe3, 0x6a, 0x24, 0x99, + 0x8c, 0x9c, 0x45, 0x28, 0xa4, 0xb0, 0x4a, 0x70, 0x68, 0x36, 0x5c, 0xe1, 0x0a, 0xa8, 0x74, 0xd5, + 0x97, 0x86, 0x4d, 0xe2, 0x0a, 0xe1, 0xfa, 0xbc, 0x0b, 0xa7, 0x51, 0x3c, 0xed, 0x4e, 0xe2, 0x90, + 0x49, 0x4f, 0x04, 0x19, 0xdf, 0xbe, 0xca, 0x59, 0x90, 0x68, 0x74, 0xef, 0x73, 0x19, 0x97, 0x86, + 0xca, 0xda, 0xda, 0xc5, 0xe6, 0x09, 0xf3, 0x7d, 0x2a, 0xbd, 0x39, 0xb7, 0x51, 0x1b, 0x75, 0xaa, + 0xbd, 0x6d, 0x47, 0x37, 0x3a, 0x79, 0xa3, 0xb3, 0x9f, 0x19, 0xf7, 0x2b, 0xcb, 0xb3, 0x96, 0xf1, + 0xf5, 0x7b, 0x0b, 0x0d, 0x2a, 0xaa, 0xeb, 0xd0, 0x9b, 0x73, 0xeb, 0x21, 0x6e, 0x4c, 0xb9, 0x1c, + 0xcf, 0xf8, 0x84, 0x46, 0x3c, 0xf4, 0x78, 0x44, 0xc7, 0x22, 0x0e, 0xa4, 0xbd, 0xd5, 0x46, 0x9d, + 0xe2, 0xc0, 0xca, 0xd8, 0x10, 0xd0, 0x9e, 0x22, 0x96, 0x83, 0x6f, 0xe6, 0x1d, 0xe3, 0x59, 0x1c, + 0x1c, 0xd3, 0x51, 0x22, 0x79, 0x64, 0x17, 0xa0, 0xe1, 0x46, 0x86, 0xf6, 0x14, 0xe9, 0x2b, 0x60, + 0x3d, 0xc0, 0xb9, 0x0b, 0x9d, 0x30, 0xc9, 0x32, 0x79, 0x11, 0xe4, 0xf5, 0x8c, 0xec, 0x33, 0xc9, + 0xb4, 0x7a, 0x17, 0xd7, 0xf8, 0x47, 0x19, 0x32, 0x3a, 0xf5, 0xb8, 0x3f, 0x89, 0xec, 0x52, 0xbb, + 0xd0, 0xa9, 0xf6, 0xee, 0x38, 0x7a, 0xae, 0x90, 0xda, 0x39, 0x50, 0x82, 0xe7, 0xc0, 0x0f, 0x02, + 0x19, 0x26, 0x83, 0x2a, 0xbf, 0xac, 0xfc, 0x9e, 0x08, 0xee, 0x97, 0x27, 0x2a, 0xff, 0x91, 0x08, + 0x2e, 0x98, 0x25, 0xea, 0xe1, 0x5b, 0x17, 0x33, 0x60, 0xf3, 0x85, 0x7f, 0x31, 0x84, 0x6b, 0xd0, + 0x92, 0xc7, 0x1d, 0x6a, 0xa6, 0x7b, 0xee, 0x62, 0xd3, 0xf7, 0xe6, 0x9e, 0xa4, 0x33, 0x4f, 0xda, + 0x95, 0x36, 0xea, 0x98, 0xfd, 0xe2, 0xf2, 0x4c, 0x8d, 0x16, 0xca, 0x2f, 0x3d, 0x69, 0xed, 0xe0, + 0xeb, 0xd1, 0xc2, 0xf7, 0x24, 0x7d, 0x1f, 0xc3, 0xf8, 0x6c, 0x13, 0xec, 0x6a, 0x50, 0x7c, 0xab, + 0x6b, 0xd6, 0x11, 0xbe, 0xad, 0x70, 0x42, 0x23, 0x29, 0x42, 0xe6, 0x72, 0x7a, 0xb9, 0x4f, 0xfc, + 0xff, 0xfb, 0x6c, 0x80, 0xc7, 0x50, 0x5b, 0xbc, 0xcb, 0x77, 0xfb, 0x1a, 0xdf, 0x57, 0xae, 0x9c, + 0xba, 0x4c, 0xf2, 0x13, 0x96, 0x50, 0x29, 0x62, 0x48, 0xb9, 0x10, 0x91, 0xf4, 0x02, 0x37, 0x8f, + 0x59, 0x85, 0x7b, 0xb5, 0x41, 0xfb, 0x42, 0x4b, 0x0f, 0xb5, 0xf2, 0x4d, 0x26, 0xd4, 0x99, 0x5f, + 0xe1, 0x9d, 0x7f, 0xfa, 0x65, 0xab, 0xad, 0x81, 0x5d, 0xeb, 0xef, 0x76, 0xb0, 0xe9, 0xe6, 0x33, + 0x5c, 0xbf, 0xba, 0x48, 0xab, 0x8e, 0x0b, 0xc7, 0x3c, 0x81, 0x97, 0x6c, 0x0e, 0xd4, 0xa7, 0xd5, + 0xc0, 0xa5, 0x0f, 0xcc, 0x8f, 0x39, 0x3c, 0x48, 0x73, 0xa0, 0x0f, 0x4f, 0xb6, 0x1e, 0xa3, 0xfe, + 0xd3, 0xd5, 0x9a, 0x18, 0xa7, 0x6b, 0x62, 0x9c, 0xaf, 0x09, 0xfa, 0x94, 0x12, 0xf4, 0x2d, 0x25, + 0x68, 0x99, 0x12, 0xb4, 0x4a, 0x09, 0xfa, 0x91, 0x12, 0xf4, 0x33, 0x25, 0xc6, 0x79, 0x4a, 0xd0, + 0x97, 0x0d, 0x31, 0x56, 0x1b, 0x62, 0x9c, 0x6e, 0x88, 0x71, 0xa4, 0xff, 0xc9, 0x51, 0x19, 0xa6, + 0xf9, 0xe8, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0x16, 0xa4, 0x90, 0x1b, 0xb0, 0x03, 0x00, 0x00, } func (this *Stats) Equal(that interface{}) bool { @@ -253,13 +276,19 @@ func (this *Stats) Equal(that interface{}) bool { if this.QueryStorageWallTime != that1.QueryStorageWallTime { return false } + if this.StoreGatewayTouchedPostingsCount != that1.StoreGatewayTouchedPostingsCount { + return false + } + if this.StoreGatewayTouchedPostingBytes != that1.StoreGatewayTouchedPostingBytes { + return false + } return true } func (this *Stats) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 14) + s := make([]string, 0, 16) s = append(s, "&stats.Stats{") s = append(s, "WallTime: "+fmt.Sprintf("%#v", this.WallTime)+",\n") s = append(s, "FetchedSeriesCount: "+fmt.Sprintf("%#v", this.FetchedSeriesCount)+",\n") @@ -283,6 +312,8 @@ func (this *Stats) GoString() string { s = append(s, "LimitHit: "+fmt.Sprintf("%#v", this.LimitHit)+",\n") s = append(s, "SplitQueries: "+fmt.Sprintf("%#v", this.SplitQueries)+",\n") s = append(s, "QueryStorageWallTime: "+fmt.Sprintf("%#v", this.QueryStorageWallTime)+",\n") + s = append(s, "StoreGatewayTouchedPostingsCount: "+fmt.Sprintf("%#v", this.StoreGatewayTouchedPostingsCount)+",\n") + s = append(s, "StoreGatewayTouchedPostingBytes: "+fmt.Sprintf("%#v", this.StoreGatewayTouchedPostingBytes)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -314,6 +345,16 @@ func (m *Stats) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.StoreGatewayTouchedPostingBytes != 0 { + i = encodeVarintStats(dAtA, i, uint64(m.StoreGatewayTouchedPostingBytes)) + i-- + dAtA[i] = 0x60 + } + if m.StoreGatewayTouchedPostingsCount != 0 { + i = encodeVarintStats(dAtA, i, uint64(m.StoreGatewayTouchedPostingsCount)) + i-- + dAtA[i] = 0x58 + } n1, err1 := github_com_gogo_protobuf_types.StdDurationMarshalTo(m.QueryStorageWallTime, dAtA[i-github_com_gogo_protobuf_types.SizeOfStdDuration(m.QueryStorageWallTime):]) if err1 != nil { return 0, err1 @@ -440,6 +481,12 @@ func (m *Stats) Size() (n int) { } l = github_com_gogo_protobuf_types.SizeOfStdDuration(m.QueryStorageWallTime) n += 1 + l + sovStats(uint64(l)) + if m.StoreGatewayTouchedPostingsCount != 0 { + n += 1 + sovStats(uint64(m.StoreGatewayTouchedPostingsCount)) + } + if m.StoreGatewayTouchedPostingBytes != 0 { + n += 1 + sovStats(uint64(m.StoreGatewayTouchedPostingBytes)) + } return n } @@ -474,6 +521,8 @@ func (this *Stats) String() string { `LimitHit:` + fmt.Sprintf("%v", this.LimitHit) + `,`, `SplitQueries:` + fmt.Sprintf("%v", this.SplitQueries) + `,`, `QueryStorageWallTime:` + strings.Replace(strings.Replace(fmt.Sprintf("%v", this.QueryStorageWallTime), "Duration", "duration.Duration", 1), `&`, ``, 1) + `,`, + `StoreGatewayTouchedPostingsCount:` + fmt.Sprintf("%v", this.StoreGatewayTouchedPostingsCount) + `,`, + `StoreGatewayTouchedPostingBytes:` + fmt.Sprintf("%v", this.StoreGatewayTouchedPostingBytes) + `,`, `}`, }, "") return s @@ -854,6 +903,44 @@ func (m *Stats) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StoreGatewayTouchedPostingsCount", wireType) + } + m.StoreGatewayTouchedPostingsCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStats + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StoreGatewayTouchedPostingsCount |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 12: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StoreGatewayTouchedPostingBytes", wireType) + } + m.StoreGatewayTouchedPostingBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowStats + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StoreGatewayTouchedPostingBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipStats(dAtA[iNdEx:]) diff --git a/pkg/querier/stats/stats.proto b/pkg/querier/stats/stats.proto index 989f71ecef..0bb3beedac 100644 --- a/pkg/querier/stats/stats.proto +++ b/pkg/querier/stats/stats.proto @@ -33,4 +33,10 @@ message Stats { uint64 split_queries = 9; // The sum of wall time spent in the querier to fetch and merge data from storage. google.protobuf.Duration query_storage_wall_time = 10 [(gogoproto.stdduration) = true, (gogoproto.nullable) = false]; + // The total number of postings touched in store gateway for a specific query. + // Only successful requests from querier to store gateway are included. + uint64 store_gateway_touched_postings_count = 11; + // The total size of postings touched in store gateway for a specific query, in bytes. + // Only successful requests from querier to store gateway are included. + uint64 store_gateway_touched_posting_bytes = 12; } diff --git a/pkg/querier/stats/stats_test.go b/pkg/querier/stats/stats_test.go index 54a7b80da5..5d938a9419 100644 --- a/pkg/querier/stats/stats_test.go +++ b/pkg/querier/stats/stats_test.go @@ -133,6 +133,42 @@ func TestStats_AddFetchedSamples(t *testing.T) { }) } +func TestStats_AddStoreGatewayTouchedPostings(t *testing.T) { + t.Parallel() + t.Run("add and load touched postings", func(t *testing.T) { + stats, _ := ContextWithEmptyStats(context.Background()) + stats.AddStoreGatewayTouchedPostings(4096) + stats.AddStoreGatewayTouchedPostings(4096) + + assert.Equal(t, uint64(8192), stats.LoadStoreGatewayTouchedPostings()) + }) + + t.Run("add and load touched postings nil receiver", func(t *testing.T) { + var stats *QueryStats + stats.AddStoreGatewayTouchedPostings(4096) + + assert.Equal(t, uint64(0), stats.LoadStoreGatewayTouchedPostings()) + }) +} + +func TestStats_AddStoreGatewayTouchedPostingBytes(t *testing.T) { + t.Parallel() + t.Run("add and load touched postings bytes", func(t *testing.T) { + stats, _ := ContextWithEmptyStats(context.Background()) + stats.AddStoreGatewayTouchedPostingBytes(4096) + stats.AddStoreGatewayTouchedPostingBytes(4096) + + assert.Equal(t, uint64(8192), stats.LoadStoreGatewayTouchedPostingBytes()) + }) + + t.Run("add and load touched posting bytes nil receiver", func(t *testing.T) { + var stats *QueryStats + stats.AddStoreGatewayTouchedPostingBytes(4096) + + assert.Equal(t, uint64(0), stats.LoadStoreGatewayTouchedPostingBytes()) + }) +} + func TestStats_StorageWallTime(t *testing.T) { t.Run("add and load query storage wall time", func(t *testing.T) { stats, _ := ContextWithEmptyStats(context.Background()) @@ -159,6 +195,8 @@ func TestStats_Merge(t *testing.T) { stats1.AddFetchedSeries(50) stats1.AddFetchedChunkBytes(42) stats1.AddFetchedDataBytes(100) + stats1.AddStoreGatewayTouchedPostings(200) + stats1.AddStoreGatewayTouchedPostingBytes(300) stats1.AddFetchedChunks(105) stats1.AddFetchedSamples(109) stats1.AddExtraFields("a", "b") @@ -170,6 +208,8 @@ func TestStats_Merge(t *testing.T) { stats2.AddFetchedSeries(60) stats2.AddFetchedChunkBytes(100) stats2.AddFetchedDataBytes(101) + stats1.AddStoreGatewayTouchedPostings(201) + stats1.AddStoreGatewayTouchedPostingBytes(301) stats2.AddFetchedChunks(102) stats2.AddFetchedSamples(103) stats2.AddExtraFields("c", "d") @@ -183,6 +223,8 @@ func TestStats_Merge(t *testing.T) { assert.Equal(t, uint64(201), stats1.LoadFetchedDataBytes()) assert.Equal(t, uint64(207), stats1.LoadFetchedChunks()) assert.Equal(t, uint64(212), stats1.LoadFetchedSamples()) + assert.Equal(t, uint64(401), stats1.LoadStoreGatewayTouchedPostings()) + assert.Equal(t, uint64(601), stats1.LoadStoreGatewayTouchedPostingBytes()) checkExtraFields(t, []interface{}{"a", "b", "c", "d"}, stats1.LoadExtraFields()) })