diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index cd8a472ebd..83e4a6cc11 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -17,7 +17,7 @@ jobs: lint: runs-on: ubuntu-20.04 container: - image: quay.io/cortexproject/build-image:master-779dcf4ba + image: quay.io/cortexproject/build-image:master-582c03a76 steps: - name: Checkout Repo uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 @@ -46,7 +46,7 @@ jobs: test: runs-on: ubuntu-20.04 container: - image: quay.io/cortexproject/build-image:master-779dcf4ba + image: quay.io/cortexproject/build-image:master-582c03a76 steps: - name: Checkout Repo uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 @@ -89,7 +89,7 @@ jobs: build: runs-on: ubuntu-20.04 container: - image: quay.io/cortexproject/build-image:master-779dcf4ba + image: quay.io/cortexproject/build-image:master-582c03a76 steps: - name: Checkout Repo uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 @@ -224,14 +224,14 @@ jobs: run: | touch build-image/.uptodate MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations - make BUILD_IMAGE=quay.io/cortexproject/build-image:master-779dcf4ba TTY='' configs-integration-test + make BUILD_IMAGE=quay.io/cortexproject/build-image:master-582c03a76 TTY='' configs-integration-test deploy_website: needs: [build, test] if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex' runs-on: ubuntu-20.04 container: - image: quay.io/cortexproject/build-image:master-779dcf4ba + image: quay.io/cortexproject/build-image:master-582c03a76 steps: - name: Checkout Repo uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 @@ -273,7 +273,7 @@ jobs: if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex' runs-on: ubuntu-20.04 container: - image: quay.io/cortexproject/build-image:master-779dcf4ba + image: quay.io/cortexproject/build-image:master-582c03a76 steps: - name: Checkout Repo uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index b748b3d5db..41dfac4a91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,10 @@ ## master / unreleased +* [CHANGE] Enable Compactor and Alertmanager in target all. #6204 * [FEATURE] Ruler: Experimental: Add `ruler.frontend-address` to allow query to query frontends instead of ingesters. #6151 * [FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via `-ruler.enable-ha-evaluation` flag. #6129 +* [ENHANCEMENT] Query Frontend: Add info field to query response. #6207 * [ENHANCEMENT] Query Frontend: Add peakSample in query stats response. #6188 * [ENHANCEMENT] Ruler: Add new ruler metric `cortex_ruler_rule_groups_in_store` that is the total rule groups per tenant in store, which can be used to compare with `cortex_prometheus_rule_group_rules` to count the number of rule groups that are not loaded by a ruler. #5869 * [ENHANCEMENT] Ingester/Ring: New `READONLY` status on ring to be used by Ingester. New ingester API to change mode of ingester #6163 @@ -11,6 +13,7 @@ * [ENHANCEMENT] Ingester: Add new API `/ingester/all_user_stats` which shows loaded blocks, active timeseries and ingestion rate for a specific ingester. #6178 * [ENHANCEMENT] Distributor: Add new `cortex_reduced_resolution_histogram_samples_total` metric to track the number of histogram samples which resolution was reduced. #6182 * [ENHANCEMENT] StoreGateway: Implement metadata API limit in queryable. #6195 +* [ENHANCEMENT] Ingester: Add matchers to ingester LabelNames() and LabelNamesStream() RPC. #6209 ## 1.18.0 2024-09-03 diff --git a/Makefile b/Makefile index 80d5396b48..c3666f7af3 100644 --- a/Makefile +++ b/Makefile @@ -115,7 +115,7 @@ build-image/$(UPTODATE): build-image/* SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E") BUILD_IN_CONTAINER := true BUILD_IMAGE ?= $(IMAGE_PREFIX)build-image -LATEST_BUILD_IMAGE_TAG ?= master-779dcf4ba +LATEST_BUILD_IMAGE_TAG ?= master-582c03a76 # TTY is parameterized to allow Google Cloud Builder to run builds, # as it currently disallows TTY devices. This value needs to be overridden diff --git a/build-image/Dockerfile b/build-image/Dockerfile index 78d832acf5..c416ffae89 100644 --- a/build-image/Dockerfile +++ b/build-image/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.22.5-bullseye +FROM golang:1.22.7-bullseye ARG goproxyValue ENV GOPROXY=${goproxyValue} RUN apt-get update && apt-get install -y curl file jq unzip protobuf-compiler libprotobuf-dev && \ diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 8cd0b9f008..41d0f433bf 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -109,6 +109,10 @@ querier: # CLI flag: -querier.ingester-metadata-streaming [ingester_metadata_streaming: | default = true] + # Use LabelNames ingester RPCs with match params. + # CLI flag: -querier.ingester-label-names-with-matchers + [ingester_label_names_with_matchers: | default = false] + # Maximum number of samples a single query can load into memory. # CLI flag: -querier.max-samples [max_samples: | default = 50000000] diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 46bf10e4f6..a8ff9ef928 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3701,6 +3701,10 @@ The `querier_config` configures the Cortex querier. # CLI flag: -querier.ingester-metadata-streaming [ingester_metadata_streaming: | default = true] +# Use LabelNames ingester RPCs with match params. +# CLI flag: -querier.ingester-label-names-with-matchers +[ingester_label_names_with_matchers: | default = false] + # Maximum number of samples a single query can load into memory. # CLI flag: -querier.max-samples [max_samples: | default = 50000000] diff --git a/integration/api_endpoints_test.go b/integration/api_endpoints_test.go index 9cd9a79aa5..6da0971b49 100644 --- a/integration/api_endpoints_test.go +++ b/integration/api_endpoints_test.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -22,10 +23,19 @@ func TestIndexAPIEndpoint(t *testing.T) { require.NoError(t, err) defer s.Close() + configOverrides := map[string]string{ + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), + } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + // Start Cortex in single binary mode, reading the config from file. require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) - cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095) + cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, configOverrides, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex1)) // GET / should succeed @@ -44,10 +54,19 @@ func TestConfigAPIEndpoint(t *testing.T) { require.NoError(t, err) defer s.Close() + configOverrides := map[string]string{ + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), + } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + // Start Cortex in single binary mode, reading the config from file. require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) - cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095) + cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, configOverrides, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex1)) // Get config from /config API endpoint. @@ -62,6 +81,7 @@ func TestConfigAPIEndpoint(t *testing.T) { // Start again Cortex in single binary with the exported config // and ensure it starts (pass the readiness probe). require.NoError(t, writeFileToSharedDir(s, cortexConfigFile, body)) - cortex2 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-2", cortexConfigFile, nil, "", 9009, 9095) + configOverrides["-alertmanager.cluster.peers"] = cortex1.HTTPEndpoint() + cortex2 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-2", cortexConfigFile, configOverrides, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex2)) } diff --git a/integration/getting_started_single_process_config_local_test.go b/integration/getting_started_single_process_config_local_test.go index 34eec0d661..0a7434c5a3 100644 --- a/integration/getting_started_single_process_config_local_test.go +++ b/integration/getting_started_single_process_config_local_test.go @@ -4,6 +4,7 @@ package integration import ( + "path/filepath" "testing" "time" @@ -24,7 +25,14 @@ func TestGettingStartedSingleProcessConfigWithFilesystem(t *testing.T) { // Start Cortex components. require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) - flags := map[string]string{} + flags := map[string]string{ + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), + } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex)) diff --git a/integration/getting_started_single_process_config_test.go b/integration/getting_started_single_process_config_test.go index 1cdf689790..624928abf0 100644 --- a/integration/getting_started_single_process_config_test.go +++ b/integration/getting_started_single_process_config_test.go @@ -5,6 +5,7 @@ package integration import ( "fmt" + "path/filepath" "testing" "time" @@ -38,7 +39,13 @@ func TestGettingStartedSingleProcessConfigWithBlocksStorage(t *testing.T) { "-blocks-storage.s3.bucket-name": bucketName, "-blocks-storage.s3.endpoint": fmt.Sprintf("%s-minio-9000:9000", networkName), "-blocks-storage.s3.insecure": "true", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex)) diff --git a/integration/getting_started_with_gossiped_ring_test.go b/integration/getting_started_with_gossiped_ring_test.go index 3b9239310b..4bcd8a090b 100644 --- a/integration/getting_started_with_gossiped_ring_test.go +++ b/integration/getting_started_with_gossiped_ring_test.go @@ -5,6 +5,7 @@ package integration import ( "fmt" + "path/filepath" "testing" "time" @@ -45,8 +46,15 @@ func TestGettingStartedWithGossipedRing(t *testing.T) { "-blocks-storage.s3.insecure": "true", "-store-gateway.sharding-ring.wait-stability-min-duration": "0", // start quickly "-store-gateway.sharding-ring.wait-stability-max-duration": "0", // start quickly + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + // This cortex will fail to join the cluster configured in yaml file. That's fine. cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", "config1.yaml", e2e.MergeFlags(flags, map[string]string{ "-ingester.lifecycler.addr": networkName + "-cortex-1", // Ingester's hostname in docker setup diff --git a/integration/integration_memberlist_single_binary_test.go b/integration/integration_memberlist_single_binary_test.go index ce62772dc7..68bd343ee0 100644 --- a/integration/integration_memberlist_single_binary_test.go +++ b/integration/integration_memberlist_single_binary_test.go @@ -43,6 +43,9 @@ func testSingleBinaryEnv(t *testing.T, tlsEnabled bool, flags map[string]string) require.NoError(t, err) defer s.Close() + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + // Start dependencies minio := e2edb.NewMinio(9000, bucketName) // Look ma, no Consul! @@ -116,16 +119,21 @@ func testSingleBinaryEnv(t *testing.T, tlsEnabled bool, flags map[string]string) } func newSingleBinary(name string, servername string, join string, testFlags map[string]string) *e2ecortex.CortexService { - flags := map[string]string{ - "-ingester.final-sleep": "0s", - "-ingester.join-after": "0s", // join quickly - "-ingester.min-ready-duration": "0s", - "-ingester.num-tokens": "512", - "-ingester.observe-period": "5s", // to avoid conflicts in tokens - "-ring.store": "memberlist", - "-memberlist.bind-port": "8000", - "-memberlist.left-ingesters-timeout": "600s", // effectively disable - } + flags := mergeFlags( + AlertmanagerLocalFlags(), + map[string]string{ + "-ingester.final-sleep": "0s", + "-ingester.join-after": "0s", // join quickly + "-ingester.min-ready-duration": "0s", + "-ingester.num-tokens": "512", + "-ingester.observe-period": "5s", // to avoid conflicts in tokens + "-ring.store": "memberlist", + "-memberlist.bind-port": "8000", + "-memberlist.left-ingesters-timeout": "600s", // effectively disable + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + }, + ) if join != "" { flags["-memberlist.join"] = join @@ -158,6 +166,9 @@ func TestSingleBinaryWithMemberlistScaling(t *testing.T) { require.NoError(t, err) defer s.Close() + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + minio := e2edb.NewMinio(9000, bucketName) require.NoError(t, s.StartAndWaitReady(minio)) diff --git a/integration/otlp_test.go b/integration/otlp_test.go index 9a603e8cb0..da689e7b67 100644 --- a/integration/otlp_test.go +++ b/integration/otlp_test.go @@ -6,6 +6,7 @@ package integration import ( "fmt" "math/rand" + "path/filepath" "testing" "time" @@ -41,7 +42,13 @@ func TestOTLP(t *testing.T) { "-blocks-storage.s3.endpoint": fmt.Sprintf("%s-minio-9000:9000", networkName), "-blocks-storage.s3.insecure": "true", "-blocks-storage.tsdb.enable-native-histograms": "true", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095) require.NoError(t, s.StartAndWaitReady(cortex)) diff --git a/integration/querier_test.go b/integration/querier_test.go index a5e92e299e..a261a6d2c6 100644 --- a/integration/querier_test.go +++ b/integration/querier_test.go @@ -228,7 +228,7 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) { // then the metric only appears in one store gateway instance. require.NoError(t, storeGateways.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"cortex_bucket_store_blocks_loaded"}, e2e.SkipMissingMetrics)) } else { - require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(float64(2*storeGateways.NumInstances())), "cortex_bucket_store_blocks_loaded")) + require.NoError(t, storeGateways.WaitSumMetricsWithOptions(e2e.Equals(float64(2*storeGateways.NumInstances())), []string{"cortex_bucket_store_blocks_loaded"}, e2e.WaitMissingMetrics)) } // Check how many tenants have been discovered and synced by store-gateways. @@ -365,26 +365,33 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) { // Configure the blocks storage to frequently compact TSDB head // and ship blocks to the storage. - flags := mergeFlags(BlocksStorageFlags(), map[string]string{ - "-blocks-storage.tsdb.block-ranges-period": blockRangePeriod.String(), - "-blocks-storage.tsdb.ship-interval": "1s", - "-blocks-storage.bucket-store.sync-interval": "1s", - "-blocks-storage.tsdb.retention-period": ((blockRangePeriod * 2) - 1).String(), - "-blocks-storage.bucket-store.index-cache.backend": testCfg.indexCacheBackend, - "-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled), - "-querier.query-store-for-labels-enabled": "true", - "-querier.thanos-engine": strconv.FormatBool(thanosEngine), - // Ingester. - "-ring.store": "consul", - "-consul.hostname": consul.NetworkHTTPEndpoint(), - // Distributor. - "-distributor.replication-factor": strconv.FormatInt(seriesReplicationFactor, 10), - // Store-gateway. - "-store-gateway.sharding-enabled": strconv.FormatBool(testCfg.blocksShardingEnabled), - "-store-gateway.sharding-ring.store": "consul", - "-store-gateway.sharding-ring.consul.hostname": consul.NetworkHTTPEndpoint(), - "-store-gateway.sharding-ring.replication-factor": "1", - }) + flags := mergeFlags( + BlocksStorageFlags(), + AlertmanagerLocalFlags(), + map[string]string{ + "-blocks-storage.tsdb.block-ranges-period": blockRangePeriod.String(), + "-blocks-storage.tsdb.ship-interval": "1s", + "-blocks-storage.bucket-store.sync-interval": "1s", + "-blocks-storage.tsdb.retention-period": ((blockRangePeriod * 2) - 1).String(), + "-blocks-storage.bucket-store.index-cache.backend": testCfg.indexCacheBackend, + "-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled), + "-querier.query-store-for-labels-enabled": "true", + "-querier.thanos-engine": strconv.FormatBool(thanosEngine), + // Ingester. + "-ring.store": "consul", + "-consul.hostname": consul.NetworkHTTPEndpoint(), + // Distributor. + "-distributor.replication-factor": strconv.FormatInt(seriesReplicationFactor, 10), + // Store-gateway. + "-store-gateway.sharding-enabled": strconv.FormatBool(testCfg.blocksShardingEnabled), + "-store-gateway.sharding-ring.store": "consul", + "-store-gateway.sharding-ring.consul.hostname": consul.NetworkHTTPEndpoint(), + "-store-gateway.sharding-ring.replication-factor": "1", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + }, + ) + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs/user-1.yaml", []byte(cortexAlertmanagerUserConfigYaml))) // Add the cache address to the flags. if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached { diff --git a/integration/query_fuzz_test.go b/integration/query_fuzz_test.go index 058a94dbd7..292f94421d 100644 --- a/integration/query_fuzz_test.go +++ b/integration/query_fuzz_test.go @@ -45,25 +45,32 @@ func TestVerticalShardingFuzz(t *testing.T) { consul2 := e2edb.NewConsulWithName("consul2") require.NoError(t, s.StartAndWaitReady(consul1, consul2)) - flags := map[string]string{ - "-store.engine": blocksStorageEngine, - "-blocks-storage.backend": "filesystem", - "-blocks-storage.tsdb.head-compaction-interval": "4m", - "-blocks-storage.tsdb.block-ranges-period": "2h", - "-blocks-storage.tsdb.ship-interval": "1h", - "-blocks-storage.bucket-store.sync-interval": "15m", - "-blocks-storage.tsdb.retention-period": "2h", - "-blocks-storage.bucket-store.index-cache.backend": tsdb.IndexCacheBackendInMemory, - "-blocks-storage.bucket-store.bucket-index.enabled": "true", - "-querier.query-store-for-labels-enabled": "true", - // Ingester. - "-ring.store": "consul", - "-consul.hostname": consul1.NetworkHTTPEndpoint(), - // Distributor. - "-distributor.replication-factor": "1", - // Store-gateway. - "-store-gateway.sharding-enabled": "false", - } + flags := mergeFlags( + AlertmanagerLocalFlags(), + map[string]string{ + "-store.engine": blocksStorageEngine, + "-blocks-storage.backend": "filesystem", + "-blocks-storage.tsdb.head-compaction-interval": "4m", + "-blocks-storage.tsdb.block-ranges-period": "2h", + "-blocks-storage.tsdb.ship-interval": "1h", + "-blocks-storage.bucket-store.sync-interval": "15m", + "-blocks-storage.tsdb.retention-period": "2h", + "-blocks-storage.bucket-store.index-cache.backend": tsdb.IndexCacheBackendInMemory, + "-blocks-storage.bucket-store.bucket-index.enabled": "true", + "-querier.query-store-for-labels-enabled": "true", + // Ingester. + "-ring.store": "consul", + "-consul.hostname": consul1.NetworkHTTPEndpoint(), + // Distributor. + "-distributor.replication-factor": "1", + // Store-gateway. + "-store-gateway.sharding-enabled": "false", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + }, + ) + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) path1 := path.Join(s.SharedDir(), "cortex-1") path2 := path.Join(s.SharedDir(), "cortex-2") diff --git a/integration/ruler_test.go b/integration/ruler_test.go index e4c5ad5439..23815ea4ee 100644 --- a/integration/ruler_test.go +++ b/integration/ruler_test.go @@ -154,7 +154,13 @@ func TestRulerAPISingleBinary(t *testing.T) { "-ruler-storage.local.directory": filepath.Join(e2e.ContainerSharedDir, "ruler_configs"), "-ruler.poll-interval": "2s", "-ruler.rule-path": filepath.Join(e2e.ContainerSharedDir, "rule_tmp/"), + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) // Start Cortex components. require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile)) diff --git a/integration/runtime_config_test.go b/integration/runtime_config_test.go index 7f9f0ceb30..f8f722084b 100644 --- a/integration/runtime_config_test.go +++ b/integration/runtime_config_test.go @@ -40,6 +40,10 @@ func TestLoadRuntimeConfigFromStorageBackend(t *testing.T) { name: "no storage backend provided", flags: map[string]string{ "-runtime-config.file": filePath, + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), }, }, { @@ -47,9 +51,15 @@ func TestLoadRuntimeConfigFromStorageBackend(t *testing.T) { flags: map[string]string{ "-runtime-config.file": filePath, "-runtime-config.backend": "filesystem", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), }, }, } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) for i, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -79,7 +89,14 @@ func TestLoadRuntimeConfigFromCloudStorage(t *testing.T) { "-runtime-config.s3.insecure": "true", "-runtime-config.file": configFileName, "-runtime-config.reload-period": "2s", + // alert manager + "-alertmanager.web.external-url": "http://localhost/alertmanager", + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), } + // make alert manager config dir + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{})) + // create s3 storage backend minio := e2edb.NewMinio(9000, bucketName) require.NoError(t, s.StartAndWaitReady(minio)) diff --git a/pkg/cortex/cortex_test.go b/pkg/cortex/cortex_test.go index 891f860898..bac7c0021c 100644 --- a/pkg/cortex/cortex_test.go +++ b/pkg/cortex/cortex_test.go @@ -12,10 +12,6 @@ import ( "testing" "time" - "github.com/cortexproject/cortex/pkg/cortex/storage" - "github.com/cortexproject/cortex/pkg/ruler/rulestore" - "github.com/cortexproject/cortex/pkg/ruler/rulestore/local" - "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -24,14 +20,20 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" + "github.com/cortexproject/cortex/pkg/alertmanager" + "github.com/cortexproject/cortex/pkg/alertmanager/alertstore" + "github.com/cortexproject/cortex/pkg/cortex/storage" "github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb" "github.com/cortexproject/cortex/pkg/ingester" "github.com/cortexproject/cortex/pkg/ring" "github.com/cortexproject/cortex/pkg/ring/kv" + "github.com/cortexproject/cortex/pkg/ruler/rulestore" + "github.com/cortexproject/cortex/pkg/ruler/rulestore/local" "github.com/cortexproject/cortex/pkg/scheduler/schedulerpb" "github.com/cortexproject/cortex/pkg/storage/bucket" "github.com/cortexproject/cortex/pkg/storage/bucket/s3" "github.com/cortexproject/cortex/pkg/storage/tsdb" + "github.com/cortexproject/cortex/pkg/util/flagext" "github.com/cortexproject/cortex/pkg/util/services" ) @@ -92,10 +94,23 @@ func TestCortex(t *testing.T) { Directory: os.TempDir(), }, }, - - Target: []string{All, Compactor}, + AlertmanagerStorage: alertstore.Config{ + Config: bucket.Config{ + Backend: "local", + }, + }, + Target: []string{All}, } + externalURL := flagext.URLValue{} + err := externalURL.Set("http://localhost/alertmanager") + require.NoError(t, err) + + multiAlertmanagerCfg := &alertmanager.MultitenantAlertmanagerConfig{} + flagext.DefaultValues(multiAlertmanagerCfg) + multiAlertmanagerCfg.ExternalURL = externalURL + cfg.Alertmanager = *multiAlertmanagerCfg + c, err := New(cfg) require.NoError(t, err) @@ -114,8 +129,9 @@ func TestCortex(t *testing.T) { require.NotNil(t, serviceMap[Ring]) require.NotNil(t, serviceMap[DistributorService]) - // check that compactor is configured which is not part of Target=All + // check compactor and alertmanager are configured. require.NotNil(t, serviceMap[Compactor]) + require.NotNil(t, serviceMap[AlertManager]) } func TestConfigValidation(t *testing.T) { diff --git a/pkg/cortex/modules.go b/pkg/cortex/modules.go index e32a7437ad..b8d39daeba 100644 --- a/pkg/cortex/modules.go +++ b/pkg/cortex/modules.go @@ -788,7 +788,7 @@ func (t *Cortex) setupModuleManager() error { TenantDeletion: {API, Overrides}, Purger: {TenantDeletion}, TenantFederation: {Queryable}, - All: {QueryFrontend, Querier, Ingester, Distributor, Purger, StoreGateway, Ruler}, + All: {QueryFrontend, Querier, Ingester, Distributor, Purger, StoreGateway, Ruler, Compactor, AlertManager}, } if t.Cfg.ExternalPusher != nil && t.Cfg.ExternalQueryable != nil { deps[Ruler] = []string{Overrides, RulerStorage} diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 351187ba83..699035e672 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -874,7 +874,13 @@ func (d *Distributor) prepareSeriesKeys(ctx context.Context, req *cortexpb.Write d.validateMetrics.DiscardedSamples.WithLabelValues( validation.DroppedByRelabelConfiguration, userID, - ).Add(float64(len(ts.Samples))) + ).Add(float64(len(ts.Samples) + len(ts.Histograms))) + + // all labels are gone, exemplars will be discarded + d.validateMetrics.DiscardedExemplars.WithLabelValues( + validation.DroppedByRelabelConfiguration, + userID, + ).Add(float64(len(ts.Exemplars))) continue } ts.Labels = cortexpb.FromLabelsToLabelAdapters(l) @@ -892,11 +898,15 @@ func (d *Distributor) prepareSeriesKeys(ctx context.Context, req *cortexpb.Write } if len(ts.Labels) == 0 { - d.validateMetrics.DiscardedExemplars.WithLabelValues( + d.validateMetrics.DiscardedSamples.WithLabelValues( validation.DroppedByUserConfigurationOverride, userID, - ).Add(float64(len(ts.Samples))) + ).Add(float64(len(ts.Samples) + len(ts.Histograms))) + d.validateMetrics.DiscardedExemplars.WithLabelValues( + validation.DroppedByUserConfigurationOverride, + userID, + ).Add(float64(len(ts.Exemplars))) continue } @@ -1101,7 +1111,7 @@ func (d *Distributor) LabelValuesForLabelNameStream(ctx context.Context, from, t }, matchers...) } -func (d *Distributor) LabelNamesCommon(ctx context.Context, from, to model.Time, hints *storage.LabelHints, f func(ctx context.Context, rs ring.ReplicationSet, req *ingester_client.LabelNamesRequest) ([]interface{}, error)) ([]string, error) { +func (d *Distributor) LabelNamesCommon(ctx context.Context, from, to model.Time, hints *storage.LabelHints, f func(ctx context.Context, rs ring.ReplicationSet, req *ingester_client.LabelNamesRequest) ([]interface{}, error), matchers ...*labels.Matcher) ([]string, error) { span, ctx := opentracing.StartSpanFromContext(ctx, "Distributor.LabelNames", opentracing.Tags{ "start": from.Unix(), "end": to.Unix(), @@ -1113,11 +1123,11 @@ func (d *Distributor) LabelNamesCommon(ctx context.Context, from, to model.Time, } limit := getLimitFromLabelHints(hints) - req := &ingester_client.LabelNamesRequest{ - StartTimestampMs: int64(from), - EndTimestampMs: int64(to), - Limit: int64(limit), + req, err := ingester_client.ToLabelNamesRequest(from, to, limit, matchers) + if err != nil { + return nil, err } + resps, err := f(ctx, replicationSet, req) if err != nil { return nil, err @@ -1142,7 +1152,7 @@ func (d *Distributor) LabelNamesCommon(ctx context.Context, from, to model.Time, return r, nil } -func (d *Distributor) LabelNamesStream(ctx context.Context, from, to model.Time, hints *storage.LabelHints) ([]string, error) { +func (d *Distributor) LabelNamesStream(ctx context.Context, from, to model.Time, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { return d.LabelNamesCommon(ctx, from, to, hints, func(ctx context.Context, rs ring.ReplicationSet, req *ingester_client.LabelNamesRequest) ([]interface{}, error) { return d.ForReplicationSet(ctx, rs, d.cfg.ZoneResultsQuorumMetadata, func(ctx context.Context, client ingester_client.IngesterClient) (interface{}, error) { stream, err := client.LabelNamesStream(ctx, req) @@ -1164,11 +1174,11 @@ func (d *Distributor) LabelNamesStream(ctx context.Context, from, to model.Time, return allLabelNames, nil }) - }) + }, matchers...) } // LabelNames returns all the label names. -func (d *Distributor) LabelNames(ctx context.Context, from, to model.Time, hint *storage.LabelHints) ([]string, error) { +func (d *Distributor) LabelNames(ctx context.Context, from, to model.Time, hint *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { return d.LabelNamesCommon(ctx, from, to, hint, func(ctx context.Context, rs ring.ReplicationSet, req *ingester_client.LabelNamesRequest) ([]interface{}, error) { return d.ForReplicationSet(ctx, rs, d.cfg.ZoneResultsQuorumMetadata, func(ctx context.Context, client ingester_client.IngesterClient) (interface{}, error) { resp, err := client.LabelNames(ctx, req) @@ -1177,7 +1187,7 @@ func (d *Distributor) LabelNames(ctx context.Context, from, to model.Time, hint } return resp.LabelNames, nil }) - }) + }, matchers...) } // MetricsForLabelMatchers gets the metrics that match said matchers diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index e5eda834ca..28cf8b1dc6 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -1490,6 +1490,7 @@ func TestDistributor_Push_LabelRemoval(t *testing.T) { expectedSeries labels.Labels removeReplica bool removeLabels []string + exemplars []cortexpb.Exemplar } cases := []testcase{ @@ -1536,6 +1537,20 @@ func TestDistributor_Push_LabelRemoval(t *testing.T) { {Name: "cluster", Value: "one"}, }, }, + // No labels left. + { + removeReplica: true, + removeLabels: []string{"cluster"}, + inputSeries: labels.Labels{ + {Name: "cluster", Value: "one"}, + {Name: "__replica__", Value: "two"}, + }, + expectedSeries: labels.Labels{}, + exemplars: []cortexpb.Exemplar{ + {Labels: cortexpb.FromLabelsToLabelAdapters(labels.FromStrings("test", "a")), Value: 1, TimestampMs: 0}, + {Labels: cortexpb.FromLabelsToLabelAdapters(labels.FromStrings("test", "b")), Value: 1, TimestampMs: 0}, + }, + }, } for _, tc := range cases { @@ -1546,6 +1561,15 @@ func TestDistributor_Push_LabelRemoval(t *testing.T) { limits.DropLabels = tc.removeLabels limits.AcceptHASamples = tc.removeReplica + expectedDiscardedSamples := 0 + expectedDiscardedExemplars := 0 + if tc.expectedSeries.Len() == 0 { + expectedDiscardedSamples = 1 + expectedDiscardedExemplars = len(tc.exemplars) + // Allow series with no labels to ingest + limits.EnforceMetricName = false + } + ds, ingesters, _, _ := prepare(t, prepConfig{ numIngesters: 2, happyIngesters: 2, @@ -1556,14 +1580,24 @@ func TestDistributor_Push_LabelRemoval(t *testing.T) { // Push the series to the distributor req := mockWriteRequest([]labels.Labels{tc.inputSeries}, 1, 1, histogram) + req.Timeseries[0].Exemplars = tc.exemplars _, err = ds[0].Push(ctx, req) require.NoError(t, err) + actualDiscardedSamples := testutil.ToFloat64(ds[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.DroppedByUserConfigurationOverride, "user")) + actualDiscardedExemplars := testutil.ToFloat64(ds[0].validateMetrics.DiscardedExemplars.WithLabelValues(validation.DroppedByUserConfigurationOverride, "user")) + require.Equal(t, float64(expectedDiscardedSamples), actualDiscardedSamples) + require.Equal(t, float64(expectedDiscardedExemplars), actualDiscardedExemplars) + // Since each test pushes only 1 series, we do expect the ingester // to have received exactly 1 series for i := range ingesters { timeseries := ingesters[i].series() - assert.Equal(t, 1, len(timeseries)) + expectedSeries := 1 + if tc.expectedSeries.Len() == 0 { + expectedSeries = 0 + } + assert.Equal(t, expectedSeries, len(timeseries)) for _, v := range timeseries { assert.Equal(t, tc.expectedSeries, cortexpb.FromLabelAdaptersToLabels(v.Labels)) } @@ -3777,39 +3811,43 @@ func TestDistributor_Push_RelabelDropWillExportMetricOfDroppedSamples(t *testing flagext.DefaultValues(&limits) limits.MetricRelabelConfigs = metricRelabelConfigs - ds, ingesters, regs, _ := prepare(t, prepConfig{ - numIngesters: 2, - happyIngesters: 2, - numDistributors: 1, - shardByAllLabels: true, - limits: &limits, - }) - - // Push the series to the distributor - req := mockWriteRequest(inputSeries, 1, 1, false) - ctx := user.InjectOrgID(context.Background(), "userDistributorPushRelabelDropWillExportMetricOfDroppedSamples") - _, err = ds[0].Push(ctx, req) - require.NoError(t, err) + for _, histogramEnabled := range []bool{false, true} { + ds, ingesters, _, _ := prepare(t, prepConfig{ + numIngesters: 2, + happyIngesters: 2, + numDistributors: 1, + shardByAllLabels: true, + limits: &limits, + }) - // Since each test pushes only 1 series, we do expect the ingester - // to have received exactly 1 series - for i := range ingesters { - timeseries := ingesters[i].series() - assert.Equal(t, 1, len(timeseries)) - } + // Push the series to the distributor + id := "user" + req := mockWriteRequest(inputSeries, 1, 1, histogramEnabled) + req.Timeseries[0].Exemplars = []cortexpb.Exemplar{ + {Labels: cortexpb.FromLabelsToLabelAdapters(labels.FromStrings("test", "a")), Value: 1, TimestampMs: 0}, + {Labels: cortexpb.FromLabelsToLabelAdapters(labels.FromStrings("test", "b")), Value: 1, TimestampMs: 0}, + } + ctx := user.InjectOrgID(context.Background(), id) + _, err = ds[0].Push(ctx, req) + require.NoError(t, err) - metrics := []string{"cortex_distributor_received_samples_total", "cortex_discarded_samples_total"} + for i := range ingesters { + timeseries := ingesters[i].series() + assert.Equal(t, 1, len(timeseries)) + } - expectedMetrics := ` - # HELP cortex_discarded_samples_total The total number of samples that were discarded. - # TYPE cortex_discarded_samples_total counter - cortex_discarded_samples_total{reason="relabel_configuration",user="userDistributorPushRelabelDropWillExportMetricOfDroppedSamples"} 1 - # HELP cortex_distributor_received_samples_total The total number of received samples, excluding rejected and deduped samples. - # TYPE cortex_distributor_received_samples_total counter - cortex_distributor_received_samples_total{type="float",user="userDistributorPushRelabelDropWillExportMetricOfDroppedSamples"} 1 - cortex_distributor_received_samples_total{type="histogram",user="userDistributorPushRelabelDropWillExportMetricOfDroppedSamples"} 0 - ` - require.NoError(t, testutil.GatherAndCompare(regs[0], strings.NewReader(expectedMetrics), metrics...)) + require.Equal(t, testutil.ToFloat64(ds[0].validateMetrics.DiscardedSamples.WithLabelValues(validation.DroppedByRelabelConfiguration, id)), float64(1)) + require.Equal(t, testutil.ToFloat64(ds[0].validateMetrics.DiscardedExemplars.WithLabelValues(validation.DroppedByRelabelConfiguration, id)), float64(2)) + receivedFloatSamples := testutil.ToFloat64(ds[0].receivedSamples.WithLabelValues(id, "float")) + receivedHistogramSamples := testutil.ToFloat64(ds[0].receivedSamples.WithLabelValues(id, "histogram")) + if histogramEnabled { + require.Equal(t, receivedFloatSamples, float64(0)) + require.Equal(t, receivedHistogramSamples, float64(1)) + } else { + require.Equal(t, receivedFloatSamples, float64(1)) + require.Equal(t, receivedHistogramSamples, float64(0)) + } + } } func countMockIngestersCalls(ingesters []*mockIngester, name string) int { diff --git a/pkg/ingester/client/compat.go b/pkg/ingester/client/compat.go index 6e4a81d634..1a8e178641 100644 --- a/pkg/ingester/client/compat.go +++ b/pkg/ingester/client/compat.go @@ -220,6 +220,36 @@ func FromLabelValuesRequest(req *LabelValuesRequest) (string, int64, int64, int, return req.LabelName, req.StartTimestampMs, req.EndTimestampMs, int(req.Limit), matchers, nil } +// ToLabelNamesRequest builds a LabelNamesRequest proto +func ToLabelNamesRequest(from, to model.Time, limit int, matchers []*labels.Matcher) (*LabelNamesRequest, error) { + ms, err := toLabelMatchers(matchers) + if err != nil { + return nil, err + } + + return &LabelNamesRequest{ + StartTimestampMs: int64(from), + EndTimestampMs: int64(to), + Matchers: &LabelMatchers{Matchers: ms}, + Limit: int64(limit), + }, nil +} + +// FromLabelNamesRequest unpacks a LabelNamesRequest proto +func FromLabelNamesRequest(req *LabelNamesRequest) (int64, int64, int, []*labels.Matcher, error) { + var err error + var matchers []*labels.Matcher + + if req.Matchers != nil { + matchers, err = FromLabelMatchers(req.Matchers.Matchers) + if err != nil { + return 0, 0, 0, nil, err + } + } + + return req.StartTimestampMs, req.EndTimestampMs, int(req.Limit), matchers, nil +} + func toLabelMatchers(matchers []*labels.Matcher) ([]*LabelMatcher, error) { result := make([]*LabelMatcher, 0, len(matchers)) for _, matcher := range matchers { diff --git a/pkg/ingester/client/ingester.pb.go b/pkg/ingester/client/ingester.pb.go index 23f1fbdf5b..374348afae 100644 --- a/pkg/ingester/client/ingester.pb.go +++ b/pkg/ingester/client/ingester.pb.go @@ -559,9 +559,10 @@ func (m *LabelValuesStreamResponse) GetLabelValues() []string { } type LabelNamesRequest struct { - StartTimestampMs int64 `protobuf:"varint,1,opt,name=start_timestamp_ms,json=startTimestampMs,proto3" json:"start_timestamp_ms,omitempty"` - EndTimestampMs int64 `protobuf:"varint,2,opt,name=end_timestamp_ms,json=endTimestampMs,proto3" json:"end_timestamp_ms,omitempty"` - Limit int64 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` + StartTimestampMs int64 `protobuf:"varint,1,opt,name=start_timestamp_ms,json=startTimestampMs,proto3" json:"start_timestamp_ms,omitempty"` + EndTimestampMs int64 `protobuf:"varint,2,opt,name=end_timestamp_ms,json=endTimestampMs,proto3" json:"end_timestamp_ms,omitempty"` + Limit int64 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` + Matchers *LabelMatchers `protobuf:"bytes,4,opt,name=matchers,proto3" json:"matchers,omitempty"` } func (m *LabelNamesRequest) Reset() { *m = LabelNamesRequest{} } @@ -617,6 +618,13 @@ func (m *LabelNamesRequest) GetLimit() int64 { return 0 } +func (m *LabelNamesRequest) GetMatchers() *LabelMatchers { + if m != nil { + return m.Matchers + } + return nil +} + type LabelNamesResponse struct { LabelNames []string `protobuf:"bytes,1,rep,name=label_names,json=labelNames,proto3" json:"label_names,omitempty"` } @@ -1476,90 +1484,91 @@ func init() { func init() { proto.RegisterFile("ingester.proto", fileDescriptor_60f6df4f3586b478) } var fileDescriptor_60f6df4f3586b478 = []byte{ - // 1328 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0x4b, 0x6f, 0xd4, 0x56, - 0x14, 0x1e, 0x67, 0x1e, 0xc9, 0x9c, 0x79, 0x30, 0xb9, 0x09, 0x64, 0x30, 0xc5, 0x01, 0x23, 0xda, - 0xa8, 0x2d, 0x09, 0xa4, 0xad, 0x04, 0x7d, 0xa1, 0x04, 0x02, 0x04, 0x08, 0x01, 0x27, 0xd0, 0xaa, - 0x6a, 0x65, 0x39, 0x33, 0x97, 0xc4, 0xc5, 0x8f, 0xc1, 0xf7, 0x1a, 0x41, 0x57, 0x45, 0xfd, 0x01, - 0xed, 0xb2, 0xdb, 0xee, 0xfa, 0x53, 0x58, 0xb2, 0xe8, 0x02, 0x75, 0x81, 0xca, 0x20, 0x55, 0x5d, - 0xd2, 0x7f, 0x50, 0xf9, 0x3e, 0xfc, 0x8a, 0xf3, 0x40, 0x82, 0xee, 0x7c, 0xcf, 0xf9, 0xce, 0xb9, - 0xe7, 0x79, 0xcf, 0x31, 0xb4, 0x6d, 0x6f, 0x13, 0x13, 0x8a, 0x83, 0xd9, 0x41, 0xe0, 0x53, 0x1f, - 0xd5, 0x7a, 0x7e, 0x40, 0xf1, 0x43, 0x75, 0x72, 0xd3, 0xdf, 0xf4, 0x19, 0x69, 0x2e, 0xfa, 0xe2, - 0x5c, 0xf5, 0xdc, 0xa6, 0x4d, 0xb7, 0xc2, 0x8d, 0xd9, 0x9e, 0xef, 0xce, 0x71, 0xe0, 0x20, 0xf0, - 0xbf, 0xc7, 0x3d, 0x2a, 0x4e, 0x73, 0x83, 0x7b, 0x9b, 0x92, 0xb1, 0x21, 0x3e, 0xb8, 0xa8, 0xfe, - 0x05, 0x34, 0x0c, 0x6c, 0xf5, 0x0d, 0x7c, 0x3f, 0xc4, 0x84, 0xa2, 0x59, 0x18, 0xbd, 0x1f, 0xe2, - 0xc0, 0xc6, 0xa4, 0xab, 0x1c, 0x2b, 0xcf, 0x34, 0xe6, 0x27, 0x67, 0x05, 0xfc, 0x56, 0x88, 0x83, - 0x47, 0x02, 0x66, 0x48, 0x90, 0x7e, 0x1e, 0x9a, 0x5c, 0x9c, 0x0c, 0x7c, 0x8f, 0x60, 0x34, 0x07, - 0xa3, 0x01, 0x26, 0xa1, 0x43, 0xa5, 0xfc, 0xc1, 0x9c, 0x3c, 0xc7, 0x19, 0x12, 0xa5, 0x5f, 0x83, - 0x56, 0x86, 0x83, 0x3e, 0x05, 0xa0, 0xb6, 0x8b, 0x49, 0x91, 0x11, 0x83, 0x8d, 0xd9, 0x75, 0xdb, - 0xc5, 0x6b, 0x8c, 0xb7, 0x58, 0x79, 0xf2, 0x7c, 0xba, 0x64, 0xa4, 0xd0, 0xfa, 0xaf, 0x0a, 0x34, - 0xd3, 0x76, 0xa2, 0x0f, 0x01, 0x11, 0x6a, 0x05, 0xd4, 0x64, 0x20, 0x6a, 0xb9, 0x03, 0xd3, 0x8d, - 0x94, 0x2a, 0x33, 0x65, 0xa3, 0xc3, 0x38, 0xeb, 0x92, 0xb1, 0x42, 0xd0, 0x0c, 0x74, 0xb0, 0xd7, - 0xcf, 0x62, 0x47, 0x18, 0xb6, 0x8d, 0xbd, 0x7e, 0x1a, 0x79, 0x1a, 0xc6, 0x5c, 0x8b, 0xf6, 0xb6, - 0x70, 0x40, 0xba, 0xe5, 0x6c, 0x9c, 0xae, 0x5b, 0x1b, 0xd8, 0x59, 0xe1, 0x4c, 0x23, 0x46, 0xe9, - 0xbf, 0x29, 0x30, 0xb9, 0xf4, 0x10, 0xbb, 0x03, 0xc7, 0x0a, 0xfe, 0x17, 0x13, 0xcf, 0x6c, 0x33, - 0xf1, 0x60, 0x91, 0x89, 0x24, 0x65, 0xe3, 0xb7, 0x30, 0xc1, 0x4c, 0x5b, 0xa3, 0x01, 0xb6, 0xdc, - 0x38, 0x23, 0xe7, 0xa1, 0xd1, 0xdb, 0x0a, 0xbd, 0x7b, 0x99, 0x94, 0x4c, 0x49, 0x65, 0x49, 0x42, - 0x2e, 0x44, 0x20, 0x91, 0x95, 0xb4, 0xc4, 0xd5, 0xca, 0xd8, 0x48, 0xa7, 0xac, 0xaf, 0xc1, 0xc1, - 0x5c, 0x00, 0xde, 0x40, 0xc6, 0xff, 0x50, 0x00, 0x31, 0x77, 0xee, 0x58, 0x4e, 0x88, 0x89, 0x0c, - 0xea, 0x51, 0x00, 0x27, 0xa2, 0x9a, 0x9e, 0xe5, 0x62, 0x16, 0xcc, 0xba, 0x51, 0x67, 0x94, 0x1b, - 0x96, 0x8b, 0x77, 0x88, 0xf9, 0xc8, 0x6b, 0xc4, 0xbc, 0xbc, 0x67, 0xcc, 0x2b, 0xc7, 0x94, 0x7d, - 0xc4, 0x1c, 0x4d, 0x42, 0xd5, 0xb1, 0x5d, 0x9b, 0x76, 0xab, 0x4c, 0x23, 0x3f, 0xe8, 0x67, 0x61, - 0x22, 0xe3, 0x95, 0x88, 0xd4, 0x71, 0x68, 0x72, 0xb7, 0x1e, 0x30, 0x3a, 0x8b, 0x55, 0xdd, 0x68, - 0x38, 0x09, 0x54, 0xff, 0x12, 0x0e, 0xa7, 0x24, 0x73, 0x99, 0xdc, 0x87, 0xfc, 0x63, 0x05, 0xc6, - 0xaf, 0xcb, 0x40, 0x91, 0xb7, 0x5d, 0xa4, 0xb1, 0xf7, 0xe5, 0xb4, 0xf7, 0x9f, 0x88, 0x9c, 0x0a, - 0x13, 0x84, 0xf1, 0xd3, 0xd0, 0x48, 0x72, 0x2a, 0x6d, 0x87, 0x38, 0xa9, 0x44, 0xff, 0x0c, 0xba, - 0x89, 0x58, 0xce, 0xf3, 0x3d, 0x85, 0x11, 0x74, 0x6e, 0x13, 0x1c, 0xac, 0x51, 0x8b, 0x4a, 0xaf, - 0xf5, 0xc7, 0x23, 0x30, 0x9e, 0x22, 0x0a, 0x55, 0x27, 0xe5, 0xe3, 0x6c, 0xfb, 0x9e, 0x19, 0x58, - 0x94, 0xd7, 0x97, 0x62, 0xb4, 0x62, 0xaa, 0x61, 0x51, 0x1c, 0x95, 0xa0, 0x17, 0xba, 0xa6, 0xa8, - 0xea, 0xc8, 0xfd, 0x8a, 0x51, 0xf7, 0x42, 0x97, 0x97, 0x72, 0x14, 0x51, 0x6b, 0x60, 0x9b, 0x39, - 0x4d, 0x65, 0xa6, 0xa9, 0x63, 0x0d, 0xec, 0xe5, 0x8c, 0xb2, 0x59, 0x98, 0x08, 0x42, 0x07, 0xe7, - 0xe1, 0x15, 0x06, 0x1f, 0x8f, 0x58, 0x59, 0xfc, 0x09, 0x68, 0x59, 0x3d, 0x6a, 0x3f, 0xc0, 0xf2, - 0xfe, 0x2a, 0xbb, 0xbf, 0xc9, 0x89, 0xc2, 0x84, 0x13, 0xd0, 0x72, 0x7c, 0xab, 0x8f, 0xfb, 0xe6, - 0x86, 0xe3, 0xf7, 0xee, 0x91, 0x6e, 0x8d, 0x83, 0x38, 0x71, 0x91, 0xd1, 0xf4, 0xef, 0x60, 0x22, - 0x0a, 0xc1, 0xf2, 0xc5, 0x6c, 0x10, 0xa6, 0x60, 0x34, 0x24, 0x38, 0x30, 0xed, 0xbe, 0xe8, 0xae, - 0x5a, 0x74, 0x5c, 0xee, 0xa3, 0x53, 0x50, 0xe9, 0x5b, 0xd4, 0x62, 0x0e, 0x37, 0xe6, 0x0f, 0xcb, - 0xf2, 0xdf, 0x16, 0x46, 0x83, 0xc1, 0xf4, 0xcb, 0x80, 0x22, 0x16, 0xc9, 0x6a, 0x3f, 0x03, 0x55, - 0x12, 0x11, 0xc4, 0x63, 0x70, 0x24, 0xad, 0x25, 0x67, 0x89, 0xc1, 0x91, 0xfa, 0x13, 0x05, 0xb4, - 0x15, 0x4c, 0x03, 0xbb, 0x47, 0x2e, 0xf9, 0x41, 0xb6, 0xdb, 0xde, 0x72, 0x11, 0x9f, 0x85, 0xa6, - 0x6c, 0x67, 0x93, 0x60, 0xba, 0xfb, 0x6b, 0xdb, 0x90, 0xd0, 0x35, 0x4c, 0x93, 0xf2, 0xaf, 0xa4, - 0xcb, 0xff, 0x1a, 0x4c, 0xef, 0xe8, 0x89, 0x08, 0xd0, 0x0c, 0xd4, 0x5c, 0x06, 0x11, 0x11, 0xea, - 0x24, 0xcf, 0x25, 0x17, 0x35, 0x04, 0x5f, 0xbf, 0x05, 0x27, 0x77, 0x50, 0x96, 0xeb, 0x90, 0xfd, - 0xab, 0xec, 0xc2, 0x21, 0xa1, 0x72, 0x05, 0x53, 0x2b, 0x4a, 0xa3, 0x6c, 0x98, 0x55, 0x98, 0xda, - 0xc6, 0x11, 0xea, 0x3f, 0x86, 0x31, 0x57, 0xd0, 0xc4, 0x05, 0xdd, 0xfc, 0x05, 0xb1, 0x4c, 0x8c, - 0xd4, 0xff, 0x55, 0xe0, 0x40, 0x6e, 0xc0, 0x44, 0x89, 0xb9, 0x1b, 0xf8, 0xae, 0x29, 0x37, 0xa4, - 0xa4, 0x06, 0xdb, 0x11, 0x7d, 0x59, 0x90, 0x97, 0xfb, 0xe9, 0x22, 0x1d, 0xc9, 0x14, 0xa9, 0x07, - 0x35, 0xd6, 0xfa, 0x72, 0x32, 0x4e, 0x24, 0xa6, 0xb0, 0x10, 0xdd, 0xb4, 0xec, 0x60, 0x71, 0x21, - 0x1a, 0x36, 0x7f, 0x3e, 0x9f, 0x7e, 0xad, 0xe5, 0x8a, 0xcb, 0x2f, 0xf4, 0xad, 0x01, 0xc5, 0x81, - 0x21, 0x6e, 0x41, 0x1f, 0x40, 0x8d, 0xcf, 0xc3, 0x6e, 0x85, 0xdd, 0xd7, 0x92, 0xb5, 0x91, 0x1e, - 0x99, 0x02, 0xa2, 0xff, 0xac, 0x40, 0x95, 0x7b, 0xfa, 0xb6, 0x0a, 0x56, 0x85, 0x31, 0xec, 0xf5, - 0xfc, 0xbe, 0xed, 0x6d, 0xb2, 0x17, 0xa7, 0x6a, 0xc4, 0x67, 0x84, 0x44, 0xff, 0x46, 0x15, 0xd9, - 0x14, 0x4d, 0xba, 0x00, 0xad, 0x4c, 0xe5, 0x64, 0xd6, 0x1f, 0x65, 0x5f, 0xeb, 0x8f, 0x09, 0xcd, - 0x34, 0x07, 0x9d, 0x84, 0x0a, 0x7d, 0x34, 0xe0, 0x4f, 0x67, 0x7b, 0x7e, 0x5c, 0x4a, 0x33, 0xf6, - 0xfa, 0xa3, 0x01, 0x36, 0x18, 0x3b, 0xb2, 0x86, 0x4d, 0x70, 0x9e, 0x3e, 0xf6, 0x1d, 0x35, 0x0d, - 0x1b, 0x5f, 0xcc, 0xf4, 0xba, 0xc1, 0x0f, 0xfa, 0x4f, 0x0a, 0xb4, 0x93, 0x4a, 0xb9, 0x64, 0x3b, - 0xf8, 0x4d, 0x14, 0x8a, 0x0a, 0x63, 0x77, 0x6d, 0x07, 0x33, 0x1b, 0xf8, 0x75, 0xf1, 0xb9, 0x28, - 0x52, 0xef, 0x5f, 0x85, 0x7a, 0xec, 0x02, 0xaa, 0x43, 0x75, 0xe9, 0xd6, 0xed, 0x85, 0xeb, 0x9d, - 0x12, 0x6a, 0x41, 0xfd, 0xc6, 0xea, 0xba, 0xc9, 0x8f, 0x0a, 0x3a, 0x00, 0x0d, 0x63, 0xe9, 0xf2, - 0xd2, 0xd7, 0xe6, 0xca, 0xc2, 0xfa, 0x85, 0x2b, 0x9d, 0x11, 0x84, 0xa0, 0xcd, 0x09, 0x37, 0x56, - 0x05, 0xad, 0x3c, 0xff, 0xf7, 0x28, 0x8c, 0x49, 0x1b, 0xd1, 0x39, 0xa8, 0xdc, 0x0c, 0xc9, 0x16, - 0x3a, 0x94, 0x54, 0xea, 0x57, 0x81, 0x4d, 0xb1, 0xe8, 0x3c, 0x75, 0x6a, 0x1b, 0x9d, 0xf7, 0x9d, - 0x5e, 0x42, 0x17, 0xa1, 0x91, 0xda, 0xea, 0x50, 0xe1, 0x42, 0xaf, 0x1e, 0xc9, 0x50, 0xb3, 0x4f, - 0x83, 0x5e, 0x3a, 0xad, 0xa0, 0x55, 0x68, 0x33, 0x96, 0x5c, 0xe1, 0x08, 0x7a, 0x47, 0x8a, 0x14, - 0xad, 0xb5, 0xea, 0xd1, 0x1d, 0xb8, 0xb1, 0x59, 0x57, 0xa0, 0x91, 0x5a, 0x54, 0x90, 0x9a, 0x29, - 0xa0, 0xcc, 0x36, 0x97, 0x18, 0x57, 0xb0, 0x13, 0xe9, 0x25, 0x74, 0x47, 0x6c, 0x2c, 0xe9, 0x95, - 0x67, 0x57, 0x7d, 0xc7, 0x0b, 0x78, 0x05, 0x2e, 0x2f, 0x01, 0x24, 0xfb, 0x04, 0x3a, 0x9c, 0x11, - 0x4a, 0x6f, 0x47, 0xaa, 0x5a, 0xc4, 0x8a, 0xcd, 0x5b, 0x83, 0x4e, 0x7e, 0x2d, 0xd9, 0x4d, 0xd9, - 0xb1, 0xed, 0xac, 0x02, 0xdb, 0x16, 0xa1, 0x1e, 0x8f, 0x54, 0xd4, 0x2d, 0x98, 0xb2, 0x5c, 0xd9, - 0xce, 0xf3, 0x57, 0x2f, 0xa1, 0x4b, 0xd0, 0x5c, 0x70, 0x9c, 0xfd, 0xa8, 0x51, 0xd3, 0x1c, 0x92, - 0xd7, 0xe3, 0xc4, 0xaf, 0x7e, 0x7e, 0xc4, 0xa0, 0x77, 0xe3, 0xc6, 0xde, 0x75, 0x34, 0xab, 0xef, - 0xed, 0x89, 0x8b, 0x6f, 0xfb, 0x01, 0x8e, 0xee, 0x3a, 0xd0, 0xf6, 0x7d, 0xe7, 0xa9, 0x3d, 0x70, - 0x05, 0x51, 0x5f, 0x87, 0x03, 0xb9, 0xf9, 0x86, 0xb4, 0x9c, 0x96, 0xdc, 0x48, 0x54, 0xa7, 0x77, - 0xe4, 0x4b, 0xbd, 0x8b, 0x9f, 0x3f, 0x7d, 0xa1, 0x95, 0x9e, 0xbd, 0xd0, 0x4a, 0xaf, 0x5e, 0x68, - 0xca, 0x8f, 0x43, 0x4d, 0xf9, 0x7d, 0xa8, 0x29, 0x4f, 0x86, 0x9a, 0xf2, 0x74, 0xa8, 0x29, 0x7f, - 0x0d, 0x35, 0xe5, 0x9f, 0xa1, 0x56, 0x7a, 0x35, 0xd4, 0x94, 0x5f, 0x5e, 0x6a, 0xa5, 0xa7, 0x2f, - 0xb5, 0xd2, 0xb3, 0x97, 0x5a, 0xe9, 0x9b, 0x5a, 0xcf, 0xb1, 0xb1, 0x47, 0x37, 0x6a, 0xec, 0x3f, - 0xfe, 0xa3, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0xc5, 0xfc, 0x4c, 0xf5, 0x32, 0x10, 0x00, 0x00, + // 1339 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0x4b, 0x6f, 0x14, 0xc7, + 0x13, 0xdf, 0xf1, 0x3e, 0xec, 0xad, 0x7d, 0xb0, 0x6e, 0x1b, 0xbc, 0x0c, 0x7f, 0xc6, 0x30, 0x88, + 0x7f, 0xac, 0x24, 0xd8, 0xe0, 0x24, 0x12, 0xe4, 0x85, 0x6c, 0x30, 0x60, 0xc0, 0x18, 0xc6, 0x86, + 0x44, 0x51, 0xa2, 0xd1, 0x78, 0xb7, 0xb1, 0x27, 0xcc, 0x63, 0x99, 0xee, 0x41, 0x90, 0x53, 0xa2, + 0x7c, 0x80, 0xe4, 0x98, 0x6b, 0x6e, 0xf9, 0x00, 0xf9, 0x10, 0x1c, 0x39, 0xe4, 0x80, 0x72, 0x40, + 0x61, 0x91, 0xa2, 0x1c, 0xc9, 0x37, 0x88, 0xa6, 0x1f, 0xf3, 0xf2, 0xf8, 0x41, 0x04, 0xb9, 0xed, + 0x54, 0xfd, 0xaa, 0xba, 0xea, 0xd7, 0x55, 0x5d, 0xb5, 0xd0, 0xb6, 0xbd, 0x4d, 0x4c, 0x28, 0x0e, + 0x66, 0x07, 0x81, 0x4f, 0x7d, 0x54, 0xeb, 0xf9, 0x01, 0xc5, 0x0f, 0xd5, 0xc9, 0x4d, 0x7f, 0xd3, + 0x67, 0xa2, 0xb9, 0xe8, 0x17, 0xd7, 0xaa, 0xe7, 0x36, 0x6d, 0xba, 0x15, 0x6e, 0xcc, 0xf6, 0x7c, + 0x77, 0x8e, 0x03, 0x07, 0x81, 0xff, 0x35, 0xee, 0x51, 0xf1, 0x35, 0x37, 0xb8, 0xb7, 0x29, 0x15, + 0x1b, 0xe2, 0x07, 0x37, 0xd5, 0x3f, 0x81, 0x86, 0x81, 0xad, 0xbe, 0x81, 0xef, 0x87, 0x98, 0x50, + 0x34, 0x0b, 0xa3, 0xf7, 0x43, 0x1c, 0xd8, 0x98, 0x74, 0x95, 0x63, 0xe5, 0x99, 0xc6, 0xfc, 0xe4, + 0xac, 0x80, 0xdf, 0x0a, 0x71, 0xf0, 0x48, 0xc0, 0x0c, 0x09, 0xd2, 0xcf, 0x43, 0x93, 0x9b, 0x93, + 0x81, 0xef, 0x11, 0x8c, 0xe6, 0x60, 0x34, 0xc0, 0x24, 0x74, 0xa8, 0xb4, 0x3f, 0x98, 0xb3, 0xe7, + 0x38, 0x43, 0xa2, 0xf4, 0x6b, 0xd0, 0xca, 0x68, 0xd0, 0x87, 0x00, 0xd4, 0x76, 0x31, 0x29, 0x0a, + 0x62, 0xb0, 0x31, 0xbb, 0x6e, 0xbb, 0x78, 0x8d, 0xe9, 0x16, 0x2b, 0x8f, 0x9f, 0x4d, 0x97, 0x8c, + 0x14, 0x5a, 0xff, 0x49, 0x81, 0x66, 0x3a, 0x4e, 0xf4, 0x2e, 0x20, 0x42, 0xad, 0x80, 0x9a, 0x0c, + 0x44, 0x2d, 0x77, 0x60, 0xba, 0x91, 0x53, 0x65, 0xa6, 0x6c, 0x74, 0x98, 0x66, 0x5d, 0x2a, 0x56, + 0x08, 0x9a, 0x81, 0x0e, 0xf6, 0xfa, 0x59, 0xec, 0x08, 0xc3, 0xb6, 0xb1, 0xd7, 0x4f, 0x23, 0x4f, + 0xc3, 0x98, 0x6b, 0xd1, 0xde, 0x16, 0x0e, 0x48, 0xb7, 0x9c, 0xe5, 0xe9, 0xba, 0xb5, 0x81, 0x9d, + 0x15, 0xae, 0x34, 0x62, 0x94, 0xfe, 0xb3, 0x02, 0x93, 0x4b, 0x0f, 0xb1, 0x3b, 0x70, 0xac, 0xe0, + 0x3f, 0x09, 0xf1, 0xcc, 0xb6, 0x10, 0x0f, 0x16, 0x85, 0x48, 0x52, 0x31, 0x7e, 0x09, 0x13, 0x2c, + 0xb4, 0x35, 0x1a, 0x60, 0xcb, 0x8d, 0x6f, 0xe4, 0x3c, 0x34, 0x7a, 0x5b, 0xa1, 0x77, 0x2f, 0x73, + 0x25, 0x53, 0xd2, 0x59, 0x72, 0x21, 0x17, 0x22, 0x90, 0xb8, 0x95, 0xb4, 0xc5, 0xd5, 0xca, 0xd8, + 0x48, 0xa7, 0xac, 0xaf, 0xc1, 0xc1, 0x1c, 0x01, 0xaf, 0xe1, 0xc6, 0x7f, 0x53, 0x00, 0xb1, 0x74, + 0xee, 0x58, 0x4e, 0x88, 0x89, 0x24, 0xf5, 0x28, 0x80, 0x13, 0x49, 0x4d, 0xcf, 0x72, 0x31, 0x23, + 0xb3, 0x6e, 0xd4, 0x99, 0xe4, 0x86, 0xe5, 0xe2, 0x1d, 0x38, 0x1f, 0x79, 0x05, 0xce, 0xcb, 0x7b, + 0x72, 0x5e, 0x39, 0xa6, 0xec, 0x83, 0x73, 0x34, 0x09, 0x55, 0xc7, 0x76, 0x6d, 0xda, 0xad, 0x32, + 0x8f, 0xfc, 0x43, 0x3f, 0x0b, 0x13, 0x99, 0xac, 0x04, 0x53, 0xc7, 0xa1, 0xc9, 0xd3, 0x7a, 0xc0, + 0xe4, 0x8c, 0xab, 0xba, 0xd1, 0x70, 0x12, 0xa8, 0xfe, 0x29, 0x1c, 0x4e, 0x59, 0xe6, 0x6e, 0x72, + 0x1f, 0xf6, 0xbf, 0x2a, 0x30, 0x7e, 0x5d, 0x12, 0x45, 0xde, 0x74, 0x91, 0xc6, 0xd9, 0x97, 0x53, + 0xd9, 0xff, 0x0b, 0x1a, 0xf5, 0x0f, 0x44, 0x19, 0x88, 0xa8, 0x45, 0xbe, 0xd3, 0xd0, 0x48, 0xca, + 0x40, 0xa6, 0x0b, 0x71, 0x1d, 0x10, 0xfd, 0x23, 0xe8, 0x26, 0x66, 0x39, 0xb2, 0xf6, 0x34, 0x46, + 0xd0, 0xb9, 0x4d, 0x70, 0xb0, 0x46, 0x2d, 0x2a, 0x89, 0xd2, 0xbf, 0x1b, 0x81, 0xf1, 0x94, 0x50, + 0xb8, 0x3a, 0x29, 0xdf, 0x73, 0xdb, 0xf7, 0xcc, 0xc0, 0xa2, 0xbc, 0x24, 0x15, 0xa3, 0x15, 0x4b, + 0x0d, 0x8b, 0xe2, 0xa8, 0x6a, 0xbd, 0xd0, 0x35, 0x45, 0x23, 0x44, 0x8c, 0x55, 0x8c, 0xba, 0x17, + 0xba, 0xbc, 0xfa, 0xa3, 0x4b, 0xb0, 0x06, 0xb6, 0x99, 0xf3, 0x54, 0x66, 0x9e, 0x3a, 0xd6, 0xc0, + 0x5e, 0xce, 0x38, 0x9b, 0x85, 0x89, 0x20, 0x74, 0x70, 0x1e, 0x5e, 0x61, 0xf0, 0xf1, 0x48, 0x95, + 0xc5, 0x9f, 0x80, 0x96, 0xd5, 0xa3, 0xf6, 0x03, 0x2c, 0xcf, 0xaf, 0xb2, 0xf3, 0x9b, 0x5c, 0x28, + 0x42, 0x38, 0x01, 0x2d, 0xc7, 0xb7, 0xfa, 0xb8, 0x6f, 0x6e, 0x38, 0x7e, 0xef, 0x1e, 0xe9, 0xd6, + 0x38, 0x88, 0x0b, 0x17, 0x99, 0x4c, 0xff, 0x0a, 0x26, 0x22, 0x0a, 0x96, 0x2f, 0x66, 0x49, 0x98, + 0x82, 0xd1, 0x90, 0xe0, 0xc0, 0xb4, 0xfb, 0xa2, 0x21, 0x6b, 0xd1, 0xe7, 0x72, 0x1f, 0x9d, 0x82, + 0x4a, 0xdf, 0xa2, 0x16, 0x4b, 0xb8, 0x31, 0x7f, 0x58, 0x5e, 0xf5, 0x36, 0x1a, 0x0d, 0x06, 0xd3, + 0x2f, 0x03, 0x8a, 0x54, 0x24, 0xeb, 0xfd, 0x0c, 0x54, 0x49, 0x24, 0x10, 0xef, 0xc7, 0x91, 0xb4, + 0x97, 0x5c, 0x24, 0x06, 0x47, 0xea, 0x8f, 0x15, 0xd0, 0x56, 0x30, 0x0d, 0xec, 0x1e, 0xb9, 0xe4, + 0x07, 0xd9, 0xca, 0x7a, 0xc3, 0x75, 0x7f, 0x16, 0x9a, 0xb2, 0x74, 0x4d, 0x82, 0xe9, 0xee, 0x0f, + 0x74, 0x43, 0x42, 0xd7, 0x30, 0x4d, 0x3a, 0xa6, 0x92, 0x7e, 0x2f, 0xae, 0xc1, 0xf4, 0x8e, 0x99, + 0x08, 0x82, 0x66, 0xa0, 0xe6, 0x32, 0x88, 0x60, 0xa8, 0x93, 0xbc, 0xb0, 0xdc, 0xd4, 0x10, 0x7a, + 0xfd, 0x16, 0x9c, 0xdc, 0xc1, 0x59, 0xae, 0x43, 0xf6, 0xef, 0xb2, 0x0b, 0x87, 0x84, 0xcb, 0x15, + 0x4c, 0xad, 0xe8, 0x1a, 0x65, 0xc3, 0xac, 0xc2, 0xd4, 0x36, 0x8d, 0x70, 0xff, 0x3e, 0x8c, 0xb9, + 0x42, 0x26, 0x0e, 0xe8, 0xe6, 0x0f, 0x88, 0x6d, 0x62, 0xa4, 0xfe, 0xb7, 0x02, 0x07, 0x72, 0x33, + 0x29, 0xba, 0x98, 0xbb, 0x81, 0xef, 0x9a, 0x72, 0xa9, 0x4a, 0x6a, 0xb0, 0x1d, 0xc9, 0x97, 0x85, + 0x78, 0xb9, 0x9f, 0x2e, 0xd2, 0x91, 0x4c, 0x91, 0x7a, 0x50, 0x63, 0xad, 0x2f, 0x87, 0xe9, 0x44, + 0x12, 0x0a, 0xa3, 0xe8, 0xa6, 0x65, 0x07, 0x8b, 0x0b, 0xd1, 0x7c, 0xfa, 0xfd, 0xd9, 0xf4, 0x2b, + 0xed, 0x63, 0xdc, 0x7e, 0xa1, 0x6f, 0x0d, 0x28, 0x0e, 0x0c, 0x71, 0x0a, 0x7a, 0x07, 0x6a, 0x7c, + 0x84, 0x76, 0x2b, 0xec, 0xbc, 0x96, 0xac, 0x8d, 0xf4, 0x94, 0x15, 0x10, 0xfd, 0x07, 0x05, 0xaa, + 0x3c, 0xd3, 0x37, 0x55, 0xb0, 0x2a, 0x8c, 0x61, 0xaf, 0xe7, 0xf7, 0x6d, 0x6f, 0x93, 0xbd, 0x38, + 0x55, 0x23, 0xfe, 0x46, 0x48, 0xf4, 0x6f, 0x54, 0x91, 0x4d, 0xd1, 0xa4, 0x0b, 0xd0, 0xca, 0x54, + 0x4e, 0x66, 0x63, 0x52, 0xf6, 0xb5, 0x31, 0x99, 0xd0, 0x4c, 0x6b, 0xd0, 0x49, 0xa8, 0xd0, 0x47, + 0x03, 0xfe, 0x74, 0xb6, 0xe7, 0xc7, 0xa5, 0x35, 0x53, 0xaf, 0x3f, 0x1a, 0x60, 0x83, 0xa9, 0xa3, + 0x68, 0xd8, 0xd0, 0xe7, 0xd7, 0xc7, 0x7e, 0x47, 0x4d, 0xc3, 0x26, 0x1e, 0x0b, 0xbd, 0x6e, 0xf0, + 0x0f, 0xfd, 0x7b, 0x05, 0xda, 0x49, 0xa5, 0x5c, 0xb2, 0x1d, 0xfc, 0x3a, 0x0a, 0x45, 0x85, 0xb1, + 0xbb, 0xb6, 0x83, 0x59, 0x0c, 0xfc, 0xb8, 0xf8, 0xbb, 0x88, 0xa9, 0xb7, 0xaf, 0x42, 0x3d, 0x4e, + 0x01, 0xd5, 0xa1, 0xba, 0x74, 0xeb, 0xf6, 0xc2, 0xf5, 0x4e, 0x09, 0xb5, 0xa0, 0x7e, 0x63, 0x75, + 0xdd, 0xe4, 0x9f, 0x0a, 0x3a, 0x00, 0x0d, 0x63, 0xe9, 0xf2, 0xd2, 0xe7, 0xe6, 0xca, 0xc2, 0xfa, + 0x85, 0x2b, 0x9d, 0x11, 0x84, 0xa0, 0xcd, 0x05, 0x37, 0x56, 0x85, 0xac, 0x3c, 0xff, 0xe7, 0x28, + 0x8c, 0xc9, 0x18, 0xd1, 0x39, 0xa8, 0xdc, 0x0c, 0xc9, 0x16, 0x3a, 0x94, 0x54, 0xea, 0x67, 0x81, + 0x4d, 0xb1, 0xe8, 0x3c, 0x75, 0x6a, 0x9b, 0x9c, 0xf7, 0x9d, 0x5e, 0x42, 0x17, 0xa1, 0x91, 0x5a, + 0x04, 0x51, 0xe1, 0x7f, 0x00, 0xf5, 0x48, 0x46, 0x9a, 0x7d, 0x1a, 0xf4, 0xd2, 0x69, 0x05, 0xad, + 0x42, 0x9b, 0xa9, 0xe4, 0xd6, 0x47, 0xd0, 0xff, 0xa4, 0x49, 0xd1, 0x26, 0xac, 0x1e, 0xdd, 0x41, + 0x1b, 0x87, 0x75, 0x05, 0x1a, 0xa9, 0xdd, 0x06, 0xa9, 0x99, 0x02, 0xca, 0x2c, 0x80, 0x49, 0x70, + 0x05, 0x6b, 0x94, 0x5e, 0x42, 0x77, 0xc4, 0x92, 0x93, 0xde, 0x92, 0x76, 0xf5, 0x77, 0xbc, 0x40, + 0x57, 0x90, 0xf2, 0x12, 0x40, 0xb2, 0x4f, 0xa0, 0xc3, 0x19, 0xa3, 0xf4, 0x42, 0xa5, 0xaa, 0x45, + 0xaa, 0x38, 0xbc, 0x35, 0xe8, 0xe4, 0xd7, 0x92, 0xdd, 0x9c, 0x1d, 0xdb, 0xae, 0x2a, 0x88, 0x6d, + 0x11, 0xea, 0xf1, 0x48, 0x45, 0xdd, 0x82, 0x29, 0xcb, 0x9d, 0xed, 0x3c, 0x7f, 0xf5, 0x12, 0xba, + 0x04, 0xcd, 0x05, 0xc7, 0xd9, 0x8f, 0x1b, 0x35, 0xad, 0x21, 0x79, 0x3f, 0x4e, 0xfc, 0xea, 0xe7, + 0x47, 0x0c, 0xfa, 0x7f, 0xdc, 0xd8, 0xbb, 0x8e, 0x66, 0xf5, 0xad, 0x3d, 0x71, 0xf1, 0x69, 0xdf, + 0xc0, 0xd1, 0x5d, 0x07, 0xda, 0xbe, 0xcf, 0x3c, 0xb5, 0x07, 0xae, 0x80, 0xf5, 0x75, 0x38, 0x90, + 0x9b, 0x6f, 0x48, 0xcb, 0x79, 0xc9, 0x8d, 0x44, 0x75, 0x7a, 0x47, 0xbd, 0xf4, 0xbb, 0xf8, 0xf1, + 0x93, 0xe7, 0x5a, 0xe9, 0xe9, 0x73, 0xad, 0xf4, 0xf2, 0xb9, 0xa6, 0x7c, 0x3b, 0xd4, 0x94, 0x5f, + 0x86, 0x9a, 0xf2, 0x78, 0xa8, 0x29, 0x4f, 0x86, 0x9a, 0xf2, 0xc7, 0x50, 0x53, 0xfe, 0x1a, 0x6a, + 0xa5, 0x97, 0x43, 0x4d, 0xf9, 0xf1, 0x85, 0x56, 0x7a, 0xf2, 0x42, 0x2b, 0x3d, 0x7d, 0xa1, 0x95, + 0xbe, 0xa8, 0xf5, 0x1c, 0x1b, 0x7b, 0x74, 0xa3, 0xc6, 0xfe, 0xfa, 0xbf, 0xf7, 0x4f, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x84, 0xf7, 0x8d, 0x61, 0x65, 0x10, 0x00, 0x00, } func (x MatchType) String() string { @@ -1906,6 +1915,9 @@ func (this *LabelNamesRequest) Equal(that interface{}) bool { if this.Limit != that1.Limit { return false } + if !this.Matchers.Equal(that1.Matchers) { + return false + } return true } func (this *LabelNamesResponse) Equal(that interface{}) bool { @@ -2536,11 +2548,14 @@ func (this *LabelNamesRequest) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 7) + s := make([]string, 0, 8) s = append(s, "&client.LabelNamesRequest{") s = append(s, "StartTimestampMs: "+fmt.Sprintf("%#v", this.StartTimestampMs)+",\n") s = append(s, "EndTimestampMs: "+fmt.Sprintf("%#v", this.EndTimestampMs)+",\n") s = append(s, "Limit: "+fmt.Sprintf("%#v", this.Limit)+",\n") + if this.Matchers != nil { + s = append(s, "Matchers: "+fmt.Sprintf("%#v", this.Matchers)+",\n") + } s = append(s, "}") return strings.Join(s, "") } @@ -3756,6 +3771,18 @@ func (m *LabelNamesRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.Matchers != nil { + { + size, err := m.Matchers.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintIngester(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + } if m.Limit != 0 { i = encodeVarintIngester(dAtA, i, uint64(m.Limit)) i-- @@ -4621,6 +4648,10 @@ func (m *LabelNamesRequest) Size() (n int) { if m.Limit != 0 { n += 1 + sovIngester(uint64(m.Limit)) } + if m.Matchers != nil { + l = m.Matchers.Size() + n += 1 + l + sovIngester(uint64(l)) + } return n } @@ -5068,6 +5099,7 @@ func (this *LabelNamesRequest) String() string { `StartTimestampMs:` + fmt.Sprintf("%v", this.StartTimestampMs) + `,`, `EndTimestampMs:` + fmt.Sprintf("%v", this.EndTimestampMs) + `,`, `Limit:` + fmt.Sprintf("%v", this.Limit) + `,`, + `Matchers:` + strings.Replace(this.Matchers.String(), "LabelMatchers", "LabelMatchers", 1) + `,`, `}`, }, "") return s @@ -6412,6 +6444,42 @@ func (m *LabelNamesRequest) Unmarshal(dAtA []byte) error { break } } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Matchers", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIngester + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthIngester + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthIngester + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Matchers == nil { + m.Matchers = &LabelMatchers{} + } + if err := m.Matchers.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipIngester(dAtA[iNdEx:]) diff --git a/pkg/ingester/client/ingester.proto b/pkg/ingester/client/ingester.proto index 34b9662264..68f343693e 100644 --- a/pkg/ingester/client/ingester.proto +++ b/pkg/ingester/client/ingester.proto @@ -82,6 +82,7 @@ message LabelNamesRequest { int64 start_timestamp_ms = 1; int64 end_timestamp_ms = 2; int64 limit = 3; + LabelMatchers matchers = 4; } message LabelNamesResponse { diff --git a/pkg/ingester/http_admin.go b/pkg/ingester/http_admin.go index a0d2ca98c5..084e132db4 100644 --- a/pkg/ingester/http_admin.go +++ b/pkg/ingester/http_admin.go @@ -43,7 +43,7 @@ const tpl = ` {{ range .Stats }} {{ .UserID }} - {{ .UserStats.LoadBlocks }} + {{ .UserStats.LoadedBlocks }} {{ .UserStats.NumSeries }} {{ .UserStats.ActiveSeries }} {{ printf "%.2f" .UserStats.IngestionRate }} diff --git a/pkg/ingester/http_admin_test.go b/pkg/ingester/http_admin_test.go new file mode 100644 index 0000000000..bb49b42cdc --- /dev/null +++ b/pkg/ingester/http_admin_test.go @@ -0,0 +1,37 @@ +package ingester + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUserStatsPageRendered(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/ingester/all_user_stats", nil) + res := httptest.NewRecorder() + userStats := []UserIDStats{ + { + UserID: "123", + UserStats: UserStats{ + IngestionRate: 11.11, + NumSeries: 2222, + APIIngestionRate: 33.33, + RuleIngestionRate: 44.44, + ActiveSeries: 5555, + LoadedBlocks: 6666, + }, + }, + } + AllUserStatsRender(res, req, userStats, 3) + assert.Equal(t, http.StatusOK, res.Code) + body := res.Body.String() + assert.Regexp(t, "", body) + assert.Regexp(t, "", body) + assert.Regexp(t, "", body) + assert.Regexp(t, "", body) + assert.Regexp(t, "", body) + assert.Regexp(t, "", body) + assert.Regexp(t, "", body) +} diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 82e4bff755..1652b7ec32 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1591,6 +1591,11 @@ func (i *Ingester) labelNamesCommon(ctx context.Context, req *client.LabelNamesR return nil, cleanup, err } + startTimestampMs, endTimestampMs, limit, matchers, err := client.FromLabelNamesRequest(req) + if err != nil { + return nil, cleanup, err + } + userID, err := tenant.TenantID(ctx) if err != nil { return nil, cleanup, err @@ -1601,13 +1606,11 @@ func (i *Ingester) labelNamesCommon(ctx context.Context, req *client.LabelNamesR return &client.LabelNamesResponse{}, cleanup, nil } - mint, maxt, err := metadataQueryRange(req.StartTimestampMs, req.EndTimestampMs, db, i.cfg.QueryIngestersWithin) + mint, maxt, err := metadataQueryRange(startTimestampMs, endTimestampMs, db, i.cfg.QueryIngestersWithin) if err != nil { return nil, cleanup, err } - limit := int(req.Limit) - q, err := db.Querier(mint, maxt) if err != nil { return nil, cleanup, err @@ -1622,7 +1625,7 @@ func (i *Ingester) labelNamesCommon(ctx context.Context, req *client.LabelNamesR return nil, cleanup, err } defer c() - names, _, err := q.LabelNames(ctx, &storage.LabelHints{Limit: limit}) + names, _, err := q.LabelNames(ctx, &storage.LabelHints{Limit: limit}, matchers...) if err != nil { return nil, cleanup, err } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index c631db582b..3b28bfc3fe 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -2302,6 +2302,7 @@ func Test_Ingester_LabelValues(t *testing.T) { tests := map[string]struct { limit int64 + match []*labels.Matcher }{ "should return all label values if no limit is set": { limit: 0, @@ -5206,6 +5207,7 @@ func Test_Ingester_ModeHandler(t *testing.T) { mode string expectedState ring.InstanceState expectedResponse int + expectedIsReady bool }{ "should change to READONLY mode": { method: "POST", @@ -5213,6 +5215,7 @@ func Test_Ingester_ModeHandler(t *testing.T) { requestUrl: "/mode?mode=reAdOnLy", expectedState: ring.READONLY, expectedResponse: http.StatusOK, + expectedIsReady: true, }, "should change mode on GET method": { method: "GET", @@ -5220,6 +5223,7 @@ func Test_Ingester_ModeHandler(t *testing.T) { requestUrl: "/mode?mode=READONLY", expectedState: ring.READONLY, expectedResponse: http.StatusOK, + expectedIsReady: true, }, "should change mode on POST method via body": { method: "POST", @@ -5228,6 +5232,7 @@ func Test_Ingester_ModeHandler(t *testing.T) { requestBody: strings.NewReader("mode=readonly"), expectedState: ring.READONLY, expectedResponse: http.StatusOK, + expectedIsReady: true, }, "should change to ACTIVE mode": { method: "POST", @@ -5235,6 +5240,7 @@ func Test_Ingester_ModeHandler(t *testing.T) { requestUrl: "/mode?mode=active", expectedState: ring.ACTIVE, expectedResponse: http.StatusOK, + expectedIsReady: true, }, "should fail to unknown mode": { method: "POST", @@ -5242,6 +5248,7 @@ func Test_Ingester_ModeHandler(t *testing.T) { requestUrl: "/mode?mode=NotSupported", expectedState: ring.ACTIVE, expectedResponse: http.StatusBadRequest, + expectedIsReady: true, }, "should maintain in readonly": { method: "POST", @@ -5249,6 +5256,7 @@ func Test_Ingester_ModeHandler(t *testing.T) { requestUrl: "/mode?mode=READONLY", expectedState: ring.READONLY, expectedResponse: http.StatusOK, + expectedIsReady: true, }, "should maintain in active": { method: "POST", @@ -5256,6 +5264,7 @@ func Test_Ingester_ModeHandler(t *testing.T) { requestUrl: "/mode?mode=ACTIVE", expectedState: ring.ACTIVE, expectedResponse: http.StatusOK, + expectedIsReady: true, }, "should fail mode READONLY if LEAVING state": { method: "POST", @@ -5263,18 +5272,21 @@ func Test_Ingester_ModeHandler(t *testing.T) { requestUrl: "/mode?mode=READONLY", expectedState: ring.LEAVING, expectedResponse: http.StatusBadRequest, + expectedIsReady: false, }, "should fail with malformatted request": { method: "GET", initialState: ring.ACTIVE, requestUrl: "/mode?mod;e=READONLY", expectedResponse: http.StatusBadRequest, + expectedIsReady: true, }, } for testName, testData := range tests { t.Run(testName, func(t *testing.T) { cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.MinReadyDuration = 0 i, err := prepareIngesterWithBlocksStorage(t, cfg, prometheus.NewRegistry()) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) @@ -5304,6 +5316,13 @@ func Test_Ingester_ModeHandler(t *testing.T) { require.Equal(t, testData.expectedResponse, response.Code) require.Equal(t, testData.expectedState, i.lifecycler.GetState()) + + err = i.CheckReady(context.Background()) + if testData.expectedIsReady { + require.NoError(t, err) + } else { + require.NotNil(t, err) + } }) } } diff --git a/pkg/querier/distributor_queryable.go b/pkg/querier/distributor_queryable.go index a2f5a50f46..8ae7c49106 100644 --- a/pkg/querier/distributor_queryable.go +++ b/pkg/querier/distributor_queryable.go @@ -29,27 +29,29 @@ type Distributor interface { QueryExemplars(ctx context.Context, from, to model.Time, matchers ...[]*labels.Matcher) (*client.ExemplarQueryResponse, error) LabelValuesForLabelName(ctx context.Context, from, to model.Time, label model.LabelName, hint *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) LabelValuesForLabelNameStream(ctx context.Context, from, to model.Time, label model.LabelName, hint *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) - LabelNames(context.Context, model.Time, model.Time, *storage.LabelHints) ([]string, error) - LabelNamesStream(context.Context, model.Time, model.Time, *storage.LabelHints) ([]string, error) + LabelNames(context.Context, model.Time, model.Time, *storage.LabelHints, ...*labels.Matcher) ([]string, error) + LabelNamesStream(context.Context, model.Time, model.Time, *storage.LabelHints, ...*labels.Matcher) ([]string, error) MetricsForLabelMatchers(ctx context.Context, from, through model.Time, hint *storage.SelectHints, matchers ...*labels.Matcher) ([]model.Metric, error) MetricsForLabelMatchersStream(ctx context.Context, from, through model.Time, hint *storage.SelectHints, matchers ...*labels.Matcher) ([]model.Metric, error) MetricsMetadata(ctx context.Context) ([]scrape.MetricMetadata, error) } -func newDistributorQueryable(distributor Distributor, streamingMetdata bool, iteratorFn chunkIteratorFunc, queryIngestersWithin time.Duration) QueryableWithFilter { +func newDistributorQueryable(distributor Distributor, streamingMetdata bool, labelNamesWithMatchers bool, iteratorFn chunkIteratorFunc, queryIngestersWithin time.Duration) QueryableWithFilter { return distributorQueryable{ - distributor: distributor, - streamingMetdata: streamingMetdata, - iteratorFn: iteratorFn, - queryIngestersWithin: queryIngestersWithin, + distributor: distributor, + streamingMetdata: streamingMetdata, + labelNamesWithMatchers: labelNamesWithMatchers, + iteratorFn: iteratorFn, + queryIngestersWithin: queryIngestersWithin, } } type distributorQueryable struct { - distributor Distributor - streamingMetdata bool - iteratorFn chunkIteratorFunc - queryIngestersWithin time.Duration + distributor Distributor + streamingMetdata bool + labelNamesWithMatchers bool + iteratorFn chunkIteratorFunc + queryIngestersWithin time.Duration } func (d distributorQueryable) Querier(mint, maxt int64) (storage.Querier, error) { @@ -58,6 +60,7 @@ func (d distributorQueryable) Querier(mint, maxt int64) (storage.Querier, error) mint: mint, maxt: maxt, streamingMetadata: d.streamingMetdata, + labelNamesMatchers: d.labelNamesWithMatchers, chunkIterFn: d.iteratorFn, queryIngestersWithin: d.queryIngestersWithin, }, nil @@ -72,6 +75,7 @@ type distributorQuerier struct { distributor Distributor mint, maxt int64 streamingMetadata bool + labelNamesMatchers bool chunkIterFn chunkIteratorFunc queryIngestersWithin time.Duration } @@ -180,7 +184,7 @@ func (q *distributorQuerier) LabelValues(ctx context.Context, name string, hints } func (q *distributorQuerier) LabelNames(ctx context.Context, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, annotations.Annotations, error) { - if len(matchers) > 0 { + if len(matchers) > 0 && !q.labelNamesMatchers { return q.labelNamesWithMatchers(ctx, hints, matchers...) } @@ -193,9 +197,9 @@ func (q *distributorQuerier) LabelNames(ctx context.Context, hints *storage.Labe ) if q.streamingMetadata { - ln, err = q.distributor.LabelNamesStream(ctx, model.Time(q.mint), model.Time(q.maxt), hints) + ln, err = q.distributor.LabelNamesStream(ctx, model.Time(q.mint), model.Time(q.maxt), hints, matchers...) } else { - ln, err = q.distributor.LabelNames(ctx, model.Time(q.mint), model.Time(q.maxt), hints) + ln, err = q.distributor.LabelNames(ctx, model.Time(q.mint), model.Time(q.maxt), hints, matchers...) } return ln, nil, err diff --git a/pkg/querier/distributor_queryable_test.go b/pkg/querier/distributor_queryable_test.go index c5decf75e9..1914c49e88 100644 --- a/pkg/querier/distributor_queryable_test.go +++ b/pkg/querier/distributor_queryable_test.go @@ -89,7 +89,7 @@ func TestDistributorQuerier_SelectShouldHonorQueryIngestersWithin(t *testing.T) distributor.On("MetricsForLabelMatchersStream", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]model.Metric{}, nil) ctx := user.InjectOrgID(context.Background(), "test") - queryable := newDistributorQueryable(distributor, streamingMetadataEnabled, nil, testData.queryIngestersWithin) + queryable := newDistributorQueryable(distributor, streamingMetadataEnabled, true, nil, testData.queryIngestersWithin) querier, err := queryable.Querier(testData.queryMinT, testData.queryMaxT) require.NoError(t, err) @@ -128,7 +128,7 @@ func TestDistributorQueryableFilter(t *testing.T) { t.Parallel() d := &MockDistributor{} - dq := newDistributorQueryable(d, false, nil, 1*time.Hour) + dq := newDistributorQueryable(d, false, true, nil, 1*time.Hour) now := time.Now() @@ -172,7 +172,7 @@ func TestIngesterStreaming(t *testing.T) { nil) ctx := user.InjectOrgID(context.Background(), "0") - queryable := newDistributorQueryable(d, true, batch.NewChunkMergeIterator, 0) + queryable := newDistributorQueryable(d, true, true, batch.NewChunkMergeIterator, 0) querier, err := queryable.Querier(mint, maxt) require.NoError(t, err) @@ -202,31 +202,42 @@ func TestDistributorQuerier_LabelNames(t *testing.T) { someMatchers := []*labels.Matcher{labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")} labelNames := []string{"foo", "job"} - for _, streamingEnabled := range []bool{false, true} { - streamingEnabled := streamingEnabled - t.Run("with matchers", func(t *testing.T) { - t.Parallel() - - metrics := []model.Metric{ - {"foo": "bar"}, - {"job": "baz"}, - {"job": "baz", "foo": "boom"}, - } - d := &MockDistributor{} - d.On("MetricsForLabelMatchers", mock.Anything, model.Time(mint), model.Time(maxt), mock.Anything, someMatchers). - Return(metrics, nil) - d.On("MetricsForLabelMatchersStream", mock.Anything, model.Time(mint), model.Time(maxt), mock.Anything, someMatchers). - Return(metrics, nil) - - queryable := newDistributorQueryable(d, streamingEnabled, nil, 0) - querier, err := queryable.Querier(mint, maxt) - require.NoError(t, err) - - ctx := context.Background() - names, warnings, err := querier.LabelNames(ctx, nil, someMatchers...) - require.NoError(t, err) - assert.Empty(t, warnings) - assert.Equal(t, labelNames, names) - }) + for _, labelNamesWithMatchers := range []bool{false, true} { + for _, streamingEnabled := range []bool{false, true} { + streamingEnabled := streamingEnabled + labelNamesWithMatchers := labelNamesWithMatchers + t.Run("with matchers", func(t *testing.T) { + t.Parallel() + + metrics := []model.Metric{ + {"foo": "bar"}, + {"job": "baz"}, + {"job": "baz", "foo": "boom"}, + } + d := &MockDistributor{} + + if labelNamesWithMatchers { + d.On("LabelNames", mock.Anything, model.Time(mint), model.Time(maxt), mock.Anything, someMatchers). + Return(labelNames, nil) + d.On("LabelNamesStream", mock.Anything, model.Time(mint), model.Time(maxt), mock.Anything, someMatchers). + Return(labelNames, nil) + } else { + d.On("MetricsForLabelMatchers", mock.Anything, model.Time(mint), model.Time(maxt), mock.Anything, someMatchers). + Return(metrics, nil) + d.On("MetricsForLabelMatchersStream", mock.Anything, model.Time(mint), model.Time(maxt), mock.Anything, someMatchers). + Return(metrics, nil) + } + + queryable := newDistributorQueryable(d, streamingEnabled, labelNamesWithMatchers, nil, 0) + querier, err := queryable.Querier(mint, maxt) + require.NoError(t, err) + + ctx := context.Background() + names, warnings, err := querier.LabelNames(ctx, nil, someMatchers...) + require.NoError(t, err) + assert.Empty(t, warnings) + assert.Equal(t, labelNames, names) + }) + } } } diff --git a/pkg/querier/querier.go b/pkg/querier/querier.go index 64ab768424..4f6b28d6ff 100644 --- a/pkg/querier/querier.go +++ b/pkg/querier/querier.go @@ -41,13 +41,14 @@ import ( // Config contains the configuration require to create a querier type Config struct { - MaxConcurrent int `yaml:"max_concurrent"` - Timeout time.Duration `yaml:"timeout"` - IngesterStreaming bool `yaml:"ingester_streaming" doc:"hidden"` - IngesterMetadataStreaming bool `yaml:"ingester_metadata_streaming"` - MaxSamples int `yaml:"max_samples"` - QueryIngestersWithin time.Duration `yaml:"query_ingesters_within"` - EnablePerStepStats bool `yaml:"per_step_stats_enabled"` + MaxConcurrent int `yaml:"max_concurrent"` + Timeout time.Duration `yaml:"timeout"` + IngesterStreaming bool `yaml:"ingester_streaming" doc:"hidden"` + IngesterMetadataStreaming bool `yaml:"ingester_metadata_streaming"` + IngesterLabelNamesWithMatchers bool `yaml:"ingester_label_names_with_matchers"` + MaxSamples int `yaml:"max_samples"` + QueryIngestersWithin time.Duration `yaml:"query_ingesters_within"` + EnablePerStepStats bool `yaml:"per_step_stats_enabled"` // Use compression for metrics query API or instant and range query APIs. ResponseCompression string `yaml:"response_compression"` @@ -110,6 +111,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.IntVar(&cfg.MaxConcurrent, "querier.max-concurrent", 20, "The maximum number of concurrent queries.") f.DurationVar(&cfg.Timeout, "querier.timeout", 2*time.Minute, "The timeout for a query.") f.BoolVar(&cfg.IngesterMetadataStreaming, "querier.ingester-metadata-streaming", true, "Deprecated (This feature will be always on after v1.18): Use streaming RPCs for metadata APIs from ingester.") + f.BoolVar(&cfg.IngesterLabelNamesWithMatchers, "querier.ingester-label-names-with-matchers", false, "Use LabelNames ingester RPCs with match params.") f.IntVar(&cfg.MaxSamples, "querier.max-samples", 50e6, "Maximum number of samples a single query can load into memory.") f.DurationVar(&cfg.QueryIngestersWithin, "querier.query-ingesters-within", 0, "Maximum lookback beyond which queries are not sent to ingester. 0 means all queries are sent to ingester.") f.BoolVar(&cfg.EnablePerStepStats, "querier.per-step-stats-enabled", false, "Enable returning samples stats per steps in query response.") @@ -165,7 +167,7 @@ func getChunksIteratorFunction(_ Config) chunkIteratorFunc { func New(cfg Config, limits *validation.Overrides, distributor Distributor, stores []QueryableWithFilter, reg prometheus.Registerer, logger log.Logger) (storage.SampleAndChunkQueryable, storage.ExemplarQueryable, promql.QueryEngine) { iteratorFunc := getChunksIteratorFunction(cfg) - distributorQueryable := newDistributorQueryable(distributor, cfg.IngesterMetadataStreaming, iteratorFunc, cfg.QueryIngestersWithin) + distributorQueryable := newDistributorQueryable(distributor, cfg.IngesterMetadataStreaming, cfg.IngesterLabelNamesWithMatchers, iteratorFunc, cfg.QueryIngestersWithin) ns := make([]QueryableWithFilter, len(stores)) for ix, s := range stores { diff --git a/pkg/querier/querier_test.go b/pkg/querier/querier_test.go index 17b4be651d..ddea8e182a 100644 --- a/pkg/querier/querier_test.go +++ b/pkg/querier/querier_test.go @@ -296,7 +296,7 @@ func TestShouldSortSeriesIfQueryingMultipleQueryables(t *testing.T) { } distributor.On("QueryStream", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(&unorderedResponse, nil) - distributorQueryable := newDistributorQueryable(distributor, cfg.IngesterMetadataStreaming, batch.NewChunkMergeIterator, cfg.QueryIngestersWithin) + distributorQueryable := newDistributorQueryable(distributor, cfg.IngesterMetadataStreaming, cfg.IngesterLabelNamesWithMatchers, batch.NewChunkMergeIterator, cfg.QueryIngestersWithin) tCases := []struct { name string @@ -442,7 +442,7 @@ func TestLimits(t *testing.T) { response: &streamResponse, } - distributorQueryableStreaming := newDistributorQueryable(distributor, cfg.IngesterMetadataStreaming, batch.NewChunkMergeIterator, cfg.QueryIngestersWithin) + distributorQueryableStreaming := newDistributorQueryable(distributor, cfg.IngesterMetadataStreaming, cfg.IngesterLabelNamesWithMatchers, batch.NewChunkMergeIterator, cfg.QueryIngestersWithin) tCases := []struct { name string @@ -1119,8 +1119,8 @@ func TestQuerier_ValidateQueryTimeRange_MaxQueryLookback(t *testing.T) { t.Run("label names", func(t *testing.T) { distributor := &MockDistributor{} - distributor.On("LabelNames", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil) - distributor.On("LabelNamesStream", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil) + distributor.On("LabelNames", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil) + distributor.On("LabelNamesStream", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]string{}, nil) queryable, _, _ := New(cfg, overrides, distributor, queryables, nil, log.NewNopLogger()) q, err := queryable.Querier(util.TimeToMillis(testData.queryStartTime), util.TimeToMillis(testData.queryEndTime)) @@ -1309,10 +1309,10 @@ func (m *errDistributor) LabelValuesForLabelName(context.Context, model.Time, mo func (m *errDistributor) LabelValuesForLabelNameStream(context.Context, model.Time, model.Time, model.LabelName, *storage.LabelHints, ...*labels.Matcher) ([]string, error) { return nil, errDistributorError } -func (m *errDistributor) LabelNames(context.Context, model.Time, model.Time, *storage.LabelHints) ([]string, error) { +func (m *errDistributor) LabelNames(context.Context, model.Time, model.Time, *storage.LabelHints, ...*labels.Matcher) ([]string, error) { return nil, errDistributorError } -func (m *errDistributor) LabelNamesStream(context.Context, model.Time, model.Time, *storage.LabelHints) ([]string, error) { +func (m *errDistributor) LabelNamesStream(context.Context, model.Time, model.Time, *storage.LabelHints, ...*labels.Matcher) ([]string, error) { return nil, errDistributorError } func (m *errDistributor) MetricsForLabelMatchers(ctx context.Context, from, through model.Time, hints *storage.SelectHints, matchers ...*labels.Matcher) ([]model.Metric, error) { @@ -1362,11 +1362,11 @@ func (d *emptyDistributor) LabelValuesForLabelNameStream(context.Context, model. return nil, nil } -func (d *emptyDistributor) LabelNames(context.Context, model.Time, model.Time, *storage.LabelHints) ([]string, error) { +func (d *emptyDistributor) LabelNames(context.Context, model.Time, model.Time, *storage.LabelHints, ...*labels.Matcher) ([]string, error) { return nil, nil } -func (d *emptyDistributor) LabelNamesStream(context.Context, model.Time, model.Time, *storage.LabelHints) ([]string, error) { +func (d *emptyDistributor) LabelNamesStream(context.Context, model.Time, model.Time, *storage.LabelHints, ...*labels.Matcher) ([]string, error) { return nil, nil } diff --git a/pkg/querier/testutils.go b/pkg/querier/testutils.go index 67917b0c18..478e61ff0c 100644 --- a/pkg/querier/testutils.go +++ b/pkg/querier/testutils.go @@ -41,12 +41,12 @@ func (m *MockDistributor) LabelValuesForLabelNameStream(ctx context.Context, fro args := m.Called(ctx, from, to, lbl, hints, matchers) return args.Get(0).([]string), args.Error(1) } -func (m *MockDistributor) LabelNames(ctx context.Context, from, to model.Time, hints *storage.LabelHints) ([]string, error) { - args := m.Called(ctx, from, to, hints) +func (m *MockDistributor) LabelNames(ctx context.Context, from, to model.Time, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { + args := m.Called(ctx, from, to, hints, matchers) return args.Get(0).([]string), args.Error(1) } -func (m *MockDistributor) LabelNamesStream(ctx context.Context, from, to model.Time, hints *storage.LabelHints) ([]string, error) { - args := m.Called(ctx, from, to, hints) +func (m *MockDistributor) LabelNamesStream(ctx context.Context, from, to model.Time, hints *storage.LabelHints, matchers ...*labels.Matcher) ([]string, error) { + args := m.Called(ctx, from, to, hints, matchers) return args.Get(0).([]string), args.Error(1) } func (m *MockDistributor) MetricsForLabelMatchers(ctx context.Context, from, to model.Time, hints *storage.SelectHints, matchers ...*labels.Matcher) ([]model.Metric, error) { diff --git a/pkg/querier/tripperware/instantquery/instant_query_test.go b/pkg/querier/tripperware/instantquery/instant_query_test.go index b35e455102..0f562909bd 100644 --- a/pkg/querier/tripperware/instantquery/instant_query_test.go +++ b/pkg/querier/tripperware/instantquery/instant_query_test.go @@ -527,6 +527,15 @@ func TestMergeResponse(t *testing.T) { }, expectedResp: `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"foo"},"value":[1,"1"]},{"metric":{"__name__":"up","job":"bar"},"value":[1,"2"]}]},"warnings":["warning1","warning2","warning3"]}`, }, + { + name: "merge with infos.", + req: &tripperware.PrometheusRequest{Query: "topk(10, up) by(job)"}, + resps: []string{ + `{"status":"success","infos":["info1","info2"],"data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"foo"},"value":[1,"1"]}]}}`, + `{"status":"success","infos":["info1","info3"],"data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"bar"},"value":[1,"2"]}]}}`, + }, + expectedResp: `{"status":"success","data":{"resultType":"vector","result":[{"metric":{"__name__":"up","job":"foo"},"value":[1,"1"]},{"metric":{"__name__":"up","job":"bar"},"value":[1,"2"]}]},"infos":["info1","info2","info3"]}`, + }, { name: "merge two responses with stats", req: defaultReq, diff --git a/pkg/querier/tripperware/merge.go b/pkg/querier/tripperware/merge.go index 79d022a8b7..8c45ebd0b6 100644 --- a/pkg/querier/tripperware/merge.go +++ b/pkg/querier/tripperware/merge.go @@ -42,11 +42,15 @@ func MergeResponse(ctx context.Context, sumStats bool, req Request, responses .. } promResponses := make([]*PrometheusResponse, 0, len(responses)) warnings := make([][]string, 0, len(responses)) + infos := make([][]string, 0, len(responses)) for _, resp := range responses { promResponses = append(promResponses, resp.(*PrometheusResponse)) if w := resp.(*PrometheusResponse).Warnings; w != nil { warnings = append(warnings, w) } + if i := resp.(*PrometheusResponse).Infos; i != nil { + infos = append(infos, i) + } } // Check if it is a range query. Range query passed req as nil since @@ -96,6 +100,7 @@ func MergeResponse(ctx context.Context, sumStats bool, req Request, responses .. Status: StatusSuccess, Data: data, Warnings: strutil.MergeUnsortedSlices(warnings...), + Infos: strutil.MergeUnsortedSlices(infos...), } return res, nil } diff --git a/pkg/querier/tripperware/query.pb.go b/pkg/querier/tripperware/query.pb.go index fa2564c41a..76fb276e0e 100644 --- a/pkg/querier/tripperware/query.pb.go +++ b/pkg/querier/tripperware/query.pb.go @@ -38,6 +38,7 @@ type PrometheusResponse struct { Error string `protobuf:"bytes,4,opt,name=Error,proto3" json:"error,omitempty"` Headers []*PrometheusResponseHeader `protobuf:"bytes,5,rep,name=Headers,proto3" json:"-"` Warnings []string `protobuf:"bytes,6,rep,name=Warnings,proto3" json:"warnings,omitempty"` + Infos []string `protobuf:"bytes,7,rep,name=Infos,proto3" json:"infos,omitempty"` } func (m *PrometheusResponse) Reset() { *m = PrometheusResponse{} } @@ -114,6 +115,13 @@ func (m *PrometheusResponse) GetWarnings() []string { return nil } +func (m *PrometheusResponse) GetInfos() []string { + if m != nil { + return m.Infos + } + return nil +} + type PrometheusData struct { ResultType string `protobuf:"bytes,1,opt,name=ResultType,proto3" json:"resultType"` Result PrometheusQueryResult `protobuf:"bytes,2,opt,name=Result,proto3" json:"result"` @@ -986,81 +994,83 @@ func init() { func init() { proto.RegisterFile("query.proto", fileDescriptor_5c6ac9b241082464) } var fileDescriptor_5c6ac9b241082464 = []byte{ - // 1174 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x56, 0xcd, 0x6f, 0x1b, 0x45, - 0x14, 0xf7, 0xfa, 0x63, 0xe3, 0x3c, 0xa7, 0x49, 0x99, 0xf4, 0xc3, 0x29, 0x65, 0xd7, 0xac, 0x40, - 0x0a, 0x82, 0x3a, 0x52, 0x2a, 0x40, 0x80, 0xa8, 0xc8, 0x42, 0x20, 0x14, 0x4a, 0xd2, 0x49, 0x54, - 0x24, 0x2e, 0xd5, 0xd8, 0x1e, 0x9c, 0x25, 0x5e, 0xef, 0x76, 0x76, 0xb6, 0x89, 0x39, 0xf1, 0x17, - 0x20, 0x2e, 0x5c, 0x90, 0x38, 0x70, 0xe3, 0xc0, 0x1f, 0x92, 0x63, 0x8e, 0x15, 0x12, 0x0b, 0x71, - 0x2e, 0x68, 0x4f, 0xfd, 0x13, 0xd0, 0x7c, 0xac, 0xbd, 0x4e, 0x9c, 0x44, 0x3d, 0x71, 0x71, 0x76, - 0xde, 0xfb, 0xbd, 0xaf, 0xdf, 0xbc, 0xf7, 0x26, 0x50, 0x7b, 0x12, 0x53, 0x36, 0x68, 0x86, 0x2c, - 0xe0, 0x01, 0xaa, 0x71, 0xe6, 0x85, 0x21, 0x65, 0xfb, 0x84, 0xd1, 0x5b, 0xd7, 0xba, 0x41, 0x37, - 0x90, 0xf2, 0x15, 0xf1, 0xa5, 0x20, 0xb7, 0xac, 0x6e, 0x10, 0x74, 0x7b, 0x74, 0x45, 0x9e, 0x5a, - 0xf1, 0xb7, 0x2b, 0x9d, 0x98, 0x11, 0xee, 0x05, 0x7d, 0xad, 0x5f, 0x3a, 0xad, 0x27, 0x7d, 0xed, - 0xfd, 0xd6, 0x7b, 0x5d, 0x8f, 0xef, 0xc6, 0xad, 0x66, 0x3b, 0xf0, 0x57, 0xda, 0x01, 0xe3, 0xf4, - 0x20, 0x64, 0xc1, 0x77, 0xb4, 0xcd, 0xf5, 0x69, 0x25, 0xdc, 0xeb, 0x66, 0x8a, 0x96, 0xfe, 0x50, - 0xa6, 0xce, 0xdf, 0x45, 0x40, 0x5b, 0x2c, 0xf0, 0x29, 0xdf, 0xa5, 0x71, 0x84, 0x69, 0x14, 0x06, - 0xfd, 0x88, 0x22, 0x07, 0xcc, 0x6d, 0x4e, 0x78, 0x1c, 0xd5, 0x8d, 0x86, 0xb1, 0x3c, 0xeb, 0x42, - 0x9a, 0xd8, 0x66, 0x24, 0x25, 0x58, 0x6b, 0xd0, 0x67, 0x50, 0xfe, 0x84, 0x70, 0x52, 0x2f, 0x36, - 0x8c, 0xe5, 0xda, 0xea, 0xcb, 0xcd, 0x5c, 0x89, 0xcd, 0xb1, 0x4b, 0x01, 0x71, 0x6f, 0x1c, 0x26, - 0x76, 0x21, 0x4d, 0xec, 0xf9, 0x0e, 0xe1, 0xe4, 0xad, 0xc0, 0xf7, 0x38, 0xf5, 0x43, 0x3e, 0xc0, - 0xd2, 0x01, 0x7a, 0x1b, 0x66, 0xd7, 0x19, 0x0b, 0xd8, 0xce, 0x20, 0xa4, 0xf5, 0x92, 0x8c, 0x77, - 0x33, 0x4d, 0xec, 0x45, 0x9a, 0x09, 0x73, 0x16, 0x63, 0x24, 0x7a, 0x03, 0x2a, 0xf2, 0x50, 0x2f, - 0x4b, 0x93, 0xc5, 0x34, 0xb1, 0x17, 0xa4, 0x49, 0x0e, 0xae, 0x10, 0xe8, 0x53, 0x98, 0xd9, 0xa0, - 0xa4, 0x43, 0x59, 0x54, 0xaf, 0x34, 0x4a, 0xcb, 0xb5, 0xd5, 0xd7, 0xcf, 0xc9, 0x36, 0x23, 0x40, - 0xa1, 0xdd, 0x4a, 0x9a, 0xd8, 0xc6, 0x1d, 0x9c, 0x19, 0xa3, 0x55, 0xa8, 0x7e, 0x4d, 0x58, 0xdf, - 0xeb, 0x77, 0xa3, 0xba, 0xd9, 0x28, 0x2d, 0xcf, 0xba, 0x37, 0xd2, 0xc4, 0x46, 0xfb, 0x5a, 0x96, - 0x0b, 0x3c, 0xc2, 0x39, 0x7f, 0x19, 0x30, 0x3f, 0x49, 0x07, 0x6a, 0x02, 0x60, 0x1a, 0xc5, 0x3d, - 0x2e, 0x2b, 0x56, 0x0c, 0xcf, 0xa7, 0x89, 0x0d, 0x6c, 0x24, 0xc5, 0x39, 0x04, 0xba, 0x0f, 0xa6, - 0x3a, 0x69, 0xae, 0x9d, 0x73, 0xb2, 0x7f, 0x28, 0x3a, 0x4e, 0x21, 0xdd, 0x79, 0x4d, 0xb9, 0xa9, - 0x7c, 0x62, 0xed, 0x01, 0x6d, 0x42, 0x45, 0xdc, 0x63, 0x24, 0x89, 0xae, 0xad, 0xbe, 0x76, 0x09, - 0x11, 0xe2, 0xae, 0x23, 0xc5, 0xad, 0x34, 0xcb, 0x73, 0x2b, 0x05, 0xce, 0x1e, 0xcc, 0x7f, 0x4c, - 0xda, 0xbb, 0xb4, 0x33, 0x6a, 0x9e, 0x25, 0x28, 0xed, 0xd1, 0x81, 0xae, 0x6b, 0x26, 0x4d, 0x6c, - 0x71, 0xc4, 0xe2, 0x07, 0xdd, 0x83, 0x19, 0x7a, 0xc0, 0x69, 0x9f, 0x47, 0xf5, 0xa2, 0xbc, 0x88, - 0xc5, 0x89, 0xf8, 0xeb, 0x52, 0xe7, 0x2e, 0xe8, 0xdc, 0x33, 0x2c, 0xce, 0x3e, 0x9c, 0x3f, 0x0c, - 0x30, 0x15, 0x08, 0xd9, 0xb2, 0x10, 0xc6, 0x65, 0x9c, 0x92, 0x3b, 0x9b, 0x26, 0xb6, 0x12, 0x60, - 0xf5, 0x47, 0xa4, 0x41, 0xfb, 0x1d, 0x49, 0x59, 0x49, 0xa5, 0x41, 0xfb, 0x1d, 0x2c, 0x7e, 0x50, - 0x03, 0xaa, 0x9c, 0x91, 0x36, 0x7d, 0xec, 0x75, 0x74, 0xf7, 0x64, 0x37, 0x2d, 0xc5, 0x9f, 0x77, - 0xd0, 0x3d, 0xa8, 0x32, 0x5d, 0x4f, 0xbd, 0x22, 0x99, 0xba, 0xd6, 0x54, 0x03, 0xd8, 0xcc, 0x06, - 0xb0, 0xb9, 0xd6, 0x1f, 0xb8, 0x73, 0x69, 0x62, 0x8f, 0x90, 0x78, 0xf4, 0x75, 0xbf, 0x5c, 0x2d, - 0x5d, 0x2d, 0x3b, 0xbf, 0x14, 0x61, 0x6e, 0x9b, 0xf8, 0x61, 0x8f, 0x6e, 0x73, 0x46, 0x89, 0x8f, - 0x0e, 0xc0, 0xec, 0x91, 0x16, 0xed, 0x89, 0xb9, 0x52, 0xe5, 0x67, 0x63, 0xd9, 0xfc, 0x52, 0xc8, - 0xb7, 0x88, 0xc7, 0xdc, 0x2f, 0x44, 0xf9, 0x7f, 0x26, 0xf6, 0x0b, 0x8d, 0xb5, 0xb2, 0x5f, 0xeb, - 0x90, 0x90, 0x53, 0x26, 0xee, 0xdd, 0xa7, 0x9c, 0x79, 0x6d, 0xac, 0xe3, 0xa1, 0xf7, 0x61, 0x26, - 0x92, 0x99, 0x64, 0xcc, 0x5f, 0x1d, 0x87, 0x56, 0x29, 0x8e, 0x5b, 0xe6, 0x29, 0xe9, 0xc5, 0x34, - 0xc2, 0x99, 0x01, 0xda, 0x01, 0xd8, 0xf5, 0x22, 0x1e, 0x74, 0x19, 0xf1, 0x45, 0xe3, 0x08, 0xf3, - 0xc6, 0xc4, 0xc5, 0x29, 0x0f, 0x1b, 0x19, 0x48, 0x96, 0x81, 0xb4, 0xbb, 0x9c, 0x2d, 0xce, 0x7d, - 0x3b, 0xdf, 0xc3, 0xe2, 0x14, 0x33, 0xf4, 0x2a, 0xcc, 0x71, 0xcf, 0xa7, 0x11, 0x27, 0x7e, 0xf8, - 0xd8, 0x57, 0x0b, 0xa8, 0x84, 0x6b, 0x23, 0xd9, 0x83, 0x08, 0x7d, 0x04, 0xb3, 0x23, 0x3f, 0x7a, - 0x24, 0x6e, 0x5f, 0x94, 0x8e, 0x5b, 0x16, 0xa9, 0xe0, 0xb1, 0x91, 0xf3, 0x04, 0x16, 0x4e, 0x61, - 0xd0, 0x35, 0xa8, 0xb4, 0x83, 0xb8, 0xaf, 0xfa, 0xc9, 0xc0, 0xea, 0x80, 0xae, 0x42, 0x29, 0x8a, - 0x55, 0x10, 0x03, 0x8b, 0x4f, 0xf4, 0x0e, 0xcc, 0xb4, 0xe2, 0xf6, 0x1e, 0xe5, 0x19, 0x13, 0x93, - 0xa1, 0xc7, 0x41, 0x25, 0x08, 0x67, 0x60, 0x27, 0x82, 0x85, 0x53, 0x3a, 0x64, 0x01, 0xb4, 0x82, - 0xb8, 0xdf, 0x21, 0xcc, 0xa3, 0xaa, 0xd0, 0x0a, 0xce, 0x49, 0x44, 0x4a, 0xbd, 0x60, 0x9f, 0x32, - 0x1d, 0x5e, 0x1d, 0x84, 0x34, 0x16, 0xe1, 0xe4, 0x04, 0x1b, 0x58, 0x1d, 0xc6, 0xe9, 0x97, 0x73, - 0xe9, 0x3b, 0x3e, 0xdc, 0x3c, 0x67, 0xa6, 0x11, 0x1e, 0x37, 0x84, 0x21, 0x29, 0x7c, 0xf3, 0xb2, - 0x55, 0xa0, 0xd0, 0x6a, 0x23, 0xd4, 0xc4, 0x78, 0x6a, 0xfb, 0x51, 0xa3, 0x38, 0x3f, 0x17, 0xc1, - 0xba, 0xd8, 0x10, 0x6d, 0xc2, 0x75, 0x1e, 0x70, 0xd2, 0x93, 0xbb, 0x8a, 0xb4, 0x7a, 0x99, 0x56, - 0x8f, 0xf1, 0x52, 0x9a, 0xd8, 0xd3, 0x01, 0x78, 0xba, 0x18, 0xfd, 0x66, 0xc0, 0xed, 0xa9, 0x9a, - 0x2d, 0xca, 0xb6, 0x39, 0x0d, 0x75, 0xbb, 0x7f, 0x70, 0x49, 0x75, 0xa7, 0xad, 0x65, 0xb6, 0xda, - 0x85, 0xdb, 0x48, 0x13, 0xfb, 0xc2, 0x20, 0xf8, 0x42, 0xad, 0xe3, 0xc1, 0x0b, 0x46, 0x14, 0xd7, - 0x29, 0xa7, 0x50, 0xb7, 0xbf, 0x3a, 0x9c, 0x99, 0x8d, 0xe2, 0x99, 0xd9, 0x70, 0x76, 0xa0, 0x7e, - 0xde, 0x73, 0x86, 0x96, 0xa0, 0xfc, 0x15, 0xf1, 0xb3, 0x17, 0x47, 0xaf, 0x3c, 0x29, 0x42, 0xaf, - 0x80, 0xf9, 0x48, 0x4e, 0xbd, 0xa4, 0x6b, 0xa4, 0xd4, 0x42, 0xe7, 0x57, 0x03, 0xae, 0x4f, 0x7d, - 0x67, 0xd0, 0x1d, 0x30, 0x9f, 0xd2, 0x36, 0x0f, 0x98, 0xee, 0xa2, 0xc9, 0x85, 0xfe, 0x48, 0xaa, - 0x36, 0x0a, 0x58, 0x83, 0xd0, 0x6d, 0xa8, 0x32, 0xb2, 0xef, 0x0e, 0x38, 0x55, 0xd9, 0xcf, 0x6d, - 0x14, 0xf0, 0x48, 0x22, 0x9c, 0xf9, 0x84, 0x33, 0xef, 0x40, 0xbf, 0x4e, 0x93, 0xce, 0x1e, 0x48, - 0x95, 0x70, 0xa6, 0x40, 0x6e, 0x15, 0xf4, 0xeb, 0xe6, 0x7c, 0x08, 0xa6, 0x0a, 0x85, 0xee, 0xe6, - 0xdb, 0xfa, 0xec, 0x0b, 0xa3, 0x57, 0x9d, 0x5a, 0x08, 0xa3, 0xbe, 0xfd, 0xb1, 0x08, 0xa6, 0xd2, - 0xfc, 0x8f, 0x1b, 0xfa, 0x5d, 0x30, 0x55, 0x3e, 0x7a, 0xa5, 0x9d, 0x5d, 0xd0, 0x57, 0x0e, 0x13, - 0xdb, 0x10, 0xef, 0x9c, 0xec, 0x06, 0xac, 0xe1, 0xe8, 0x61, 0x7e, 0x1d, 0x2a, 0xe2, 0x2e, 0xdf, - 0xce, 0x2f, 0x69, 0x5f, 0x63, 0xd3, 0xfc, 0x7e, 0xdc, 0x04, 0x53, 0xb1, 0x8d, 0xd6, 0xe1, 0x4a, - 0x94, 0x7b, 0xc1, 0x32, 0x5a, 0x96, 0xa6, 0x04, 0x50, 0x08, 0xcd, 0xed, 0xa4, 0x95, 0xbb, 0x76, - 0x74, 0x6c, 0x15, 0x9e, 0x1d, 0x5b, 0x85, 0xe7, 0xc7, 0x96, 0xf1, 0xc3, 0xd0, 0x32, 0x7e, 0x1f, - 0x5a, 0xc6, 0xe1, 0xd0, 0x32, 0x8e, 0x86, 0x96, 0xf1, 0xcf, 0xd0, 0x32, 0xfe, 0x1d, 0x5a, 0x85, - 0xe7, 0x43, 0xcb, 0xf8, 0xe9, 0xc4, 0x2a, 0x1c, 0x9d, 0x58, 0x85, 0x67, 0x27, 0x56, 0xe1, 0x9b, - 0xfc, 0xbf, 0xcd, 0x2d, 0x53, 0x3e, 0xbc, 0x77, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x26, 0xdf, - 0x3a, 0xdc, 0x59, 0x0b, 0x00, 0x00, + // 1207 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x56, 0x4b, 0x6f, 0x1b, 0xd5, + 0x17, 0xf7, 0xf8, 0x31, 0x76, 0x8e, 0xd3, 0xa4, 0xff, 0x9b, 0x3e, 0x9c, 0xfe, 0xcb, 0x8c, 0x19, + 0x81, 0x14, 0x04, 0x75, 0x44, 0x10, 0x54, 0x80, 0x54, 0xd1, 0x81, 0x42, 0x5a, 0x28, 0x6d, 0x6f, + 0xaa, 0x22, 0xb1, 0xa9, 0xae, 0xed, 0x5b, 0x67, 0x88, 0xe7, 0xd1, 0x3b, 0x77, 0x9a, 0x98, 0x15, + 0x6b, 0x16, 0x88, 0x35, 0x12, 0x0b, 0x76, 0x2c, 0xf8, 0x20, 0x59, 0x76, 0x59, 0x21, 0x31, 0x22, + 0xce, 0x06, 0xcd, 0xaa, 0x1f, 0x01, 0xdd, 0xc7, 0xd8, 0xe3, 0xc4, 0x49, 0xd4, 0x15, 0x1b, 0x67, + 0xce, 0x39, 0xbf, 0xf3, 0xbc, 0xe7, 0x11, 0x68, 0x3e, 0x4d, 0x28, 0x1b, 0x75, 0x22, 0x16, 0xf2, + 0x10, 0x35, 0x39, 0xf3, 0xa2, 0x88, 0xb2, 0x5d, 0xc2, 0xe8, 0x95, 0x0b, 0x83, 0x70, 0x10, 0x4a, + 0xfe, 0xba, 0xf8, 0x52, 0x90, 0x2b, 0xd6, 0x20, 0x0c, 0x07, 0x43, 0xba, 0x2e, 0xa9, 0x6e, 0xf2, + 0x64, 0xbd, 0x9f, 0x30, 0xc2, 0xbd, 0x30, 0xd0, 0xf2, 0xd5, 0xa3, 0x72, 0x12, 0x68, 0xeb, 0x57, + 0x3e, 0x1c, 0x78, 0x7c, 0x3b, 0xe9, 0x76, 0x7a, 0xa1, 0xbf, 0xde, 0x0b, 0x19, 0xa7, 0x7b, 0x11, + 0x0b, 0xbf, 0xa3, 0x3d, 0xae, 0xa9, 0xf5, 0x68, 0x67, 0x90, 0x0b, 0xba, 0xfa, 0x43, 0xa9, 0x3a, + 0x3f, 0x56, 0x00, 0xdd, 0x67, 0xa1, 0x4f, 0xf9, 0x36, 0x4d, 0x62, 0x4c, 0xe3, 0x28, 0x0c, 0x62, + 0x8a, 0x1c, 0x30, 0xb7, 0x38, 0xe1, 0x49, 0xdc, 0x32, 0xda, 0xc6, 0xda, 0x82, 0x0b, 0x59, 0x6a, + 0x9b, 0xb1, 0xe4, 0x60, 0x2d, 0x41, 0x5f, 0x40, 0xf5, 0x33, 0xc2, 0x49, 0xab, 0xdc, 0x36, 0xd6, + 0x9a, 0x1b, 0xff, 0xef, 0x14, 0x52, 0xec, 0x4c, 0x4d, 0x0a, 0x88, 0x7b, 0x69, 0x3f, 0xb5, 0x4b, + 0x59, 0x6a, 0x2f, 0xf5, 0x09, 0x27, 0xef, 0x84, 0xbe, 0xc7, 0xa9, 0x1f, 0xf1, 0x11, 0x96, 0x06, + 0xd0, 0xfb, 0xb0, 0x70, 0x8b, 0xb1, 0x90, 0x3d, 0x1c, 0x45, 0xb4, 0x55, 0x91, 0xfe, 0x2e, 0x67, + 0xa9, 0xbd, 0x42, 0x73, 0x66, 0x41, 0x63, 0x8a, 0x44, 0x6f, 0x41, 0x4d, 0x12, 0xad, 0xaa, 0x54, + 0x59, 0xc9, 0x52, 0x7b, 0x59, 0xaa, 0x14, 0xe0, 0x0a, 0x81, 0x3e, 0x87, 0xfa, 0x26, 0x25, 0x7d, + 0xca, 0xe2, 0x56, 0xad, 0x5d, 0x59, 0x6b, 0x6e, 0xbc, 0x79, 0x42, 0xb4, 0x79, 0x01, 0x14, 0xda, + 0xad, 0x65, 0xa9, 0x6d, 0x5c, 0xc3, 0xb9, 0x32, 0xda, 0x80, 0xc6, 0x37, 0x84, 0x05, 0x5e, 0x30, + 0x88, 0x5b, 0x66, 0xbb, 0xb2, 0xb6, 0xe0, 0x5e, 0xca, 0x52, 0x1b, 0xed, 0x6a, 0x5e, 0xc1, 0xf1, + 0x04, 0x27, 0xc2, 0xbc, 0x1d, 0x3c, 0x09, 0xe3, 0x56, 0x5d, 0x2a, 0xc8, 0x30, 0x3d, 0xc1, 0x28, + 0x86, 0x29, 0x11, 0xce, 0x5f, 0x06, 0x2c, 0xcd, 0x56, 0x0e, 0x75, 0x00, 0x30, 0x8d, 0x93, 0x21, + 0x97, 0xc5, 0x51, 0x8f, 0xb1, 0x94, 0xa5, 0x36, 0xb0, 0x09, 0x17, 0x17, 0x10, 0xe8, 0x0e, 0x98, + 0x8a, 0xd2, 0xcf, 0xe2, 0x9c, 0x90, 0xe8, 0x03, 0xd1, 0x9c, 0x0a, 0xe9, 0x2e, 0xe9, 0xd7, 0x31, + 0x95, 0x4d, 0xac, 0x2d, 0xa0, 0x7b, 0x50, 0x13, 0x4f, 0x1e, 0xcb, 0x37, 0x69, 0x6e, 0xbc, 0x71, + 0x46, 0xcd, 0x44, 0x5b, 0xc4, 0x2a, 0x3f, 0xa9, 0x56, 0xcc, 0x4f, 0x32, 0x9c, 0x1d, 0x58, 0xfa, + 0x94, 0xf4, 0xb6, 0x69, 0x7f, 0xd2, 0x67, 0xab, 0x50, 0xd9, 0xa1, 0x23, 0x9d, 0x57, 0x3d, 0x4b, + 0x6d, 0x41, 0x62, 0xf1, 0x83, 0x6e, 0x40, 0x9d, 0xee, 0x71, 0x1a, 0xf0, 0xb8, 0x55, 0x96, 0x6f, + 0xb6, 0x32, 0xe3, 0xff, 0x96, 0x94, 0xb9, 0xcb, 0x3a, 0xf6, 0x1c, 0x8b, 0xf3, 0x0f, 0xe7, 0x0f, + 0x03, 0x4c, 0x05, 0x42, 0xb6, 0x4c, 0x84, 0x71, 0xe9, 0xa7, 0xe2, 0x2e, 0x64, 0xa9, 0xad, 0x18, + 0x58, 0xfd, 0x11, 0x61, 0xd0, 0xa0, 0x2f, 0x4b, 0x56, 0x51, 0x61, 0xd0, 0xa0, 0x8f, 0xc5, 0x0f, + 0x6a, 0x43, 0x83, 0x33, 0xd2, 0xa3, 0x8f, 0xbd, 0xbe, 0x6e, 0xb4, 0xbc, 0x29, 0x24, 0xfb, 0x76, + 0x1f, 0xdd, 0x80, 0x06, 0xd3, 0xf9, 0xb4, 0x6a, 0xb2, 0x52, 0x17, 0x3a, 0x6a, 0x56, 0x3b, 0xf9, + 0xac, 0x76, 0x6e, 0x06, 0x23, 0x77, 0x31, 0x4b, 0xed, 0x09, 0x12, 0x4f, 0xbe, 0xee, 0x54, 0x1b, + 0x95, 0xf3, 0x55, 0xe7, 0x97, 0x32, 0x2c, 0x6e, 0x11, 0x3f, 0x1a, 0xd2, 0x2d, 0xce, 0x28, 0xf1, + 0xd1, 0x1e, 0x98, 0x43, 0xd2, 0xa5, 0x43, 0x31, 0x82, 0x2a, 0xfd, 0x7c, 0x82, 0x3b, 0x5f, 0x09, + 0xfe, 0x7d, 0xe2, 0x31, 0xf7, 0x4b, 0x91, 0xfe, 0x9f, 0xa9, 0xfd, 0x4a, 0x1b, 0x40, 0xe9, 0xdf, + 0xec, 0x93, 0x88, 0x53, 0x26, 0xde, 0xdd, 0xa7, 0x9c, 0x79, 0x3d, 0xac, 0xfd, 0xa1, 0x8f, 0xa0, + 0x1e, 0xcb, 0x48, 0xf2, 0xca, 0x9f, 0x9f, 0xba, 0x56, 0x21, 0x4e, 0x5b, 0xe6, 0x19, 0x19, 0x26, + 0x34, 0xc6, 0xb9, 0x02, 0x7a, 0x08, 0xb0, 0xed, 0xc5, 0x3c, 0x1c, 0x30, 0xe2, 0x8b, 0xc6, 0x11, + 0xea, 0xed, 0x99, 0x87, 0x53, 0x16, 0x36, 0x73, 0x90, 0x4c, 0x03, 0x69, 0x73, 0x05, 0x5d, 0x5c, + 0xf8, 0x76, 0xbe, 0x87, 0x95, 0x39, 0x6a, 0xe8, 0x75, 0x58, 0xe4, 0x9e, 0x4f, 0x63, 0x4e, 0xfc, + 0xe8, 0xb1, 0xaf, 0x76, 0x55, 0x05, 0x37, 0x27, 0xbc, 0xbb, 0x31, 0xfa, 0x04, 0x16, 0x26, 0x76, + 0xf4, 0x48, 0x5c, 0x3d, 0x2d, 0x1c, 0xb7, 0x2a, 0x42, 0xc1, 0x53, 0x25, 0xe7, 0x29, 0x2c, 0x1f, + 0xc1, 0xa0, 0x0b, 0x50, 0xeb, 0x85, 0x49, 0xa0, 0xfa, 0xc9, 0xc0, 0x8a, 0x40, 0xe7, 0xa1, 0x12, + 0x27, 0xca, 0x89, 0x81, 0xc5, 0x27, 0xfa, 0x00, 0xea, 0xdd, 0xa4, 0xb7, 0x43, 0x79, 0x5e, 0x89, + 0x59, 0xd7, 0x53, 0xa7, 0x12, 0x84, 0x73, 0xb0, 0x13, 0xc3, 0xf2, 0x11, 0x19, 0xb2, 0x00, 0xba, + 0x61, 0x12, 0xf4, 0x09, 0xf3, 0xa8, 0x4a, 0xb4, 0x86, 0x0b, 0x1c, 0x11, 0xd2, 0x30, 0xdc, 0xa5, + 0x4c, 0xbb, 0x57, 0x84, 0xe0, 0x26, 0xc2, 0x9d, 0x9c, 0x60, 0x03, 0x2b, 0x62, 0x1a, 0x7e, 0xb5, + 0x10, 0xbe, 0xe3, 0xc3, 0xe5, 0x13, 0x66, 0x1a, 0xe1, 0x69, 0x43, 0x18, 0xb2, 0x84, 0x6f, 0x9f, + 0xb5, 0x0a, 0x14, 0x5a, 0x6d, 0x84, 0xa6, 0x18, 0x4f, 0xad, 0x3f, 0x69, 0x14, 0x67, 0xbf, 0x0c, + 0xd6, 0xe9, 0x8a, 0xe8, 0x1e, 0x5c, 0xe4, 0x21, 0x27, 0x43, 0xb9, 0xab, 0x48, 0x77, 0x98, 0x4b, + 0xf5, 0x18, 0xaf, 0x66, 0xa9, 0x3d, 0x1f, 0x80, 0xe7, 0xb3, 0xd1, 0x6f, 0x06, 0x5c, 0x9d, 0x2b, + 0xb9, 0x4f, 0xd9, 0x16, 0xa7, 0x91, 0x6e, 0xf7, 0x8f, 0xcf, 0xc8, 0xee, 0xa8, 0xb6, 0x8c, 0x56, + 0x9b, 0x70, 0xdb, 0x59, 0x6a, 0x9f, 0xea, 0x04, 0x9f, 0x2a, 0x45, 0xef, 0x42, 0x33, 0xa2, 0x64, + 0x27, 0x4f, 0xb5, 0x22, 0x53, 0x5d, 0xce, 0x52, 0xbb, 0xc8, 0xc6, 0x45, 0xc2, 0xf1, 0xe0, 0x15, + 0x83, 0x14, 0x1d, 0x20, 0x07, 0x57, 0x4f, 0x8c, 0x22, 0x8e, 0x8d, 0x53, 0xf9, 0xd8, 0x38, 0x39, + 0x0f, 0xa1, 0x75, 0xd2, 0xb1, 0x44, 0xab, 0x50, 0xfd, 0x9a, 0xf8, 0xf9, 0x91, 0xd2, 0x5b, 0x52, + 0xb2, 0xd0, 0x6b, 0x60, 0x3e, 0x92, 0x8b, 0x42, 0x56, 0x78, 0x22, 0xd4, 0x4c, 0xe7, 0x57, 0x03, + 0x2e, 0xce, 0x3d, 0x4d, 0xe8, 0x1a, 0x98, 0xcf, 0x68, 0x8f, 0x87, 0x4c, 0x37, 0xde, 0xec, 0x0d, + 0x78, 0x24, 0x45, 0x9b, 0x25, 0xac, 0x41, 0xe8, 0x2a, 0x34, 0x18, 0xd9, 0x75, 0x47, 0x9c, 0xaa, + 0xe8, 0x17, 0x37, 0x4b, 0x78, 0xc2, 0x11, 0xc6, 0x7c, 0xc2, 0x99, 0xb7, 0xa7, 0x0f, 0xda, 0xac, + 0xb1, 0xbb, 0x52, 0x24, 0x8c, 0x29, 0x90, 0xdb, 0x00, 0x7d, 0x10, 0x9d, 0xeb, 0x60, 0x2a, 0x57, + 0xe8, 0x5a, 0x71, 0x12, 0x8e, 0x1f, 0x25, 0x55, 0xeb, 0x69, 0x93, 0xff, 0x54, 0x06, 0x53, 0xf1, + 0xfe, 0xc3, 0x75, 0x7e, 0x1d, 0x4c, 0x15, 0x8f, 0xde, 0x7f, 0xc7, 0xb7, 0xf9, 0xb9, 0xfd, 0xd4, + 0x36, 0xc4, 0x51, 0x94, 0x7d, 0x80, 0x35, 0x1c, 0x3d, 0x28, 0xee, 0x4e, 0x55, 0xb2, 0xb3, 0x57, + 0xf9, 0xff, 0xb4, 0xad, 0xa9, 0x6a, 0x71, 0x99, 0xde, 0x03, 0x53, 0xd5, 0x19, 0xdd, 0x82, 0x73, + 0x71, 0xe1, 0xdc, 0xe5, 0x65, 0x59, 0x9d, 0xe3, 0x40, 0x21, 0xf4, 0x66, 0x9e, 0xd5, 0x72, 0x6f, + 0x3e, 0x3f, 0xb0, 0x4a, 0x2f, 0x0e, 0xac, 0xd2, 0xcb, 0x03, 0xcb, 0xf8, 0x61, 0x6c, 0x19, 0xbf, + 0x8f, 0x2d, 0x63, 0x7f, 0x6c, 0x19, 0xcf, 0xc7, 0x96, 0xf1, 0xf7, 0xd8, 0x32, 0xfe, 0x19, 0x5b, + 0xa5, 0x97, 0x63, 0xcb, 0xf8, 0xf9, 0xd0, 0x2a, 0x3d, 0x3f, 0xb4, 0x4a, 0x2f, 0x0e, 0xad, 0xd2, + 0xb7, 0xc5, 0x7f, 0xc7, 0xbb, 0xa6, 0xbc, 0xd2, 0xef, 0xfd, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x31, + 0xdc, 0x8d, 0x85, 0xb1, 0x0b, 0x00, 0x00, } func (this *PrometheusResponse) Equal(that interface{}) bool { @@ -1110,6 +1120,14 @@ func (this *PrometheusResponse) Equal(that interface{}) bool { return false } } + if len(this.Infos) != len(that1.Infos) { + return false + } + for i := range this.Infos { + if this.Infos[i] != that1.Infos[i] { + return false + } + } return true } func (this *PrometheusData) Equal(that interface{}) bool { @@ -1664,7 +1682,7 @@ func (this *PrometheusResponse) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 10) + s := make([]string, 0, 11) s = append(s, "&tripperware.PrometheusResponse{") s = append(s, "Status: "+fmt.Sprintf("%#v", this.Status)+",\n") s = append(s, "Data: "+strings.Replace(this.Data.GoString(), `&`, ``, 1)+",\n") @@ -1674,6 +1692,7 @@ func (this *PrometheusResponse) GoString() string { s = append(s, "Headers: "+fmt.Sprintf("%#v", this.Headers)+",\n") } s = append(s, "Warnings: "+fmt.Sprintf("%#v", this.Warnings)+",\n") + s = append(s, "Infos: "+fmt.Sprintf("%#v", this.Infos)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -1945,6 +1964,15 @@ func (m *PrometheusResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.Infos) > 0 { + for iNdEx := len(m.Infos) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Infos[iNdEx]) + copy(dAtA[i:], m.Infos[iNdEx]) + i = encodeVarintQuery(dAtA, i, uint64(len(m.Infos[iNdEx]))) + i-- + dAtA[i] = 0x3a + } + } if len(m.Warnings) > 0 { for iNdEx := len(m.Warnings) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Warnings[iNdEx]) @@ -2767,6 +2795,12 @@ func (m *PrometheusResponse) Size() (n int) { n += 1 + l + sovQuery(uint64(l)) } } + if len(m.Infos) > 0 { + for _, s := range m.Infos { + l = len(s) + n += 1 + l + sovQuery(uint64(l)) + } + } return n } @@ -3105,6 +3139,7 @@ func (this *PrometheusResponse) String() string { `Error:` + fmt.Sprintf("%v", this.Error) + `,`, `Headers:` + repeatedStringForHeaders + `,`, `Warnings:` + fmt.Sprintf("%v", this.Warnings) + `,`, + `Infos:` + fmt.Sprintf("%v", this.Infos) + `,`, `}`, }, "") return s @@ -3576,6 +3611,38 @@ func (m *PrometheusResponse) Unmarshal(dAtA []byte) error { } m.Warnings = append(m.Warnings, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Infos", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthQuery + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthQuery + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Infos = append(m.Infos, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQuery(dAtA[iNdEx:]) diff --git a/pkg/querier/tripperware/query.proto b/pkg/querier/tripperware/query.proto index ee03100db1..013b73022a 100644 --- a/pkg/querier/tripperware/query.proto +++ b/pkg/querier/tripperware/query.proto @@ -19,6 +19,7 @@ message PrometheusResponse { string Error = 4 [(gogoproto.jsontag) = "error,omitempty"]; repeated tripperware.PrometheusResponseHeader Headers = 5 [(gogoproto.jsontag) = "-"]; repeated string Warnings = 6 [(gogoproto.jsontag) = "warnings,omitempty"]; + repeated string Infos = 7 [(gogoproto.jsontag) = "infos,omitempty"]; } message PrometheusData { diff --git a/pkg/querier/tripperware/queryrange/query_range_middlewares_test.go b/pkg/querier/tripperware/queryrange/query_range_middlewares_test.go index c4479e01ca..8ba197865f 100644 --- a/pkg/querier/tripperware/queryrange/query_range_middlewares_test.go +++ b/pkg/querier/tripperware/queryrange/query_range_middlewares_test.go @@ -34,6 +34,8 @@ func TestRoundTrip(t *testing.T) { _, err = w.Write([]byte(responseBody)) } else if r.RequestURI == queryWithWarnings { _, err = w.Write([]byte(responseBodyWithWarnings)) + } else if r.RequestURI == queryWithInfos { + _, err = w.Write([]byte(responseBodyWithInfos)) } else { _, err = w.Write([]byte("bar")) } diff --git a/pkg/querier/tripperware/queryrange/query_range_test.go b/pkg/querier/tripperware/queryrange/query_range_test.go index 8e9c73e04d..f69457d209 100644 --- a/pkg/querier/tripperware/queryrange/query_range_test.go +++ b/pkg/querier/tripperware/queryrange/query_range_test.go @@ -91,6 +91,9 @@ func TestRequest(t *testing.T) { func TestResponse(t *testing.T) { t.Parallel() + r := *parsedResponse + rWithWarnings := *parsedResponseWithWarnings + rWithInfos := *parsedResponseWithInfos testCases := []struct { promBody *tripperware.PrometheusResponse jsonBody string @@ -125,9 +128,15 @@ func TestResponse(t *testing.T) { jsonBody: responseBody, isProtobuf: true, }, + { + promBody: &r, + jsonBody: responseBody, + isProtobuf: false, + }, { promBody: &tripperware.PrometheusResponse{ - Status: "success", + Status: "success", + Warnings: []string{"test-warn"}, Data: tripperware.PrometheusData{ ResultType: model.ValMatrix.String(), Result: tripperware.PrometheusQueryResult{ @@ -149,13 +158,18 @@ func TestResponse(t *testing.T) { }, }, }, - cancelCtxBeforeDecode: true, - expectedDecodeErr: context.Canceled, - isProtobuf: true, + jsonBody: responseBodyWithWarnings, + isProtobuf: true, + }, + { + promBody: &rWithWarnings, + jsonBody: responseBodyWithWarnings, + isProtobuf: false, }, { promBody: &tripperware.PrometheusResponse{ Status: "success", + Infos: []string{"PromQL info: metric might not be a counter, name does not end in _total/_sum/_count/_bucket: \"go_gc_gogc_percent\" (1:6)"}, Data: tripperware.PrometheusData{ ResultType: model.ValMatrix.String(), Result: tripperware.PrometheusQueryResult{ @@ -177,7 +191,12 @@ func TestResponse(t *testing.T) { }, }, }, - jsonBody: responseBody, + jsonBody: responseBodyWithInfos, + isProtobuf: true, + }, + { + promBody: &rWithInfos, + jsonBody: responseBodyWithInfos, isProtobuf: false, }, { @@ -206,40 +225,11 @@ func TestResponse(t *testing.T) { }, cancelCtxBeforeDecode: true, expectedDecodeErr: context.Canceled, - isProtobuf: false, - }, - { - promBody: &tripperware.PrometheusResponse{ - Status: "success", - Warnings: []string{"test-warn"}, - Data: tripperware.PrometheusData{ - ResultType: model.ValMatrix.String(), - Result: tripperware.PrometheusQueryResult{ - Result: &tripperware.PrometheusQueryResult_Matrix{ - Matrix: &tripperware.Matrix{ - SampleStreams: []tripperware.SampleStream{ - { - Labels: []cortexpb.LabelAdapter{ - {Name: "foo", Value: "bar"}, - }, - Samples: []cortexpb.Sample{ - {Value: 137, TimestampMs: 1536673680000}, - {Value: 137, TimestampMs: 1536673780000}, - }, - }, - }, - }, - }, - }, - }, - }, - jsonBody: responseBodyWithWarnings, - isProtobuf: true, + isProtobuf: true, }, { promBody: &tripperware.PrometheusResponse{ - Status: "success", - Warnings: []string{"test-warn"}, + Status: "success", Data: tripperware.PrometheusData{ ResultType: model.ValMatrix.String(), Result: tripperware.PrometheusQueryResult{ @@ -261,8 +251,9 @@ func TestResponse(t *testing.T) { }, }, }, - jsonBody: responseBodyWithWarnings, - isProtobuf: false, + cancelCtxBeforeDecode: true, + expectedDecodeErr: context.Canceled, + isProtobuf: false, }, } for i, tc := range testCases { @@ -705,6 +696,34 @@ func TestMergeAPIResponses(t *testing.T) { }, }, }, + { + name: "Merge response with infos.", + input: []tripperware.Response{ + mustParse(t, `{"status":"success","infos":["info1","info2"],"data":{"resultType":"matrix","result":[{"metric":{"a":"b","c":"d"},"values":[[1,"1"]]}]}}`), + mustParse(t, `{"status":"success","infos":["info1","info3"],"data":{"resultType":"matrix","result":[{"metric":{"a":"b","c":"d"},"values":[[1,"1"]]}]}}`), + }, + expected: &tripperware.PrometheusResponse{ + Status: StatusSuccess, + Infos: []string{"info1", "info2", "info3"}, + Data: tripperware.PrometheusData{ + ResultType: matrix, + Result: tripperware.PrometheusQueryResult{ + Result: &tripperware.PrometheusQueryResult_Matrix{ + Matrix: &tripperware.Matrix{ + SampleStreams: []tripperware.SampleStream{ + { + Labels: []cortexpb.LabelAdapter{{Name: "a", Value: "b"}, {Name: "c", Value: "d"}}, + Samples: []cortexpb.Sample{ + {Value: 1, TimestampMs: 1000}, + }, + }, + }, + }, + }, + }, + }, + }, + }, { name: "Merging of samples where there is complete overlap.", input: []tripperware.Response{ diff --git a/pkg/querier/tripperware/queryrange/queryrange.pb.go b/pkg/querier/tripperware/queryrange/queryrange.pb.go index 77290ee97a..1edbd8df03 100644 --- a/pkg/querier/tripperware/queryrange/queryrange.pb.go +++ b/pkg/querier/tripperware/queryrange/queryrange.pb.go @@ -33,6 +33,7 @@ type PrometheusResponse struct { Error string `protobuf:"bytes,4,opt,name=Error,proto3" json:"error,omitempty"` Headers []*tripperware.PrometheusResponseHeader `protobuf:"bytes,5,rep,name=Headers,proto3" json:"-"` Warnings []string `protobuf:"bytes,6,rep,name=Warnings,proto3" json:"warnings,omitempty"` + Infos []string `protobuf:"bytes,7,rep,name=Infos,proto3" json:"infos,omitempty"` } func (m *PrometheusResponse) Reset() { *m = PrometheusResponse{} } @@ -109,6 +110,13 @@ func (m *PrometheusResponse) GetWarnings() []string { return nil } +func (m *PrometheusResponse) GetInfos() []string { + if m != nil { + return m.Infos + } + return nil +} + type PrometheusData struct { ResultType string `protobuf:"bytes,1,opt,name=ResultType,proto3" json:"resultType"` Result []tripperware.SampleStream `protobuf:"bytes,2,rep,name=Result,proto3" json:"result"` @@ -176,37 +184,38 @@ func init() { func init() { proto.RegisterFile("queryrange.proto", fileDescriptor_79b02382e213d0b2) } var fileDescriptor_79b02382e213d0b2 = []byte{ - // 470 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0x41, 0x6f, 0xd3, 0x30, - 0x14, 0xc7, 0xe3, 0x76, 0x0d, 0xd4, 0x95, 0x0a, 0xf2, 0xd0, 0x08, 0x3d, 0x38, 0x55, 0x05, 0x52, - 0x91, 0x20, 0x91, 0x8a, 0xb8, 0x43, 0x80, 0x89, 0x1b, 0x28, 0x45, 0x42, 0xe2, 0xe6, 0x75, 0x4f, - 0x59, 0x60, 0xa9, 0x8d, 0xed, 0x68, 0xf4, 0xc6, 0x47, 0xe0, 0x63, 0xf0, 0x51, 0x76, 0xec, 0x81, - 0xc3, 0x4e, 0x16, 0x4d, 0x2f, 0x28, 0xa7, 0x7d, 0x04, 0x14, 0x27, 0x5b, 0x33, 0x71, 0xe0, 0x96, - 0xf7, 0xcf, 0xef, 0xff, 0xec, 0xf7, 0xf7, 0xc3, 0x77, 0xbf, 0xe6, 0x20, 0x57, 0x92, 0x2d, 0x13, - 0x08, 0x84, 0xe4, 0x9a, 0x13, 0xbc, 0x53, 0x46, 0xf7, 0x12, 0x9e, 0x70, 0x2b, 0x87, 0xd5, 0x57, - 0x4d, 0x8c, 0x5e, 0x25, 0xa9, 0x3e, 0xc9, 0x8f, 0x82, 0x05, 0xcf, 0xc2, 0x05, 0x97, 0x1a, 0xbe, - 0x09, 0xc9, 0x3f, 0xc3, 0x42, 0x37, 0x55, 0x28, 0xbe, 0x24, 0x61, 0xd5, 0x25, 0x05, 0x19, 0x6a, - 0x99, 0x0a, 0x01, 0xf2, 0x8c, 0x49, 0xb0, 0xda, 0xaa, 0x6e, 0x32, 0x31, 0x1d, 0x4c, 0xde, 0x4b, - 0x9e, 0x81, 0x3e, 0x81, 0x5c, 0xc5, 0xa0, 0x04, 0x5f, 0x2a, 0x20, 0x13, 0xec, 0xce, 0x35, 0xd3, - 0xb9, 0xf2, 0xd0, 0x18, 0x4d, 0xfb, 0x11, 0x2e, 0x8d, 0xef, 0x2a, 0xab, 0xc4, 0xcd, 0x1f, 0x72, - 0x88, 0xf7, 0x5e, 0x33, 0xcd, 0xbc, 0xce, 0x18, 0x4d, 0x07, 0xb3, 0x51, 0xd0, 0x1a, 0x61, 0xd7, - 0xb1, 0x22, 0xa2, 0x83, 0x73, 0xe3, 0x3b, 0xa5, 0xf1, 0x87, 0xc7, 0x4c, 0xb3, 0x27, 0x3c, 0x4b, - 0x35, 0x64, 0x42, 0xaf, 0x62, 0xeb, 0x27, 0xcf, 0x71, 0xff, 0x8d, 0x94, 0x5c, 0x7e, 0x58, 0x09, - 0xf0, 0xba, 0xf6, 0xb8, 0xfb, 0xa5, 0xf1, 0xf7, 0xe1, 0x4a, 0x6c, 0x39, 0x76, 0x24, 0x79, 0x8c, - 0x7b, 0xb6, 0xf0, 0xf6, 0xac, 0x65, 0xbf, 0x34, 0xfe, 0x1d, 0x6b, 0x69, 0xe1, 0x35, 0x41, 0x0e, - 0xf1, 0xad, 0xb7, 0xc0, 0x8e, 0x41, 0x2a, 0xaf, 0x37, 0xee, 0x4e, 0x07, 0xb3, 0x47, 0x41, 0x2b, - 0x8f, 0xe0, 0xdf, 0xf9, 0x6b, 0x3a, 0xea, 0x95, 0xc6, 0x47, 0x4f, 0xe3, 0x2b, 0x33, 0x99, 0xe1, - 0xdb, 0x1f, 0x99, 0x5c, 0xa6, 0xcb, 0x44, 0x79, 0xee, 0xb8, 0x3b, 0xed, 0x47, 0x07, 0xa5, 0xf1, - 0xc9, 0x59, 0xa3, 0xb5, 0x0e, 0xbe, 0xe6, 0x26, 0xbf, 0x10, 0x1e, 0xde, 0x8c, 0x83, 0x04, 0x18, - 0xc7, 0xa0, 0xf2, 0x53, 0x6d, 0x27, 0xae, 0x03, 0x1e, 0x96, 0xc6, 0xc7, 0xf2, 0x5a, 0x8d, 0x5b, - 0x04, 0x79, 0x89, 0xdd, 0xba, 0xf2, 0x3a, 0xf6, 0xf6, 0x0f, 0x6e, 0xdc, 0x7e, 0xce, 0x32, 0x71, - 0x0a, 0x73, 0x2d, 0x81, 0x65, 0xd1, 0xb0, 0x49, 0xda, 0xad, 0x5b, 0xc5, 0x8d, 0x91, 0xbc, 0xc3, - 0xbd, 0xea, 0xf5, 0x94, 0xcd, 0x77, 0x30, 0x7b, 0xf8, 0x9f, 0xf9, 0xab, 0x17, 0x56, 0x75, 0xa4, - 0xd6, 0xd6, 0x8e, 0xd4, 0x0a, 0xd1, 0x8b, 0xf5, 0x86, 0x3a, 0x17, 0x1b, 0xea, 0x5c, 0x6e, 0x28, - 0xfa, 0x5e, 0x50, 0xf4, 0xb3, 0xa0, 0xe8, 0xbc, 0xa0, 0x68, 0x5d, 0x50, 0xf4, 0xbb, 0xa0, 0xe8, - 0x4f, 0x41, 0x9d, 0xcb, 0x82, 0xa2, 0x1f, 0x5b, 0xea, 0xac, 0xb7, 0xd4, 0xb9, 0xd8, 0x52, 0xe7, - 0x53, 0x6b, 0xa9, 0x8f, 0x5c, 0xbb, 0x80, 0xcf, 0xfe, 0x06, 0x00, 0x00, 0xff, 0xff, 0xc5, 0x7b, - 0x72, 0x81, 0xfb, 0x02, 0x00, 0x00, + // 488 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x52, 0x31, 0x6f, 0xd3, 0x40, + 0x14, 0xf6, 0x35, 0x8d, 0x4b, 0x2e, 0x52, 0x40, 0x57, 0x54, 0x4c, 0x86, 0x73, 0x14, 0x81, 0x14, + 0x24, 0xb0, 0xa5, 0x20, 0x76, 0x30, 0x50, 0xc1, 0x04, 0x72, 0x90, 0x90, 0xd8, 0xae, 0xe9, 0xc3, + 0x35, 0xd4, 0x3e, 0x73, 0x77, 0x56, 0xc9, 0xc6, 0xca, 0xc6, 0xcf, 0xe0, 0xa7, 0x74, 0xcc, 0xc0, + 0xd0, 0xc9, 0x22, 0xce, 0x82, 0x3c, 0xf5, 0x27, 0x20, 0x9f, 0xdd, 0xe6, 0x22, 0x06, 0x36, 0xbf, + 0xcf, 0xdf, 0xf7, 0xee, 0x7d, 0xef, 0x7b, 0xf8, 0xd6, 0x97, 0x1c, 0xc4, 0x42, 0xb0, 0x34, 0x02, + 0x2f, 0x13, 0x5c, 0x71, 0x82, 0x37, 0xc8, 0xf0, 0x76, 0xc4, 0x23, 0xae, 0x61, 0xbf, 0xfe, 0x6a, + 0x18, 0xc3, 0xe7, 0x51, 0xac, 0x4e, 0xf2, 0x23, 0x6f, 0xce, 0x13, 0x7f, 0xce, 0x85, 0x82, 0xaf, + 0x99, 0xe0, 0x9f, 0x60, 0xae, 0xda, 0xca, 0xcf, 0x3e, 0x47, 0x7e, 0xdd, 0x25, 0x06, 0xe1, 0x2b, + 0x11, 0x67, 0x19, 0x88, 0x33, 0x26, 0x40, 0x63, 0x8b, 0xa6, 0xc9, 0xf8, 0x7b, 0x07, 0x93, 0xb7, + 0x82, 0x27, 0xa0, 0x4e, 0x20, 0x97, 0x21, 0xc8, 0x8c, 0xa7, 0x12, 0xc8, 0x18, 0xdb, 0x33, 0xc5, + 0x54, 0x2e, 0x1d, 0x34, 0x42, 0x93, 0x5e, 0x80, 0xab, 0xc2, 0xb5, 0xa5, 0x46, 0xc2, 0xf6, 0x0f, + 0x39, 0xc4, 0xbb, 0x2f, 0x98, 0x62, 0xce, 0xce, 0x08, 0x4d, 0xfa, 0xd3, 0xa1, 0x67, 0x58, 0xd8, + 0x74, 0xac, 0x19, 0xc1, 0xc1, 0x79, 0xe1, 0x5a, 0x55, 0xe1, 0x0e, 0x8e, 0x99, 0x62, 0x0f, 0x79, + 0x12, 0x2b, 0x48, 0x32, 0xb5, 0x08, 0xb5, 0x9e, 0x3c, 0xc1, 0xbd, 0x97, 0x42, 0x70, 0xf1, 0x6e, + 0x91, 0x81, 0xd3, 0xd1, 0xcf, 0xdd, 0xa9, 0x0a, 0x77, 0x1f, 0xae, 0x40, 0x43, 0xb1, 0x61, 0x92, + 0x07, 0xb8, 0xab, 0x0b, 0x67, 0x57, 0x4b, 0xf6, 0xab, 0xc2, 0xbd, 0xa9, 0x25, 0x06, 0xbd, 0x61, + 0x90, 0x43, 0xbc, 0xf7, 0x0a, 0xd8, 0x31, 0x08, 0xe9, 0x74, 0x47, 0x9d, 0x49, 0x7f, 0x7a, 0xdf, + 0x33, 0xf6, 0xe1, 0xfd, 0xeb, 0xbf, 0x61, 0x07, 0xdd, 0xaa, 0x70, 0xd1, 0xa3, 0xf0, 0x4a, 0x4c, + 0xa6, 0xf8, 0xc6, 0x7b, 0x26, 0xd2, 0x38, 0x8d, 0xa4, 0x63, 0x8f, 0x3a, 0x93, 0x5e, 0x70, 0x50, + 0x15, 0x2e, 0x39, 0x6b, 0x31, 0xe3, 0xe1, 0x6b, 0x5e, 0x3d, 0xe6, 0xeb, 0xf4, 0x23, 0x97, 0xce, + 0x9e, 0x16, 0xe8, 0x31, 0xe3, 0x1a, 0x30, 0xc7, 0xd4, 0x8c, 0xf1, 0x2f, 0x84, 0x07, 0xdb, 0x9b, + 0x23, 0x1e, 0xc6, 0x21, 0xc8, 0xfc, 0x54, 0xe9, 0xe5, 0x34, 0x59, 0x0c, 0xaa, 0xc2, 0xc5, 0xe2, + 0x1a, 0x0d, 0x0d, 0x06, 0x79, 0x86, 0xed, 0xa6, 0x72, 0x76, 0xb4, 0xd1, 0xbb, 0x5b, 0x46, 0x67, + 0x2c, 0xc9, 0x4e, 0x61, 0xa6, 0x04, 0xb0, 0x24, 0x18, 0xb4, 0xa1, 0xd8, 0x4d, 0xab, 0xb0, 0x15, + 0x92, 0x37, 0xb8, 0x5b, 0x07, 0x2d, 0x75, 0x14, 0xfd, 0xe9, 0xbd, 0xff, 0xac, 0xaa, 0x3e, 0x06, + 0xd9, 0xd8, 0xd2, 0x32, 0xd3, 0x96, 0x06, 0x82, 0xa7, 0xcb, 0x15, 0xb5, 0x2e, 0x56, 0xd4, 0xba, + 0x5c, 0x51, 0xf4, 0xad, 0xa4, 0xe8, 0x67, 0x49, 0xd1, 0x79, 0x49, 0xd1, 0xb2, 0xa4, 0xe8, 0x77, + 0x49, 0xd1, 0x9f, 0x92, 0x5a, 0x97, 0x25, 0x45, 0x3f, 0xd6, 0xd4, 0x5a, 0xae, 0xa9, 0x75, 0xb1, + 0xa6, 0xd6, 0x07, 0xe3, 0xfe, 0x8f, 0x6c, 0x7d, 0xab, 0x8f, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, + 0x7b, 0xf7, 0x21, 0xc5, 0x26, 0x03, 0x00, 0x00, } func (this *PrometheusResponse) Equal(that interface{}) bool { @@ -256,6 +265,14 @@ func (this *PrometheusResponse) Equal(that interface{}) bool { return false } } + if len(this.Infos) != len(that1.Infos) { + return false + } + for i := range this.Infos { + if this.Infos[i] != that1.Infos[i] { + return false + } + } return true } func (this *PrometheusData) Equal(that interface{}) bool { @@ -297,7 +314,7 @@ func (this *PrometheusResponse) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 10) + s := make([]string, 0, 11) s = append(s, "&queryrange.PrometheusResponse{") s = append(s, "Status: "+fmt.Sprintf("%#v", this.Status)+",\n") s = append(s, "Data: "+strings.Replace(this.Data.GoString(), `&`, ``, 1)+",\n") @@ -307,6 +324,7 @@ func (this *PrometheusResponse) GoString() string { s = append(s, "Headers: "+fmt.Sprintf("%#v", this.Headers)+",\n") } s = append(s, "Warnings: "+fmt.Sprintf("%#v", this.Warnings)+",\n") + s = append(s, "Infos: "+fmt.Sprintf("%#v", this.Infos)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -358,6 +376,15 @@ func (m *PrometheusResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.Infos) > 0 { + for iNdEx := len(m.Infos) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Infos[iNdEx]) + copy(dAtA[i:], m.Infos[iNdEx]) + i = encodeVarintQueryrange(dAtA, i, uint64(len(m.Infos[iNdEx]))) + i-- + dAtA[i] = 0x3a + } + } if len(m.Warnings) > 0 { for iNdEx := len(m.Warnings) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Warnings[iNdEx]) @@ -514,6 +541,12 @@ func (m *PrometheusResponse) Size() (n int) { n += 1 + l + sovQueryrange(uint64(l)) } } + if len(m.Infos) > 0 { + for _, s := range m.Infos { + l = len(s) + n += 1 + l + sovQueryrange(uint64(l)) + } + } return n } @@ -562,6 +595,7 @@ func (this *PrometheusResponse) String() string { `Error:` + fmt.Sprintf("%v", this.Error) + `,`, `Headers:` + repeatedStringForHeaders + `,`, `Warnings:` + fmt.Sprintf("%v", this.Warnings) + `,`, + `Infos:` + fmt.Sprintf("%v", this.Infos) + `,`, `}`, }, "") return s @@ -815,6 +849,38 @@ func (m *PrometheusResponse) Unmarshal(dAtA []byte) error { } m.Warnings = append(m.Warnings, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Infos", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQueryrange + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthQueryrange + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthQueryrange + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Infos = append(m.Infos, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipQueryrange(dAtA[iNdEx:]) diff --git a/pkg/querier/tripperware/queryrange/queryrange.proto b/pkg/querier/tripperware/queryrange/queryrange.proto index 8d2e024d13..1bdbc238b3 100644 --- a/pkg/querier/tripperware/queryrange/queryrange.proto +++ b/pkg/querier/tripperware/queryrange/queryrange.proto @@ -17,6 +17,7 @@ message PrometheusResponse { string Error = 4 [(gogoproto.jsontag) = "error,omitempty"]; repeated tripperware.PrometheusResponseHeader Headers = 5 [(gogoproto.jsontag) = "-"]; repeated string Warnings = 6 [(gogoproto.jsontag) = "warnings,omitempty"]; + repeated string Infos = 7 [(gogoproto.jsontag) = "infos,omitempty"]; } message PrometheusData { diff --git a/pkg/querier/tripperware/queryrange/results_cache.go b/pkg/querier/tripperware/queryrange/results_cache.go index 76757e72bb..b3a474ba8e 100644 --- a/pkg/querier/tripperware/queryrange/results_cache.go +++ b/pkg/querier/tripperware/queryrange/results_cache.go @@ -101,6 +101,7 @@ func (PrometheusResponseExtractor) Extract(start, end int64, from tripperware.Re }, Headers: promRes.Headers, Warnings: promRes.Warnings, + Infos: promRes.Infos, } } @@ -116,6 +117,7 @@ func (PrometheusResponseExtractor) ResponseWithoutHeaders(resp tripperware.Respo Stats: promRes.Data.Stats, }, Warnings: promRes.Warnings, + Infos: promRes.Infos, } } @@ -130,6 +132,7 @@ func (PrometheusResponseExtractor) ResponseWithoutStats(resp tripperware.Respons }, Headers: promRes.Headers, Warnings: promRes.Warnings, + Infos: promRes.Infos, } } @@ -600,6 +603,7 @@ func convertToTripperwarePrometheusResponse(resp tripperware.Response) tripperwa Error: r.Error, Headers: r.Headers, Warnings: r.Warnings, + Infos: r.Infos, } } @@ -621,6 +625,7 @@ func convertFromTripperwarePrometheusResponse(resp tripperware.Response) tripper Error: r.Error, Headers: r.Headers, Warnings: r.Warnings, + Infos: r.Infos, } } diff --git a/pkg/querier/tripperware/queryrange/results_cache_test.go b/pkg/querier/tripperware/queryrange/results_cache_test.go index 8325a71f1b..51b68531b5 100644 --- a/pkg/querier/tripperware/queryrange/results_cache_test.go +++ b/pkg/querier/tripperware/queryrange/results_cache_test.go @@ -26,8 +26,10 @@ import ( const ( query = "/api/v1/query_range?end=1536716898&query=sum%28container_memory_rss%29+by+%28namespace%29&start=1536673680&stats=all&step=120" queryWithWarnings = "/api/v1/query_range?end=1536716898&query=sumsum%28warnings%29&start=1536673680&stats=all&step=120&warnings=true" + queryWithInfos = "/api/v1/query_range?end=1536716898&query=rate%28go_gc_gogc_percent%5B5m%5D%29&start=1536673680&stats=all&step=120" responseBody = `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"foo":"bar"},"values":[[1536673680,"137"],[1536673780,"137"]]}]}}` responseBodyWithWarnings = `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"foo":"bar"},"values":[[1536673680,"137"],[1536673780,"137"]]}]},"warnings":["test-warn"]}` + responseBodyWithInfos = `{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"foo":"bar"},"values":[[1536673680,"137"],[1536673780,"137"]]}]},"infos":["PromQL info: metric might not be a counter, name does not end in _total/_sum/_count/_bucket: \"go_gc_gogc_percent\" (1:6)"]}` ) var ( @@ -71,6 +73,7 @@ var ( Values: []string{tripperware.ApplicationProtobuf}, }, } + parsedResponse = &tripperware.PrometheusResponse{ Status: "success", Data: tripperware.PrometheusData{ @@ -94,6 +97,56 @@ var ( }, }, } + + parsedResponseWithWarnings = &tripperware.PrometheusResponse{ + Status: "success", + Warnings: []string{"test-warn"}, + Data: tripperware.PrometheusData{ + ResultType: model.ValMatrix.String(), + Result: tripperware.PrometheusQueryResult{ + Result: &tripperware.PrometheusQueryResult_Matrix{ + Matrix: &tripperware.Matrix{ + SampleStreams: []tripperware.SampleStream{ + { + Labels: []cortexpb.LabelAdapter{ + {Name: "foo", Value: "bar"}, + }, + Samples: []cortexpb.Sample{ + {Value: 137, TimestampMs: 1536673680000}, + {Value: 137, TimestampMs: 1536673780000}, + }, + }, + }, + }, + }, + }, + }, + } + + parsedResponseWithInfos = &tripperware.PrometheusResponse{ + Status: "success", + Infos: []string{"PromQL info: metric might not be a counter, name does not end in _total/_sum/_count/_bucket: \"go_gc_gogc_percent\" (1:6)"}, + Data: tripperware.PrometheusData{ + ResultType: model.ValMatrix.String(), + Result: tripperware.PrometheusQueryResult{ + Result: &tripperware.PrometheusQueryResult_Matrix{ + Matrix: &tripperware.Matrix{ + SampleStreams: []tripperware.SampleStream{ + { + Labels: []cortexpb.LabelAdapter{ + {Name: "foo", Value: "bar"}, + }, + Samples: []cortexpb.Sample{ + {Value: 137, TimestampMs: 1536673680000}, + {Value: 137, TimestampMs: 1536673780000}, + }, + }, + }, + }, + }, + }, + }, + } ) func mkAPIResponse(start, end, step int64) tripperware.Response { diff --git a/pkg/ring/lifecycler.go b/pkg/ring/lifecycler.go index 6e55aba13e..e9b9970e1a 100644 --- a/pkg/ring/lifecycler.go +++ b/pkg/ring/lifecycler.go @@ -468,7 +468,7 @@ func (i *Lifecycler) loop(ctx context.Context) error { // We are jittering for at least half of the time and max the time of the heartbeat. // If we jitter too soon, we can have problems of concurrency with autoJoin leaving the instance on ACTIVE without tokens time.AfterFunc(time.Duration(uint64(i.cfg.HeartbeatPeriod/2)+uint64(mathrand.Int63())%uint64(i.cfg.HeartbeatPeriod/2)), func() { - i.heartbeat() + i.heartbeat(ctx) heartbeatTicker.Reset(i.cfg.HeartbeatPeriod) }) defer heartbeatTicker.Stop() @@ -530,7 +530,7 @@ func (i *Lifecycler) loop(ctx context.Context) error { } case <-heartbeatTickerChan: - i.heartbeat() + i.heartbeat(ctx) case f := <-i.actorChan: f() @@ -541,9 +541,11 @@ func (i *Lifecycler) loop(ctx context.Context) error { } } -func (i *Lifecycler) heartbeat() { +func (i *Lifecycler) heartbeat(ctx context.Context) { i.lifecyclerMetrics.consulHeartbeats.Inc() - if err := i.updateConsul(context.Background()); err != nil { + ctx, cancel := context.WithTimeout(ctx, i.cfg.HeartbeatPeriod) + defer cancel() + if err := i.updateConsul(ctx); err != nil { level.Error(i.logger).Log("msg", "failed to write to the KV store, sleeping", "ring", i.RingName, "err", err) } } diff --git a/pkg/ring/model.go b/pkg/ring/model.go index 503ab63e69..3f0e6944e2 100644 --- a/pkg/ring/model.go +++ b/pkg/ring/model.go @@ -153,7 +153,7 @@ func (i *InstanceDesc) IsReady(storageLastUpdated time.Time, heartbeatTimeout ti if !i.IsHeartbeatHealthy(heartbeatTimeout, storageLastUpdated) { return fmt.Errorf("instance %s past heartbeat timeout", i.Addr) } - if i.State != ACTIVE { + if i.State != ACTIVE && i.State != READONLY { return fmt.Errorf("instance %s in state %v", i.Addr, i.State) } return nil diff --git a/pkg/ring/model_test.go b/pkg/ring/model_test.go index 05a255aacd..896aef5689 100644 --- a/pkg/ring/model_test.go +++ b/pkg/ring/model_test.go @@ -275,6 +275,20 @@ func TestDesc_Ready(t *testing.T) { if err := r.IsReady(now, 10*time.Second); err != nil { t.Fatal("expected ready, got", err) } + + r = &Desc{ + Ingesters: map[string]InstanceDesc{ + "ing1": { + Tokens: []uint32{100, 200, 300}, + State: READONLY, + Timestamp: now.Unix(), + }, + }, + } + + if err := r.IsReady(now, 10*time.Second); err != nil { + t.Fatal("expected readonly ingester as ready, but got", err) + } } func TestDesc_getTokensByZone(t *testing.T) { diff --git a/pkg/ruler/frontend_decoder.go b/pkg/ruler/frontend_decoder.go index 832a511a4d..c8e653a05f 100644 --- a/pkg/ruler/frontend_decoder.go +++ b/pkg/ruler/frontend_decoder.go @@ -9,6 +9,8 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql" + + "github.com/cortexproject/cortex/pkg/util/api" ) const ( @@ -19,13 +21,8 @@ type JsonDecoder struct{} type Warning []string func (j JsonDecoder) Decode(body []byte) (promql.Vector, Warning, error) { - var response struct { - Status string `json:"status"` - Data json.RawMessage `json:"data"` - ErrorType string `json:"errorType"` - Warnings Warning `json:"warnings"` - Error string `json:"error"` - } + var response api.Response + if err := json.NewDecoder(bytes.NewReader(body)).Decode(&response); err != nil { return nil, nil, err } @@ -37,8 +34,12 @@ func (j JsonDecoder) Decode(body []byte) (promql.Vector, Warning, error) { Result json.RawMessage `json:"result"` }{} - if err := json.Unmarshal(response.Data, &data); err != nil { + if responseDataBytes, err := json.Marshal(response.Data); err != nil { return nil, response.Warnings, err + } else { + if err = json.Unmarshal(responseDataBytes, &data); err != nil { + return nil, response.Warnings, err + } } switch data.Type { diff --git a/pkg/util/api/response.go b/pkg/util/api/response.go index 7e7c7713a2..c58baf60b9 100644 --- a/pkg/util/api/response.go +++ b/pkg/util/api/response.go @@ -23,6 +23,7 @@ type Response struct { ErrorType v1.ErrorType `json:"errorType,omitempty"` Error string `json:"error,omitempty"` Warnings []string `json:"warnings,omitempty"` + Infos []string `json:"infos,omitempty"` } // RespondFromGRPCError writes gRPC error in Prometheus response format.