Merge remote-tracking branch 'upstream/master'

Signed-off-by: Ahmed Hassan <[email protected]>

.github/workflows/test-build-deploy.yml

@@ -17,7 +17,7 @@ jobs:
   lint:
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:master-779dcf4ba
+      image: quay.io/cortexproject/build-image:master-582c03a76
     steps:
       - name: Checkout Repo
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
@@ -46,7 +46,7 @@ jobs:
   test:
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:master-779dcf4ba
+      image: quay.io/cortexproject/build-image:master-582c03a76
     steps:
       - name: Checkout Repo
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
@@ -89,7 +89,7 @@ jobs:
   build:
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:master-779dcf4ba
+      image: quay.io/cortexproject/build-image:master-582c03a76
     steps:
       - name: Checkout Repo
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
@@ -224,14 +224,14 @@ jobs:
         run: |
           touch build-image/.uptodate
           MIGRATIONS_DIR=$(pwd)/cmd/cortex/migrations
-          make BUILD_IMAGE=quay.io/cortexproject/build-image:master-779dcf4ba TTY='' configs-integration-test
+          make BUILD_IMAGE=quay.io/cortexproject/build-image:master-582c03a76 TTY='' configs-integration-test
 
   deploy_website:
     needs: [build, test]
     if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:master-779dcf4ba
+      image: quay.io/cortexproject/build-image:master-582c03a76
     steps:
       - name: Checkout Repo
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0
@@ -273,7 +273,7 @@ jobs:
     if: (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) && github.repository == 'cortexproject/cortex'
     runs-on: ubuntu-20.04
     container:
-      image: quay.io/cortexproject/build-image:master-779dcf4ba
+      image: quay.io/cortexproject/build-image:master-582c03a76
     steps:
       - name: Checkout Repo
         uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2.7.0

CHANGELOG.md

@@ -2,15 +2,18 @@
 
 ## master / unreleased
 
+* [CHANGE] Enable Compactor and Alertmanager in target all. #6204
 * [FEATURE] Ruler: Experimental: Add `ruler.frontend-address` to allow query to query frontends instead of ingesters. #6151
 * [FEATURE] Ruler: Minimize chances of missed rule group evaluations that can occur due to OOM kills, bad underlying nodes, or due to an unhealthy ruler that appears in the ring as healthy. This feature is enabled via `-ruler.enable-ha-evaluation` flag. #6129
+* [ENHANCEMENT] Query Frontend: Add info field to query response. #6207
 * [ENHANCEMENT] Query Frontend: Add peakSample in query stats response. #6188
 * [ENHANCEMENT] Ruler: Add new ruler metric `cortex_ruler_rule_groups_in_store` that is the total rule groups per tenant in store, which can be used to compare with `cortex_prometheus_rule_group_rules` to count the number of rule groups that are not loaded by a ruler. #5869
 * [ENHANCEMENT] Ingester/Ring: New `READONLY` status on ring to be used by Ingester. New ingester API to change mode of ingester #6163
 * [ENHANCEMENT] Ruler: Add query statistics metrics when --ruler.query-stats-enabled=true. #6173
 * [ENHANCEMENT] Ingester: Add new API `/ingester/all_user_stats` which shows loaded blocks, active timeseries and ingestion rate for a specific ingester. #6178
 * [ENHANCEMENT] Distributor: Add new `cortex_reduced_resolution_histogram_samples_total` metric to track the number of histogram samples which resolution was reduced. #6182
 * [ENHANCEMENT] StoreGateway: Implement metadata API limit in queryable. #6195
+* [ENHANCEMENT] Ingester: Add matchers to ingester LabelNames() and LabelNamesStream() RPC. #6209
 
 ## 1.18.0 2024-09-03
 

Makefile

@@ -115,7 +115,7 @@ build-image/$(UPTODATE): build-image/*
 SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E")
 BUILD_IN_CONTAINER := true
 BUILD_IMAGE ?= $(IMAGE_PREFIX)build-image
-LATEST_BUILD_IMAGE_TAG ?= master-779dcf4ba
+LATEST_BUILD_IMAGE_TAG ?= master-582c03a76
 
 # TTY is parameterized to allow Google Cloud Builder to run builds,
 # as it currently disallows TTY devices. This value needs to be overridden

build-image/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22.5-bullseye
+FROM golang:1.22.7-bullseye
 ARG goproxyValue
 ENV GOPROXY=${goproxyValue}
 RUN apt-get update && apt-get install -y curl file jq unzip protobuf-compiler libprotobuf-dev && \

docs/blocks-storage/querier.md

@@ -109,6 +109,10 @@ querier:
   # CLI flag: -querier.ingester-metadata-streaming
   [ingester_metadata_streaming: <boolean> | default = true]
 
+  # Use LabelNames ingester RPCs with match params.
+  # CLI flag: -querier.ingester-label-names-with-matchers
+  [ingester_label_names_with_matchers: <boolean> | default = false]
+
   # Maximum number of samples a single query can load into memory.
   # CLI flag: -querier.max-samples
   [max_samples: <int> | default = 50000000]

docs/configuration/config-file-reference.md

@@ -3701,6 +3701,10 @@ The `querier_config` configures the Cortex querier.
 # CLI flag: -querier.ingester-metadata-streaming
 [ingester_metadata_streaming: <boolean> | default = true]
 
+# Use LabelNames ingester RPCs with match params.
+# CLI flag: -querier.ingester-label-names-with-matchers
+[ingester_label_names_with_matchers: <boolean> | default = false]
+
 # Maximum number of samples a single query can load into memory.
 # CLI flag: -querier.max-samples
 [max_samples: <int> | default = 50000000]

integration/api_endpoints_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -22,10 +23,19 @@ func TestIndexAPIEndpoint(t *testing.T) {
 	require.NoError(t, err)
 	defer s.Close()
 
+	configOverrides := map[string]string{
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
+	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
+
 	// Start Cortex in single binary mode, reading the config from file.
 	require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile))
 
-	cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095)
+	cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, configOverrides, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex1))
 
 	// GET / should succeed
@@ -44,10 +54,19 @@ func TestConfigAPIEndpoint(t *testing.T) {
 	require.NoError(t, err)
 	defer s.Close()
 
+	configOverrides := map[string]string{
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
+	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
+
 	// Start Cortex in single binary mode, reading the config from file.
 	require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile))
 
-	cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, nil, "", 9009, 9095)
+	cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, configOverrides, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex1))
 
 	// Get config from /config API endpoint.
@@ -62,6 +81,7 @@ func TestConfigAPIEndpoint(t *testing.T) {
 	// Start again Cortex in single binary with the exported config
 	// and ensure it starts (pass the readiness probe).
 	require.NoError(t, writeFileToSharedDir(s, cortexConfigFile, body))
-	cortex2 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-2", cortexConfigFile, nil, "", 9009, 9095)
+	configOverrides["-alertmanager.cluster.peers"] = cortex1.HTTPEndpoint()
+	cortex2 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-2", cortexConfigFile, configOverrides, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex2))
 }

integration/getting_started_single_process_config_local_test.go

@@ -4,6 +4,7 @@
 package integration
 
 import (
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -24,7 +25,14 @@ func TestGettingStartedSingleProcessConfigWithFilesystem(t *testing.T) {
 	// Start Cortex components.
 	require.NoError(t, copyFileToSharedDir(s, "docs/configuration/single-process-config-blocks-local.yaml", cortexConfigFile))
 
-	flags := map[string]string{}
+	flags := map[string]string{
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
+	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
 
 	cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex))

integration/getting_started_single_process_config_test.go

@@ -5,6 +5,7 @@ package integration
 
 import (
 	"fmt"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -38,7 +39,13 @@ func TestGettingStartedSingleProcessConfigWithBlocksStorage(t *testing.T) {
 		"-blocks-storage.s3.bucket-name":       bucketName,
 		"-blocks-storage.s3.endpoint":          fmt.Sprintf("%s-minio-9000:9000", networkName),
 		"-blocks-storage.s3.insecure":          "true",
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
 	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
 
 	cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex))

integration/getting_started_with_gossiped_ring_test.go

@@ -5,6 +5,7 @@ package integration
 
 import (
 	"fmt"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -45,8 +46,15 @@ func TestGettingStartedWithGossipedRing(t *testing.T) {
 		"-blocks-storage.s3.insecure":                              "true",
 		"-store-gateway.sharding-ring.wait-stability-min-duration": "0", // start quickly
 		"-store-gateway.sharding-ring.wait-stability-max-duration": "0", // start quickly
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
 	}
 
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
+
 	// This cortex will fail to join the cluster configured in yaml file. That's fine.
 	cortex1 := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", "config1.yaml", e2e.MergeFlags(flags, map[string]string{
 		"-ingester.lifecycler.addr": networkName + "-cortex-1", // Ingester's hostname in docker setup

integration/integration_memberlist_single_binary_test.go

@@ -43,6 +43,9 @@ func testSingleBinaryEnv(t *testing.T, tlsEnabled bool, flags map[string]string)
 	require.NoError(t, err)
 	defer s.Close()
 
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
+
 	// Start dependencies
 	minio := e2edb.NewMinio(9000, bucketName)
 	// Look ma, no Consul!
@@ -116,16 +119,21 @@ func testSingleBinaryEnv(t *testing.T, tlsEnabled bool, flags map[string]string)
 }
 
 func newSingleBinary(name string, servername string, join string, testFlags map[string]string) *e2ecortex.CortexService {
-	flags := map[string]string{
-		"-ingester.final-sleep":              "0s",
-		"-ingester.join-after":               "0s", // join quickly
-		"-ingester.min-ready-duration":       "0s",
-		"-ingester.num-tokens":               "512",
-		"-ingester.observe-period":           "5s", // to avoid conflicts in tokens
-		"-ring.store":                        "memberlist",
-		"-memberlist.bind-port":              "8000",
-		"-memberlist.left-ingesters-timeout": "600s", // effectively disable
-	}
+	flags := mergeFlags(
+		AlertmanagerLocalFlags(),
+		map[string]string{
+			"-ingester.final-sleep":              "0s",
+			"-ingester.join-after":               "0s", // join quickly
+			"-ingester.min-ready-duration":       "0s",
+			"-ingester.num-tokens":               "512",
+			"-ingester.observe-period":           "5s", // to avoid conflicts in tokens
+			"-ring.store":                        "memberlist",
+			"-memberlist.bind-port":              "8000",
+			"-memberlist.left-ingesters-timeout": "600s", // effectively disable
+			// alert manager
+			"-alertmanager.web.external-url": "http://localhost/alertmanager",
+		},
+	)
 
 	if join != "" {
 		flags["-memberlist.join"] = join
@@ -158,6 +166,9 @@ func TestSingleBinaryWithMemberlistScaling(t *testing.T) {
 	require.NoError(t, err)
 	defer s.Close()
 
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
+
 	minio := e2edb.NewMinio(9000, bucketName)
 	require.NoError(t, s.StartAndWaitReady(minio))
 

integration/otlp_test.go

@@ -6,6 +6,7 @@ package integration
 import (
 	"fmt"
 	"math/rand"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -41,7 +42,13 @@ func TestOTLP(t *testing.T) {
 		"-blocks-storage.s3.endpoint":                   fmt.Sprintf("%s-minio-9000:9000", networkName),
 		"-blocks-storage.s3.insecure":                   "true",
 		"-blocks-storage.tsdb.enable-native-histograms": "true",
+		// alert manager
+		"-alertmanager.web.external-url":   "http://localhost/alertmanager",
+		"-alertmanager-storage.backend":    "local",
+		"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
 	}
+	// make alert manager config dir
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs", []byte{}))
 
 	cortex := e2ecortex.NewSingleBinaryWithConfigFile("cortex-1", cortexConfigFile, flags, "", 9009, 9095)
 	require.NoError(t, s.StartAndWaitReady(cortex))

integration/querier_test.go

@@ -228,7 +228,7 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
 					// then the metric only appears in one store gateway instance.
 					require.NoError(t, storeGateways.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"cortex_bucket_store_blocks_loaded"}, e2e.SkipMissingMetrics))
 				} else {
-					require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(float64(2*storeGateways.NumInstances())), "cortex_bucket_store_blocks_loaded"))
+					require.NoError(t, storeGateways.WaitSumMetricsWithOptions(e2e.Equals(float64(2*storeGateways.NumInstances())), []string{"cortex_bucket_store_blocks_loaded"}, e2e.WaitMissingMetrics))
 				}
 
 				// Check how many tenants have been discovered and synced by store-gateways.
@@ -365,26 +365,33 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) {
 
 				// Configure the blocks storage to frequently compact TSDB head
 				// and ship blocks to the storage.
-				flags := mergeFlags(BlocksStorageFlags(), map[string]string{
-					"-blocks-storage.tsdb.block-ranges-period":          blockRangePeriod.String(),
-					"-blocks-storage.tsdb.ship-interval":                "1s",
-					"-blocks-storage.bucket-store.sync-interval":        "1s",
-					"-blocks-storage.tsdb.retention-period":             ((blockRangePeriod * 2) - 1).String(),
-					"-blocks-storage.bucket-store.index-cache.backend":  testCfg.indexCacheBackend,
-					"-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled),
-					"-querier.query-store-for-labels-enabled":           "true",
-					"-querier.thanos-engine":                            strconv.FormatBool(thanosEngine),
-					// Ingester.
-					"-ring.store":      "consul",
-					"-consul.hostname": consul.NetworkHTTPEndpoint(),
-					// Distributor.
-					"-distributor.replication-factor": strconv.FormatInt(seriesReplicationFactor, 10),
-					// Store-gateway.
-					"-store-gateway.sharding-enabled":                 strconv.FormatBool(testCfg.blocksShardingEnabled),
-					"-store-gateway.sharding-ring.store":              "consul",
-					"-store-gateway.sharding-ring.consul.hostname":    consul.NetworkHTTPEndpoint(),
-					"-store-gateway.sharding-ring.replication-factor": "1",
-				})
+				flags := mergeFlags(
+					BlocksStorageFlags(),
+					AlertmanagerLocalFlags(),
+					map[string]string{
+						"-blocks-storage.tsdb.block-ranges-period":          blockRangePeriod.String(),
+						"-blocks-storage.tsdb.ship-interval":                "1s",
+						"-blocks-storage.bucket-store.sync-interval":        "1s",
+						"-blocks-storage.tsdb.retention-period":             ((blockRangePeriod * 2) - 1).String(),
+						"-blocks-storage.bucket-store.index-cache.backend":  testCfg.indexCacheBackend,
+						"-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled),
+						"-querier.query-store-for-labels-enabled":           "true",
+						"-querier.thanos-engine":                            strconv.FormatBool(thanosEngine),
+						// Ingester.
+						"-ring.store":      "consul",
+						"-consul.hostname": consul.NetworkHTTPEndpoint(),
+						// Distributor.
+						"-distributor.replication-factor": strconv.FormatInt(seriesReplicationFactor, 10),
+						// Store-gateway.
+						"-store-gateway.sharding-enabled":                 strconv.FormatBool(testCfg.blocksShardingEnabled),
+						"-store-gateway.sharding-ring.store":              "consul",
+						"-store-gateway.sharding-ring.consul.hostname":    consul.NetworkHTTPEndpoint(),
+						"-store-gateway.sharding-ring.replication-factor": "1",
+						// alert manager
+						"-alertmanager.web.external-url": "http://localhost/alertmanager",
+					},
+				)
+				require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs/user-1.yaml", []byte(cortexAlertmanagerUserConfigYaml)))
 
 				// Add the cache address to the flags.
 				if testCfg.indexCacheBackend == tsdb.IndexCacheBackendMemcached {