Propagate store-gateway ring config correctly (#2706)

Note: The replication factor before was always hardcoded to 3. In order
to retain same behaviour, the flag
`-store-gateway.sharding-ring.replication-factor` needs to be set on
querier and store-gateways.

cmd/pyroscope/help-all.txt.tmpl

@@ -1002,7 +1002,7 @@ Usage of ./pyroscope:
   -store-gateway.sharding-ring.prefix string
     	The prefix for the keys in the store. Should end with a /. (default "collectors/")
   -store-gateway.sharding-ring.replication-factor int
-    	The replication factor to use when sharding blocks. This option needs be set both on the store-gateway, querier and ruler when running in microservices mode. (default 1)
+    	The replication factor to use when sharding blocks. This option needs be set both on the store-gateway and querier when running in microservices mode. (default 1)
   -store-gateway.sharding-ring.store string
     	Backend storage to use for the ring. Supported values are: consul, etcd, inmemory, memberlist, multi. (default "memberlist")
   -store-gateway.sharding-ring.tokens-file-path string
@@ -1014,7 +1014,7 @@ Usage of ./pyroscope:
   -store-gateway.sharding-ring.wait-stability-min-duration duration
     	Minimum time to wait for ring stability at startup, if set to positive value.
   -store-gateway.sharding-ring.zone-awareness-enabled
-    	True to enable zone-awareness and replicate blocks across different availability zones. This option needs be set both on the store-gateway, querier and ruler when running in microservices mode.
+    	True to enable zone-awareness and replicate blocks across different availability zones. This option needs be set both on the store-gateway and querier when running in microservices mode.
   -store-gateway.tenant-shard-size int
     	The tenant's shard size, used when store-gateway sharding is enabled. Value of 0 disables shuffle sharding for the tenant, that is all tenant blocks are sharded across all store-gateway replicas.
   -target comma-separated-list-of-strings

cmd/pyroscope/help.txt.tmpl

@@ -354,7 +354,7 @@ Usage of ./pyroscope:
   -store-gateway.sharding-ring.unregister-on-shutdown
     	Unregister from the ring upon clean shutdown. (default true)
   -store-gateway.sharding-ring.zone-awareness-enabled
-    	True to enable zone-awareness and replicate blocks across different availability zones. This option needs be set both on the store-gateway, querier and ruler when running in microservices mode.
+    	True to enable zone-awareness and replicate blocks across different availability zones. This option needs be set both on the store-gateway and querier when running in microservices mode.
   -store-gateway.tenant-shard-size int
     	The tenant's shard size, used when store-gateway sharding is enabled. Value of 0 disables shuffle sharding for the tenant, that is all tenant blocks are sharded across all store-gateway replicas.
   -target comma-separated-list-of-strings

docs/sources/configure-server/reference-configuration-parameters/index.md

@@ -1212,8 +1212,7 @@ sharding_ring:
   [instance_enable_ipv6: <boolean> | default = false]
 
   # The replication factor to use when sharding blocks. This option needs be set
-  # both on the store-gateway, querier and ruler when running in microservices
-  # mode.
+  # both on the store-gateway and querier when running in microservices mode.
   # CLI flag: -store-gateway.sharding-ring.replication-factor
   [replication_factor: <int> | default = 1]
 
@@ -1223,8 +1222,8 @@ sharding_ring:
   [tokens_file_path: <string> | default = ""]
 
   # True to enable zone-awareness and replicate blocks across different
-  # availability zones. This option needs be set both on the store-gateway,
-  # querier and ruler when running in microservices mode.
+  # availability zones. This option needs be set both on the store-gateway and
+  # querier when running in microservices mode.
   # CLI flag: -store-gateway.sharding-ring.zone-awareness-enabled
   [zone_awareness_enabled: <boolean> | default = false]
 

operations/pyroscope/helm/pyroscope/rendered/micro-services.yaml

@@ -2065,6 +2065,7 @@ spec:
             - "-config.file=/etc/pyroscope/config.yaml"
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
+            - "-store-gateway.sharding-ring.replication-factor=3"
           ports:
             - name: http2
               containerPort: 4040
@@ -2158,6 +2159,7 @@ spec:
             - "-config.file=/etc/pyroscope/config.yaml"
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
+            - "-store-gateway.sharding-ring.replication-factor=3"
           ports:
             - name: http2
               containerPort: 4040
@@ -2251,6 +2253,7 @@ spec:
             - "-config.file=/etc/pyroscope/config.yaml"
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
+            - "-store-gateway.sharding-ring.replication-factor=3"
           ports:
             - name: http2
               containerPort: 4040
@@ -2344,6 +2347,7 @@ spec:
             - "-config.file=/etc/pyroscope/config.yaml"
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
+            - "-store-gateway.sharding-ring.replication-factor=3"
           ports:
             - name: http2
               containerPort: 4040
@@ -2616,6 +2620,7 @@ spec:
             - "-config.file=/etc/pyroscope/config.yaml"
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
+            - "-store-gateway.sharding-ring.replication-factor=3"
           ports:
             - name: http2
               containerPort: 4040
@@ -2715,6 +2720,7 @@ spec:
             - "-config.file=/etc/pyroscope/config.yaml"
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
+            - "-store-gateway.sharding-ring.replication-factor=3"
           ports:
             - name: http2
               containerPort: 4040
@@ -2811,6 +2817,7 @@ spec:
             - "-config.file=/etc/pyroscope/config.yaml"
             - "-runtime-config.file=/etc/pyroscope/overrides/overrides.yaml"
             - "-log.level=debug"
+            - "-store-gateway.sharding-ring.replication-factor=3"
           ports:
             - name: http2
               containerPort: 4040

operations/pyroscope/helm/pyroscope/values-micro-services.yaml

@@ -3,6 +3,8 @@
 # Declare variables to be passed into your templates.
 
 pyroscope:
+  extraArgs:
+    store-gateway.sharding-ring.replication-factor: "3"
   components:
     querier:
       kind: Deployment

operations/pyroscope/jsonnet/values-micro-services.json

@@ -102,6 +102,9 @@
           }
         }
       }
+    },
+    "extraArgs": {
+      "store-gateway.sharding-ring.replication-factor": "3"
     }
   }
 }

pkg/querier/store_gateway_querier.go

@@ -63,7 +63,7 @@ func NewStoreGatewayQuerier(
 	reg prometheus.Registerer,
 	clientsOptions ...connect.ClientOption,
 ) (*StoreGatewayQuerier, error) {
-	storesRingCfg := gatewayCfg.ShardingRing.Ring.ToRingConfig()
+	storesRingCfg := gatewayCfg.ShardingRing.ToRingConfig()
 	storesRingBackend, err := kv.NewClient(
 		storesRingCfg.KVStore,
 		ring.GetCodec(),

pkg/storegateway/gateway.go

@@ -129,13 +129,13 @@ func newStoreGateway(gatewayCfg Config, storageBucket phlareobj.Bucket, ringStor
 	delegate = ring.NewTokensPersistencyDelegate(gatewayCfg.ShardingRing.TokensFilePath, ring.JOINING, delegate, logger)
 	delegate = ring.NewAutoForgetDelegate(ringAutoForgetUnhealthyPeriods*gatewayCfg.ShardingRing.Ring.HeartbeatTimeout, delegate, logger)
 
-	g.ringLifecycler, err = ring.NewBasicLifecycler(lifecyclerCfg, RingNameForServer, RingKey, ringStore, delegate, logger, prometheus.WrapRegistererWithPrefix("cortex_", reg))
+	g.ringLifecycler, err = ring.NewBasicLifecycler(lifecyclerCfg, RingNameForServer, RingKey, ringStore, delegate, logger, prometheus.WrapRegistererWithPrefix("pyroscope_", reg))
 	if err != nil {
 		return nil, errors.Wrap(err, "create ring lifecycler")
 	}
 
-	ringCfg := gatewayCfg.ShardingRing.Ring.ToRingConfig()
-	g.ring, err = ring.NewWithStoreClientAndStrategy(ringCfg, RingNameForServer, RingKey, ringStore, ring.NewIgnoreUnhealthyInstancesReplicationStrategy(), prometheus.WrapRegistererWithPrefix("cortex_", reg), logger)
+	ringCfg := gatewayCfg.ShardingRing.ToRingConfig()
+	g.ring, err = ring.NewWithStoreClientAndStrategy(ringCfg, RingNameForServer, RingKey, ringStore, ring.NewIgnoreUnhealthyInstancesReplicationStrategy(), prometheus.WrapRegistererWithPrefix("pyroscope_", reg), logger)
 	if err != nil {
 		return nil, errors.Wrap(err, "create ring client")
 	}

pkg/storegateway/gateway_ring.go

@@ -28,7 +28,7 @@ const (
 
 	// sharedOptionWithRingClient is a message appended to all config options that should be also
 	// set on the components running the store-gateway ring client.
-	sharedOptionWithRingClient = " This option needs be set both on the store-gateway, querier and ruler when running in microservices mode."
+	sharedOptionWithRingClient = " This option needs be set both on the store-gateway and querier when running in microservices mode."
 )
 
 var (
@@ -70,6 +70,12 @@ type RingConfig struct {
 	RingCheckPeriod time.Duration `yaml:"-"`
 }
 
+func (cfg *RingConfig) ToRingConfig() ring.Config {
+	ringCfg := cfg.Ring.ToRingConfig()
+	ringCfg.ReplicationFactor = cfg.ReplicationFactor
+	return ringCfg
+}
+
 // RegisterFlags adds the flags required to config this to the given FlagSet
 func (cfg *RingConfig) RegisterFlags(f *flag.FlagSet, logger log.Logger) {
 	ringFlagsPrefix := "store-gateway.sharding-ring."