Adding a jitter on the head compaction and changing the max head comp… (

#5919) * Adding a jitter on the head compaction and changing the max head compaction interval to 30 min Signed-off-by: alanprot <[email protected]> * Doc Signed-off-by: alanprot <[email protected]> * changelog Signed-off-by: alanprot <[email protected]> * fix test Signed-off-by: alanprot <[email protected]> * whitenoise Signed-off-by: alanprot <[email protected]> --------- Signed-off-by: alanprot <[email protected]>
cortexproject · May 3, 2024 · 04cc770 · 04cc770
1 parent 7ff681a
commit 04cc770
Show file tree

Hide file tree

Showing 7 changed files with 22 additions and 10 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,7 @@
 ## master / unreleased
 * [ENHANCEMENT] Query Frontend/Querier: Added store gateway postings touched count and touched size in Querier stats and log in Query Frontend. #5892
 * [ENHANCEMENT] Query Frontend/Querier: Returns `warnings` on prometheus query responses. #5916
+* [ENHANCEMENT] Ingester: Allowing to configure `-blocks-storage.tsdb.head-compaction-interval` flag up to 30 min and add a jitter on the first head compaction. #5919
 * [CHANGE] Upgrade Dockerfile Node version from 14x to 18x. #5906
 
 ## 1.17.0 2024-04-30

diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md
@@ -1372,8 +1372,9 @@ blocks_storage:
     [ship_concurrency: <int> | default = 10]
 
     # How frequently does Cortex try to compact TSDB head. Block is only created
-    # if data covers smallest block range. Must be greater than 0 and max 5
-    # minutes.
+    # if data covers smallest block range. Must be greater than 0 and max 30
+    # minutes. Note that up to 50% jitter is added to the value for the first
+    # compaction to avoid ingesters compacting concurrently.
     # CLI flag: -blocks-storage.tsdb.head-compaction-interval
     [head_compaction_interval: <duration> | default = 1m]
 

diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md
@@ -1491,8 +1491,9 @@ blocks_storage:
     [ship_concurrency: <int> | default = 10]
 
     # How frequently does Cortex try to compact TSDB head. Block is only created
-    # if data covers smallest block range. Must be greater than 0 and max 5
-    # minutes.
+    # if data covers smallest block range. Must be greater than 0 and max 30
+    # minutes. Note that up to 50% jitter is added to the value for the first
+    # compaction to avoid ingesters compacting concurrently.
     # CLI flag: -blocks-storage.tsdb.head-compaction-interval
     [head_compaction_interval: <duration> | default = 1m]
 

diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md
@@ -1924,8 +1924,9 @@ tsdb:
   [ship_concurrency: <int> | default = 10]
 
   # How frequently does Cortex try to compact TSDB head. Block is only created
-  # if data covers smallest block range. Must be greater than 0 and max 5
-  # minutes.
+  # if data covers smallest block range. Must be greater than 0 and max 30
+  # minutes. Note that up to 50% jitter is added to the value for the first
+  # compaction to avoid ingesters compacting concurrently.
   # CLI flag: -blocks-storage.tsdb.head-compaction-interval
   [head_compaction_interval: <duration> | default = 1m]
 

diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go
@@ -70,6 +70,7 @@ const (
 
 	// Jitter applied to the idle timeout to prevent compaction in all ingesters concurrently.
 	compactionIdleTimeoutJitter = 0.25
+	initialHeadCompactionJitter = 0.5
 
 	instanceIngestionRateTickInterval = time.Second
 
@@ -2404,13 +2405,20 @@ func (i *Ingester) shipBlocks(ctx context.Context, allowed *util.AllowedTenants)
 }
 
 func (i *Ingester) compactionLoop(ctx context.Context) error {
-	ticker := time.NewTicker(i.cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval)
+	// Apply a jitter on the first head compaction
+	firstHeadCompaction := true
+	ticker := time.NewTicker(util.DurationWithPositiveJitter(i.cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval, initialHeadCompactionJitter))
 	defer ticker.Stop()
 
 	for ctx.Err() == nil {
 		select {
 		case <-ticker.C:
 			i.compactBlocks(ctx, false, nil)
+			// Reset the ticker to run the configured interval on the first head compaction
+			if firstHeadCompaction {
+				ticker.Reset(i.cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval)
+				firstHeadCompaction = false
+			}
 
 		case req := <-i.TSDBState.forceCompactTrigger:
 			i.compactBlocks(ctx, true, req.users)

diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go
@@ -173,7 +173,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) {
 	f.DurationVar(&cfg.ShipInterval, "blocks-storage.tsdb.ship-interval", 1*time.Minute, "How frequently the TSDB blocks are scanned and new ones are shipped to the storage. 0 means shipping is disabled.")
 	f.IntVar(&cfg.ShipConcurrency, "blocks-storage.tsdb.ship-concurrency", 10, "Maximum number of tenants concurrently shipping blocks to the storage.")
 	f.IntVar(&cfg.MaxTSDBOpeningConcurrencyOnStartup, "blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup", 10, "limit the number of concurrently opening TSDB's on startup")
-	f.DurationVar(&cfg.HeadCompactionInterval, "blocks-storage.tsdb.head-compaction-interval", 1*time.Minute, "How frequently does Cortex try to compact TSDB head. Block is only created if data covers smallest block range. Must be greater than 0 and max 5 minutes.")
+	f.DurationVar(&cfg.HeadCompactionInterval, "blocks-storage.tsdb.head-compaction-interval", 1*time.Minute, "How frequently does Cortex try to compact TSDB head. Block is only created if data covers smallest block range. Must be greater than 0 and max 30 minutes. Note that up to 50% jitter is added to the value for the first compaction to avoid ingesters compacting concurrently.")
 	f.IntVar(&cfg.HeadCompactionConcurrency, "blocks-storage.tsdb.head-compaction-concurrency", 5, "Maximum number of tenants concurrently compacting TSDB head into a new block")
 	f.DurationVar(&cfg.HeadCompactionIdleTimeout, "blocks-storage.tsdb.head-compaction-idle-timeout", 1*time.Hour, "If TSDB head is idle for this duration, it is compacted. Note that up to 25% jitter is added to the value to avoid ingesters compacting concurrently. 0 means disabled.")
 	f.IntVar(&cfg.HeadChunksWriteBufferSize, "blocks-storage.tsdb.head-chunks-write-buffer-size-bytes", chunks.DefaultWriteBufferSize, "The write buffer size used by the head chunks mapper. Lower values reduce memory utilisation on clusters with a large number of tenants at the cost of increased disk I/O operations.")
@@ -198,7 +198,7 @@ func (cfg *TSDBConfig) Validate() error {
 		return errInvalidOpeningConcurrency
 	}
 
-	if cfg.HeadCompactionInterval <= 0 || cfg.HeadCompactionInterval > 5*time.Minute {
+	if cfg.HeadCompactionInterval <= 0 || cfg.HeadCompactionInterval > 30*time.Minute {
 		return errInvalidCompactionInterval
 	}
 

diff --git a/pkg/storage/tsdb/config_test.go b/pkg/storage/tsdb/config_test.go
@@ -63,7 +63,7 @@ func TestConfig_Validate(t *testing.T) {
 		},
 		"should fail on too high compaction interval": {
 			setup: func(cfg *BlocksStorageConfig) {
-				cfg.TSDB.HeadCompactionInterval = 10 * time.Minute
+				cfg.TSDB.HeadCompactionInterval = 40 * time.Minute
 			},
 			expectedErr: errInvalidCompactionInterval,
 		},