From 04cc770fc53cdb5a0c3ceb6e21687b05d1fc2080 Mon Sep 17 00:00:00 2001 From: Alan Protasio Date: Thu, 2 May 2024 17:36:14 -0700 Subject: [PATCH] =?UTF-8?q?Adding=20a=20jitter=20on=20the=20head=20compact?= =?UTF-8?q?ion=20and=20changing=20the=20max=20head=20comp=E2=80=A6=20(#591?= =?UTF-8?q?9)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adding a jitter on the head compaction and changing the max head compaction interval to 30 min Signed-off-by: alanprot * Doc Signed-off-by: alanprot * changelog Signed-off-by: alanprot * fix test Signed-off-by: alanprot * whitenoise Signed-off-by: alanprot --------- Signed-off-by: alanprot --- CHANGELOG.md | 1 + docs/blocks-storage/querier.md | 5 +++-- docs/blocks-storage/store-gateway.md | 5 +++-- docs/configuration/config-file-reference.md | 5 +++-- pkg/ingester/ingester.go | 10 +++++++++- pkg/storage/tsdb/config.go | 4 ++-- pkg/storage/tsdb/config_test.go | 2 +- 7 files changed, 22 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c764a8761..b7d680d1c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## master / unreleased * [ENHANCEMENT] Query Frontend/Querier: Added store gateway postings touched count and touched size in Querier stats and log in Query Frontend. #5892 * [ENHANCEMENT] Query Frontend/Querier: Returns `warnings` on prometheus query responses. #5916 +* [ENHANCEMENT] Ingester: Allowing to configure `-blocks-storage.tsdb.head-compaction-interval` flag up to 30 min and add a jitter on the first head compaction. #5919 * [CHANGE] Upgrade Dockerfile Node version from 14x to 18x. #5906 ## 1.17.0 2024-04-30 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 3b0c998eeb..0d5831ee7d 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -1372,8 +1372,9 @@ blocks_storage: [ship_concurrency: | default = 10] # How frequently does Cortex try to compact TSDB head. Block is only created - # if data covers smallest block range. Must be greater than 0 and max 5 - # minutes. + # if data covers smallest block range. Must be greater than 0 and max 30 + # minutes. Note that up to 50% jitter is added to the value for the first + # compaction to avoid ingesters compacting concurrently. # CLI flag: -blocks-storage.tsdb.head-compaction-interval [head_compaction_interval: | default = 1m] diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 9ecd0215d1..bb88559db5 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -1491,8 +1491,9 @@ blocks_storage: [ship_concurrency: | default = 10] # How frequently does Cortex try to compact TSDB head. Block is only created - # if data covers smallest block range. Must be greater than 0 and max 5 - # minutes. + # if data covers smallest block range. Must be greater than 0 and max 30 + # minutes. Note that up to 50% jitter is added to the value for the first + # compaction to avoid ingesters compacting concurrently. # CLI flag: -blocks-storage.tsdb.head-compaction-interval [head_compaction_interval: | default = 1m] diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 30bbee288c..1ad826306e 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -1924,8 +1924,9 @@ tsdb: [ship_concurrency: | default = 10] # How frequently does Cortex try to compact TSDB head. Block is only created - # if data covers smallest block range. Must be greater than 0 and max 5 - # minutes. + # if data covers smallest block range. Must be greater than 0 and max 30 + # minutes. Note that up to 50% jitter is added to the value for the first + # compaction to avoid ingesters compacting concurrently. # CLI flag: -blocks-storage.tsdb.head-compaction-interval [head_compaction_interval: | default = 1m] diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 1b552c330e..96c6e4397b 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -70,6 +70,7 @@ const ( // Jitter applied to the idle timeout to prevent compaction in all ingesters concurrently. compactionIdleTimeoutJitter = 0.25 + initialHeadCompactionJitter = 0.5 instanceIngestionRateTickInterval = time.Second @@ -2404,13 +2405,20 @@ func (i *Ingester) shipBlocks(ctx context.Context, allowed *util.AllowedTenants) } func (i *Ingester) compactionLoop(ctx context.Context) error { - ticker := time.NewTicker(i.cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval) + // Apply a jitter on the first head compaction + firstHeadCompaction := true + ticker := time.NewTicker(util.DurationWithPositiveJitter(i.cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval, initialHeadCompactionJitter)) defer ticker.Stop() for ctx.Err() == nil { select { case <-ticker.C: i.compactBlocks(ctx, false, nil) + // Reset the ticker to run the configured interval on the first head compaction + if firstHeadCompaction { + ticker.Reset(i.cfg.BlocksStorageConfig.TSDB.HeadCompactionInterval) + firstHeadCompaction = false + } case req := <-i.TSDBState.forceCompactTrigger: i.compactBlocks(ctx, true, req.users) diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 2d2458bd12..644b855c97 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -173,7 +173,7 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) { f.DurationVar(&cfg.ShipInterval, "blocks-storage.tsdb.ship-interval", 1*time.Minute, "How frequently the TSDB blocks are scanned and new ones are shipped to the storage. 0 means shipping is disabled.") f.IntVar(&cfg.ShipConcurrency, "blocks-storage.tsdb.ship-concurrency", 10, "Maximum number of tenants concurrently shipping blocks to the storage.") f.IntVar(&cfg.MaxTSDBOpeningConcurrencyOnStartup, "blocks-storage.tsdb.max-tsdb-opening-concurrency-on-startup", 10, "limit the number of concurrently opening TSDB's on startup") - f.DurationVar(&cfg.HeadCompactionInterval, "blocks-storage.tsdb.head-compaction-interval", 1*time.Minute, "How frequently does Cortex try to compact TSDB head. Block is only created if data covers smallest block range. Must be greater than 0 and max 5 minutes.") + f.DurationVar(&cfg.HeadCompactionInterval, "blocks-storage.tsdb.head-compaction-interval", 1*time.Minute, "How frequently does Cortex try to compact TSDB head. Block is only created if data covers smallest block range. Must be greater than 0 and max 30 minutes. Note that up to 50% jitter is added to the value for the first compaction to avoid ingesters compacting concurrently.") f.IntVar(&cfg.HeadCompactionConcurrency, "blocks-storage.tsdb.head-compaction-concurrency", 5, "Maximum number of tenants concurrently compacting TSDB head into a new block") f.DurationVar(&cfg.HeadCompactionIdleTimeout, "blocks-storage.tsdb.head-compaction-idle-timeout", 1*time.Hour, "If TSDB head is idle for this duration, it is compacted. Note that up to 25% jitter is added to the value to avoid ingesters compacting concurrently. 0 means disabled.") f.IntVar(&cfg.HeadChunksWriteBufferSize, "blocks-storage.tsdb.head-chunks-write-buffer-size-bytes", chunks.DefaultWriteBufferSize, "The write buffer size used by the head chunks mapper. Lower values reduce memory utilisation on clusters with a large number of tenants at the cost of increased disk I/O operations.") @@ -198,7 +198,7 @@ func (cfg *TSDBConfig) Validate() error { return errInvalidOpeningConcurrency } - if cfg.HeadCompactionInterval <= 0 || cfg.HeadCompactionInterval > 5*time.Minute { + if cfg.HeadCompactionInterval <= 0 || cfg.HeadCompactionInterval > 30*time.Minute { return errInvalidCompactionInterval } diff --git a/pkg/storage/tsdb/config_test.go b/pkg/storage/tsdb/config_test.go index e7cb5dc502..d8c19849d2 100644 --- a/pkg/storage/tsdb/config_test.go +++ b/pkg/storage/tsdb/config_test.go @@ -63,7 +63,7 @@ func TestConfig_Validate(t *testing.T) { }, "should fail on too high compaction interval": { setup: func(cfg *BlocksStorageConfig) { - cfg.TSDB.HeadCompactionInterval = 10 * time.Minute + cfg.TSDB.HeadCompactionInterval = 40 * time.Minute }, expectedErr: errInvalidCompactionInterval, },