feat: Add CompactPointsPerBlock config opt

etc/config.sample.toml

@@ -124,6 +124,11 @@
   # will allow TSM compactions to write to disk.
   # compact-throughput-burst = "48m"
 
+  # CompactPointsPerBlock is the points per block to be used when aggressive
+  # compaction is used. There are certain cases where TSM files do not get
+  # fully compacted. This adjusts an internal parameter that alleviates that.
+  # compact-points-per-block = 10000
+
   # If true, then the mmap advise value MADV_WILLNEED will be provided to the kernel with respect to
   # TSM files. This setting has been found to be problematic on some kernels, and defaults to off.
   # It might help users who have slow disks in some cases.

tsdb/config.go

@@ -52,9 +52,9 @@ const (
 	// block in a TSM file
 	DefaultMaxPointsPerBlock = 1000
 
-	// AggressiveMaxPointsPerBlock is used when we want to further compact blocks
+	// DefaultAggressiveMaxPointsPerBlock is used when we want to further compact blocks
 	// it is 100 times the default amount of points we use per block
-	AggressiveMaxPointsPerBlock = DefaultMaxPointsPerBlock * 100
+	DefaultAggressiveMaxPointsPerBlock = DefaultMaxPointsPerBlock * 10
 
 	// DefaultMaxSeriesPerDatabase is the maximum number of series a node can hold per database.
 	// This limit only applies to the "inmem" index.
@@ -92,7 +92,7 @@ var SingleGenerationReasonText string = SingleGenerationReason()
 // when checked for full compaction.
 // 1048576000 is a magic number for bytes per gigabyte.
 func SingleGenerationReason() string {
-	return fmt.Sprintf("not fully compacted and not idle because single generation with more than 2 files under %d GB and more than 1 file(s) under aggressive compaction points per block count (%d points)", int(MaxTSMFileSize/1048576000), AggressiveMaxPointsPerBlock)
+	return fmt.Sprintf("not fully compacted and not idle because single generation with more than 2 files under %d GB and more than 1 file(s) under aggressive compaction points per block count (%d points)", int(MaxTSMFileSize/1048576000), DefaultAggressiveMaxPointsPerBlock)
 }
 
 // Config holds the configuration for the tsbd package.
@@ -128,6 +128,7 @@ type Config struct {
 	CompactFullWriteColdDuration   toml.Duration `toml:"compact-full-write-cold-duration"`
 	CompactThroughput              toml.Size     `toml:"compact-throughput"`
 	CompactThroughputBurst         toml.Size     `toml:"compact-throughput-burst"`
+	CompactPointsPerBlock          toml.Size     `toml:"compact-points-per-block"`
 
 	// Options for ingress metrics
 	IngressMetricByMeasurement bool `toml:"ingress-metric-by-measurement-enabled"`
@@ -197,6 +198,7 @@ func NewConfig() Config {
 		CompactFullWriteColdDuration:   toml.Duration(DefaultCompactFullWriteColdDuration),
 		CompactThroughput:              toml.Size(DefaultCompactThroughput),
 		CompactThroughputBurst:         toml.Size(DefaultCompactThroughputBurst),
+		CompactPointsPerBlock:          toml.Size(DefaultAggressiveMaxPointsPerBlock),
 
 		MaxSeriesPerDatabase:     DefaultMaxSeriesPerDatabase,
 		MaxValuesPerTag:          DefaultMaxValuesPerTag,

tsdb/engine/tsm1/compact.go

@@ -253,7 +253,7 @@ func (c *DefaultPlanner) FullyCompacted() (bool, string) {
 			aggressivePointsPerBlockCount := 0
 			filesUnderMaxTsmSizeCount := 0
 			for _, tsmFile := range gens[0].files {
-				if c.FileStore.BlockCount(tsmFile.Path, 1) >= tsdb.AggressiveMaxPointsPerBlock {
+				if c.FileStore.BlockCount(tsmFile.Path, 1) >= tsdb.DefaultAggressiveMaxPointsPerBlock {
 					aggressivePointsPerBlockCount++
 				}
 				if tsmFile.Size < tsdb.MaxTSMFileSize {

tsdb/engine/tsm1/compact_test.go

@@ -2400,7 +2400,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		// > 2 GB total group size
 		// 50% of files are at aggressive max block size
 		{
-			"Small group size with single generation 50% at DefaultMaxPointsPerBlock and 50% at AggressiveMaxPointsPerBlock",
+			"Small group size with single generation 50% at DefaultMaxPointsPerBlock and 50% at DefaultAggressiveMaxPointsPerBlock",
 			[]tsm1.FileStat{
 				{
 					Path: "01-05.tsm1",
@@ -2436,10 +2436,10 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 				},
 			},
 			[]int{
-				tsdb.AggressiveMaxPointsPerBlock,
-				tsdb.AggressiveMaxPointsPerBlock,
-				tsdb.AggressiveMaxPointsPerBlock,
-				tsdb.AggressiveMaxPointsPerBlock,
+				tsdb.DefaultAggressiveMaxPointsPerBlock,
+				tsdb.DefaultAggressiveMaxPointsPerBlock,
+				tsdb.DefaultAggressiveMaxPointsPerBlock,
+				tsdb.DefaultAggressiveMaxPointsPerBlock,
 				tsdb.DefaultMaxPointsPerBlock,
 				tsdb.DefaultMaxPointsPerBlock,
 				tsdb.DefaultMaxPointsPerBlock,
@@ -2464,7 +2464,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 					Size: 450 * 1024 * 1024,
 				},
 			}, []int{
-			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
 			tsdb.DefaultMaxPointsPerBlock,
 			tsdb.DefaultMaxPointsPerBlock,
 		},
@@ -2582,16 +2582,16 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 					Size: 400 * 1024 * 1024,
 				},
 			}, []int{
-			tsdb.AggressiveMaxPointsPerBlock,
-			tsdb.AggressiveMaxPointsPerBlock,
-			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
 
-			tsdb.AggressiveMaxPointsPerBlock,
-			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
 			tsdb.DefaultMaxPointsPerBlock,
 
-			tsdb.AggressiveMaxPointsPerBlock,
-			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
 			tsdb.DefaultMaxPointsPerBlock,
 
 			tsdb.DefaultMaxPointsPerBlock,
@@ -2674,7 +2674,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 			// This test is added to account for a single generation that has a group size
 			// over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
 			// It should not compact any further.
-			"TSM files at AggressiveMaxPointsPerBlock",
+			"TSM files at DefaultAggressiveMaxPointsPerBlock",
 			[]tsm1.FileStat{
 				{
 					Path: "01-13.tsm1",
@@ -2685,8 +2685,8 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 					Size: 691 * 1024 * 1024,
 				},
 			}, []int{
-			tsdb.AggressiveMaxPointsPerBlock,
-			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
 		}, "", 0,
 		},
 		{
@@ -2705,7 +2705,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 					Size: 691 * 1024 * 1024,
 				},
 			}, []int{
-			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
 			tsdb.DefaultMaxPointsPerBlock,
 		},
 			"",
@@ -2714,7 +2714,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		{
 			// This test is added to account for a single generation that has a group size
 			// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction.
-			"Group size over 2 with multiple files under 2GB and at AggressiveMaxPointsPerBlock",
+			"Group size over 2 with multiple files under 2GB and at DefaultAggressiveMaxPointsPerBlock",
 			[]tsm1.FileStat{
 				{
 					Path: "01-13.tsm1",
@@ -2729,9 +2729,9 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 					Size: 450 * 1024 * 1024,
 				},
 			}, []int{
-			tsdb.AggressiveMaxPointsPerBlock,
-			tsdb.AggressiveMaxPointsPerBlock,
-			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
+			tsdb.DefaultAggressiveMaxPointsPerBlock,
 		}, "", 0,
 		},
 	}

tsdb/engine/tsm1/engine.go

@@ -2167,14 +2167,14 @@ func (e *Engine) compact(wg *sync.WaitGroup) {
 						level3Groups = level3Groups[1:]
 					}
 				case 4:
-					// This is a heuristic. 100_000 points per block is suitable for when we have a
+					// This is a heuristic. The 10_000 points per block default is suitable for when we have a
 					// single generation with multiple files at max block size under 2 GB.
 					if genLen == 1 {
 						// Log TSM files that will have an increased points per block count.
 						for _, f := range level4Groups[0] {
-							e.logger.Info("TSM optimized compaction on single generation running, increasing total points per block to 100_000.", zap.String("path", f))
+							e.logger.Info("TSM optimized compaction on single generation running, increasing total points per block.", zap.String("path", f), zap.Int("points-per-block", tsdb.DefaultAggressiveMaxPointsPerBlock))
 						}
-						e.Compactor.Size = tsdb.AggressiveMaxPointsPerBlock
+						e.Compactor.Size = tsdb.DefaultAggressiveMaxPointsPerBlock
 					} else {
 						e.Compactor.Size = tsdb.DefaultMaxPointsPerBlock
 					}