keep 10k recent tx in history (#10702)

erigon-lib/state/aggregator.go

@@ -159,7 +159,7 @@ func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint6
 		hist: histCfg{
 			iiCfg:             iiCfg{salt: salt, dirs: dirs, db: db},
 			withLocalityIndex: false, withExistenceIndex: false, compression: CompressNone, historyLargeValues: false,
-			dontProduceHistoryFiles: true,
+			snapshotsDisabled: true,
 		},
 		replaceKeysInValues:         a.commitmentValuesTransform,
 		restrictSubsetFileDeletions: a.commitmentValuesTransform,
@@ -189,7 +189,7 @@ func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint6
 	if err := a.registerII(kv.TracesToIdxPos, salt, dirs, db, aggregationStep, kv.FileTracesToIdx, kv.TblTracesToKeys, kv.TblTracesToIdx, logger); err != nil {
 		return nil, err
 	}
-	a.LimitRecentHistoryWithoutFiles(aggregationStep / 2)
+	a.KeepRecentTxnsOfHistoriesWithDisabledSnapshots(100_000) // ~1k blocks of history
 	a.recalcVisibleFiles()
 
 	if dbg.NoSync() {
@@ -1565,14 +1565,14 @@ func (a *Aggregator) cleanAfterMerge(in MergedFilesV3) {
 	}
 }
 
-// LimitRecentHistoryWithoutFiles limits amount of recent transactions protected from prune in domains history.
+// KeepRecentTxnsOfHistoriesWithDisabledSnapshots limits amount of recent transactions protected from prune in domains history.
 // Affects only domains with dontProduceHistoryFiles=true.
 // Usually equal to one a.aggregationStep, but could be set to step/2 or step/4 to reduce size of history tables.
 // when we exec blocks from snapshots we can set it to 0, because no re-org on those blocks are possible
-func (a *Aggregator) LimitRecentHistoryWithoutFiles(recentTxs uint64) *Aggregator {
+func (a *Aggregator) KeepRecentTxnsOfHistoriesWithDisabledSnapshots(recentTxs uint64) *Aggregator {
 	for _, d := range a.d {
-		if d != nil && d.History.dontProduceHistoryFiles {
-			d.History.keepRecentTxInDB = recentTxs
+		if d != nil && d.History.snapshotsDisabled {
+			d.History.keepRecentTxnInDB = recentTxs
 		}
 	}
 	return a

erigon-lib/state/history.go

@@ -88,9 +88,9 @@ type History struct {
 	//   vals: key1+key2+txNum -> value (not DupSort)
 	historyLargeValues bool // can't use DupSort optimization (aka. prefix-compression) if values size > 4kb
 
-	dontProduceHistoryFiles bool   // don't produce .v and .ef files. old data will be pruned anyway.
-	historyDisabled         bool   // skip all write operations to this History (even in DB)
-	keepRecentTxInDB        uint64 // When dontProduceHistoryFiles=true, keepRecentTxInDB is used to keep this amount of tx in db before pruning
+	snapshotsDisabled bool   // don't produce .v and .ef files, keep in db table. old data will be pruned anyway.
+	historyDisabled   bool   // skip all write operations to this History (even in DB)
+	keepRecentTxnInDB uint64 // When dontProduceHistoryFiles=true, keepRecentTxInDB is used to keep this amount of tx in db before pruning
 }
 
 type histCfg struct {
@@ -105,21 +105,21 @@ type histCfg struct {
 	withLocalityIndex  bool
 	withExistenceIndex bool // move to iiCfg
 
-	dontProduceHistoryFiles bool   // don't produce .v and .ef files. old data will be pruned anyway.
-	keepTxInDB              uint64 // When dontProduceHistoryFiles=true, keepTxInDB is used to keep this amount of tx in db before pruning
+	snapshotsDisabled bool   // don't produce .v and .ef files. old data will be pruned anyway.
+	keepTxInDB        uint64 // When dontProduceHistoryFiles=true, keepTxInDB is used to keep this amount of tx in db before pruning
 }
 
 func NewHistory(cfg histCfg, aggregationStep uint64, filenameBase, indexKeysTable, indexTable, historyValsTable string, integrityCheck func(fromStep, toStep uint64) bool, logger log.Logger) (*History, error) {
 	h := History{
-		dirtyFiles:              btree2.NewBTreeGOptions[*filesItem](filesItemLess, btree2.Options{Degree: 128, NoLocks: false}),
-		historyValsTable:        historyValsTable,
-		compression:             cfg.compression,
-		compressWorkers:         1,
-		indexList:               withHashMap,
-		integrityCheck:          integrityCheck,
-		historyLargeValues:      cfg.historyLargeValues,
-		dontProduceHistoryFiles: cfg.dontProduceHistoryFiles,
-		keepRecentTxInDB:        cfg.keepTxInDB,
+		dirtyFiles:         btree2.NewBTreeGOptions[*filesItem](filesItemLess, btree2.Options{Degree: 128, NoLocks: false}),
+		historyValsTable:   historyValsTable,
+		compression:        cfg.compression,
+		compressWorkers:    1,
+		indexList:          withHashMap,
+		integrityCheck:     integrityCheck,
+		historyLargeValues: cfg.historyLargeValues,
+		snapshotsDisabled:  cfg.snapshotsDisabled,
+		keepRecentTxnInDB:  cfg.keepTxInDB,
 	}
 	h._visibleFiles = []ctxItem{}
 	var err error
@@ -573,7 +573,7 @@ func (c HistoryCollation) Close() {
 
 // [txFrom; txTo)
 func (h *History) collate(ctx context.Context, step, txFrom, txTo uint64, roTx kv.Tx) (HistoryCollation, error) {
-	if h.dontProduceHistoryFiles {
+	if h.snapshotsDisabled {
 		return HistoryCollation{}, nil
 	}
 
@@ -794,7 +794,7 @@ func (h *History) reCalcVisibleFiles() {
 // buildFiles performs potentially resource intensive operations of creating
 // static files and their indices
 func (h *History) buildFiles(ctx context.Context, step uint64, collation HistoryCollation, ps *background.ProgressSet) (HistoryFiles, error) {
-	if h.dontProduceHistoryFiles {
+	if h.snapshotsDisabled {
 		return HistoryFiles{}, nil
 	}
 	var (
@@ -893,7 +893,7 @@ func (h *History) buildFiles(ctx context.Context, step uint64, collation History
 }
 
 func (h *History) integrateDirtyFiles(sf HistoryFiles, txNumFrom, txNumTo uint64) {
-	if h.dontProduceHistoryFiles {
+	if h.snapshotsDisabled {
 		return
 	}
 
@@ -1008,11 +1008,11 @@ func (ht *HistoryRoTx) canPruneUntil(tx kv.Tx, untilTx uint64) (can bool, txTo u
 	//		ht.h.filenameBase, untilTx, ht.h.dontProduceHistoryFiles, txTo, minIdxTx, maxIdxTx, ht.h.keepRecentTxInDB, minIdxTx < txTo)
 	//}()
 
-	if ht.h.dontProduceHistoryFiles {
-		if ht.h.keepRecentTxInDB >= maxIdxTx {
+	if ht.h.snapshotsDisabled {
+		if ht.h.keepRecentTxnInDB >= maxIdxTx {
 			return false, 0
 		}
-		txTo = min(maxIdxTx-ht.h.keepRecentTxInDB, untilTx) // bound pruning
+		txTo = min(maxIdxTx-ht.h.keepRecentTxnInDB, untilTx) // bound pruning
 	} else {
 		canPruneIdx := ht.iit.CanPrune(tx)
 		if !canPruneIdx {
@@ -1114,7 +1114,7 @@ func (ht *HistoryRoTx) Prune(ctx context.Context, rwTx kv.RwTx, txFrom, txTo, li
 	}
 	mxPruneSizeHistory.AddInt(pruned)
 
-	if !forced && ht.h.dontProduceHistoryFiles {
+	if !forced && ht.h.snapshotsDisabled {
 		forced = true // or index.CanPrune will return false cuz no snapshots made
 	}
 

erigon-lib/state/history_test.go

@@ -425,7 +425,7 @@ func TestHistoryCanPrune(t *testing.T) {
 	}
 	t.Run("withFiles", func(t *testing.T) {
 		db, h := testDbAndHistory(t, true, logger)
-		h.dontProduceHistoryFiles = false
+		h.snapshotsDisabled = false
 
 		defer db.Close()
 		writeKey(t, h, db)
@@ -460,8 +460,8 @@ func TestHistoryCanPrune(t *testing.T) {
 	})
 	t.Run("withoutFiles", func(t *testing.T) {
 		db, h := testDbAndHistory(t, false, logger)
-		h.dontProduceHistoryFiles = true
-		h.keepRecentTxInDB = stepKeepInDB * h.aggregationStep
+		h.snapshotsDisabled = true
+		h.keepRecentTxnInDB = stepKeepInDB * h.aggregationStep
 
 		defer db.Close()
 

erigon-lib/state/merge.go

@@ -74,7 +74,7 @@ func (ii *InvertedIndex) endIndexedTxNumMinimax(needFrozen bool) uint64 {
 }
 
 func (h *History) endTxNumMinimax() uint64 {
-	if h.dontProduceHistoryFiles {
+	if h.snapshotsDisabled {
 		return math.MaxUint64
 	}
 	minimax := h.InvertedIndex.endTxNumMinimax()
@@ -99,7 +99,7 @@ func (ap *Appendable) endTxNumMinimax() uint64 {
 }
 func (h *History) endIndexedTxNumMinimax(needFrozen bool) uint64 {
 	var _max uint64
-	if h.dontProduceHistoryFiles && h.dirtyFiles.Len() == 0 {
+	if h.snapshotsDisabled && h.dirtyFiles.Len() == 0 {
 		_max = math.MaxUint64
 	}
 	h.dirtyFiles.Walk(func(items []*filesItem) bool {