diff --git a/erigon-lib/state/aggregator.go b/erigon-lib/state/aggregator.go index 79688980f92..5020798a74d 100644 --- a/erigon-lib/state/aggregator.go +++ b/erigon-lib/state/aggregator.go @@ -159,7 +159,7 @@ func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint6 hist: histCfg{ iiCfg: iiCfg{salt: salt, dirs: dirs, db: db}, withLocalityIndex: false, withExistenceIndex: false, compression: CompressNone, historyLargeValues: false, - dontProduceHistoryFiles: true, + snapshotsDisabled: true, }, replaceKeysInValues: a.commitmentValuesTransform, restrictSubsetFileDeletions: a.commitmentValuesTransform, @@ -189,7 +189,7 @@ func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint6 if err := a.registerII(kv.TracesToIdxPos, salt, dirs, db, aggregationStep, kv.FileTracesToIdx, kv.TblTracesToKeys, kv.TblTracesToIdx, logger); err != nil { return nil, err } - a.LimitRecentHistoryWithoutFiles(aggregationStep / 2) + a.KeepRecentTxnsOfHistoriesWithDisabledSnapshots(100_000) // ~1k blocks of history a.recalcVisibleFiles() if dbg.NoSync() { @@ -1565,14 +1565,14 @@ func (a *Aggregator) cleanAfterMerge(in MergedFilesV3) { } } -// LimitRecentHistoryWithoutFiles limits amount of recent transactions protected from prune in domains history. +// KeepRecentTxnsOfHistoriesWithDisabledSnapshots limits amount of recent transactions protected from prune in domains history. // Affects only domains with dontProduceHistoryFiles=true. // Usually equal to one a.aggregationStep, but could be set to step/2 or step/4 to reduce size of history tables. // when we exec blocks from snapshots we can set it to 0, because no re-org on those blocks are possible -func (a *Aggregator) LimitRecentHistoryWithoutFiles(recentTxs uint64) *Aggregator { +func (a *Aggregator) KeepRecentTxnsOfHistoriesWithDisabledSnapshots(recentTxs uint64) *Aggregator { for _, d := range a.d { - if d != nil && d.History.dontProduceHistoryFiles { - d.History.keepRecentTxInDB = recentTxs + if d != nil && d.History.snapshotsDisabled { + d.History.keepRecentTxnInDB = recentTxs } } return a diff --git a/erigon-lib/state/history.go b/erigon-lib/state/history.go index 71fb9b1785d..40f29e7ae2a 100644 --- a/erigon-lib/state/history.go +++ b/erigon-lib/state/history.go @@ -88,9 +88,9 @@ type History struct { // vals: key1+key2+txNum -> value (not DupSort) historyLargeValues bool // can't use DupSort optimization (aka. prefix-compression) if values size > 4kb - dontProduceHistoryFiles bool // don't produce .v and .ef files. old data will be pruned anyway. - historyDisabled bool // skip all write operations to this History (even in DB) - keepRecentTxInDB uint64 // When dontProduceHistoryFiles=true, keepRecentTxInDB is used to keep this amount of tx in db before pruning + snapshotsDisabled bool // don't produce .v and .ef files, keep in db table. old data will be pruned anyway. + historyDisabled bool // skip all write operations to this History (even in DB) + keepRecentTxnInDB uint64 // When dontProduceHistoryFiles=true, keepRecentTxInDB is used to keep this amount of tx in db before pruning } type histCfg struct { @@ -105,21 +105,21 @@ type histCfg struct { withLocalityIndex bool withExistenceIndex bool // move to iiCfg - dontProduceHistoryFiles bool // don't produce .v and .ef files. old data will be pruned anyway. - keepTxInDB uint64 // When dontProduceHistoryFiles=true, keepTxInDB is used to keep this amount of tx in db before pruning + snapshotsDisabled bool // don't produce .v and .ef files. old data will be pruned anyway. + keepTxInDB uint64 // When dontProduceHistoryFiles=true, keepTxInDB is used to keep this amount of tx in db before pruning } func NewHistory(cfg histCfg, aggregationStep uint64, filenameBase, indexKeysTable, indexTable, historyValsTable string, integrityCheck func(fromStep, toStep uint64) bool, logger log.Logger) (*History, error) { h := History{ - dirtyFiles: btree2.NewBTreeGOptions[*filesItem](filesItemLess, btree2.Options{Degree: 128, NoLocks: false}), - historyValsTable: historyValsTable, - compression: cfg.compression, - compressWorkers: 1, - indexList: withHashMap, - integrityCheck: integrityCheck, - historyLargeValues: cfg.historyLargeValues, - dontProduceHistoryFiles: cfg.dontProduceHistoryFiles, - keepRecentTxInDB: cfg.keepTxInDB, + dirtyFiles: btree2.NewBTreeGOptions[*filesItem](filesItemLess, btree2.Options{Degree: 128, NoLocks: false}), + historyValsTable: historyValsTable, + compression: cfg.compression, + compressWorkers: 1, + indexList: withHashMap, + integrityCheck: integrityCheck, + historyLargeValues: cfg.historyLargeValues, + snapshotsDisabled: cfg.snapshotsDisabled, + keepRecentTxnInDB: cfg.keepTxInDB, } h._visibleFiles = []ctxItem{} var err error @@ -573,7 +573,7 @@ func (c HistoryCollation) Close() { // [txFrom; txTo) func (h *History) collate(ctx context.Context, step, txFrom, txTo uint64, roTx kv.Tx) (HistoryCollation, error) { - if h.dontProduceHistoryFiles { + if h.snapshotsDisabled { return HistoryCollation{}, nil } @@ -794,7 +794,7 @@ func (h *History) reCalcVisibleFiles() { // buildFiles performs potentially resource intensive operations of creating // static files and their indices func (h *History) buildFiles(ctx context.Context, step uint64, collation HistoryCollation, ps *background.ProgressSet) (HistoryFiles, error) { - if h.dontProduceHistoryFiles { + if h.snapshotsDisabled { return HistoryFiles{}, nil } var ( @@ -893,7 +893,7 @@ func (h *History) buildFiles(ctx context.Context, step uint64, collation History } func (h *History) integrateDirtyFiles(sf HistoryFiles, txNumFrom, txNumTo uint64) { - if h.dontProduceHistoryFiles { + if h.snapshotsDisabled { return } @@ -1008,11 +1008,11 @@ func (ht *HistoryRoTx) canPruneUntil(tx kv.Tx, untilTx uint64) (can bool, txTo u // ht.h.filenameBase, untilTx, ht.h.dontProduceHistoryFiles, txTo, minIdxTx, maxIdxTx, ht.h.keepRecentTxInDB, minIdxTx < txTo) //}() - if ht.h.dontProduceHistoryFiles { - if ht.h.keepRecentTxInDB >= maxIdxTx { + if ht.h.snapshotsDisabled { + if ht.h.keepRecentTxnInDB >= maxIdxTx { return false, 0 } - txTo = min(maxIdxTx-ht.h.keepRecentTxInDB, untilTx) // bound pruning + txTo = min(maxIdxTx-ht.h.keepRecentTxnInDB, untilTx) // bound pruning } else { canPruneIdx := ht.iit.CanPrune(tx) if !canPruneIdx { @@ -1114,7 +1114,7 @@ func (ht *HistoryRoTx) Prune(ctx context.Context, rwTx kv.RwTx, txFrom, txTo, li } mxPruneSizeHistory.AddInt(pruned) - if !forced && ht.h.dontProduceHistoryFiles { + if !forced && ht.h.snapshotsDisabled { forced = true // or index.CanPrune will return false cuz no snapshots made } diff --git a/erigon-lib/state/history_test.go b/erigon-lib/state/history_test.go index 52e9103686b..c92c09edf55 100644 --- a/erigon-lib/state/history_test.go +++ b/erigon-lib/state/history_test.go @@ -425,7 +425,7 @@ func TestHistoryCanPrune(t *testing.T) { } t.Run("withFiles", func(t *testing.T) { db, h := testDbAndHistory(t, true, logger) - h.dontProduceHistoryFiles = false + h.snapshotsDisabled = false defer db.Close() writeKey(t, h, db) @@ -460,8 +460,8 @@ func TestHistoryCanPrune(t *testing.T) { }) t.Run("withoutFiles", func(t *testing.T) { db, h := testDbAndHistory(t, false, logger) - h.dontProduceHistoryFiles = true - h.keepRecentTxInDB = stepKeepInDB * h.aggregationStep + h.snapshotsDisabled = true + h.keepRecentTxnInDB = stepKeepInDB * h.aggregationStep defer db.Close() diff --git a/erigon-lib/state/merge.go b/erigon-lib/state/merge.go index d1a73aac021..10c1ac155d4 100644 --- a/erigon-lib/state/merge.go +++ b/erigon-lib/state/merge.go @@ -74,7 +74,7 @@ func (ii *InvertedIndex) endIndexedTxNumMinimax(needFrozen bool) uint64 { } func (h *History) endTxNumMinimax() uint64 { - if h.dontProduceHistoryFiles { + if h.snapshotsDisabled { return math.MaxUint64 } minimax := h.InvertedIndex.endTxNumMinimax() @@ -99,7 +99,7 @@ func (ap *Appendable) endTxNumMinimax() uint64 { } func (h *History) endIndexedTxNumMinimax(needFrozen bool) uint64 { var _max uint64 - if h.dontProduceHistoryFiles && h.dirtyFiles.Len() == 0 { + if h.snapshotsDisabled && h.dirtyFiles.Len() == 0 { _max = math.MaxUint64 } h.dirtyFiles.Walk(func(items []*filesItem) bool {