metrics: add some trace metrics for epoch meta;

trie/trie: fix some update epoch bugs;
state/state_object: fix epoch update issue;
state/statedb: fix oom issue;

core/blockchain.go

@@ -1654,6 +1654,10 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 		// If node is running in path mode, skip explicit gc operation
 		// which is unnecessary in this mode.
 		if bc.triedb.Scheme() == rawdb.PathScheme {
+			err := bc.triedb.CommitEpochMeta(block.Root())
+			if err != nil {
+				return err
+			}
 			return nil
 		}
 
@@ -1668,10 +1672,12 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 			// Full but not archive node, do proper garbage collection
 			triedb.Reference(block.Root(), common.Hash{}) // metadata reference to keep trie alive
 			bc.triegc.Push(block.Root(), -int64(block.NumberU64()))
-			err := triedb.CommitEpochMeta(block.Root())
-			if err != nil {
-				return err
-			}
+			// TODO(0xbundler): when opt commit later, remove it.
+			go triedb.CommitEpochMeta(block.Root())
+			//err := triedb.CommitEpochMeta(block.Root())
+			//if err != nil {
+			//	return err
+			//}
 
 			if current := block.NumberU64(); current > bc.triesInMemory {
 				// If we exceeded our memory allowance, flush matured singleton nodes to disk

core/state/state_expiry.go

@@ -112,6 +112,9 @@ func batchFetchExpiredFromRemote(expiryMeta *stateExpiryMeta, addr common.Addres
 			keysStr[i] = common.Bytes2Hex(key[:])
 		}
 	}
+	if len(prefixKeysStr) == 0 {
+		return ret, nil
+	}
 
 	// cannot revive locally, fetch remote proof
 	proofs, err := expiryMeta.fullStateDB.GetStorageReviveProof(expiryMeta.originalRoot, addr, root, prefixKeysStr, keysStr)

core/state/state_object.go

@@ -169,7 +169,6 @@ func (s *stateObject) getTrie() (Trie, error) {
 			s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root)
 		}
 		if s.trie == nil {
-			// TODO(0xbundler): if any change to open a storage trie in state expiry feature?
 			tr, err := s.db.db.OpenStorageTrie(s.db.originalRoot, s.address, s.data.Root)
 			if err != nil {
 				return nil, err
@@ -320,6 +319,7 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash {
 		// handle state expiry situation
 		if s.db.EnableExpire() {
 			if enErr, ok := err.(*trie.ExpiredNodeError); ok {
+				log.Debug("GetCommittedState expired in trie", "addr", s.address, "key", key, "err", err)
 				val, err = s.fetchExpiredFromRemote(enErr.Path, key, false)
 			}
 			// TODO(0xbundler): add epoch record cache for prevent frequency access epoch update, may implement later
@@ -486,18 +486,21 @@ func (s *stateObject) updateTrie() (Trie, error) {
 				if _, err = fetchExpiredStorageFromRemote(s.db.expiryMeta, s.address, s.data.Root, tr, enErr.Path, key); err != nil {
 					s.db.setError(fmt.Errorf("state object pendingFutureReviveState fetchExpiredStorageFromRemote err, contract: %v, key: %v, path: %v, err: %v", s.address, key, enErr.Path, err))
 				}
+				log.Debug("updateTrie pendingFutureReviveState", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "tr.epoch", tr.Epoch(), "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s))
 			}
 		}
 		for key, value := range dirtyStorage {
 			if len(value) == 0 {
 				if err := tr.DeleteStorage(s.address, key[:]); err != nil {
 					s.db.setError(fmt.Errorf("state object update trie DeleteStorage err, contract: %v, key: %v, err: %v", s.address, key, err))
 				}
+				log.Debug("updateTrie DeleteStorage", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "value", value, "tr.epoch", tr.Epoch(), "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s))
 				s.db.StorageDeleted += 1
 			} else {
 				if err := tr.UpdateStorage(s.address, key[:], value); err != nil {
 					s.db.setError(fmt.Errorf("state object update trie UpdateStorage err, contract: %v, key: %v, err: %v", s.address, key, err))
 				}
+				log.Debug("updateTrie UpdateStorage", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "value", value, "tr.epoch", tr.Epoch(), "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s))
 				s.db.StorageUpdated += 1
 			}
 			// Cache the items for preloading
@@ -527,13 +530,16 @@ func (s *stateObject) updateTrie() (Trie, error) {
 
 			// rlp-encoded value to be used by the snapshot
 			var snapshotVal []byte
-			// Encoding []byte cannot fail, ok to ignore the error.
-			if s.db.EnableExpire() {
-				snapshotVal, _ = snapshot.EncodeValueToRLPBytes(snapshot.NewValueWithEpoch(s.db.Epoch(), value))
-			} else {
-				snapshotVal, _ = rlp.EncodeToBytes(value)
+			if len(value) != 0 {
+				// Encoding []byte cannot fail, ok to ignore the error.
+				if s.db.EnableExpire() {
+					snapshotVal, _ = snapshot.EncodeValueToRLPBytes(snapshot.NewValueWithEpoch(s.db.Epoch(), value))
+				} else {
+					snapshotVal, _ = rlp.EncodeToBytes(value)
+				}
 			}
 			storage[khash] = snapshotVal // snapshotVal will be nil if it's deleted
+			log.Debug("updateTrie UpdateSnapShot", "contract", s.address, "key", key, "epoch", s.db.Epoch(), "value", snapshotVal, "tr.epoch", tr.Epoch(), "tr", fmt.Sprintf("%p", tr), "ins", fmt.Sprintf("%p", s))
 
 			// Track the original value of slot only if it's mutated first time
 			prev := s.originStorage[key]
@@ -905,6 +911,7 @@ func (s *stateObject) getExpirySnapStorage(key common.Hash) ([]byte, error, erro
 		return val.GetVal(), nil, nil
 	}
 
+	log.Debug("GetCommittedState expired in snapshot", "addr", s.address, "key", key, "val", val, "enc", enc, "err", err)
 	// handle from remoteDB, if got err just setError, or return to revert in consensus version.
 	valRaw, err := s.fetchExpiredFromRemote(nil, key, true)
 	if err != nil {

core/state/statedb.go

@@ -18,6 +18,7 @@
 package state
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -1800,6 +1801,7 @@ func (s *StateDB) Commit(block uint64, failPostCommitFunc func(), postCommitFunc
 	if root == (common.Hash{}) {
 		root = types.EmptyRootHash
 	}
+	//log.Info("state commit", "nodes", stringfyEpochMeta(nodes.FlattenEpochMeta()))
 	//origin := s.originalRoot
 	//if origin == (common.Hash{}) {
 	//	origin = types.EmptyRootHash
@@ -1824,6 +1826,16 @@ func (s *StateDB) Commit(block uint64, failPostCommitFunc func(), postCommitFunc
 	return root, diffLayer, nil
 }
 
+func stringfyEpochMeta(meta map[common.Hash]map[string][]byte) string {
+	buf := bytes.NewBuffer(nil)
+	for hash, m := range meta {
+		for k, v := range m {
+			buf.WriteString(fmt.Sprintf("%v: %v|%v, ", hash, []byte(k), common.Bytes2Hex(v)))
+		}
+	}
+	return buf.String()
+}
+
 func (s *StateDB) SnapToDiffLayer() ([]common.Address, []types.DiffAccount, []types.DiffStorage) {
 	destructs := make([]common.Address, 0, len(s.stateObjectsDestruct))
 	for account := range s.stateObjectsDestruct {

trie/epochmeta/difflayer.go

@@ -17,6 +17,7 @@ const (
 	// MaxEpochMetaDiffDepth default is 128 layers
 	MaxEpochMetaDiffDepth        = 128
 	journalVersion        uint64 = 1
+	enableBloomFilter            = false
 )
 
 var (
@@ -95,7 +96,6 @@ func (h storageBloomHasher) Sum64() uint64 {
 		binary.BigEndian.Uint64([]byte(h.path[bloomStorageHasherOffset:bloomStorageHasherOffset+8]))
 }
 
-// TODO(0xbundler): add bloom filter?
 type diffLayer struct {
 	blockNumber *big.Int
 	blockRoot   common.Hash
@@ -114,23 +114,25 @@ func newEpochMetaDiffLayer(blockNumber *big.Int, blockRoot common.Hash, parent s
 		nodeSet:     nodeSet,
 	}
 
-	switch p := parent.(type) {
-	case *diffLayer:
-		dl.origin = p.origin
-		dl.diffed, _ = p.diffed.Copy()
-	case *diskLayer:
-		dl.origin = p
-		dl.diffed, _ = bloomfilter.New(uint64(bloomSize), uint64(bloomFuncs))
-	default:
-		panic("newEpochMetaDiffLayer got wrong snapshot type")
-	}
-
-	// Iterate over all the accounts and storage metas and index them
-	for accountHash, metas := range dl.nodeSet {
-		for path := range metas {
-			dl.diffed.Add(storageBloomHasher{accountHash, path})
+	if enableBloomFilter {
+		switch p := parent.(type) {
+		case *diffLayer:
+			dl.origin = p.origin
+			dl.diffed, _ = p.diffed.Copy()
+		case *diskLayer:
+			dl.origin = p
+			dl.diffed, _ = bloomfilter.New(uint64(bloomSize), uint64(bloomFuncs))
+		default:
+			panic("newEpochMetaDiffLayer got wrong snapshot type")
+		}
+		// Iterate over all the accounts and storage metas and index them
+		for accountHash, metas := range dl.nodeSet {
+			for path := range metas {
+				dl.diffed.Add(storageBloomHasher{accountHash, path})
+			}
 		}
 	}
+
 	return dl
 }
 
@@ -141,7 +143,7 @@ func (s *diffLayer) Root() common.Hash {
 // EpochMeta find target epoch meta from diff layer or disk layer
 func (s *diffLayer) EpochMeta(addrHash common.Hash, path string) ([]byte, error) {
 	// if the diff chain not contain the meta or staled, try get from disk layer
-	if !s.diffed.Contains(storageBloomHasher{addrHash, path}) {
+	if s.diffed != nil && !s.diffed.Contains(storageBloomHasher{addrHash, path}) {
 		return s.origin.EpochMeta(addrHash, path)
 	}
 

trie/epochmeta/snapshot.go

@@ -142,7 +142,7 @@ func (s *SnapshotTree) Cap(blockRoot common.Hash) error {
 			diff.resetParent(newDiskLayer)
 		}
 	}
-	log.Info("SnapshotTree cap", "layers", len(s.layers), "children", len(s.children), "flatten", len(flatten))
+	log.Debug("epochmeta snap tree cap", "root", blockRoot, "layers", len(s.layers), "flatten", len(flatten))
 	return nil
 }
 
@@ -172,6 +172,7 @@ func (s *SnapshotTree) Update(parentRoot common.Hash, blockNumber *big.Int, bloc
 
 	s.layers[blockRoot] = snap
 	s.children[parentRoot] = append(s.children[parentRoot], blockRoot)
+	log.Debug("epochmeta snap tree update", "root", blockRoot, "number", blockNumber, "layers", len(s.layers))
 	return nil
 }