Skip to content

Commit

Permalink
Merge pull request #6216 from multiversx/update-trie-from-batch
Browse files Browse the repository at this point in the history 
add batched changes to trie
  • Loading branch information
@BeniaminDrasovean
BeniaminDrasovean authored Dec 5, 2024
2 parents cdda030 + 4d35897 commit c44697a
Show file tree
Hide file tree
Showing 15 changed files with 1,413 additions and 243 deletions.
6 changes: 3 additions & 3 deletions common/interface.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,8 +375,8 @@ type ExecutionOrderGetter interface {
// TrieBatcher defines the methods needed for a trie batcher
type TrieBatcher interface {
BatchHandler
GetSortedDataForInsertion() ([]string, map[string]core.TrieData)
GetSortedDataForRemoval() []string
GetSortedDataForInsertion() []core.TrieData
GetSortedDataForRemoval() []core.TrieData
IsInterfaceNil() bool
}

Expand All @@ -390,7 +390,7 @@ type TrieBatchManager interface {

// BatchHandler is the interface for the batch handler
type BatchHandler interface {
Add(key []byte, data core.TrieData)
Add(data core.TrieData)
MarkForRemoval(key []byte)
Get(key []byte) ([]byte, bool)
}
2 changes: 2 additions & 0 deletions integrationTests/vm/txsFee/migrateDataTrie_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,8 @@ func TestMigrateDataTrieBuiltInFunc(t *testing.T) {

err = testContext.Accounts.SaveAccount(acc)
require.Nil(t, err)
_, err = testContext.Accounts.Commit()
require.Nil(t, err)

acc = getAccount(t, testContext, sndAddr)

Expand Down
202 changes: 149 additions & 53 deletions trie/branchNode.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -443,61 +443,139 @@ func (bn *branchNode) getNext(key []byte, db common.TrieStorageInteractor) (node
return bn.children[childPos], key, nil
}

func (bn *branchNode) insert(newData core.TrieData, db common.TrieStorageInteractor) (node, [][]byte, error) {
func (bn *branchNode) insert(newData []core.TrieData, db common.TrieStorageInteractor) (node, [][]byte, error) {
emptyHashes := make([][]byte, 0)
err := bn.isEmptyOrNil()
if err != nil {
return nil, emptyHashes, fmt.Errorf("insert error %w", err)
}

if len(newData.Key) == 0 {
return nil, emptyHashes, ErrValueTooShort
dataForInsertion, err := splitDataForChildren(newData)
if err != nil {
return nil, emptyHashes, err
}
childPos := newData.Key[firstByte]
if childPosOutOfRange(childPos) {
return nil, emptyHashes, ErrChildPosOutOfRange
modifiedHashes := make([][]byte, 0)
bnHasBeenModified := false

for childPos := range dataForInsertion {
if len(dataForInsertion[childPos]) == 0 {
continue
}
err = resolveIfCollapsed(bn, byte(childPos), db)
if err != nil {
return nil, emptyHashes, err
}

if bn.children[childPos] == nil {
newModifiedHashes, err := bn.insertOnNilChild(dataForInsertion[childPos], byte(childPos), db)
if err != nil {
return nil, emptyHashes, err
}
modifiedHashes = append(modifiedHashes, newModifiedHashes...)
bnHasBeenModified = true

continue
}

dirty, newModifiedHashes, err := bn.insertOnExistingChild(dataForInsertion[childPos], byte(childPos), db)
if err != nil {
return nil, emptyHashes, err
}
if dirty {
bnHasBeenModified = true
}
modifiedHashes = append(modifiedHashes, newModifiedHashes...)
}

newData.Key = newData.Key[1:]
err = resolveIfCollapsed(bn, childPos, db)
if err != nil {
return nil, emptyHashes, err
if bnHasBeenModified {
return bn, modifiedHashes, nil
}

if bn.children[childPos] == nil {
return bn.insertOnNilChild(newData, childPos)
return nil, emptyHashes, nil
}

// the prerequisite for this to work is that the data is already sorted
func splitDataForChildren(newSortedData []core.TrieData) ([][]core.TrieData, error) {
if len(newSortedData) == 0 {
return nil, ErrValueTooShort
}
childrenData := make([][]core.TrieData, nrOfChildren)

startIndex := 0
childPos := byte(0)
prevChildPos := byte(0)
for i := range newSortedData {
if len(newSortedData[i].Key) == 0 {
return nil, ErrValueTooShort
}
childPos = newSortedData[i].Key[firstByte]
if childPosOutOfRange(childPos) {
return nil, ErrChildPosOutOfRange
}
newSortedData[i].Key = newSortedData[i].Key[1:]

if i == 0 {
prevChildPos = childPos
continue
}

if childPos == prevChildPos {
continue
}

childrenData[prevChildPos] = newSortedData[startIndex:i]
startIndex = i
prevChildPos = childPos
}

return bn.insertOnExistingChild(newData, childPos, db)
childrenData[childPos] = newSortedData[startIndex:]
return childrenData, nil
}

func (bn *branchNode) insertOnNilChild(newData core.TrieData, childPos byte) (node, [][]byte, error) {
newLn, err := newLeafNode(newData, bn.marsh, bn.hasher)
if err != nil {
return nil, [][]byte{}, err
func (bn *branchNode) insertOnNilChild(newData []core.TrieData, childPos byte, db common.TrieStorageInteractor) ([][]byte, error) {
if len(newData) == 0 {
return [][]byte{}, ErrValueTooShort
}

var newNode node
modifiedHashes := make([][]byte, 0)
modifiedHashes, err = bn.modifyNodeAfterInsert(modifiedHashes, childPos, newLn)

newNode, err := newLeafNode(newData[0], bn.marsh, bn.hasher)
if err != nil {
return [][]byte{}, err
}

if len(newData) > 1 {
newNode, modifiedHashes, err = newNode.insert(newData[1:], db)
if check.IfNil(newNode) || err != nil {
return [][]byte{}, err
}
}

modifiedHashes, err = bn.modifyNodeAfterInsert(modifiedHashes, childPos, newNode)
if err != nil {
return nil, [][]byte{}, err
return [][]byte{}, err
}

return bn, modifiedHashes, nil
return modifiedHashes, nil
}

func (bn *branchNode) insertOnExistingChild(newData core.TrieData, childPos byte, db common.TrieStorageInteractor) (node, [][]byte, error) {
func (bn *branchNode) insertOnExistingChild(newData []core.TrieData, childPos byte, db common.TrieStorageInteractor) (bool, [][]byte, error) {
newNode, modifiedHashes, err := bn.children[childPos].insert(newData, db)
if check.IfNil(newNode) || err != nil {
return nil, [][]byte{}, err
if err != nil {
return false, [][]byte{}, err
}

if check.IfNil(newNode) {
return false, [][]byte{}, nil
}

modifiedHashes, err = bn.modifyNodeAfterInsert(modifiedHashes, childPos, newNode)
if err != nil {
return nil, [][]byte{}, err
return false, [][]byte{}, err
}

return bn, modifiedHashes, nil
return true, modifiedHashes, nil
}

func (bn *branchNode) modifyNodeAfterInsert(modifiedHashes [][]byte, childPos byte, newNode node) ([][]byte, error) {
Expand All @@ -518,45 +596,65 @@ func (bn *branchNode) modifyNodeAfterInsert(modifiedHashes [][]byte, childPos by
return modifiedHashes, nil
}

func (bn *branchNode) delete(key []byte, db common.TrieStorageInteractor) (bool, node, [][]byte, error) {
func (bn *branchNode) delete(data []core.TrieData, db common.TrieStorageInteractor) (bool, node, [][]byte, error) {
emptyHashes := make([][]byte, 0)
err := bn.isEmptyOrNil()
if err != nil {
return false, nil, emptyHashes, fmt.Errorf("delete error %w", err)
}
if len(key) == 0 {
return false, nil, emptyHashes, ErrValueTooShort
}
childPos := key[firstByte]
if childPosOutOfRange(childPos) {
return false, nil, emptyHashes, ErrChildPosOutOfRange
}
key = key[1:]
err = resolveIfCollapsed(bn, childPos, db)

dataForRemoval, err := splitDataForChildren(data)
if err != nil {
return false, nil, emptyHashes, err
}
modifiedHashes := make([][]byte, 0)
oldHash := make([]byte, len(bn.hash))
copy(oldHash, bn.hash)
hasBeenModified := false

if bn.children[childPos] == nil {
return false, bn, emptyHashes, nil
}
for childPos := range dataForRemoval {
if len(dataForRemoval[childPos]) == 0 {
continue
}
err = resolveIfCollapsed(bn, byte(childPos), db)
if err != nil {
return false, nil, emptyHashes, err
}

if bn.children[childPos] == nil {
continue
}

dirty, newNode, oldHashes, err := bn.children[childPos].delete(dataForRemoval[childPos], db)
if err != nil {
return false, bn, emptyHashes, err
}
if !dirty {
continue
}

hasBeenModified = true
err = bn.setNewChild(byte(childPos), newNode)
if err != nil {
return false, nil, emptyHashes, err
}

dirty, newNode, oldHashes, err := bn.children[childPos].delete(key, db)
if !dirty || err != nil {
return false, bn, emptyHashes, err
modifiedHashes = append(modifiedHashes, oldHashes...)
}

if !bn.dirty {
oldHashes = append(oldHashes, bn.hash)
if !hasBeenModified {
return false, bn, emptyHashes, nil
}

err = bn.setNewChild(childPos, newNode)
if err != nil {
return false, nil, emptyHashes, err
if len(oldHash) != 0 {
modifiedHashes = append(modifiedHashes, oldHash)
}
bn.dirty = true

numChildren, pos := getChildPosition(bn)

if numChildren == 0 {
return true, nil, modifiedHashes, nil
}
if numChildren == 1 {
err = resolveIfCollapsed(bn, byte(pos), db)
if err != nil {
Expand All @@ -569,21 +667,19 @@ func (bn *branchNode) delete(key []byte, db common.TrieStorageInteractor) (bool,
}

var newChildHash bool
newNode, newChildHash, err = bn.children[pos].reduceNode(pos)
newNode, newChildHash, err := bn.children[pos].reduceNode(pos)
if err != nil {
return false, nil, emptyHashes, err
}

if newChildHash && !bn.children[pos].isDirty() {
oldHashes = append(oldHashes, bn.children[pos].getHash())
modifiedHashes = append(modifiedHashes, bn.children[pos].getHash())
}

return true, newNode, oldHashes, nil
return true, newNode, modifiedHashes, nil
}

bn.dirty = dirty

return true, bn, oldHashes, nil
return true, bn, modifiedHashes, nil
}

func (bn *branchNode) setNewChild(childPos byte, newNode node) error {
Expand Down
Loading

0 comments on commit c44697a

Please sign in to comment.