From 95875777f834a0db980231bcb76b0c411b51014f Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Tue, 22 Jun 2021 16:03:18 +0530 Subject: [PATCH 01/11] Add new major branch to github action. --- .github/workflows/on-master.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/on-master.yaml b/.github/workflows/on-master.yaml index 9546cdb0db76..c8c6759256f3 100644 --- a/.github/workflows/on-master.yaml +++ b/.github/workflows/on-master.yaml @@ -3,6 +3,7 @@ name: Docker Build and publish to Github on: push: branches: + - v1.10.4-statediff - v1.10.3-statediff - v1.10.2-statediff - v1.10.1-statediff From bc2280b654cbbc28bb2afb2b061045d2be5dce24 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Fri, 25 Jun 2021 09:28:00 +0530 Subject: [PATCH 02/11] Fix failing test. --- go.mod | 3 ++ go.sum | 9 ++++ statediff/builder_test.go | 47 +++++++++++---------- statediff/testhelpers/constant.go | 17 ++++++++ statediff/testhelpers/helpers.go | 24 +++++------ statediff/testhelpers/mocks/service_test.go | 6 ++- statediff/testhelpers/test_data.go | 3 +- 7 files changed, 71 insertions(+), 38 deletions(-) create mode 100644 statediff/testhelpers/constant.go diff --git a/go.mod b/go.mod index 158e6a89728f..db3018b1aee9 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/go-stack/stack v1.8.0 github.com/golang/protobuf v1.4.3 github.com/golang/snappy v0.0.3 + github.com/google/go-cmp v0.5.4 // indirect github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa github.com/google/uuid v1.1.5 github.com/gorilla/websocket v1.4.2 @@ -61,8 +62,10 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 github.com/prometheus/tsdb v0.7.1 + github.com/r3labs/diff/v2 v2.13.1 // indirect github.com/rjeczalik/notify v0.9.1 github.com/rs/cors v1.7.0 + github.com/sergi/go-diff v1.2.0 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 github.com/stretchr/testify v1.7.0 diff --git a/go.sum b/go.sum index c73d3f9ce2ee..8ccf7cbba86e 100644 --- a/go.sum +++ b/go.sum @@ -398,6 +398,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/r3labs/diff/v2 v2.13.1 h1:o0mlcYwUKTeL6jVblu42vCnXS7gfArPKYJCs4qkwOw8= +github.com/r3labs/diff/v2 v2.13.1/go.mod h1:I8noH9Fc2fjSaMxqF3G2lhDdC0b+JXCfyx85tWFM9kc= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= @@ -408,6 +410,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -428,6 +432,7 @@ github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= @@ -440,6 +445,8 @@ github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZF github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= +github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc h1:9lDbC6Rz4bwmou+oE6Dt4Cb2BGMur5eR/GYptkKUVHo= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= @@ -636,6 +643,8 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 7ad49f565fed..6cbf8b31379a 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -44,12 +44,6 @@ var ( minerAddress = common.HexToAddress("0x0") minerLeafKey = testhelpers.AddressToLeafKey(minerAddress) - balanceChange10000 = int64(10000) - balanceChange1000 = int64(1000) - block1BankBalance = int64(99990000) - block1Account1Balance = int64(10000) - block2Account2Balance = int64(1000) - slot0 = common.HexToHash("0") slot1 = common.HexToHash("1") slot2 = common.HexToHash("2") @@ -129,7 +123,7 @@ var ( minerAccountAtBlock1, _ = rlp.EncodeToBytes(state.Account{ Nonce: 0, - Balance: big.NewInt(miningReward), + Balance: big.NewInt(2000002625000000000), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -139,7 +133,7 @@ var ( }) minerAccountAtBlock2, _ = rlp.EncodeToBytes(state.Account{ Nonce: 0, - Balance: big.NewInt(miningReward + miningReward), + Balance: big.NewInt(4000111203461610525), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -150,7 +144,7 @@ var ( account1AtBlock1, _ = rlp.EncodeToBytes(state.Account{ Nonce: 0, - Balance: big.NewInt(balanceChange10000), + Balance: testhelpers.Block1Account1Balance, CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -160,7 +154,7 @@ var ( }) account1AtBlock2, _ = rlp.EncodeToBytes(state.Account{ Nonce: 2, - Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000), + Balance: big.NewInt(999555797000009000), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -170,7 +164,7 @@ var ( }) account1AtBlock5, _ = rlp.EncodeToBytes(state.Account{ Nonce: 2, - Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000 + miningReward), + Balance: big.NewInt(2999566008847709960), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -180,7 +174,7 @@ var ( }) account1AtBlock6, _ = rlp.EncodeToBytes(state.Account{ Nonce: 3, - Balance: big.NewInt(block1Account1Balance - balanceChange1000 + balanceChange1000 + miningReward), + Balance: big.NewInt(2999537516847709960), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -191,7 +185,7 @@ var ( account2AtBlock2, _ = rlp.EncodeToBytes(state.Account{ Nonce: 0, - Balance: big.NewInt(balanceChange1000), + Balance: big.NewInt(1000), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -201,7 +195,7 @@ var ( }) account2AtBlock3, _ = rlp.EncodeToBytes(state.Account{ Nonce: 0, - Balance: big.NewInt(block2Account2Balance + miningReward), + Balance: big.NewInt(2000013574009435976), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -211,7 +205,7 @@ var ( }) account2AtBlock4, _ = rlp.EncodeToBytes(state.Account{ Nonce: 0, - Balance: big.NewInt(block2Account2Balance + miningReward*2), + Balance: big.NewInt(4000048088163070348), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -221,7 +215,7 @@ var ( }) account2AtBlock6, _ = rlp.EncodeToBytes(state.Account{ Nonce: 0, - Balance: big.NewInt(block2Account2Balance + miningReward*3), + Balance: big.NewInt(6000063293259748636), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -240,9 +234,11 @@ var ( common.Hex2Bytes("2000bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), bankAccountAtBlock0, }) + + block1BankBalance = big.NewInt(testhelpers.TestBankFunds.Int64() - testhelpers.BalanceChange10000 - testhelpers.GasFees) bankAccountAtBlock1, _ = rlp.EncodeToBytes(state.Account{ Nonce: 1, - Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - balanceChange10000), + Balance: block1BankBalance, CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -250,9 +246,11 @@ var ( common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"), bankAccountAtBlock1, }) + + block2BankBalance = block1BankBalance.Int64() - testhelpers.BalanceChange1Ether - testhelpers.GasFees bankAccountAtBlock2, _ = rlp.EncodeToBytes(state.Account{ Nonce: 2, - Balance: big.NewInt(block1BankBalance - balanceChange1000), + Balance: big.NewInt(block2BankBalance), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -262,7 +260,7 @@ var ( }) bankAccountAtBlock3, _ = rlp.EncodeToBytes(state.Account{ Nonce: 3, - Balance: big.NewInt(99989000), + Balance: big.NewInt(999914255999990000), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -272,7 +270,7 @@ var ( }) bankAccountAtBlock4, _ = rlp.EncodeToBytes(state.Account{ Nonce: 6, - Balance: big.NewInt(99989000), + Balance: big.NewInt(999826859999990000), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -282,7 +280,7 @@ var ( }) bankAccountAtBlock5, _ = rlp.EncodeToBytes(state.Account{ Nonce: 7, - Balance: big.NewInt(99989000), + Balance: big.NewInt(999805027999990000), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -1706,10 +1704,12 @@ func TestBuilderWithRemovedAccountAndStorageWithoutIntermediateNodes(t *testing. if err != nil { t.Error(err) } + expectedStateDiffRlp, err := rlp.EncodeToBytes(test.expected) if err != nil { t.Error(err) } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { @@ -1740,7 +1740,7 @@ var ( bankAccountAtBlock01, _ = rlp.EncodeToBytes(state.Account{ Nonce: 1, - Balance: big.NewInt(testhelpers.TestBankFunds.Int64() + miningReward), + Balance: big.NewInt(3999629697375000000), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -1750,7 +1750,7 @@ var ( }) bankAccountAtBlock02, _ = rlp.EncodeToBytes(state.Account{ Nonce: 2, - Balance: big.NewInt(testhelpers.TestBankFunds.Int64() + miningReward*2), + Balance: big.NewInt(5999607323457344852), CodeHash: testhelpers.NullCodeHash.Bytes(), Root: testhelpers.EmptyContractRoot, }) @@ -1926,6 +1926,7 @@ func TestBuilderWithMovedAccount(t *testing.T) { if err != nil { t.Error(err) } + sort.Slice(receivedStateDiffRlp, func(i, j int) bool { return receivedStateDiffRlp[i] < receivedStateDiffRlp[j] }) sort.Slice(expectedStateDiffRlp, func(i, j int) bool { return expectedStateDiffRlp[i] < expectedStateDiffRlp[j] }) if !bytes.Equal(receivedStateDiffRlp, expectedStateDiffRlp) { diff --git a/statediff/testhelpers/constant.go b/statediff/testhelpers/constant.go new file mode 100644 index 000000000000..9788549e6df2 --- /dev/null +++ b/statediff/testhelpers/constant.go @@ -0,0 +1,17 @@ +package testhelpers + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/params" +) + +var ( + BalanceChange1000 = int64(1000) + BalanceChange10000 = int64(10000) + BalanceChange1Ether = int64(params.Ether) + Block1Account1Balance = big.NewInt(BalanceChange10000) + Block2Account2Balance = big.NewInt(21000000000000) + GasFees = int64(params.GWei) * int64(params.TxGas) + ContractGasLimit = uint64(1000000) +) diff --git a/statediff/testhelpers/helpers.go b/statediff/testhelpers/helpers.go index 7fd320b25bb5..168d770af33c 100644 --- a/statediff/testhelpers/helpers.go +++ b/statediff/testhelpers/helpers.go @@ -44,7 +44,7 @@ func TestSelfDestructChainGen(i int, block *core.BlockGen) { // Block 1 is mined by Account1Addr // Account1Addr creates a new contract block.SetCoinbase(TestBankAddress) - tx, _ := types.SignTx(types.NewContractCreation(0, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, TestBankKey) + tx, _ := types.SignTx(types.NewContractCreation(0, big.NewInt(0), 1000000, big.NewInt(params.GWei), ContractCode), signer, TestBankKey) ContractAddr = crypto.CreateAddress(TestBankAddress, 0) block.AddTx(tx) case 1: @@ -52,7 +52,7 @@ func TestSelfDestructChainGen(i int, block *core.BlockGen) { // Account1Addr self-destructs the contract block.SetCoinbase(TestBankAddress) data := common.Hex2Bytes("43D726D6") - tx, _ := types.SignTx(types.NewTransaction(1, ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey) + tx, _ := types.SignTx(types.NewTransaction(1, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.GWei), data), signer, TestBankKey) block.AddTx(tx) } } @@ -62,17 +62,17 @@ func TestChainGen(i int, block *core.BlockGen) { switch i { case 0: // In block 1, the test bank sends account #1 some ether. - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(10000), params.TxGas, nil, nil), signer, TestBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(BalanceChange10000), params.TxGas, big.NewInt(params.GWei), nil), signer, TestBankKey) block.AddTx(tx) case 1: // In block 2, the test bank sends some more ether to account #1. // Account1Addr passes it on to account #2. // Account1Addr creates a test contract. - tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, TestBankKey) + tx1, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, big.NewInt(BalanceChange1Ether), params.TxGas, big.NewInt(params.GWei), nil), signer, TestBankKey) nonce := block.TxNonce(Account1Addr) - tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(1000), params.TxGas, nil, nil), signer, Account1Key) + tx2, _ := types.SignTx(types.NewTransaction(nonce, Account2Addr, big.NewInt(BalanceChange1000), params.TxGas, big.NewInt(params.GWei), nil), signer, Account1Key) nonce++ - tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), 1000000, big.NewInt(0), ContractCode), signer, Account1Key) + tx3, _ := types.SignTx(types.NewContractCreation(nonce, big.NewInt(0), ContractGasLimit, big.NewInt(params.GWei), ContractCode), signer, Account1Key) ContractAddr = crypto.CreateAddress(Account1Addr, nonce) block.AddTx(tx1) block.AddTx(tx2) @@ -84,7 +84,7 @@ func TestChainGen(i int, block *core.BlockGen) { //put function: c16431b9 //close function: 43d726d6 data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000003") - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), ContractAddr, big.NewInt(0), params.TxGasContractCreation, big.NewInt(params.GWei), data), signer, TestBankKey) block.AddTx(tx) case 3: // Block 4 has three txs from bankAccount to the contract, that transfer no value @@ -96,11 +96,11 @@ func TestChainGen(i int, block *core.BlockGen) { data3 := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000009") nonce := block.TxNonce(TestBankAddress) - tx1, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data1), signer, TestBankKey) + tx1, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data1), signer, TestBankKey) nonce++ - tx2, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data2), signer, TestBankKey) + tx2, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data2), signer, TestBankKey) nonce++ - tx3, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data3), signer, TestBankKey) + tx3, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data3), signer, TestBankKey) block.AddTx(tx1) block.AddTx(tx2) block.AddTx(tx3) @@ -111,14 +111,14 @@ func TestChainGen(i int, block *core.BlockGen) { block.SetCoinbase(Account1Addr) data := common.Hex2Bytes("C16431B900000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000") nonce := block.TxNonce(TestBankAddress) - tx, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, nil, data), signer, TestBankKey) + tx, _ := types.SignTx(types.NewTransaction(nonce, ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data), signer, TestBankKey) block.AddTx(tx) case 5: // Block 6 has a tx from Account1Key which self-destructs the contract, it transfers no value // Block 6 is mined by Account2Addr block.SetCoinbase(Account2Addr) data := common.Hex2Bytes("43D726D6") - tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(Account1Addr), ContractAddr, big.NewInt(0), 100000, nil, data), signer, Account1Key) + tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(Account1Addr), ContractAddr, big.NewInt(0), 100000, big.NewInt(params.InitialBaseFee), data), signer, Account1Key) block.AddTx(tx) } } diff --git a/statediff/testhelpers/mocks/service_test.go b/statediff/testhelpers/mocks/service_test.go index e618268fd32c..a4f2c0d49b78 100644 --- a/statediff/testhelpers/mocks/service_test.go +++ b/statediff/testhelpers/mocks/service_test.go @@ -51,7 +51,7 @@ var ( }) minerAccount, _ = rlp.EncodeToBytes(state.Account{ Nonce: uint64(0), - Balance: big.NewInt(2000000000000000000), + Balance: big.NewInt(2000002625000000000), CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), }) @@ -61,7 +61,7 @@ var ( }) bankAccount, _ = rlp.EncodeToBytes(state.Account{ Nonce: uint64(1), - Balance: big.NewInt(testhelpers.TestBankFunds.Int64() - 10000), + Balance: big.NewInt(1999978999999990000), CodeHash: common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes(), Root: common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"), }) @@ -129,6 +129,7 @@ func testSubscriptionAPI(t *testing.T) { }, } expectedStateDiffBytes, _ := rlp.EncodeToBytes(expectedStateDiff) + blockChan := make(chan *types.Block) parentBlockChain := make(chan *types.Block) serviceQuitChan := make(chan bool) @@ -145,6 +146,7 @@ func testSubscriptionAPI(t *testing.T) { Subscriptions: make(map[common.Hash]map[rpc.ID]statediff.Subscription), SubscriptionTypes: make(map[common.Hash]statediff.Params), } + mockService.Start() id := rpc.NewID() payloadChan := make(chan statediff.Payload) diff --git a/statediff/testhelpers/test_data.go b/statediff/testhelpers/test_data.go index cc5374da9730..73def50a47bb 100644 --- a/statediff/testhelpers/test_data.go +++ b/statediff/testhelpers/test_data.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -54,7 +55,7 @@ var ( TestBankKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") TestBankAddress = crypto.PubkeyToAddress(TestBankKey.PublicKey) //0x71562b71999873DB5b286dF957af199Ec94617F7 BankLeafKey = AddressToLeafKey(TestBankAddress) - TestBankFunds = big.NewInt(100000000) + TestBankFunds = big.NewInt(params.Ether * 2) Genesis = core.GenesisBlockForTesting(Testdb, TestBankAddress, TestBankFunds) Account1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") From 4f627f614f79f97ffdc7ed3d2487d61eeb654566 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Fri, 25 Jun 2021 10:16:41 +0530 Subject: [PATCH 03/11] Fix lint errors. --- statediff/builder_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/statediff/builder_test.go b/statediff/builder_test.go index 6cbf8b31379a..a99a39664dd9 100644 --- a/statediff/builder_test.go +++ b/statediff/builder_test.go @@ -40,7 +40,6 @@ var ( emptyStorage = make([]sdtypes.StorageNode, 0) block0, block1, block2, block3, block4, block5, block6 *types.Block builder statediff.Builder - miningReward = int64(2000000000000000000) minerAddress = common.HexToAddress("0x0") minerLeafKey = testhelpers.AddressToLeafKey(minerAddress) From a1a02a097d0528ca709747bdb7b01ea34c897ec4 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Tue, 29 Jun 2021 13:21:37 +0530 Subject: [PATCH 04/11] Add support for Dynamic txn(EIP-1559). --- .../00004_create_eth_header_cids_table.sql | 1 + statediff/indexer/indexer.go | 1 + statediff/indexer/indexer_test.go | 51 +++++++++++++++---- statediff/indexer/mocks/test_data.go | 40 +++++++++++++-- statediff/indexer/models/models.go | 3 +- statediff/indexer/writer.go | 8 +-- 6 files changed, 85 insertions(+), 19 deletions(-) diff --git a/statediff/db/migrations/00004_create_eth_header_cids_table.sql b/statediff/db/migrations/00004_create_eth_header_cids_table.sql index 339eb427b16a..5993f5a36e7d 100644 --- a/statediff/db/migrations/00004_create_eth_header_cids_table.sql +++ b/statediff/db/migrations/00004_create_eth_header_cids_table.sql @@ -16,6 +16,7 @@ CREATE TABLE eth.header_cids ( bloom BYTEA NOT NULL, timestamp NUMERIC NOT NULL, times_validated INTEGER NOT NULL DEFAULT 1, + base_fee BIGINT NOT NULL, UNIQUE (block_number, block_hash) ); diff --git a/statediff/indexer/indexer.go b/statediff/indexer/indexer.go index 942bf19523b6..85d0018d0595 100644 --- a/statediff/indexer/indexer.go +++ b/statediff/indexer/indexer.go @@ -229,6 +229,7 @@ func (sdi *StateDiffIndexer) processHeader(tx *sqlx.Tx, header *types.Header, he TxRoot: header.TxHash.String(), UncleRoot: header.UncleHash.String(), Timestamp: header.Time, + BaseFee: header.BaseFee.String(), }) } diff --git a/statediff/indexer/indexer_test.go b/statediff/indexer/indexer_test.go index faa178197264..9d4901e0917e 100644 --- a/statediff/indexer/indexer_test.go +++ b/statediff/indexer/indexer_test.go @@ -25,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/statediff/indexer" "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/mocks" @@ -45,11 +44,11 @@ var ( ind *indexer.StateDiffIndexer ipfsPgGet = `SELECT data FROM public.blocks WHERE key = $1` - tx1, tx2, tx3, tx4, rct1, rct2, rct3, rct4 []byte - mockBlock *types.Block - headerCID, trx1CID, trx2CID, trx3CID, trx4CID cid.Cid - rct1CID, rct2CID, rct3CID, rct4CID cid.Cid - state1CID, state2CID, storageCID cid.Cid + tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte + mockBlock *types.Block + headerCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid + rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid + state1CID, state2CID, storageCID cid.Cid ) func expectTrue(t *testing.T, value bool) { @@ -88,6 +87,11 @@ func init() { copy(tx4, buf.Bytes()) buf.Reset() + txs.EncodeIndex(4, buf) + tx5 = make([]byte, buf.Len()) + copy(tx5, buf.Bytes()) + buf.Reset() + rcts.EncodeIndex(0, buf) rct1 = make([]byte, buf.Len()) copy(rct1, buf.Bytes()) @@ -108,15 +112,22 @@ func init() { copy(rct4, buf.Bytes()) buf.Reset() + rcts.EncodeIndex(4, buf) + rct5 = make([]byte, buf.Len()) + copy(rct5, buf.Bytes()) + buf.Reset() + headerCID, _ = ipld.RawdataToCid(ipld.MEthHeader, mocks.MockHeaderRlp, multihash.KECCAK_256) trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx1, multihash.KECCAK_256) trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx2, multihash.KECCAK_256) trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx3, multihash.KECCAK_256) trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx4, multihash.KECCAK_256) + trx5CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx5, multihash.KECCAK_256) rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct1, multihash.KECCAK_256) rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct2, multihash.KECCAK_256) rct3CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct3, multihash.KECCAK_256) rct4CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct4, multihash.KECCAK_256) + rct5CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct5, multihash.KECCAK_256) state1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.ContractLeafNode, multihash.KECCAK_256) state2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.AccountLeafNode, multihash.KECCAK_256) storageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, mocks.StorageLeafNode, multihash.KECCAK_256) @@ -127,7 +138,7 @@ func setup(t *testing.T) { if err != nil { t.Fatal(err) } - ind = indexer.NewStateDiffIndexer(params.MainnetChainConfig, db) + ind = indexer.NewStateDiffIndexer(mocks.TestConfig, db) var tx *indexer.BlockTx tx, err = ind.PushBlock( mockBlock, @@ -198,11 +209,12 @@ func TestPublishAndIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - shared.ExpectEqual(t, len(trxs), 4) + shared.ExpectEqual(t, len(trxs), 5) expectTrue(t, shared.ListContainsString(trxs, trx1CID.String())) expectTrue(t, shared.ListContainsString(trxs, trx2CID.String())) expectTrue(t, shared.ListContainsString(trxs, trx3CID.String())) expectTrue(t, shared.ListContainsString(trxs, trx4CID.String())) + expectTrue(t, shared.ListContainsString(trxs, trx5CID.String())) // and published for _, c := range trxs { dc, err := cid.Decode(c) @@ -281,6 +293,17 @@ func TestPublishAndIndexer(t *testing.T) { } shared.ExpectEqual(t, model1, mocks.AccessListEntry1Model) shared.ExpectEqual(t, model2, mocks.AccessListEntry2Model) + case trx5CID.String(): + shared.ExpectEqual(t, data, tx5) + var txType *uint8 + pgStr = `SELECT tx_type FROM eth.transaction_cids WHERE cid = $1` + err = db.Get(&txType, pgStr, c) + if err != nil { + t.Fatal(err) + } + if *txType != types.DynamicFeeTxType { + t.Fatalf("expected DynamicFeeTxType (2), got %d", *txType) + } } } }) @@ -298,11 +321,12 @@ func TestPublishAndIndexer(t *testing.T) { if err != nil { t.Fatal(err) } - shared.ExpectEqual(t, len(rcts), 4) + shared.ExpectEqual(t, len(rcts), 5) expectTrue(t, shared.ListContainsString(rcts, rct1CID.String())) expectTrue(t, shared.ListContainsString(rcts, rct2CID.String())) expectTrue(t, shared.ListContainsString(rcts, rct3CID.String())) expectTrue(t, shared.ListContainsString(rcts, rct4CID.String())) + expectTrue(t, shared.ListContainsString(rcts, rct5CID.String())) // and published for _, c := range rcts { dc, err := cid.Decode(c) @@ -353,6 +377,15 @@ func TestPublishAndIndexer(t *testing.T) { t.Fatal(err) } shared.ExpectEqual(t, postState, mocks.ExpectedPostState3) + case rct5CID.String(): + shared.ExpectEqual(t, data, rct5) + var postState string + pgStr = `SELECT post_state FROM eth.receipt_cids WHERE cid = $1` + err = db.Get(&postState, pgStr, c) + if err != nil { + t.Fatal(err) + } + shared.ExpectEqual(t, postState, mocks.ExpectedPostState3) } } }) diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go index d0b693fba1d3..f132cd1fab0b 100644 --- a/statediff/indexer/mocks/test_data.go +++ b/statediff/indexer/mocks/test_data.go @@ -40,7 +40,9 @@ import ( // Test variables var ( // block data - BlockNumber = big.NewInt(12244001) + // TODO: Update this to `MainnetChainConfig` when `LondonBlock` is added + TestConfig = params.RinkebyChainConfig + BlockNumber = TestConfig.LondonBlock MockHeader = types.Header{ Time: 0, Number: new(big.Int).Set(BlockNumber), @@ -49,6 +51,7 @@ var ( ReceiptHash: common.HexToHash("0x0"), Difficulty: big.NewInt(5000000), Extra: []byte{}, + BaseFee: big.NewInt(params.InitialBaseFee), } MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts() MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts, new(trie.Trie)) @@ -186,7 +189,7 @@ func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{}) trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode) trx4 := types.NewTx(&types.AccessListTx{ - ChainID: big.NewInt(1), + ChainID: TestConfig.ChainID, Nonce: 0, GasPrice: big.NewInt(100), Gas: 50, @@ -198,8 +201,22 @@ func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common AccessListEntry2, }, }) + trx5 := types.NewTx(&types.DynamicFeeTx{ + ChainID: TestConfig.ChainID, + Nonce: 0, + GasTipCap: big.NewInt(100), + GasFeeCap: big.NewInt(100), + Gas: 50, + To: &AnotherAddress, + Value: big.NewInt(1000), + Data: []byte{}, + AccessList: types.AccessList{ + AccessListEntry1, + AccessListEntry2, + }, + }) - transactionSigner := types.NewEIP2930Signer(params.MainnetChainConfig.ChainID) + transactionSigner := types.MakeSigner(TestConfig, BlockNumber) mockCurve := elliptic.P256() mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader) if err != nil { @@ -219,13 +236,18 @@ func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common } signedTrx4, err := types.SignTx(trx4, transactionSigner, mockPrvKey) if err != nil { - println(err.Error()) log.Crit(err.Error()) } + signedTrx5, err := types.SignTx(trx5, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + senderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx if err != nil { log.Crit(err.Error()) } + // make receipts mockReceipt1 := types.NewReceipt(nil, false, 50) mockReceipt1.Logs = []*types.Log{MockLog1} @@ -244,6 +266,14 @@ func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common Logs: []*types.Log{}, TxHash: signedTrx4.Hash(), } + mockReceipt5 := &types.Receipt{ + Type: types.DynamicFeeTxType, + PostState: common.HexToHash("0x3").Bytes(), + Status: types.ReceiptStatusSuccessful, + CumulativeGasUsed: 175, + Logs: []*types.Log{}, + TxHash: signedTrx5.Hash(), + } - return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4}, senderAddr + return types.Transactions{signedTrx1, signedTrx2, signedTrx3, signedTrx4, signedTrx5}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3, mockReceipt4, mockReceipt5}, senderAddr } diff --git a/statediff/indexer/models/models.go b/statediff/indexer/models/models.go index 604cf6b624c6..2855c5889c03 100644 --- a/statediff/indexer/models/models.go +++ b/statediff/indexer/models/models.go @@ -36,6 +36,7 @@ type HeaderModel struct { Bloom []byte `db:"bloom"` Timestamp uint64 `db:"timestamp"` TimesValidated int64 `db:"times_validated"` + BaseFee string `db:"base_fee"` } // UncleModel is the db model for eth.uncle_cids @@ -63,7 +64,7 @@ type TxModel struct { Type *uint8 `db:"tx_type"` } -// AccessListEntryModel is the db model for eth.access_list_entry +// AccessListElementModel is the db model for eth.access_list_entry type AccessListElementModel struct { ID int64 `db:"id"` Index int64 `db:"index"` diff --git a/statediff/indexer/writer.go b/statediff/indexer/writer.go index e8c0a03983cb..79ffbed8c72d 100644 --- a/statediff/indexer/writer.go +++ b/statediff/indexer/writer.go @@ -44,12 +44,12 @@ func NewPostgresCIDWriter(db *postgres.DB) *PostgresCIDWriter { func (in *PostgresCIDWriter) upsertHeaderCID(tx *sqlx.Tx, header models.HeaderModel) (int64, error) { var headerID int64 - err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) - ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1) + err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, base_fee) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) + ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward, state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, base_fee) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16) RETURNING id`, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, in.db.NodeID, header.Reward, header.StateRoot, header.TxRoot, - header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, header.MhKey, 1).Scan(&headerID) + header.RctRoot, header.UncleRoot, header.Bloom, header.Timestamp, header.MhKey, 1, header.BaseFee).Scan(&headerID) if err != nil { return 0, fmt.Errorf("error upserting header_cids entry: %v", err) } From 43f2339fd9d62b1e1e6867162a5f5b33f5de5f5b Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Tue, 29 Jun 2021 13:22:13 +0530 Subject: [PATCH 05/11] Update version meta to 0.0.24 --- params/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/version.go b/params/version.go index 1255213dde24..5be034174f9e 100644 --- a/params/version.go +++ b/params/version.go @@ -24,7 +24,7 @@ const ( VersionMajor = 1 // Major version component of the current release VersionMinor = 10 // Minor version component of the current release VersionPatch = 4 // Patch version component of the current release - VersionMeta = "statediff-0.0.23" // Version metadata to append to the version string + VersionMeta = "statediff-0.0.24" // Version metadata to append to the version string ) // Version holds the textual version string. From 66ac3f30bb1dcf77828cf14ddd16680ef60729e1 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Tue, 29 Jun 2021 16:45:48 +0530 Subject: [PATCH 06/11] Verify block base fee in test. --- statediff/indexer/indexer_test.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/statediff/indexer/indexer_test.go b/statediff/indexer/indexer_test.go index 9d4901e0917e..9af8622c9d04 100644 --- a/statediff/indexer/indexer_test.go +++ b/statediff/indexer/indexer_test.go @@ -22,9 +22,8 @@ import ( "os" "testing" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/statediff/indexer" "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/mocks" @@ -166,15 +165,16 @@ func TestPublishAndIndexer(t *testing.T) { t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) { setup(t) defer tearDown(t) - pgStr := `SELECT cid, td, reward, id + pgStr := `SELECT cid, td, reward, id, base_fee FROM eth.header_cids WHERE block_number = $1` // check header was properly indexed type res struct { - CID string - TD string - Reward string - ID int + CID string + TD string + Reward string + ID int + BaseFee string `db:"base_fee"` } header := new(res) err = db.QueryRowx(pgStr, mocks.BlockNumber.Uint64()).StructScan(header) @@ -184,6 +184,7 @@ func TestPublishAndIndexer(t *testing.T) { shared.ExpectEqual(t, header.CID, headerCID.String()) shared.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String()) shared.ExpectEqual(t, header.Reward, "2000000000000021250") + shared.ExpectEqual(t, header.BaseFee, mocks.MockHeader.BaseFee.String()) dc, err := cid.Decode(header.CID) if err != nil { t.Fatal(err) From 43960d523308ee75d33d52f5dd1c499fad86777d Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Thu, 1 Jul 2021 11:34:01 +0530 Subject: [PATCH 07/11] Fix base_fee type and add backward compatible test. --- .../00004_create_eth_header_cids_table.sql | 2 +- statediff/indexer/indexer_legacy_test.go | 86 ++++++++++++ statediff/indexer/mocks/test_data.go | 128 +++++++++++++++++- 3 files changed, 210 insertions(+), 6 deletions(-) create mode 100644 statediff/indexer/indexer_legacy_test.go diff --git a/statediff/db/migrations/00004_create_eth_header_cids_table.sql b/statediff/db/migrations/00004_create_eth_header_cids_table.sql index 5993f5a36e7d..13bb4e317f85 100644 --- a/statediff/db/migrations/00004_create_eth_header_cids_table.sql +++ b/statediff/db/migrations/00004_create_eth_header_cids_table.sql @@ -16,7 +16,7 @@ CREATE TABLE eth.header_cids ( bloom BYTEA NOT NULL, timestamp NUMERIC NOT NULL, times_validated INTEGER NOT NULL DEFAULT 1, - base_fee BIGINT NOT NULL, + base_fee VARCHAR(38), UNIQUE (block_number, block_hash) ); diff --git a/statediff/indexer/indexer_legacy_test.go b/statediff/indexer/indexer_legacy_test.go new file mode 100644 index 000000000000..bef319354556 --- /dev/null +++ b/statediff/indexer/indexer_legacy_test.go @@ -0,0 +1,86 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package indexer_test + +import ( + "testing" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff/indexer" + "github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld" + "github.com/ethereum/go-ethereum/statediff/indexer/mocks" + "github.com/ethereum/go-ethereum/statediff/indexer/shared" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" +) + +var ( + legacyData = mocks.NewLegacyData() + mockLegacyBlock *types.Block + legacyHeaderCID cid.Cid +) + +func setupLegacy(t *testing.T) { + mockLegacyBlock = legacyData.MockBlock + legacyHeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, legacyData.MockHeaderRlp, multihash.KECCAK_256) + + db, err = shared.SetupDB() + require.NoError(t, err) + + ind = indexer.NewStateDiffIndexer(legacyData.Config, db) + var tx *indexer.BlockTx + tx, err = ind.PushBlock( + mockLegacyBlock, + legacyData.MockReceipts, + legacyData.MockBlock.Difficulty()) + require.NoError(t, err) + + defer tx.Close(err) + for _, node := range legacyData.StateDiffs { + err = ind.PushStateNode(tx, node) + require.NoError(t, err) + } + + shared.ExpectEqual(t, tx.BlockNumber, legacyData.BlockNumber.Uint64()) +} + +func TestPublishAndIndexerLegacy(t *testing.T) { + t.Run("Publish and index header IPLDs in a legacy tx", func(t *testing.T) { + setupLegacy(t) + defer tearDown(t) + pgStr := `SELECT cid, td, reward, id, base_fee + FROM eth.header_cids + WHERE block_number = $1` + // check header was properly indexed + type res struct { + CID string + TD string + Reward string + ID int + BaseFee string `db:"base_fee"` + } + header := new(res) + err = db.QueryRowx(pgStr, legacyData.BlockNumber.Uint64()).StructScan(header) + require.NoError(t, err) + + shared.ExpectEqual(t, header.CID, legacyHeaderCID.String()) + shared.ExpectEqual(t, header.TD, legacyData.MockBlock.Difficulty().String()) + shared.ExpectEqual(t, header.Reward, "5000000000000011250") + shared.ExpectEqual(t, header.BaseFee, legacyData.MockHeader.BaseFee.String()) + }) +} diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go index f132cd1fab0b..a8bd0430dca1 100644 --- a/statediff/indexer/mocks/test_data.go +++ b/statediff/indexer/mocks/test_data.go @@ -53,7 +53,7 @@ var ( Extra: []byte{}, BaseFee: big.NewInt(params.InitialBaseFee), } - MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts() + MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts(TestConfig, BlockNumber) MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts, new(trie.Trie)) MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header()) Address = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") @@ -182,14 +182,132 @@ type AccessListTx struct { */ +type LegacyData struct { + Config *params.ChainConfig + BlockNumber *big.Int + MockHeader types.Header + MockTransactions types.Transactions + MockReceipts types.Receipts + SenderAddr common.Address + MockBlock *types.Block + MockHeaderRlp []byte + Address []byte + AnotherAddress []byte + ContractAddress common.Address + MockContractByteCode []byte + MockLog1 *types.Log + MockLog2 *types.Log + StorageLeafKey []byte + MockStorageLeafKey []byte + StorageLeafNode []byte + ContractLeafKey []byte + ContractAccount []byte + ContractPartialPath []byte + ContractLeafNode []byte + AccountRoot string + AccountLeafNode []byte + StateDiffs []sdtypes.StateNode +} + +func NewLegacyData() *LegacyData { + config := params.MainnetChainConfig + // Block number before london fork. + blockNumber := config.EIP155Block + + mockHeader := types.Header{ + Time: 0, + Number: new(big.Int).Set(blockNumber), + Root: common.HexToHash("0x0"), + TxHash: common.HexToHash("0x0"), + ReceiptHash: common.HexToHash("0x0"), + Difficulty: big.NewInt(5000000), + Extra: []byte{}, + } + + mockTransactions, mockReceipts, senderAddr := createLegacyTransactionsAndReceipts(config, blockNumber) + mockBlock := types.NewBlock(&mockHeader, mockTransactions, nil, mockReceipts, new(trie.Trie)) + mockHeaderRlp, _ := rlp.EncodeToBytes(mockBlock.Header()) + contractAddress := crypto.CreateAddress(senderAddr, mockTransactions[2].Nonce()) + + return &LegacyData{ + Config: config, + BlockNumber: blockNumber, + MockHeader: mockHeader, + MockTransactions: mockTransactions, + MockReceipts: mockReceipts, + SenderAddr: senderAddr, + MockBlock: mockBlock, + MockHeaderRlp: mockHeaderRlp, + ContractAddress: contractAddress, + MockContractByteCode: MockContractByteCode, + MockLog1: MockLog1, + MockLog2: MockLog2, + StorageLeafKey: StorageLeafKey, + MockStorageLeafKey: MockStorageLeafKey, + StorageLeafNode: StorageLeafNode, + ContractLeafKey: ContractLeafKey, + ContractAccount: ContractAccount, + ContractPartialPath: ContractPartialPath, + ContractLeafNode: ContractLeafNode, + AccountRoot: AccountRoot, + AccountLeafNode: AccountLeafKey, + StateDiffs: StateDiffs, + } +} + +// createLegacyTransactionsAndReceipts is a helper function to generate signed mock legacy transactions and mock receipts with mock logs +func createLegacyTransactionsAndReceipts(config *params.ChainConfig, blockNumber *big.Int) (types.Transactions, types.Receipts, common.Address) { + // make transactions + trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{}) + trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{}) + trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode) + + transactionSigner := types.MakeSigner(config, blockNumber) + mockCurve := elliptic.P256() + mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader) + if err != nil { + log.Crit(err.Error()) + } + signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + signedTrx2, err := types.SignTx(trx2, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + signedTrx3, err := types.SignTx(trx3, transactionSigner, mockPrvKey) + if err != nil { + log.Crit(err.Error()) + } + + senderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx + if err != nil { + log.Crit(err.Error()) + } + + // make receipts + mockReceipt1 := types.NewReceipt(nil, false, 50) + mockReceipt1.Logs = []*types.Log{MockLog1} + mockReceipt1.TxHash = signedTrx1.Hash() + mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100) + mockReceipt2.Logs = []*types.Log{MockLog2} + mockReceipt2.TxHash = signedTrx2.Hash() + mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 75) + mockReceipt3.Logs = []*types.Log{} + mockReceipt3.TxHash = signedTrx3.Hash() + + return types.Transactions{signedTrx1, signedTrx2, signedTrx3}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3}, senderAddr +} + // createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs -func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) { +func createTransactionsAndReceipts(config *params.ChainConfig, blockNumber *big.Int) (types.Transactions, types.Receipts, common.Address) { // make transactions trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{}) trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{}) trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode) trx4 := types.NewTx(&types.AccessListTx{ - ChainID: TestConfig.ChainID, + ChainID: config.ChainID, Nonce: 0, GasPrice: big.NewInt(100), Gas: 50, @@ -202,7 +320,7 @@ func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common }, }) trx5 := types.NewTx(&types.DynamicFeeTx{ - ChainID: TestConfig.ChainID, + ChainID: config.ChainID, Nonce: 0, GasTipCap: big.NewInt(100), GasFeeCap: big.NewInt(100), @@ -216,7 +334,7 @@ func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common }, }) - transactionSigner := types.MakeSigner(TestConfig, BlockNumber) + transactionSigner := types.MakeSigner(config, blockNumber) mockCurve := elliptic.P256() mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader) if err != nil { From f0a8824f8883137e24ea42f3badc0149fccf24cc Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Thu, 1 Jul 2021 11:45:11 +0530 Subject: [PATCH 08/11] Remove type definition for AccessListElementModel --- statediff/indexer/indexer.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/statediff/indexer/indexer.go b/statediff/indexer/indexer.go index 85d0018d0595..c67442760f98 100644 --- a/statediff/indexer/indexer.go +++ b/statediff/indexer/indexer.go @@ -23,8 +23,6 @@ import ( "math/big" "time" - "github.com/lib/pq" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -335,14 +333,6 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *sqlx.Tx, args processArgs return err } - // AccessListEntryModel is the db model for eth.access_list_entry - type AccessListElementModel struct { - ID int64 `db:"id"` - Index int64 `db:"index"` - TxID int64 `db:"tx_id"` - Address string `db:"address"` - StorageKeys pq.StringArray `db:"storage_keys"` - } // index access list if this is one for j, accessListElement := range trx.AccessList() { storageKeys := make([]string, len(accessListElement.StorageKeys)) From 3eb4467799eb854b091e2d626bb4fe782be08f98 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Thu, 1 Jul 2021 21:10:12 +0530 Subject: [PATCH 09/11] Change basefee to int64/bigint. --- .../db/migrations/00004_create_eth_header_cids_table.sql | 2 +- statediff/indexer/indexer.go | 8 +++++++- statediff/indexer/indexer_legacy_test.go | 5 +++-- statediff/indexer/indexer_test.go | 4 ++-- statediff/indexer/models/models.go | 2 +- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/statediff/db/migrations/00004_create_eth_header_cids_table.sql b/statediff/db/migrations/00004_create_eth_header_cids_table.sql index 13bb4e317f85..f7231a30be57 100644 --- a/statediff/db/migrations/00004_create_eth_header_cids_table.sql +++ b/statediff/db/migrations/00004_create_eth_header_cids_table.sql @@ -16,7 +16,7 @@ CREATE TABLE eth.header_cids ( bloom BYTEA NOT NULL, timestamp NUMERIC NOT NULL, times_validated INTEGER NOT NULL DEFAULT 1, - base_fee VARCHAR(38), + base_fee BIGINT DEFAULT 0, UNIQUE (block_number, block_hash) ); diff --git a/statediff/indexer/indexer.go b/statediff/indexer/indexer.go index c67442760f98..ebb53aca8de1 100644 --- a/statediff/indexer/indexer.go +++ b/statediff/indexer/indexer.go @@ -212,6 +212,12 @@ func (sdi *StateDiffIndexer) processHeader(tx *sqlx.Tx, header *types.Header, he if err := shared.PublishIPLD(tx, headerNode); err != nil { return 0, fmt.Errorf("error publishing header IPLD: %v", err) } + + var baseFee int64 + if header.BaseFee != nil { + baseFee = header.BaseFee.Int64() + } + // index header return sdi.dbWriter.upsertHeaderCID(tx, models.HeaderModel{ CID: headerNode.Cid().String(), @@ -227,7 +233,7 @@ func (sdi *StateDiffIndexer) processHeader(tx *sqlx.Tx, header *types.Header, he TxRoot: header.TxHash.String(), UncleRoot: header.UncleHash.String(), Timestamp: header.Time, - BaseFee: header.BaseFee.String(), + BaseFee: baseFee, }) } diff --git a/statediff/indexer/indexer_legacy_test.go b/statediff/indexer/indexer_legacy_test.go index bef319354556..59b02b6e43a7 100644 --- a/statediff/indexer/indexer_legacy_test.go +++ b/statediff/indexer/indexer_legacy_test.go @@ -72,7 +72,7 @@ func TestPublishAndIndexerLegacy(t *testing.T) { TD string Reward string ID int - BaseFee string `db:"base_fee"` + BaseFee int64 `db:"base_fee"` } header := new(res) err = db.QueryRowx(pgStr, legacyData.BlockNumber.Uint64()).StructScan(header) @@ -81,6 +81,7 @@ func TestPublishAndIndexerLegacy(t *testing.T) { shared.ExpectEqual(t, header.CID, legacyHeaderCID.String()) shared.ExpectEqual(t, header.TD, legacyData.MockBlock.Difficulty().String()) shared.ExpectEqual(t, header.Reward, "5000000000000011250") - shared.ExpectEqual(t, header.BaseFee, legacyData.MockHeader.BaseFee.String()) + require.Nil(t, legacyData.MockHeader.BaseFee) + shared.ExpectEqual(t, header.BaseFee, int64(0)) }) } diff --git a/statediff/indexer/indexer_test.go b/statediff/indexer/indexer_test.go index 9af8622c9d04..8ae522e875b8 100644 --- a/statediff/indexer/indexer_test.go +++ b/statediff/indexer/indexer_test.go @@ -174,7 +174,7 @@ func TestPublishAndIndexer(t *testing.T) { TD string Reward string ID int - BaseFee string `db:"base_fee"` + BaseFee int64 `db:"base_fee"` } header := new(res) err = db.QueryRowx(pgStr, mocks.BlockNumber.Uint64()).StructScan(header) @@ -184,7 +184,7 @@ func TestPublishAndIndexer(t *testing.T) { shared.ExpectEqual(t, header.CID, headerCID.String()) shared.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String()) shared.ExpectEqual(t, header.Reward, "2000000000000021250") - shared.ExpectEqual(t, header.BaseFee, mocks.MockHeader.BaseFee.String()) + shared.ExpectEqual(t, header.BaseFee, mocks.MockHeader.BaseFee.Int64()) dc, err := cid.Decode(header.CID) if err != nil { t.Fatal(err) diff --git a/statediff/indexer/models/models.go b/statediff/indexer/models/models.go index 2855c5889c03..4dcb7492dc02 100644 --- a/statediff/indexer/models/models.go +++ b/statediff/indexer/models/models.go @@ -36,7 +36,7 @@ type HeaderModel struct { Bloom []byte `db:"bloom"` Timestamp uint64 `db:"timestamp"` TimesValidated int64 `db:"times_validated"` - BaseFee string `db:"base_fee"` + BaseFee int64 `db:"base_fee"` } // UncleModel is the db model for eth.uncle_cids From 60eed589974413aef5fcc5ebed4a62169682267f Mon Sep 17 00:00:00 2001 From: Ramil Amerzyanov Date: Wed, 23 Jun 2021 23:30:27 +0300 Subject: [PATCH 10/11] block and uncle reward in PoA network = 0 (#87) * in PoA networks there is no block and uncle rewards * bump meta version (cherry picked from commit b64ca14689689178b78f915c855fe75010d886c7) --- statediff/indexer/indexer.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/statediff/indexer/indexer.go b/statediff/indexer/indexer.go index ebb53aca8de1..255d90b73b72 100644 --- a/statediff/indexer/indexer.go +++ b/statediff/indexer/indexer.go @@ -117,7 +117,13 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip return nil, fmt.Errorf("expected number of transactions (%d), transaction trie nodes (%d), receipts (%d), and receipt trie nodes (%d)to be equal", len(txNodes), len(txTrieNodes), len(rctNodes), len(rctTrieNodes)) } // Calculate reward - reward := CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts) + var reward *big.Int + // in PoA networks block reward is 0 + if sdi.chainConfig.Clique != nil { + reward = big.NewInt(0) + } else { + reward = CalcEthBlockReward(block.Header(), block.Uncles(), block.Transactions(), receipts) + } t = time.Now() // Begin new db tx for everything tx, err := sdi.dbWriter.db.Beginx() @@ -243,7 +249,13 @@ func (sdi *StateDiffIndexer) processUncles(tx *sqlx.Tx, headerID int64, blockNum if err := shared.PublishIPLD(tx, uncleNode); err != nil { return fmt.Errorf("error publishing uncle IPLD: %v", err) } - uncleReward := CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64()) + var uncleReward *big.Int + // in PoA networks uncle reward is 0 + if sdi.chainConfig.Clique != nil { + uncleReward = big.NewInt(0) + } else { + uncleReward = CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64()) + } uncle := models.UncleModel{ CID: uncleNode.Cid().String(), MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()), From 0f14f515e01b15ceb6a5774b53bd372ae0984001 Mon Sep 17 00:00:00 2001 From: Arijit Das Date: Thu, 1 Jul 2021 22:02:08 +0530 Subject: [PATCH 11/11] Use Ropsten to test block reward. --- statediff/indexer/mocks/test_data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statediff/indexer/mocks/test_data.go b/statediff/indexer/mocks/test_data.go index a8bd0430dca1..a713b744a9ad 100644 --- a/statediff/indexer/mocks/test_data.go +++ b/statediff/indexer/mocks/test_data.go @@ -41,7 +41,7 @@ import ( var ( // block data // TODO: Update this to `MainnetChainConfig` when `LondonBlock` is added - TestConfig = params.RinkebyChainConfig + TestConfig = params.RopstenChainConfig BlockNumber = TestConfig.LondonBlock MockHeader = types.Header{ Time: 0,