From b6380d5e4eca7bd37e7465fa242c6924e69531ea Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Tue, 29 Oct 2024 10:21:05 +0000 Subject: [PATCH 01/25] Thor client (#818) * feat: add thorclient * refactor: remove roundTripper * refactor: change null check * clean: remove commented code * feat: add account revision and pending tx * fix: add licence headers and fix linter issue * refactor: rename package * refactor: change revision type to string * refactor: rename GetLogs and GetTransfers to FilterEvents and FilterTransfers * refactor: change FilterEvents and FilterTransactions request type to EventFilter * Adding common.EventWrapper to handle channel errors * tweak * update rawclient + update account tests * tidy up names * update tests * pr comments * adding raw tx * Tidy up method names and calls * options client * tweaks * pr comments * Update thorclient/common/common.go Co-authored-by: libotony * pr comments * Adding Subscriptions * Pr comments * adjust func orders * pr comments * changing subscribe to use the channel close vs multiple channels * adding go-doc * no error after unsubscribe * pr comments * checking status code is 2xx * fix: change FilterTransfers argument --------- Co-authored-by: otherview Co-authored-by: libotony --- api/accounts/accounts.go | 4 +- api/accounts/accounts_test.go | 193 ++++++---- api/accounts/types.go | 8 + api/blocks/blocks_test.go | 49 ++- api/debug/debug_test.go | 108 +++--- api/events/events_test.go | 39 +- api/node/node_test.go | 12 +- api/transactions/transactions.go | 11 +- api/transactions/transactions_test.go | 92 ++--- api/transactions/types.go | 7 +- api/transfers/transfers_test.go | 68 ++-- bft/engine_test.go | 14 +- test/datagen/address.go | 6 +- thorclient/common/common.go | 33 ++ thorclient/httpclient/client.go | 309 ++++++++++++++++ thorclient/httpclient/client_test.go | 501 ++++++++++++++++++++++++++ thorclient/httpclient/http_request.go | 75 ++++ thorclient/thorclient.go | 274 ++++++++++++++ thorclient/thorclient_test.go | 130 +++++++ thorclient/wsclient/client.go | 218 +++++++++++ thorclient/wsclient/client_test.go | 449 +++++++++++++++++++++++ 21 files changed, 2318 insertions(+), 282 deletions(-) create mode 100644 thorclient/common/common.go create mode 100644 thorclient/httpclient/client.go create mode 100644 thorclient/httpclient/client_test.go create mode 100644 thorclient/httpclient/http_request.go create mode 100644 thorclient/thorclient.go create mode 100644 thorclient/thorclient_test.go create mode 100644 thorclient/wsclient/client.go create mode 100644 thorclient/wsclient/client_test.go diff --git a/api/accounts/accounts.go b/api/accounts/accounts.go index a6d1df3ee..4bd940709 100644 --- a/api/accounts/accounts.go +++ b/api/accounts/accounts.go @@ -81,7 +81,7 @@ func (a *Accounts) handleGetCode(w http.ResponseWriter, req *http.Request) error return err } - return utils.WriteJSON(w, map[string]string{"code": hexutil.Encode(code)}) + return utils.WriteJSON(w, &GetCodeResult{Code: hexutil.Encode(code)}) } func (a *Accounts) getAccount(addr thor.Address, header *block.Header, state *state.State) (*Account, error) { @@ -164,7 +164,7 @@ func (a *Accounts) handleGetStorage(w http.ResponseWriter, req *http.Request) er if err != nil { return err } - return utils.WriteJSON(w, map[string]string{"value": storage.String()}) + return utils.WriteJSON(w, &GetStorageResult{Value: storage.String()}) } func (a *Accounts) handleCallContract(w http.ResponseWriter, req *http.Request) error { diff --git a/api/accounts/accounts_test.go b/api/accounts/accounts_test.go index d9f66fa24..cd4aac3cb 100644 --- a/api/accounts/accounts_test.go +++ b/api/accounts/accounts_test.go @@ -21,6 +21,7 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ABI "github.com/vechain/thor/v2/abi" "github.com/vechain/thor/v2/api/accounts" "github.com/vechain/thor/v2/block" @@ -31,6 +32,8 @@ import ( "github.com/vechain/thor/v2/packer" "github.com/vechain/thor/v2/state" "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/thorclient" + tccommon "github.com/vechain/thor/v2/thorclient/common" "github.com/vechain/thor/v2/tx" ) @@ -84,58 +87,64 @@ var abiJSON = `[ "type": "function" } ]` -var addr = thor.BytesToAddress([]byte("to")) -var value = big.NewInt(10000) -var storageKey = thor.Bytes32{} -var storageValue = byte(1) -var gasLimit uint64 -var genesisBlock *block.Block -var contractAddr thor.Address - -var bytecode = common.Hex2Bytes("608060405234801561001057600080fd5b50610125806100206000396000f3006080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806324b8ba5f14604e578063bb4e3f4d14607b575b600080fd5b348015605957600080fd5b506079600480360381019080803560ff16906020019092919050505060cf565b005b348015608657600080fd5b5060b3600480360381019080803560ff169060200190929190803560ff16906020019092919050505060ec565b604051808260ff1660ff16815260200191505060405180910390f35b806000806101000a81548160ff021916908360ff16021790555050565b60008183019050929150505600a165627a7a723058201584add23e31d36c569b468097fe01033525686b59bbb263fb3ab82e9553dae50029") - -var runtimeBytecode = common.Hex2Bytes("6080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806324b8ba5f14604e578063bb4e3f4d14607b575b600080fd5b348015605957600080fd5b506079600480360381019080803560ff16906020019092919050505060cf565b005b348015608657600080fd5b5060b3600480360381019080803560ff169060200190929190803560ff16906020019092919050505060ec565b604051808260ff1660ff16815260200191505060405180910390f35b806000806101000a81548160ff021916908360ff16021790555050565b60008183019050929150505600a165627a7a723058201584add23e31d36c569b468097fe01033525686b59bbb263fb3ab82e9553dae50029") - -var invalidAddr = "abc" //invlaid address -var invalidBytes32 = "0x000000000000000000000000000000000000000000000000000000000000000g" //invlaid bytes32 -var invalidNumberRevision = "4294967296" //invalid block number +const ( + storageValue = byte(1) + invalidAddr = "abc" // invalid address + invalidBytes32 = "0x000000000000000000000000000000000000000000000000000000000000000g" // invalid bytes32 + invalidNumberRevision = "4294967296" // invalid block number +) -var acc *accounts.Accounts -var ts *httptest.Server +var ( + gasLimit uint64 + addr = thor.BytesToAddress([]byte("to")) + value = big.NewInt(10000) + storageKey = thor.Bytes32{} + genesisBlock *block.Block + contractAddr thor.Address + bytecode = common.Hex2Bytes("608060405234801561001057600080fd5b50610125806100206000396000f3006080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806324b8ba5f14604e578063bb4e3f4d14607b575b600080fd5b348015605957600080fd5b506079600480360381019080803560ff16906020019092919050505060cf565b005b348015608657600080fd5b5060b3600480360381019080803560ff169060200190929190803560ff16906020019092919050505060ec565b604051808260ff1660ff16815260200191505060405180910390f35b806000806101000a81548160ff021916908360ff16021790555050565b60008183019050929150505600a165627a7a723058201584add23e31d36c569b468097fe01033525686b59bbb263fb3ab82e9553dae50029") + runtimeBytecode = common.Hex2Bytes("6080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806324b8ba5f14604e578063bb4e3f4d14607b575b600080fd5b348015605957600080fd5b506079600480360381019080803560ff16906020019092919050505060cf565b005b348015608657600080fd5b5060b3600480360381019080803560ff169060200190929190803560ff16906020019092919050505060ec565b604051808260ff1660ff16815260200191505060405180910390f35b806000806101000a81548160ff021916908360ff16021790555050565b60008183019050929150505600a165627a7a723058201584add23e31d36c569b468097fe01033525686b59bbb263fb3ab82e9553dae50029") + acc *accounts.Accounts + ts *httptest.Server + tclient *thorclient.Client +) func TestAccount(t *testing.T) { initAccountServer(t) defer ts.Close() + tclient = thorclient.New(ts.URL) for name, tt := range map[string]func(*testing.T){ - "getAccount": getAccount, - "getAccountWithNonExisitingRevision": getAccountWithNonExisitingRevision, - "getAccountWithGenesisRevision": getAccountWithGenesisRevision, - "getAccountWithFinalizedRevision": getAccountWithFinalizedRevision, - "getCode": getCode, - "getCodeWithNonExisitingRevision": getCodeWithNonExisitingRevision, - "getStorage": getStorage, - "getStorageWithNonExisitingRevision": getStorageWithNonExisitingRevision, - "deployContractWithCall": deployContractWithCall, - "callContract": callContract, - "callContractWithNonExisitingRevision": callContractWithNonExisitingRevision, - "batchCall": batchCall, - "batchCallWithNonExisitingRevision": batchCallWithNonExisitingRevision, + "getAccount": getAccount, + "getAccountWithNonExistingRevision": getAccountWithNonExistingRevision, + "getAccountWithGenesisRevision": getAccountWithGenesisRevision, + "getAccountWithFinalizedRevision": getAccountWithFinalizedRevision, + "getCode": getCode, + "getCodeWithNonExistingRevision": getCodeWithNonExistingRevision, + "getStorage": getStorage, + "getStorageWithNonExistingRevision": getStorageWithNonExistingRevision, + "deployContractWithCall": deployContractWithCall, + "callContract": callContract, + "callContractWithNonExistingRevision": callContractWithNonExistingRevision, + "batchCall": batchCall, + "batchCallWithNonExistingRevision": batchCallWithNonExistingRevision, } { t.Run(name, tt) } } func getAccount(t *testing.T) { - _, statusCode := httpGet(t, ts.URL+"/accounts/"+invalidAddr) + _, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/accounts/" + invalidAddr) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad address") - _, statusCode = httpGet(t, ts.URL+"/accounts/"+addr.String()+"?revision="+invalidNumberRevision) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPGet("/accounts/" + addr.String() + "?revision=" + invalidNumberRevision) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad revision") - //revision is optional defaut `best` - res, statusCode := httpGet(t, ts.URL+"/accounts/"+addr.String()) + //revision is optional default `best` + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/accounts/" + addr.String()) + require.NoError(t, err) var acc accounts.Account if err := json.Unmarshal(res, &acc); err != nil { t.Fatal(err) @@ -144,17 +153,19 @@ func getAccount(t *testing.T) { assert.Equal(t, http.StatusOK, statusCode, "OK") } -func getAccountWithNonExisitingRevision(t *testing.T) { +func getAccountWithNonExistingRevision(t *testing.T) { revision64Len := "0x00000000851caf3cfdb6e899cf5958bfb1ac3413d346d43539627e6be7ec1b4a" - res, statusCode := httpGet(t, ts.URL+"/accounts/"+addr.String()+"?revision="+revision64Len) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/accounts/" + addr.String() + "?revision=" + revision64Len) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad revision") assert.Equal(t, "revision: leveldb: not found\n", string(res), "revision not found") } func getAccountWithGenesisRevision(t *testing.T) { - res, statusCode := httpGet(t, ts.URL+"/accounts/"+addr.String()+"?revision="+genesisBlock.Header().ID().String()) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/accounts/" + addr.String() + "?revision=" + genesisBlock.Header().ID().String()) + require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode, "bad revision") var acc accounts.Account @@ -174,10 +185,13 @@ func getAccountWithGenesisRevision(t *testing.T) { } func getAccountWithFinalizedRevision(t *testing.T) { - soloAddress := "0xf077b491b355E64048cE21E3A6Fc4751eEeA77fa" + soloAddress := thor.MustParseAddress("0xf077b491b355E64048cE21E3A6Fc4751eEeA77fa") - genesisAccount := httpGetAccount(t, soloAddress+"?revision="+genesisBlock.Header().ID().String()) - finalizedAccount := httpGetAccount(t, soloAddress+"?revision=finalized") + genesisAccount, err := tclient.Account(&soloAddress, thorclient.Revision(genesisBlock.Header().ID().String())) + require.NoError(t, err) + + finalizedAccount, err := tclient.Account(&soloAddress, thorclient.Revision(tccommon.FinalizedRevision)) + require.NoError(t, err) genesisEnergy := (*big.Int)(&genesisAccount.Energy) finalizedEnergy := (*big.Int)(&finalizedAccount.Energy) @@ -186,14 +200,17 @@ func getAccountWithFinalizedRevision(t *testing.T) { } func getCode(t *testing.T) { - _, statusCode := httpGet(t, ts.URL+"/accounts/"+invalidAddr+"/code") + _, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/accounts/" + invalidAddr + "/code") + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad address") - _, statusCode = httpGet(t, ts.URL+"/accounts/"+contractAddr.String()+"/code?revision="+invalidNumberRevision) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPGet("/accounts/" + contractAddr.String() + "/code?revision=" + invalidNumberRevision) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad revision") //revision is optional defaut `best` - res, statusCode := httpGet(t, ts.URL+"/accounts/"+contractAddr.String()+"/code") + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/accounts/" + contractAddr.String() + "/code") + require.NoError(t, err) var code map[string]string if err := json.Unmarshal(res, &code); err != nil { t.Fatal(err) @@ -206,27 +223,32 @@ func getCode(t *testing.T) { assert.Equal(t, http.StatusOK, statusCode, "OK") } -func getCodeWithNonExisitingRevision(t *testing.T) { +func getCodeWithNonExistingRevision(t *testing.T) { revision64Len := "0x00000000851caf3cfdb6e899cf5958bfb1ac3413d346d43539627e6be7ec1b4a" - res, statusCode := httpGet(t, ts.URL+"/accounts/"+contractAddr.String()+"/code?revision="+revision64Len) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/accounts/" + contractAddr.String() + "/code?revision=" + revision64Len) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad revision") assert.Equal(t, "revision: leveldb: not found\n", string(res), "revision not found") } func getStorage(t *testing.T) { - _, statusCode := httpGet(t, ts.URL+"/accounts/"+invalidAddr+"/storage/"+storageKey.String()) + _, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/accounts/" + invalidAddr + "/storage/" + storageKey.String()) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad address") - _, statusCode = httpGet(t, ts.URL+"/accounts/"+contractAddr.String()+"/storage/"+invalidBytes32) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPGet("/accounts/" + contractAddr.String() + "/storage/" + invalidBytes32) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad storage key") - _, statusCode = httpGet(t, ts.URL+"/accounts/"+contractAddr.String()+"/storage/"+storageKey.String()+"?revision="+invalidNumberRevision) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPGet("/accounts/" + contractAddr.String() + "/storage/" + storageKey.String() + "?revision=" + invalidNumberRevision) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad revision") //revision is optional defaut `best` - res, statusCode := httpGet(t, ts.URL+"/accounts/"+contractAddr.String()+"/storage/"+storageKey.String()) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/accounts/" + contractAddr.String() + "/storage/" + storageKey.String()) + require.NoError(t, err) var value map[string]string if err := json.Unmarshal(res, &value); err != nil { t.Fatal(err) @@ -239,10 +261,11 @@ func getStorage(t *testing.T) { assert.Equal(t, http.StatusOK, statusCode, "OK") } -func getStorageWithNonExisitingRevision(t *testing.T) { +func getStorageWithNonExistingRevision(t *testing.T) { revision64Len := "0x00000000851caf3cfdb6e899cf5958bfb1ac3413d346d43539627e6be7ec1b4a" - res, statusCode := httpGet(t, ts.URL+"/accounts/"+contractAddr.String()+"/storage/"+storageKey.String()+"?revision="+revision64Len) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/accounts/" + contractAddr.String() + "/storage/" + storageKey.String() + "?revision=" + revision64Len) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad revision") assert.Equal(t, "revision: leveldb: not found\n", string(res), "revision not found") @@ -327,7 +350,8 @@ func deployContractWithCall(t *testing.T) { Gas: 10000000, Data: "abc", } - _, statusCode := httpPost(t, ts.URL+"/accounts", badBody) + _, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/accounts", badBody) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad data") reqBody := &accounts.CallData{ @@ -335,11 +359,13 @@ func deployContractWithCall(t *testing.T) { Data: hexutil.Encode(bytecode), } - _, statusCode = httpPost(t, ts.URL+"/accounts?revision="+invalidNumberRevision, reqBody) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/accounts?revision="+invalidNumberRevision, reqBody) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad revision") //revision is optional defaut `best` - res, _ := httpPost(t, ts.URL+"/accounts", reqBody) + res, _, err := tclient.RawHTTPClient().RawHTTPPost("/accounts", reqBody) + require.NoError(t, err) var output *accounts.CallResult if err := json.Unmarshal(res, &output); err != nil { t.Fatal(err) @@ -348,20 +374,24 @@ func deployContractWithCall(t *testing.T) { } func callContract(t *testing.T) { - _, statusCode := httpPost(t, ts.URL+"/accounts/"+invalidAddr, nil) + _, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/accounts/"+invalidAddr, nil) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "invalid address") malFormedBody := 123 - _, statusCode = httpPost(t, ts.URL+"/accounts", malFormedBody) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/accounts", malFormedBody) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "invalid address") - _, statusCode = httpPost(t, ts.URL+"/accounts/"+contractAddr.String(), malFormedBody) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/accounts/"+contractAddr.String(), malFormedBody) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "invalid address") badBody := &accounts.CallData{ Data: "input", } - _, statusCode = httpPost(t, ts.URL+"/accounts/"+contractAddr.String(), badBody) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/accounts/"+contractAddr.String(), badBody) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "invalid input data") a := uint8(1) @@ -378,12 +408,16 @@ func callContract(t *testing.T) { } // next revisoun should be valid - _, statusCode = httpPost(t, ts.URL+"/accounts/"+contractAddr.String()+"?revision=next", reqBody) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/accounts/"+contractAddr.String()+"?revision=next", reqBody) + require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode, "next revision should be okay") - _, statusCode = httpPost(t, ts.URL+"/accounts?revision=next", reqBody) + + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/accounts?revision=next", reqBody) + require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode, "next revision should be okay") - res, statusCode := httpPost(t, ts.URL+"/accounts/"+contractAddr.String(), reqBody) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/accounts/"+contractAddr.String(), reqBody) + require.NoError(t, err) var output *accounts.CallResult if err = json.Unmarshal(res, &output); err != nil { t.Fatal(err) @@ -401,10 +435,11 @@ func callContract(t *testing.T) { assert.Equal(t, a+b, ret) } -func callContractWithNonExisitingRevision(t *testing.T) { +func callContractWithNonExistingRevision(t *testing.T) { revision64Len := "0x00000000851caf3cfdb6e899cf5958bfb1ac3413d346d43539627e6be7ec1b4a" - res, statusCode := httpPost(t, ts.URL+"/accounts/"+contractAddr.String()+"?revision="+revision64Len, nil) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/accounts/"+contractAddr.String()+"?revision="+revision64Len, nil) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad revision") assert.Equal(t, "revision: leveldb: not found\n", string(res), "revision not found") @@ -413,7 +448,8 @@ func callContractWithNonExisitingRevision(t *testing.T) { func batchCall(t *testing.T) { // Request body is not a valid JSON malformedBody := 123 - _, statusCode := httpPost(t, ts.URL+"/accounts/*", malformedBody) + _, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/accounts/*", malformedBody) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "malformed data") // Request body is not a valid BatchCallData @@ -430,22 +466,26 @@ func batchCall(t *testing.T) { Value: nil, }}, } - _, statusCode = httpPost(t, ts.URL+"/accounts/*", badBody) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/accounts/*", badBody) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "invalid data") // Request body has an invalid blockRef badBlockRef := &accounts.BatchCallData{ BlockRef: "0x00", } - _, statusCode = httpPost(t, ts.URL+"/accounts/*", badBlockRef) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/accounts/*", badBlockRef) + require.NoError(t, err) assert.Equal(t, http.StatusInternalServerError, statusCode, "invalid blockRef") // Request body has an invalid malformed revision - _, statusCode = httpPost(t, fmt.Sprintf("%s/accounts/*?revision=%s", ts.URL, "0xZZZ"), badBody) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost(fmt.Sprintf("/accounts/*?revision=%s", "0xZZZ"), badBody) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "revision") // Request body has an invalid revision number - _, statusCode = httpPost(t, ts.URL+"/accounts/*?revision="+invalidNumberRevision, badBody) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/accounts/*?revision="+invalidNumberRevision, badBody) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "invalid revision") // Valid request @@ -473,10 +513,12 @@ func batchCall(t *testing.T) { } // 'next' revisoun should be valid - _, statusCode = httpPost(t, ts.URL+"/accounts/*?revision=next", reqBody) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/accounts/*?revision=next", reqBody) + require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode, "next revision should be okay") - res, statusCode := httpPost(t, ts.URL+"/accounts/*", reqBody) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/accounts/*", reqBody) + require.NoError(t, err) var results accounts.BatchCallResults if err = json.Unmarshal(res, &results); err != nil { t.Fatal(err) @@ -507,7 +549,8 @@ func batchCall(t *testing.T) { Expiration: 100, BlockRef: "0x00000000aabbccdd", } - _, statusCode = httpPost(t, ts.URL+"/accounts/*", fullBody) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/accounts/*", fullBody) + require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode) // Request with not enough gas @@ -521,14 +564,16 @@ func batchCall(t *testing.T) { Expiration: 100, BlockRef: "0x00000000aabbccdd", } - _, statusCode = httpPost(t, ts.URL+"/accounts/*", tooMuchGasBody) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/accounts/*", tooMuchGasBody) + require.NoError(t, err) assert.Equal(t, http.StatusForbidden, statusCode) } -func batchCallWithNonExisitingRevision(t *testing.T) { +func batchCallWithNonExistingRevision(t *testing.T) { revision64Len := "0x00000000851caf3cfdb6e899cf5958bfb1ac3413d346d43539627e6be7ec1b4a" - res, statusCode := httpPost(t, ts.URL+"/accounts/*?revision="+revision64Len, nil) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/accounts/*?revision="+revision64Len, nil) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode, "bad revision") assert.Equal(t, "revision: leveldb: not found\n", string(res), "revision not found") diff --git a/api/accounts/types.go b/api/accounts/types.go index d220232bb..02bf15041 100644 --- a/api/accounts/types.go +++ b/api/accounts/types.go @@ -29,6 +29,14 @@ type CallData struct { Caller *thor.Address `json:"caller"` } +type GetCodeResult struct { + Code string `json:"code"` +} + +type GetStorageResult struct { + Value string `json:"value"` +} + type CallResult struct { Data string `json:"data"` Events []*transactions.Event `json:"events"` diff --git a/api/blocks/blocks_test.go b/api/blocks/blocks_test.go index b8c9636fe..60b96b299 100644 --- a/api/blocks/blocks_test.go +++ b/api/blocks/blocks_test.go @@ -29,19 +29,26 @@ import ( "github.com/vechain/thor/v2/packer" "github.com/vechain/thor/v2/state" "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/thorclient" "github.com/vechain/thor/v2/tx" ) -var genesisBlock *block.Block -var blk *block.Block -var ts *httptest.Server +const ( + invalidBytes32 = "0x000000000000000000000000000000000000000000000000000000000000000g" // invalid bytes32 +) -var invalidBytes32 = "0x000000000000000000000000000000000000000000000000000000000000000g" //invlaid bytes32 +var ( + genesisBlock *block.Block + blk *block.Block + ts *httptest.Server + tclient *thorclient.Client +) func TestBlock(t *testing.T) { initBlockServer(t) defer ts.Close() + tclient = thorclient.New(ts.URL) for name, tt := range map[string]func(*testing.T){ "testBadQueryParams": testBadQueryParams, "testInvalidBlockID": testInvalidBlockID, @@ -61,14 +68,16 @@ func TestBlock(t *testing.T) { func testBadQueryParams(t *testing.T) { badQueryParams := "?expanded=1" - res, statusCode := httpGet(t, ts.URL+"/blocks/best"+badQueryParams) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/best" + badQueryParams) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode) assert.Equal(t, "expanded: should be boolean", strings.TrimSpace(string(res))) } func testGetBestBlock(t *testing.T) { - res, statusCode := httpGet(t, ts.URL+"/blocks/best") + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/best") + require.NoError(t, err) rb := new(blocks.JSONCollapsedBlock) if err := json.Unmarshal(res, &rb); err != nil { t.Fatal(err) @@ -78,7 +87,8 @@ func testGetBestBlock(t *testing.T) { } func testGetBlockByHeight(t *testing.T) { - res, statusCode := httpGet(t, ts.URL+"/blocks/1") + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/1") + require.NoError(t, err) rb := new(blocks.JSONCollapsedBlock) if err := json.Unmarshal(res, &rb); err != nil { t.Fatal(err) @@ -88,7 +98,8 @@ func testGetBlockByHeight(t *testing.T) { } func testGetFinalizedBlock(t *testing.T) { - res, statusCode := httpGet(t, ts.URL+"/blocks/finalized") + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/finalized") + require.NoError(t, err) finalized := new(blocks.JSONCollapsedBlock) if err := json.Unmarshal(res, &finalized); err != nil { t.Fatal(err) @@ -101,7 +112,8 @@ func testGetFinalizedBlock(t *testing.T) { } func testGetJustifiedBlock(t *testing.T) { - res, statusCode := httpGet(t, ts.URL+"/blocks/justified") + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/justified") + require.NoError(t, err) justified := new(blocks.JSONCollapsedBlock) require.NoError(t, json.Unmarshal(res, &justified)) @@ -111,7 +123,8 @@ func testGetJustifiedBlock(t *testing.T) { } func testGetBlockByID(t *testing.T) { - res, statusCode := httpGet(t, ts.URL+"/blocks/"+blk.Header().ID().String()) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/" + blk.Header().ID().String()) + require.NoError(t, err) rb := new(blocks.JSONCollapsedBlock) if err := json.Unmarshal(res, rb); err != nil { t.Fatal(err) @@ -121,14 +134,17 @@ func testGetBlockByID(t *testing.T) { } func testGetBlockNotFound(t *testing.T) { - res, statusCode := httpGet(t, ts.URL+"/blocks/0x00000000851caf3cfdb6e899cf5958bfb1ac3413d346d43539627e6be7ec1b4a") + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/0x00000000851caf3cfdb6e899cf5958bfb1ac3413d346d43539627e6be7ec1b4a") + require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode) assert.Equal(t, "null", strings.TrimSpace(string(res))) } func testGetExpandedBlockByID(t *testing.T) { - res, statusCode := httpGet(t, ts.URL+"/blocks/"+blk.Header().ID().String()+"?expanded=true") + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/" + blk.Header().ID().String() + "?expanded=true") + require.NoError(t, err) + rb := new(blocks.JSONExpandedBlock) if err := json.Unmarshal(res, rb); err != nil { t.Fatal(err) @@ -139,18 +155,21 @@ func testGetExpandedBlockByID(t *testing.T) { func testInvalidBlockNumber(t *testing.T) { invalidNumberRevision := "4294967296" //invalid block number - _, statusCode := httpGet(t, ts.URL+"/blocks/"+invalidNumberRevision) + _, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/" + invalidNumberRevision) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode) } func testInvalidBlockID(t *testing.T) { - _, statusCode := httpGet(t, ts.URL+"/blocks/"+invalidBytes32) + _, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/" + invalidBytes32) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode) } func testGetBlockWithRevisionNumberTooHigh(t *testing.T) { revisionNumberTooHigh := strconv.FormatUint(math.MaxUint64, 10) - res, statusCode := httpGet(t, ts.URL+"/blocks/"+revisionNumberTooHigh) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPGet("/blocks/" + revisionNumberTooHigh) + require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode) assert.Equal(t, "revision: block number out of max uint32", strings.TrimSpace(string(res))) diff --git a/api/debug/debug_test.go b/api/debug/debug_test.go index 90a0d7ac1..36f26c075 100644 --- a/api/debug/debug_test.go +++ b/api/debug/debug_test.go @@ -6,13 +6,10 @@ package debug import ( - "bytes" "context" "encoding/json" "fmt" - "io" "math/big" - "net/http" "net/http/httptest" "strings" "testing" @@ -22,6 +19,7 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/block" "github.com/vechain/thor/v2/builtin" "github.com/vechain/thor/v2/chain" @@ -32,6 +30,7 @@ import ( "github.com/vechain/thor/v2/state" "github.com/vechain/thor/v2/test/datagen" "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/thorclient" "github.com/vechain/thor/v2/tracers/logger" "github.com/vechain/thor/v2/tx" @@ -40,16 +39,20 @@ import ( _ "github.com/vechain/thor/v2/tracers/native" ) -var ts *httptest.Server -var blk *block.Block -var transaction *tx.Transaction -var debug *Debug +var ( + ts *httptest.Server + blk *block.Block + transaction *tx.Transaction + debug *Debug + tclient *thorclient.Client +) func TestDebug(t *testing.T) { initDebugServer(t) defer ts.Close() // /tracers endpoint + tclient = thorclient.New(ts.URL) for name, tt := range map[string]func(*testing.T){ "testTraceClauseWithInvalidTracerName": testTraceClauseWithInvalidTracerName, "testTraceClauseWithEmptyTracerTarget": testTraceClauseWithEmptyTracerTarget, @@ -157,12 +160,12 @@ func TestStorageRangeMaxResult(t *testing.T) { } func testTraceClauseWithInvalidTracerName(t *testing.T) { - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers", &TraceClauseOption{Name: "non-existent"}, 403) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers", &TraceClauseOption{Name: "non-existent"}, 403) assert.Contains(t, res, "unable to create custom tracer") } func testTraceClauseWithEmptyTracerTarget(t *testing.T) { - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers", &TraceClauseOption{Name: "structLogger"}, 400) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers", &TraceClauseOption{Name: "structLogger"}, 400) assert.Equal(t, "target: unsupported", strings.TrimSpace(res)) } @@ -171,7 +174,7 @@ func testTraceClauseWithBadBlockID(t *testing.T) { Name: "structLogger", Target: "badBlockId/x/x", } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers", traceClauseOption, 400) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers", traceClauseOption, 400) assert.Equal(t, "target[0]: invalid length", strings.TrimSpace(res)) } @@ -186,7 +189,7 @@ func testTraceClauseWithBadTxID(t *testing.T) { Name: "structLogger", Target: fmt.Sprintf("%s/badTxId/x", blk.Header().ID()), } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers", traceClauseOption, 400) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers", traceClauseOption, 400) assert.Equal(t, `target[1]: strconv.ParseUint: parsing "badTxId": invalid syntax`, strings.TrimSpace(res)) } @@ -196,7 +199,7 @@ func testTraceClauseWithNonExistingTx(t *testing.T) { Name: "structLogger", Target: fmt.Sprintf("%s/%s/x", blk.Header().ID(), nonExistingTxID), } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers", traceClauseOption, 403) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers", traceClauseOption, 403) assert.Equal(t, "transaction not found", strings.TrimSpace(res)) } @@ -206,7 +209,7 @@ func testTraceClauseWithBadClauseIndex(t *testing.T) { Name: "structLogger", Target: fmt.Sprintf("%s/%s/x", blk.Header().ID(), transaction.ID()), } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers", traceClauseOption, 400) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers", traceClauseOption, 400) assert.Equal(t, `target[2]: strconv.ParseUint: parsing "x": invalid syntax`, strings.TrimSpace(res)) // Clause index is out of range @@ -214,7 +217,7 @@ func testTraceClauseWithBadClauseIndex(t *testing.T) { Name: "structLogger", Target: fmt.Sprintf("%s/%s/%d", blk.Header().ID(), transaction.ID(), uint64(math.MaxUint64)), } - res = httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers", traceClauseOption, 400) + res = httpPostAndCheckResponseStatus(t, "/debug/tracers", traceClauseOption, 400) assert.Equal(t, `invalid target[2]`, strings.TrimSpace(res)) } @@ -223,7 +226,7 @@ func testTraceClauseWithCustomTracer(t *testing.T) { Target: fmt.Sprintf("%s/%s/1", blk.Header().ID(), transaction.ID()), Name: "nonExistingTracer", } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers", traceClauseOption, 403) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers", traceClauseOption, 403) assert.Contains(t, strings.TrimSpace(res), "create custom tracer: ReferenceError: nonExistingTracer is not defined") traceClauseOption = &TraceClauseOption{ @@ -236,7 +239,7 @@ func testTraceClauseWithCustomTracer(t *testing.T) { ReturnValue: "", StructLogs: nil, } - res = httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers", traceClauseOption, 200) + res = httpPostAndCheckResponseStatus(t, "/debug/tracers", traceClauseOption, 200) var parsedExecutionRes *logger.ExecutionResult if err := json.Unmarshal([]byte(res), &parsedExecutionRes); err != nil { @@ -256,7 +259,7 @@ func testTraceClause(t *testing.T) { ReturnValue: "", StructLogs: make([]logger.StructLogRes, 0), } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers", traceClauseOption, 200) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers", traceClauseOption, 200) var parsedExecutionRes *logger.ExecutionResult if err := json.Unmarshal([]byte(res), &parsedExecutionRes); err != nil { @@ -271,7 +274,7 @@ func testTraceClauseWithTxIndexOutOfBound(t *testing.T) { Target: fmt.Sprintf("%s/10/1", blk.Header().ID()), } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers", traceClauseOption, 403) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers", traceClauseOption, 403) assert.Equal(t, "tx index out of range", strings.TrimSpace(res)) } @@ -282,14 +285,14 @@ func testTraceClauseWithClauseIndexOutOfBound(t *testing.T) { Target: fmt.Sprintf("%s/%s/10", blk.Header().ID(), transaction.ID()), } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers", traceClauseOption, 403) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers", traceClauseOption, 403) assert.Equal(t, "clause index out of range", strings.TrimSpace(res)) } func testHandleTraceCallWithMalformedBodyRequest(t *testing.T) { badBodyRequest := "badBodyRequest" - httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers/call", badBodyRequest, 400) + httpPostAndCheckResponseStatus(t, "/debug/tracers/call", badBodyRequest, 400) } func testHandleTraceCallWithEmptyTraceCallOption(t *testing.T) { @@ -301,7 +304,7 @@ func testHandleTraceCallWithEmptyTraceCallOption(t *testing.T) { StructLogs: make([]logger.StructLogRes, 0), } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers/call", traceCallOption, 200) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers/call", traceCallOption, 200) var parsedExecutionRes *logger.ExecutionResult if err := json.Unmarshal([]byte(res), &parsedExecutionRes); err != nil { @@ -312,11 +315,11 @@ func testHandleTraceCallWithEmptyTraceCallOption(t *testing.T) { func testTraceCallNextBlock(t *testing.T) { traceCallOption := &TraceCallOption{Name: "structLogger"} - httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers/call?revision=next", traceCallOption, 200) + httpPostAndCheckResponseStatus(t, "/debug/tracers/call?revision=next", traceCallOption, 200) } func testHandleTraceCall(t *testing.T) { - addr := datagen.RandomAddress() + addr := datagen.RandAddress() provedWork := math.HexOrDecimal256(*big.NewInt(1000)) traceCallOption := &TraceCallOption{ Name: "structLogger", @@ -338,7 +341,7 @@ func testHandleTraceCall(t *testing.T) { StructLogs: make([]logger.StructLogRes, 0), } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers/call", traceCallOption, 200) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers/call", traceCallOption, 200) var parsedExecutionRes *logger.ExecutionResult if err := json.Unmarshal([]byte(res), &parsedExecutionRes); err != nil { @@ -363,7 +366,7 @@ func testHandleTraceCallWithValidRevisions(t *testing.T) { StructLogs: make([]logger.StructLogRes, 0), } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers/call?revision="+revision, &TraceCallOption{Name: "structLogger"}, 200) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers/call?revision="+revision, &TraceCallOption{Name: "structLogger"}, 200) var parsedExecutionRes *logger.ExecutionResult if err := json.Unmarshal([]byte(res), &parsedExecutionRes); err != nil { @@ -374,7 +377,7 @@ func testHandleTraceCallWithValidRevisions(t *testing.T) { } func testHandleTraceCallWithRevisionAsNonExistingHeight(t *testing.T) { - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers/call?revision=12345", &TraceCallOption{}, 400) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers/call?revision=12345", &TraceCallOption{}, 400) assert.Equal(t, "revision: not found", strings.TrimSpace(res)) } @@ -382,7 +385,7 @@ func testHandleTraceCallWithRevisionAsNonExistingHeight(t *testing.T) { func testHandleTraceCallWithRevisionAsNonExistingID(t *testing.T) { nonExistingRevision := "0x4500ade0d72115abfc77571aef752df45ba5e87ca81fbd67fbfc46d455b17f91" - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers/call?revision="+nonExistingRevision, &TraceCallOption{}, 400) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers/call?revision="+nonExistingRevision, &TraceCallOption{}, 400) assert.Equal(t, "revision: leveldb: not found", strings.TrimSpace(res)) } @@ -390,20 +393,20 @@ func testHandleTraceCallWithRevisionAsNonExistingID(t *testing.T) { func testHandleTraceCallWithMalfomredRevision(t *testing.T) { // Revision is a malformed byte array traceCallOption := &TraceCallOption{} - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers/call?revision=012345678901234567890123456789012345678901234567890123456789012345", traceCallOption, 400) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers/call?revision=012345678901234567890123456789012345678901234567890123456789012345", traceCallOption, 400) assert.Equal(t, "revision: invalid prefix", strings.TrimSpace(res)) // Revision is a not accepted string - res = httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers/call?revision=badRevision", traceCallOption, 400) + res = httpPostAndCheckResponseStatus(t, "/debug/tracers/call?revision=badRevision", traceCallOption, 400) assert.Equal(t, `revision: strconv.ParseUint: parsing "badRevision": invalid syntax`, strings.TrimSpace(res)) // Revision number is out of range - res = httpPostAndCheckResponseStatus(t, fmt.Sprintf("%s/debug/tracers/call?revision=%d", ts.URL, uint64(math.MaxUint64)), traceCallOption, 400) + res = httpPostAndCheckResponseStatus(t, fmt.Sprintf("/debug/tracers/call?revision=%d", uint64(math.MaxUint64)), traceCallOption, 400) assert.Equal(t, "revision: block number out of max uint32", strings.TrimSpace(res)) } func testHandleTraceCallWithInsufficientGas(t *testing.T) { - addr := datagen.RandomAddress() + addr := datagen.RandAddress() traceCallOption := &TraceCallOption{ Name: "structLogger", To: &addr, @@ -417,13 +420,13 @@ func testHandleTraceCallWithInsufficientGas(t *testing.T) { BlockRef: "0x0000000000000000", } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers/call", traceCallOption, 403) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers/call", traceCallOption, 403) assert.Equal(t, "gas: exceeds limit", strings.TrimSpace(res)) } func testHandleTraceCallWithBadBlockRef(t *testing.T) { - addr := datagen.RandomAddress() + addr := datagen.RandAddress() traceCallOption := &TraceCallOption{ Name: "structLogger", To: &addr, @@ -437,13 +440,13 @@ func testHandleTraceCallWithBadBlockRef(t *testing.T) { BlockRef: "jh000000000000000", } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers/call", traceCallOption, 500) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers/call", traceCallOption, 500) assert.Equal(t, "blockRef: hex string without 0x prefix", strings.TrimSpace(res)) } func testHandleTraceCallWithInvalidLengthBlockRef(t *testing.T) { - addr := datagen.RandomAddress() + addr := datagen.RandAddress() traceCallOption := &TraceCallOption{ Name: "structLogger", To: &addr, @@ -457,7 +460,7 @@ func testHandleTraceCallWithInvalidLengthBlockRef(t *testing.T) { BlockRef: "0x00", } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/tracers/call", traceCallOption, 500) + res := httpPostAndCheckResponseStatus(t, "/debug/tracers/call", traceCallOption, 500) assert.Equal(t, "blockRef: invalid length", strings.TrimSpace(res)) } @@ -465,19 +468,19 @@ func testHandleTraceCallWithInvalidLengthBlockRef(t *testing.T) { func testStorageRangeWithError(t *testing.T) { // Error case 1: empty StorageRangeOption opt := &StorageRangeOption{} - httpPostAndCheckResponseStatus(t, ts.URL+"/debug/storage-range", opt, 400) + httpPostAndCheckResponseStatus(t, "/debug/storage-range", opt, 400) // Error case 2: bad StorageRangeOption badBodyRequest := 123 - httpPostAndCheckResponseStatus(t, ts.URL+"/debug/storage-range", badBodyRequest, 400) + httpPostAndCheckResponseStatus(t, "/debug/storage-range", badBodyRequest, 400) badMaxResult := &StorageRangeOption{MaxResult: 1001} - httpPostAndCheckResponseStatus(t, ts.URL+"/debug/storage-range", badMaxResult, 400) + httpPostAndCheckResponseStatus(t, "/debug/storage-range", badMaxResult, 400) } func testStorageRange(t *testing.T) { opt := StorageRangeOption{ - Address: datagen.RandomAddress(), + Address: datagen.RandAddress(), KeyStart: "0x00", MaxResult: 100, Target: fmt.Sprintf("%s/%s/0", blk.Header().ID(), transaction.ID()), @@ -487,7 +490,7 @@ func testStorageRange(t *testing.T) { NextKey: nil, } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/storage-range", &opt, 200) + res := httpPostAndCheckResponseStatus(t, "/debug/storage-range", &opt, 200) var parsedExecutionRes *StorageRangeResult if err := json.Unmarshal([]byte(res), &parsedExecutionRes); err != nil { @@ -502,7 +505,7 @@ func testStorageRangeDefaultOption(t *testing.T) { Target: fmt.Sprintf("%s/%s/0", blk.Header().ID(), transaction.ID()), } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/debug/storage-range", &opt, 200) + res := httpPostAndCheckResponseStatus(t, "/debug/storage-range", &opt, 200) var storageRangeRes *StorageRangeResult if err := json.Unmarshal([]byte(res), &storageRangeRes); err != nil { @@ -584,23 +587,12 @@ func initDebugServer(t *testing.T) { } func httpPostAndCheckResponseStatus(t *testing.T, url string, obj interface{}, responseStatusCode int) string { - data, err := json.Marshal(obj) - if err != nil { - t.Fatal(err) - } - res, err := http.Post(url, "application/x-www-form-urlencoded", bytes.NewReader(data)) //#nosec G107 - if err != nil { - t.Fatal(err) - } - assert.Equal(t, responseStatusCode, res.StatusCode) - r, err := io.ReadAll(res.Body) - res.Body.Close() - if err != nil { - t.Fatal(err) - } - return string(r) -} + body, status, err := tclient.RawHTTPClient().RawHTTPPost(url, obj) + require.NoError(t, err) + require.Equal(t, responseStatusCode, status) + return string(body) +} func TestCreateTracer(t *testing.T) { debug := &Debug{} diff --git a/api/events/events_test.go b/api/events/events_test.go index b9d8c18d2..9f859780b 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -16,6 +16,7 @@ import ( "github.com/gorilla/mux" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/api/events" "github.com/vechain/thor/v2/block" "github.com/vechain/thor/v2/chain" @@ -24,16 +25,17 @@ import ( "github.com/vechain/thor/v2/muxdb" "github.com/vechain/thor/v2/state" "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/thorclient" "github.com/vechain/thor/v2/tx" ) const defaultLogLimit uint64 = 1000 -var ts *httptest.Server - var ( - addr = thor.BytesToAddress([]byte("address")) - topic = thor.BytesToBytes32([]byte("topic")) + ts *httptest.Server + addr = thor.BytesToAddress([]byte("address")) + topic = thor.BytesToBytes32([]byte("topic")) + tclient *thorclient.Client ) func TestEmptyEvents(t *testing.T) { @@ -41,6 +43,7 @@ func TestEmptyEvents(t *testing.T) { initEventServer(t, db, defaultLogLimit) defer ts.Close() + tclient = thorclient.New(ts.URL) for name, tt := range map[string]func(*testing.T){ "testEventsBadRequest": testEventsBadRequest, "testEventWithEmptyDb": testEventWithEmptyDb, @@ -55,6 +58,7 @@ func TestEvents(t *testing.T) { defer ts.Close() blocksToInsert := 5 + tclient = thorclient.New(ts.URL) insertBlocks(t, db, blocksToInsert) testEventWithBlocks(t, blocksToInsert) } @@ -65,6 +69,7 @@ func TestOption(t *testing.T) { defer ts.Close() insertBlocks(t, db, 5) + tclient = thorclient.New(ts.URL) filter := events.EventFilter{ CriteriaSet: make([]*events.EventCriteria, 0), Range: nil, @@ -72,18 +77,21 @@ func TestOption(t *testing.T) { Order: logdb.DESC, } - res, statusCode := httpPost(t, ts.URL+"/events", filter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/events", filter) + require.NoError(t, err) assert.Equal(t, "options.limit exceeds the maximum allowed value of 5", strings.Trim(string(res), "\n")) assert.Equal(t, http.StatusForbidden, statusCode) filter.Options.Limit = 5 - _, statusCode = httpPost(t, ts.URL+"/events", filter) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/events", filter) + require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode) // with nil options, should use default limit, when the filtered lower // or equal to the limit, should return the filtered events filter.Options = nil - res, statusCode = httpPost(t, ts.URL+"/events", filter) + res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/events", filter) + require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode) var tLogs []*events.FilteredEvent if err := json.Unmarshal(res, &tLogs); err != nil { @@ -94,7 +102,8 @@ func TestOption(t *testing.T) { // when the filtered events exceed the limit, should return the forbidden insertBlocks(t, db, 6) - res, statusCode = httpPost(t, ts.URL+"/events", filter) + res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/events", filter) + require.NoError(t, err) assert.Equal(t, http.StatusForbidden, statusCode) assert.Equal(t, "the number of filtered logs exceeds the maximum allowed value of 5, please use pagination", strings.Trim(string(res), "\n")) } @@ -103,10 +112,9 @@ func TestOption(t *testing.T) { func testEventsBadRequest(t *testing.T) { badBody := []byte{0x00, 0x01, 0x02} - res, err := http.Post(ts.URL+"/events", "application/x-www-form-urlencoded", bytes.NewReader(badBody)) - + _, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/events", badBody) assert.NoError(t, err) - assert.Equal(t, http.StatusBadRequest, res.StatusCode) + assert.Equal(t, http.StatusBadRequest, statusCode) } func testEventWithEmptyDb(t *testing.T) { @@ -117,7 +125,8 @@ func testEventWithEmptyDb(t *testing.T) { Order: logdb.DESC, } - res, statusCode := httpPost(t, ts.URL+"/events", emptyFilter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/events", emptyFilter) + require.NoError(t, err) var tLogs []*events.FilteredEvent if err := json.Unmarshal(res, &tLogs); err != nil { t.Fatal(err) @@ -135,7 +144,8 @@ func testEventWithBlocks(t *testing.T, expectedBlocks int) { Order: logdb.DESC, } - res, statusCode := httpPost(t, ts.URL+"/events", emptyFilter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/events", emptyFilter) + require.NoError(t, err) var tLogs []*events.FilteredEvent if err := json.Unmarshal(res, &tLogs); err != nil { t.Fatal(err) @@ -161,7 +171,8 @@ func testEventWithBlocks(t *testing.T, expectedBlocks int) { }}, } - res, statusCode = httpPost(t, ts.URL+"/events", matchingFilter) + res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/events", matchingFilter) + require.NoError(t, err) if err := json.Unmarshal(res, &tLogs); err != nil { t.Fatal(err) } diff --git a/api/node/node_test.go b/api/node/node_test.go index 9f9179ef1..fd74326d3 100644 --- a/api/node/node_test.go +++ b/api/node/node_test.go @@ -5,7 +5,6 @@ package node_test import ( - "encoding/json" "io" "net/http" "net/http/httptest" @@ -14,12 +13,14 @@ import ( "github.com/gorilla/mux" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/api/node" "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/comm" "github.com/vechain/thor/v2/genesis" "github.com/vechain/thor/v2/muxdb" "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/thorclient" "github.com/vechain/thor/v2/txpool" ) @@ -27,11 +28,10 @@ var ts *httptest.Server func TestNode(t *testing.T) { initCommServer(t) - res := httpGet(t, ts.URL+"/node/network/peers") - var peersStats map[string]string - if err := json.Unmarshal(res, &peersStats); err != nil { - t.Fatal(err) - } + tclient := thorclient.New(ts.URL) + + peersStats, err := tclient.Peers() + require.NoError(t, err) assert.Equal(t, 0, len(peersStats), "count should be zero") } diff --git a/api/transactions/transactions.go b/api/transactions/transactions.go index 85404cf21..af32cb6da 100644 --- a/api/transactions/transactions.go +++ b/api/transactions/transactions.go @@ -30,7 +30,7 @@ func New(repo *chain.Repository, pool *txpool.TxPool) *Transactions { } } -func (t *Transactions) getRawTransaction(txID thor.Bytes32, head thor.Bytes32, allowPending bool) (*rawTransaction, error) { +func (t *Transactions) getRawTransaction(txID thor.Bytes32, head thor.Bytes32, allowPending bool) (*RawTransaction, error) { chain := t.repo.NewChain(head) tx, meta, err := chain.GetTransaction(txID) if err != nil { @@ -41,7 +41,7 @@ func (t *Transactions) getRawTransaction(txID thor.Bytes32, head thor.Bytes32, a if err != nil { return nil, err } - return &rawTransaction{ + return &RawTransaction{ RawTx: RawTx{hexutil.Encode(raw)}, }, nil } @@ -59,7 +59,7 @@ func (t *Transactions) getRawTransaction(txID thor.Bytes32, head thor.Bytes32, a if err != nil { return nil, err } - return &rawTransaction{ + return &RawTransaction{ RawTx: RawTx{hexutil.Encode(raw)}, Meta: &TxMeta{ BlockID: summary.Header.ID(), @@ -133,9 +133,8 @@ func (t *Transactions) handleSendTransaction(w http.ResponseWriter, req *http.Re } return err } - return utils.WriteJSON(w, map[string]string{ - "id": tx.ID().String(), - }) + txID := tx.ID() + return utils.WriteJSON(w, &SendTxResult{ID: &txID}) } func (t *Transactions) handleGetTransactionByID(w http.ResponseWriter, req *http.Request) error { diff --git a/api/transactions/transactions_test.go b/api/transactions/transactions_test.go index 0beada359..cef22e191 100644 --- a/api/transactions/transactions_test.go +++ b/api/transactions/transactions_test.go @@ -6,12 +6,9 @@ package transactions_test import ( - "bytes" "encoding/json" "fmt" - "io" "math/big" - "net/http" "net/http/httptest" "strings" "testing" @@ -21,6 +18,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/api/transactions" "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/genesis" @@ -28,20 +26,25 @@ import ( "github.com/vechain/thor/v2/packer" "github.com/vechain/thor/v2/state" "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/thorclient" "github.com/vechain/thor/v2/tx" "github.com/vechain/thor/v2/txpool" ) -var repo *chain.Repository -var ts *httptest.Server -var transaction *tx.Transaction -var mempoolTx *tx.Transaction +var ( + repo *chain.Repository + ts *httptest.Server + transaction *tx.Transaction + mempoolTx *tx.Transaction + tclient *thorclient.Client +) func TestTransaction(t *testing.T) { initTransactionServer(t) defer ts.Close() // Send tx + tclient = thorclient.New(ts.URL) for name, tt := range map[string]func(*testing.T){ "sendTx": sendTx, "sendTxWithBadFormat": sendTxWithBadFormat, @@ -77,14 +80,14 @@ func TestTransaction(t *testing.T) { } func getTx(t *testing.T) { - res := httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+transaction.ID().String(), 200) + res := httpGetAndCheckResponseStatus(t, "/transactions/"+transaction.ID().String(), 200) var rtx *transactions.Transaction if err := json.Unmarshal(res, &rtx); err != nil { t.Fatal(err) } checkMatchingTx(t, transaction, rtx) - res = httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+transaction.ID().String()+"?raw=true", 200) + res = httpGetAndCheckResponseStatus(t, "/transactions/"+transaction.ID().String()+"?raw=true", 200) var rawTx map[string]interface{} if err := json.Unmarshal(res, &rawTx); err != nil { t.Fatal(err) @@ -97,7 +100,7 @@ func getTx(t *testing.T) { } func getTxReceipt(t *testing.T) { - r := httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+transaction.ID().String()+"/receipt", 200) + r := httpGetAndCheckResponseStatus(t, "/transactions/"+transaction.ID().String()+"/receipt", 200) var receipt *transactions.Receipt if err := json.Unmarshal(r, &receipt); err != nil { t.Fatal(err) @@ -126,7 +129,7 @@ func sendTx(t *testing.T) { t.Fatal(err) } - res := httpPostAndCheckResponseStatus(t, ts.URL+"/transactions", transactions.RawTx{Raw: hexutil.Encode(rlpTx)}, 200) + res := httpPostAndCheckResponseStatus(t, "/transactions", transactions.RawTx{Raw: hexutil.Encode(rlpTx)}, 200) var txObj map[string]string if err = json.Unmarshal(res, &txObj); err != nil { t.Fatal(err) @@ -137,15 +140,15 @@ func sendTx(t *testing.T) { func getTxWithBadID(t *testing.T) { txBadID := "0x123" - res := httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+txBadID, 400) + res := httpGetAndCheckResponseStatus(t, "/transactions/"+txBadID, 400) assert.Contains(t, string(res), "invalid length") } func txWithBadHeader(t *testing.T) { badHeaderURL := []string{ - ts.URL + "/transactions/" + transaction.ID().String() + "?head=badHead", - ts.URL + "/transactions/" + transaction.ID().String() + "/receipt?head=badHead", + "/transactions/" + transaction.ID().String() + "?head=badHead", + "/transactions/" + transaction.ID().String() + "/receipt?head=badHead", } for _, url := range badHeaderURL { @@ -157,7 +160,7 @@ func txWithBadHeader(t *testing.T) { func getReceiptWithBadID(t *testing.T) { txBadID := "0x123" - httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+txBadID+"/receipt", 400) + httpGetAndCheckResponseStatus(t, "/transactions/"+txBadID+"/receipt", 400) } func getNonExistingRawTransactionWhenTxStillInMempool(t *testing.T) { @@ -168,14 +171,14 @@ func getNonExistingRawTransactionWhenTxStillInMempool(t *testing.T) { } for _, queryParam := range queryParams { - res := httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+nonExistingTxID+queryParam, 200) + res := httpGetAndCheckResponseStatus(t, "/transactions/"+nonExistingTxID+queryParam, 200) assert.Equal(t, "null\n", string(res)) } } func getNonPendingRawTransactionWhenTxStillInMempool(t *testing.T) { - res := httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+mempoolTx.ID().String()+"?raw=true", 200) + res := httpGetAndCheckResponseStatus(t, "/transactions/"+mempoolTx.ID().String()+"?raw=true", 200) var rawTx map[string]interface{} if err := json.Unmarshal(res, &rawTx); err != nil { t.Fatal(err) @@ -185,7 +188,7 @@ func getNonPendingRawTransactionWhenTxStillInMempool(t *testing.T) { } func getRawTransactionWhenTxStillInMempool(t *testing.T) { - res := httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+mempoolTx.ID().String()+"?raw=true&pending=true", 200) + res := httpGetAndCheckResponseStatus(t, "/transactions/"+mempoolTx.ID().String()+"?raw=true&pending=true", 200) var rawTx map[string]interface{} if err := json.Unmarshal(res, &rawTx); err != nil { t.Fatal(err) @@ -200,13 +203,13 @@ func getRawTransactionWhenTxStillInMempool(t *testing.T) { } func getTransactionByIDTxNotFound(t *testing.T) { - res := httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+mempoolTx.ID().String(), 200) + res := httpGetAndCheckResponseStatus(t, "/transactions/"+mempoolTx.ID().String(), 200) assert.Equal(t, "null\n", string(res)) } func getTransactionByIDPendingTxNotFound(t *testing.T) { - res := httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+mempoolTx.ID().String()+"?pending=true", 200) + res := httpGetAndCheckResponseStatus(t, "/transactions/"+mempoolTx.ID().String()+"?pending=true", 200) var rtx *transactions.Transaction if err := json.Unmarshal(res, &rtx); err != nil { t.Fatal(err) @@ -218,7 +221,7 @@ func getTransactionByIDPendingTxNotFound(t *testing.T) { func sendTxWithBadFormat(t *testing.T) { badRawTx := transactions.RawTx{Raw: "badRawTx"} - res := httpPostAndCheckResponseStatus(t, ts.URL+"/transactions", badRawTx, 400) + res := httpPostAndCheckResponseStatus(t, "/transactions", badRawTx, 400) assert.Contains(t, string(res), hexutil.ErrMissingPrefix.Error()) } @@ -231,7 +234,7 @@ func sendTxThatCannotBeAcceptedInLocalMempool(t *testing.T) { } duplicatedRawTx := transactions.RawTx{Raw: hexutil.Encode(rlpTx)} - res := httpPostAndCheckResponseStatus(t, ts.URL+"/transactions", duplicatedRawTx, 400) + res := httpPostAndCheckResponseStatus(t, "/transactions", duplicatedRawTx, 400) assert.Contains(t, string(res), "bad tx: chain tag mismatch") } @@ -243,34 +246,27 @@ func handleGetTransactionByIDWithBadQueryParams(t *testing.T) { } for _, badQueryParam := range badQueryParams { - res := httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+transaction.ID().String()+badQueryParam, 400) + res := httpGetAndCheckResponseStatus(t, "/transactions/"+transaction.ID().String()+badQueryParam, 400) assert.Contains(t, string(res), "should be boolean") } } func handleGetTransactionByIDWithNonExistingHead(t *testing.T) { - res := httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+transaction.ID().String()+"?head=0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 400) + res := httpGetAndCheckResponseStatus(t, "/transactions/"+transaction.ID().String()+"?head=0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 400) assert.Equal(t, "head: leveldb: not found", strings.TrimSpace(string(res))) } func handleGetTransactionReceiptByIDWithNonExistingHead(t *testing.T) { - res := httpGetAndCheckResponseStatus(t, ts.URL+"/transactions/"+transaction.ID().String()+"/receipt?head=0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 400) + res := httpGetAndCheckResponseStatus(t, "/transactions/"+transaction.ID().String()+"/receipt?head=0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 400) assert.Equal(t, "head: leveldb: not found", strings.TrimSpace(string(res))) } func httpPostAndCheckResponseStatus(t *testing.T, url string, obj interface{}, responseStatusCode int) []byte { - data, err := json.Marshal(obj) - if err != nil { - t.Fatal(err) - } - res, err := http.Post(url, "application/x-www-form-urlencoded", bytes.NewReader(data)) //#nosec G107 - if err != nil { - t.Fatal(err) - } - assert.Equal(t, responseStatusCode, res.StatusCode, fmt.Sprintf("status code should be %d", responseStatusCode)) - r := parseBytesBody(t, res.Body) - res.Body.Close() - return r + body, statusCode, err := tclient.RawHTTPClient().RawHTTPPost(url, obj) + require.NoError(t, err) + assert.Equal(t, responseStatusCode, statusCode, fmt.Sprintf("status code should be %d", responseStatusCode)) + + return body } func initTransactionServer(t *testing.T) { @@ -358,23 +354,9 @@ func checkMatchingTx(t *testing.T, expectedTx *tx.Transaction, actualTx *transac } func httpGetAndCheckResponseStatus(t *testing.T, url string, responseStatusCode int) []byte { - res, err := http.Get(url) //#nosec G107 - if err != nil { - t.Fatal(err) - } - assert.Equal(t, responseStatusCode, res.StatusCode, fmt.Sprintf("status code should be %d", responseStatusCode)) - r, err := io.ReadAll(res.Body) - res.Body.Close() - if err != nil { - t.Fatal(err) - } - return r -} + body, statusCode, err := tclient.RawHTTPClient().RawHTTPGet(url) + require.NoError(t, err) + assert.Equal(t, responseStatusCode, statusCode, fmt.Sprintf("status code should be %d", responseStatusCode)) -func parseBytesBody(t *testing.T, body io.ReadCloser) []byte { - r, err := io.ReadAll(body) - if err != nil { - t.Fatal(err) - } - return r + return body } diff --git a/api/transactions/types.go b/api/transactions/types.go index 7aa3f822e..7c3a892ac 100644 --- a/api/transactions/types.go +++ b/api/transactions/types.go @@ -78,7 +78,7 @@ func (rtx *RawTx) decode() (*tx.Transaction, error) { return tx, nil } -type rawTransaction struct { +type RawTransaction struct { RawTx Meta *TxMeta `json:"meta"` } @@ -220,3 +220,8 @@ func convertReceipt(txReceipt *tx.Receipt, header *block.Header, tx *tx.Transact } return receipt, nil } + +// SendTxResult is the response to the Send Tx method +type SendTxResult struct { + ID *thor.Bytes32 `json:"id"` +} diff --git a/api/transfers/transfers_test.go b/api/transfers/transfers_test.go index 97d8d1238..b0764c85f 100644 --- a/api/transfers/transfers_test.go +++ b/api/transfers/transfers_test.go @@ -6,10 +6,7 @@ package transfers_test import ( - "bytes" - "crypto/rand" "encoding/json" - "io" "math/big" "net/http" "net/http/httptest" @@ -18,6 +15,7 @@ import ( "github.com/gorilla/mux" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/api/events" "github.com/vechain/thor/v2/api/transfers" "github.com/vechain/thor/v2/block" @@ -26,19 +24,24 @@ import ( "github.com/vechain/thor/v2/logdb" "github.com/vechain/thor/v2/muxdb" "github.com/vechain/thor/v2/state" - "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/test/datagen" + "github.com/vechain/thor/v2/thorclient" "github.com/vechain/thor/v2/tx" ) const defaultLogLimit uint64 = 1000 -var ts *httptest.Server +var ( + ts *httptest.Server + tclient *thorclient.Client +) func TestEmptyTransfers(t *testing.T) { db := createDb(t) initTransferServer(t, db, defaultLogLimit) defer ts.Close() + tclient = thorclient.New(ts.URL) testTransferBadRequest(t) testTransferWithEmptyDb(t) } @@ -48,6 +51,7 @@ func TestTransfers(t *testing.T) { initTransferServer(t, db, defaultLogLimit) defer ts.Close() + tclient = thorclient.New(ts.URL) blocksToInsert := 5 insertBlocks(t, db, blocksToInsert) @@ -60,6 +64,7 @@ func TestOption(t *testing.T) { defer ts.Close() insertBlocks(t, db, 5) + tclient = thorclient.New(ts.URL) filter := transfers.TransferFilter{ CriteriaSet: make([]*logdb.TransferCriteria, 0), Range: nil, @@ -67,18 +72,21 @@ func TestOption(t *testing.T) { Order: logdb.DESC, } - res, statusCode := httpPost(t, ts.URL+"/transfers", filter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/transfers", filter) + require.NoError(t, err) assert.Equal(t, "options.limit exceeds the maximum allowed value of 5", strings.Trim(string(res), "\n")) assert.Equal(t, http.StatusForbidden, statusCode) filter.Options.Limit = 5 - _, statusCode = httpPost(t, ts.URL+"/transfers", filter) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/transfers", filter) + require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode) // with nil options, should use default limit, when the filtered lower // or equal to the limit, should return the filtered transfers filter.Options = nil - res, statusCode = httpPost(t, ts.URL+"/transfers", filter) + res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/transfers", filter) + require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode) var tLogs []*events.FilteredEvent if err := json.Unmarshal(res, &tLogs); err != nil { @@ -89,7 +97,8 @@ func TestOption(t *testing.T) { // when the filtered transfers exceed the limit, should return the forbidden insertBlocks(t, db, 6) - res, statusCode = httpPost(t, ts.URL+"/transfers", filter) + res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/transfers", filter) + require.NoError(t, err) assert.Equal(t, http.StatusForbidden, statusCode) assert.Equal(t, "the number of filtered logs exceeds the maximum allowed value of 5, please use pagination", strings.Trim(string(res), "\n")) } @@ -98,10 +107,9 @@ func TestOption(t *testing.T) { func testTransferBadRequest(t *testing.T) { badBody := []byte{0x00, 0x01, 0x02} - res, err := http.Post(ts.URL+"/transfers", "application/x-www-form-urlencoded", bytes.NewReader(badBody)) - - assert.NoError(t, err) - assert.Equal(t, http.StatusBadRequest, res.StatusCode) + _, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/transfers", badBody) + require.NoError(t, err) + assert.Equal(t, http.StatusBadRequest, statusCode) } func testTransferWithEmptyDb(t *testing.T) { @@ -112,7 +120,8 @@ func testTransferWithEmptyDb(t *testing.T) { Order: logdb.DESC, } - res, statusCode := httpPost(t, ts.URL+"/transfers", emptyFilter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/transfers", emptyFilter) + require.NoError(t, err) var tLogs []*transfers.FilteredTransfer if err := json.Unmarshal(res, &tLogs); err != nil { t.Fatal(err) @@ -130,7 +139,8 @@ func testTransferWithBlocks(t *testing.T, expectedBlocks int) { Order: logdb.DESC, } - res, statusCode := httpPost(t, ts.URL+"/transfers", emptyFilter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/transfers", emptyFilter) + require.NoError(t, err) var tLogs []*transfers.FilteredTransfer if err := json.Unmarshal(res, &tLogs); err != nil { t.Fatal(err) @@ -190,38 +200,16 @@ func createDb(t *testing.T) *logdb.LogDB { } // Utilities functions -func randAddress() (addr thor.Address) { - rand.Read(addr[:]) - return -} - func newReceipt() *tx.Receipt { return &tx.Receipt{ Outputs: []*tx.Output{ { Transfers: tx.Transfers{{ - Sender: randAddress(), - Recipient: randAddress(), - Amount: new(big.Int).SetBytes(randAddress().Bytes()), + Sender: datagen.RandAddress(), + Recipient: datagen.RandAddress(), + Amount: new(big.Int).SetBytes(datagen.RandAddress().Bytes()), }}, }, }, } } - -func httpPost(t *testing.T, url string, body interface{}) ([]byte, int) { - data, err := json.Marshal(body) - if err != nil { - t.Fatal(err) - } - res, err := http.Post(url, "application/x-www-form-urlencoded", bytes.NewReader(data)) //#nosec G107 - if err != nil { - t.Fatal(err) - } - r, err := io.ReadAll(res.Body) - res.Body.Close() - if err != nil { - t.Fatal(err) - } - return r, res.StatusCode -} diff --git a/bft/engine_test.go b/bft/engine_test.go index 4c43aa87f..54e2e8bec 100644 --- a/bft/engine_test.go +++ b/bft/engine_test.go @@ -729,7 +729,7 @@ func TestJustifier(t *testing.T) { } for i := 0; i <= MaxBlockProposers*2/3; i++ { - vs.AddBlock(datagen.RandomAddress(), true) + vs.AddBlock(datagen.RandAddress(), true) } st := vs.Summarize() @@ -738,7 +738,7 @@ func TestJustifier(t *testing.T) { assert.True(t, st.Committed) // add vote after commits，commit/justify stays the same - vs.AddBlock(datagen.RandomAddress(), true) + vs.AddBlock(datagen.RandAddress(), true) st = vs.Summarize() assert.Equal(t, uint32(3), st.Quality) assert.True(t, st.Justified) @@ -760,7 +760,7 @@ func TestJustifier(t *testing.T) { } for i := 0; i <= MaxBlockProposers*2/3; i++ { - vs.AddBlock(datagen.RandomAddress(), false) + vs.AddBlock(datagen.RandAddress(), false) } st := vs.Summarize() @@ -785,10 +785,10 @@ func TestJustifier(t *testing.T) { // vote times COM for i := 0; i < MaxBlockProposers*2/3; i++ { - vs.AddBlock(datagen.RandomAddress(), true) + vs.AddBlock(datagen.RandAddress(), true) } - master := datagen.RandomAddress() + master := datagen.RandAddress() // master votes WIT vs.AddBlock(master, false) @@ -805,7 +805,7 @@ func TestJustifier(t *testing.T) { assert.False(t, st.Committed) // another master votes WIT - vs.AddBlock(datagen.RandomAddress(), true) + vs.AddBlock(datagen.RandAddress(), true) st = vs.Summarize() assert.True(t, st.Committed) }, @@ -821,7 +821,7 @@ func TestJustifier(t *testing.T) { t.Fatal(err) } - master := datagen.RandomAddress() + master := datagen.RandAddress() vs.AddBlock(master, true) assert.Equal(t, true, vs.votes[master]) assert.Equal(t, uint64(1), vs.comVotes) diff --git a/test/datagen/address.go b/test/datagen/address.go index 882159f6e..211a1dd34 100644 --- a/test/datagen/address.go +++ b/test/datagen/address.go @@ -11,9 +11,7 @@ import ( "github.com/vechain/thor/v2/thor" ) -func RandomAddress() thor.Address { - var addr thor.Address - +func RandAddress() (addr thor.Address) { rand.Read(addr[:]) - return addr + return } diff --git a/thorclient/common/common.go b/thorclient/common/common.go new file mode 100644 index 000000000..3bb9b0992 --- /dev/null +++ b/thorclient/common/common.go @@ -0,0 +1,33 @@ +// Copyright (c) 2024 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package common + +import ( + "errors" +) + +const ( + BestRevision = "best" + FinalizedRevision = "finalized" +) + +var ( + ErrNotFound = errors.New("not found") + ErrNot200Status = errors.New("not 200 status code") + ErrUnexpectedMsg = errors.New("unexpected message format") +) + +// EventWrapper is used to return errors from the websocket alongside the data +type EventWrapper[T any] struct { + Data T + Error error +} + +// Subscription is used to handle the active subscription +type Subscription[T any] struct { + EventChan <-chan EventWrapper[T] + Unsubscribe func() +} diff --git a/thorclient/httpclient/client.go b/thorclient/httpclient/client.go new file mode 100644 index 000000000..8f88783f5 --- /dev/null +++ b/thorclient/httpclient/client.go @@ -0,0 +1,309 @@ +// Copyright (c) 2024 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +// Package httpclient provides an HTTP client to interact with the VeChainThor blockchain. +// It offers various methods to retrieve accounts, transactions, blocks, events, and other blockchain data +// through HTTP requests. +package httpclient + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + + "github.com/vechain/thor/v2/api/accounts" + "github.com/vechain/thor/v2/api/blocks" + "github.com/vechain/thor/v2/api/events" + "github.com/vechain/thor/v2/api/node" + "github.com/vechain/thor/v2/api/transactions" + "github.com/vechain/thor/v2/api/transfers" + "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/thorclient/common" +) + +// Client represents the HTTP client for interacting with the VeChainThor blockchain. +// It manages communication via HTTP requests. +type Client struct { + url string + c *http.Client +} + +// New creates a new Client with the provided URL. +func New(url string) *Client { + return &Client{ + url: url, + c: &http.Client{}, + } +} + +// GetAccount retrieves the account details for the given address at the specified revision. +func (c *Client) GetAccount(addr *thor.Address, revision string) (*accounts.Account, error) { + url := c.url + "/accounts/" + addr.String() + if revision != "" { + url += "?revision=" + revision + } + + body, err := c.httpGET(url) + if err != nil { + return nil, fmt.Errorf("unable to retrieve account - %w", err) + } + + var account accounts.Account + if err = json.Unmarshal(body, &account); err != nil { + return nil, fmt.Errorf("unable to unmarshal account - %w", err) + } + + return &account, nil +} + +// InspectClauses performs a clause inspection on batch call data at the specified revision. +func (c *Client) InspectClauses(calldata *accounts.BatchCallData, revision string) ([]*accounts.CallResult, error) { + url := c.url + "/accounts/*" + if revision != "" { + url += "?revision=" + revision + } + body, err := c.httpPOST(url, calldata) + if err != nil { + return nil, fmt.Errorf("unable to request inspect clauses - %w", err) + } + + var inspectionRes []*accounts.CallResult + if err = json.Unmarshal(body, &inspectionRes); err != nil { + return nil, fmt.Errorf("unable to unmarshal inspection result - %w", err) + } + + return inspectionRes, nil +} + +// GetAccountCode retrieves the contract code for the given address at the specified revision. +func (c *Client) GetAccountCode(addr *thor.Address, revision string) (*accounts.GetCodeResult, error) { + url := c.url + "/accounts/" + addr.String() + "/code" + if revision != "" { + url += "?revision=" + revision + } + + body, err := c.httpGET(url) + if err != nil { + return nil, fmt.Errorf("unable to retrieve account code - %w", err) + } + + var res accounts.GetCodeResult + if err = json.Unmarshal(body, &res); err != nil { + return nil, fmt.Errorf("unable to unmarshal code - %w", err) + } + + return &res, nil +} + +// GetAccountStorage retrieves the storage value for the given address and key at the specified revision. +func (c *Client) GetAccountStorage(addr *thor.Address, key *thor.Bytes32, revision string) (*accounts.GetStorageResult, error) { + url := c.url + "/accounts/" + addr.String() + "/key/" + key.String() + if revision != "" { + url += "?revision=" + revision + } + + body, err := c.httpGET(url) + if err != nil { + return nil, fmt.Errorf("unable to retrieve account storage - %w", err) + } + + var res accounts.GetStorageResult + if err = json.Unmarshal(body, &res); err != nil { + return nil, fmt.Errorf("unable to unmarshal storage result - %w", err) + } + + return &res, nil +} + +// GetTransaction retrieves the transaction details by the transaction ID, along with options for head and pending status. +func (c *Client) GetTransaction(txID *thor.Bytes32, head string, isPending bool) (*transactions.Transaction, error) { + url := c.url + "/transactions/" + txID.String() + "?" + if isPending { + url += "pending=true&" + } + if head != "" { + url += "head=" + head + } + + body, err := c.httpGET(url) + if err != nil { + return nil, fmt.Errorf("unable to retrieve transaction - %w", err) + } + + var tx transactions.Transaction + if err = json.Unmarshal(body, &tx); err != nil { + return nil, fmt.Errorf("unable to unmarshal transaction - %w", err) + } + + return &tx, nil +} + +// GetRawTransaction retrieves the raw transaction data by the transaction ID, along with options for head and pending status. +func (c *Client) GetRawTransaction(txID *thor.Bytes32, head string, isPending bool) (*transactions.RawTransaction, error) { + url := c.url + "/transactions/" + txID.String() + "?raw=true&" + if isPending { + url += "pending=true&" + } + if head != "" { + url += "head=" + head + } + + body, err := c.httpGET(url) + if err != nil { + return nil, fmt.Errorf("unable to retrieve raw transaction - %w", err) + } + + var tx transactions.RawTransaction + if err = json.Unmarshal(body, &tx); err != nil { + return nil, fmt.Errorf("unable to unmarshal raw transaction - %w", err) + } + + return &tx, nil +} + +// GetTransactionReceipt retrieves the receipt for the given transaction ID at the specified head. +func (c *Client) GetTransactionReceipt(txID *thor.Bytes32, head string) (*transactions.Receipt, error) { + url := c.url + "/transactions/" + txID.String() + "/receipt" + if head != "" { + url += "?head=" + head + } + + body, err := c.httpGET(url) + if err != nil { + return nil, fmt.Errorf("unable to fetch receipt - %w", err) + } + + if len(body) == 0 || bytes.Equal(bytes.TrimSpace(body), []byte("null")) { + return nil, common.ErrNotFound + } + + var receipt transactions.Receipt + if err = json.Unmarshal(body, &receipt); err != nil { + return nil, fmt.Errorf("unable to unmarshal receipt - %w", err) + } + + return &receipt, nil +} + +// SendTransaction sends a raw transaction to the blockchain. +func (c *Client) SendTransaction(obj *transactions.RawTx) (*transactions.SendTxResult, error) { + body, err := c.httpPOST(c.url+"/transactions", obj) + if err != nil { + return nil, fmt.Errorf("unable to send raw transaction - %w", err) + } + + var txID transactions.SendTxResult + if err = json.Unmarshal(body, &txID); err != nil { + return nil, fmt.Errorf("unable to unmarshal send transaction result - %w", err) + } + + return &txID, nil +} + +// GetBlock retrieves a block by its block ID. +func (c *Client) GetBlock(blockID string) (*blocks.JSONCollapsedBlock, error) { + body, err := c.httpGET(c.url + "/blocks/" + blockID) + if err != nil { + return nil, fmt.Errorf("unable to retrieve block - %w", err) + } + + if len(body) == 0 || bytes.Equal(bytes.TrimSpace(body), []byte("null")) { + return nil, common.ErrNotFound + } + + var block blocks.JSONCollapsedBlock + if err = json.Unmarshal(body, &block); err != nil { + return nil, fmt.Errorf("unable to unmarshal block - %w", err) + } + + return &block, nil +} + +// GetExpandedBlock retrieves an expanded block by its revision. +func (c *Client) GetExpandedBlock(revision string) (*blocks.JSONExpandedBlock, error) { + body, err := c.httpGET(c.url + "/blocks/" + revision + "?expanded=true") + if err != nil { + return nil, fmt.Errorf("unable to retrieve expanded block - %w", err) + } + + if len(body) == 0 || bytes.Equal(bytes.TrimSpace(body), []byte("null")) { + return nil, common.ErrNotFound + } + + var block blocks.JSONExpandedBlock + if err = json.Unmarshal(body, &block); err != nil { + return nil, fmt.Errorf("unable to unmarshal expanded block - %w", err) + } + + return &block, nil +} + +// FilterEvents filters events based on the provided event filter. +func (c *Client) FilterEvents(req *events.EventFilter) ([]events.FilteredEvent, error) { + body, err := c.httpPOST(c.url+"/logs/event", req) + if err != nil { + return nil, fmt.Errorf("unable to filter events - %w", err) + } + + var filteredEvents []events.FilteredEvent + if err = json.Unmarshal(body, &filteredEvents); err != nil { + return nil, fmt.Errorf("unable to unmarshal events - %w", err) + } + + return filteredEvents, nil +} + +// FilterTransfers filters transfer based on the provided transfer filter. +func (c *Client) FilterTransfers(req *transfers.TransferFilter) ([]*transfers.FilteredTransfer, error) { + body, err := c.httpPOST(c.url+"/logs/transfer", req) + if err != nil { + return nil, fmt.Errorf("unable to retrieve transfer logs - %w", err) + } + + var filteredTransfers []*transfers.FilteredTransfer + if err = json.Unmarshal(body, &filteredTransfers); err != nil { + return nil, fmt.Errorf("unable to unmarshal transfers - %w", err) + } + + return filteredTransfers, nil +} + +// GetPeers retrieves the network peers connected to the node. +func (c *Client) GetPeers() ([]*node.PeerStats, error) { + body, err := c.httpGET(c.url + "/node/network/peers") + if err != nil { + return nil, fmt.Errorf("unable to retrieve peers - %w", err) + } + + var peers []*node.PeerStats + if err = json.Unmarshal(body, &peers); err != nil { + return nil, fmt.Errorf("unable to unmarshal peers - %w", err) + } + + return peers, nil +} + +// RawHTTPPost sends a raw HTTP POST request to the specified URL with the provided data. +func (c *Client) RawHTTPPost(url string, calldata interface{}) ([]byte, int, error) { + var data []byte + var err error + + if _, ok := calldata.([]byte); ok { + data = calldata.([]byte) + } else { + data, err = json.Marshal(calldata) + if err != nil { + return nil, 0, fmt.Errorf("unable to marshal payload - %w", err) + } + } + + return c.rawHTTPRequest("POST", c.url+url, bytes.NewBuffer(data)) +} + +// RawHTTPGet sends a raw HTTP GET request to the specified URL. +func (c *Client) RawHTTPGet(url string) ([]byte, int, error) { + return c.rawHTTPRequest("GET", c.url+url, nil) +} diff --git a/thorclient/httpclient/client_test.go b/thorclient/httpclient/client_test.go new file mode 100644 index 000000000..7e807ba55 --- /dev/null +++ b/thorclient/httpclient/client_test.go @@ -0,0 +1,501 @@ +// Copyright (c) 2024 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package httpclient + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common/math" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/vechain/thor/v2/api/accounts" + "github.com/vechain/thor/v2/api/blocks" + "github.com/vechain/thor/v2/api/events" + "github.com/vechain/thor/v2/api/node" + "github.com/vechain/thor/v2/api/transactions" + "github.com/vechain/thor/v2/api/transfers" + "github.com/vechain/thor/v2/thor" + + tccommon "github.com/vechain/thor/v2/thorclient/common" +) + +func TestClient_GetTransactionReceipt(t *testing.T) { + txID := thor.Bytes32{0x01} + expectedReceipt := &transactions.Receipt{ + GasUsed: 1000, + GasPayer: thor.Address{0x01}, + Paid: &math.HexOrDecimal256{}, + Reward: &math.HexOrDecimal256{}, + Reverted: false, + Meta: transactions.ReceiptMeta{}, + Outputs: []*transactions.Output{}, + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/transactions/"+txID.String()+"/receipt", r.URL.Path) + + receiptBytes, _ := json.Marshal(expectedReceipt) + w.Write(receiptBytes) + })) + defer ts.Close() + + client := New(ts.URL) + receipt, err := client.GetTransactionReceipt(&txID, "") + + assert.NoError(t, err) + assert.Equal(t, expectedReceipt, receipt) +} + +func TestClient_InspectClauses(t *testing.T) { + calldata := &accounts.BatchCallData{} + expectedResults := []*accounts.CallResult{{ + Data: "data", + Events: []*transactions.Event{}, + Transfers: []*transactions.Transfer{}, + GasUsed: 1000, + Reverted: false, + VMError: "no error"}} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/accounts/*", r.URL.Path) + + inspectionResBytes, _ := json.Marshal(expectedResults) + w.Write(inspectionResBytes) + })) + defer ts.Close() + + client := New(ts.URL) + results, err := client.InspectClauses(calldata, "") + + assert.NoError(t, err) + assert.Equal(t, expectedResults, results) +} + +func TestClient_SendTransaction(t *testing.T) { + rawTx := &transactions.RawTx{} + expectedResult := &transactions.SendTxResult{ID: &thor.Bytes32{0x01}} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/transactions", r.URL.Path) + + txIDBytes, _ := json.Marshal(expectedResult) + w.Write(txIDBytes) + })) + defer ts.Close() + + client := New(ts.URL) + result, err := client.SendTransaction(rawTx) + + assert.NoError(t, err) + assert.Equal(t, expectedResult, result) +} + +func TestClient_FilterTransfers(t *testing.T) { + req := &transfers.TransferFilter{} + expectedTransfers := []*transfers.FilteredTransfer{{ + Sender: thor.Address{0x01}, + Recipient: thor.Address{0x02}, + Amount: &math.HexOrDecimal256{}, + Meta: transfers.LogMeta{}, + }} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/logs/transfer", r.URL.Path) + + filteredTransfersBytes, _ := json.Marshal(expectedTransfers) + w.Write(filteredTransfersBytes) + })) + defer ts.Close() + + client := New(ts.URL) + transfers, err := client.FilterTransfers(req) + + assert.NoError(t, err) + assert.Equal(t, expectedTransfers, transfers) +} + +func TestClient_FilterEvents(t *testing.T) { + req := &events.EventFilter{} + expectedEvents := []events.FilteredEvent{{ + Address: thor.Address{0x01}, + Topics: []*thor.Bytes32{{0x01}}, + Data: "data", + Meta: events.LogMeta{}, + }} + expectedPath := "/logs/event" + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, expectedPath, r.URL.Path) + + filteredEventsBytes, _ := json.Marshal(expectedEvents) + w.Write(filteredEventsBytes) + })) + defer ts.Close() + + client := New(ts.URL) + events, err := client.FilterEvents(req) + + assert.NoError(t, err) + assert.Equal(t, expectedEvents, events) +} + +func TestClient_GetAccount(t *testing.T) { + addr := thor.Address{0x01} + expectedAccount := &accounts.Account{ + Balance: math.HexOrDecimal256{}, + Energy: math.HexOrDecimal256{}, + HasCode: false, + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/accounts/"+addr.String(), r.URL.Path) + + accountBytes, _ := json.Marshal(expectedAccount) + w.Write(accountBytes) + })) + defer ts.Close() + + client := New(ts.URL) + account, err := client.GetAccount(&addr, "") + + assert.NoError(t, err) + assert.Equal(t, expectedAccount, account) +} + +func TestClient_GetAccountCode(t *testing.T) { + addr := thor.Address{0x01} + expectedCodeRsp := &accounts.GetCodeResult{Code: hexutil.Encode([]byte{0x01, 0x03})} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/accounts/"+addr.String()+"/code", r.URL.Path) + + marshal, err := json.Marshal(expectedCodeRsp) + require.NoError(t, err) + + w.Write(marshal) + })) + defer ts.Close() + + client := New(ts.URL) + byteCode, err := client.GetAccountCode(&addr, "") + + assert.NoError(t, err) + assert.Equal(t, expectedCodeRsp.Code, byteCode.Code) +} + +func TestClient_GetStorage(t *testing.T) { + addr := thor.Address{0x01} + key := thor.Bytes32{0x01} + expectedStorageRsp := &accounts.GetStorageResult{Value: hexutil.Encode([]byte{0x01, 0x03})} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/accounts/"+addr.String()+"/key/"+key.String(), r.URL.Path) + + marshal, err := json.Marshal(expectedStorageRsp) + require.NoError(t, err) + + w.Write(marshal) + })) + defer ts.Close() + + client := New(ts.URL) + data, err := client.GetAccountStorage(&addr, &key, tccommon.BestRevision) + + assert.NoError(t, err) + assert.Equal(t, expectedStorageRsp.Value, data.Value) +} + +func TestClient_GetExpandedBlock(t *testing.T) { + blockID := "123" + expectedBlock := &blocks.JSONExpandedBlock{} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/blocks/"+blockID+"?expanded=true", r.URL.Path+"?"+r.URL.RawQuery) + + blockBytes, _ := json.Marshal(expectedBlock) + w.Write(blockBytes) + })) + defer ts.Close() + + client := New(ts.URL) + block, err := client.GetExpandedBlock(blockID) + + assert.NoError(t, err) + assert.Equal(t, expectedBlock, block) +} + +func TestClient_GetBlock(t *testing.T) { + blockID := "123" + expectedBlock := &blocks.JSONCollapsedBlock{ + JSONBlockSummary: &blocks.JSONBlockSummary{ + Number: 123456, + ID: thor.Bytes32{0x01}, + GasLimit: 1000, + Beneficiary: thor.Address{0x01}, + GasUsed: 100, + TxsRoot: thor.Bytes32{0x03}, + TxsFeatures: 1, + IsFinalized: false, + }, + Transactions: nil, + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/blocks/"+blockID, r.URL.Path) + + blockBytes, _ := json.Marshal(expectedBlock) + w.Write(blockBytes) + })) + defer ts.Close() + + client := New(ts.URL) + block, err := client.GetBlock(blockID) + + assert.NoError(t, err) + assert.Equal(t, expectedBlock, block) +} + +func TestClient_GetNilBlock(t *testing.T) { + blockID := "123" + var expectedBlock *blocks.JSONCollapsedBlock + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/blocks/"+blockID, r.URL.Path) + + w.Write([]byte("null")) + })) + defer ts.Close() + + client := New(ts.URL) + block, err := client.GetBlock(blockID) + + assert.Equal(t, tccommon.ErrNotFound, err) + assert.Equal(t, expectedBlock, block) +} + +func TestClient_GetTransaction(t *testing.T) { + txID := thor.Bytes32{0x01} + expectedTx := &transactions.Transaction{ID: txID} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/transactions/"+txID.String(), r.URL.Path) + + txBytes, _ := json.Marshal(expectedTx) + w.Write(txBytes) + })) + defer ts.Close() + + client := New(ts.URL) + tx, err := client.GetTransaction(&txID, tccommon.BestRevision, false) + + assert.NoError(t, err) + assert.Equal(t, expectedTx, tx) +} + +func TestClient_GetRawTransaction(t *testing.T) { + txID := thor.Bytes32{0x01} + expectedTx := &transactions.RawTransaction{ + Meta: &transactions.TxMeta{ + BlockID: thor.Bytes32{0x01}, + BlockNumber: 1, + BlockTimestamp: 123, + }, + RawTx: transactions.RawTx{Raw: hexutil.Encode([]byte{0x03})}, + } + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/transactions/"+txID.String(), r.URL.Path) + + txBytes, err := json.Marshal(expectedTx) + require.NoError(t, err) + + w.Write(txBytes) + })) + defer ts.Close() + + client := New(ts.URL) + tx, err := client.GetRawTransaction(&txID, tccommon.BestRevision, false) + + assert.NoError(t, err) + assert.Equal(t, expectedTx, tx) +} + +func TestClient_RawHTTPPost(t *testing.T) { + url := "/test" + calldata := map[string]interface{}{} + expectedResponse := []byte{0x01} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, url, r.URL.Path) + + w.Write(expectedResponse) + })) + defer ts.Close() + + client := New(ts.URL) + response, statusCode, err := client.RawHTTPPost(url, calldata) + + assert.NoError(t, err) + assert.Equal(t, expectedResponse, response) + assert.Equal(t, http.StatusOK, statusCode) +} + +func TestClient_RawHTTPGet(t *testing.T) { + url := "/test" + expectedResponse := []byte{0x01} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, url, r.URL.Path) + + w.Write(expectedResponse) + })) + defer ts.Close() + + client := New(ts.URL) + response, statusCode, err := client.RawHTTPGet(url) + + assert.NoError(t, err) + assert.Equal(t, expectedResponse, response) + assert.Equal(t, http.StatusOK, statusCode) +} + +func TestClient_GetPeers(t *testing.T) { + expectedPeers := []*node.PeerStats{{ + Name: "nodeA", + BestBlockID: thor.Bytes32{0x01}, + TotalScore: 1000, + PeerID: "peerId", + NetAddr: "netAddr", + Inbound: false, + Duration: 1000, + }} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/node/network/peers", r.URL.Path) + + peersBytes, _ := json.Marshal(expectedPeers) + w.Write(peersBytes) + })) + defer ts.Close() + + client := New(ts.URL) + peers, err := client.GetPeers() + + assert.NoError(t, err) + assert.Equal(t, expectedPeers, peers) +} + +func TestClient_Errors(t *testing.T) { + txID := thor.Bytes32{0x01} + blockID := "123" + addr := thor.Address{0x01} + + for _, tc := range []struct { + name string + path string + function interface{} + }{ + { + name: "TransactionReceipt", + path: "/transactions/" + txID.String() + "/receipt", + function: func(client *Client) (*transactions.Receipt, error) { return client.GetTransactionReceipt(&txID, "") }, + }, + { + name: "InspectClauses", + path: "/accounts/*", + function: func(client *Client) ([]*accounts.CallResult, error) { + return client.InspectClauses(&accounts.BatchCallData{}, "") + }, + }, + { + name: "SendTransaction", + path: "/transactions", + function: func(client *Client) (*transactions.SendTxResult, error) { + return client.SendTransaction(&transactions.RawTx{}) + }, + }, + { + name: "FilterTransfers", + path: "/logs/transfer", + function: func(client *Client) ([]*transfers.FilteredTransfer, error) { + return client.FilterTransfers(&transfers.TransferFilter{}) + }, + }, + { + name: "FilterEvents", + path: "/logs/event", + function: func(client *Client) ([]events.FilteredEvent, error) { + return client.FilterEvents(&events.EventFilter{}) + }, + }, + { + name: "Account", + path: "/accounts/" + addr.String(), + function: func(client *Client) (*accounts.Account, error) { return client.GetAccount(&addr, "") }, + }, + { + name: "GetContractByteCode", + path: "/accounts/" + addr.String() + "/code", + function: func(client *Client) (*accounts.GetCodeResult, error) { return client.GetAccountCode(&addr, "") }, + }, + { + name: "GetAccountStorage", + path: "/accounts/" + addr.String() + "/key/" + thor.Bytes32{}.String(), + function: func(client *Client) (*accounts.GetStorageResult, error) { + return client.GetAccountStorage(&addr, &thor.Bytes32{}, tccommon.BestRevision) + }, + }, + { + name: "ExpandedBlock", + path: "/blocks/" + blockID + "?expanded=true", + function: func(client *Client) (*blocks.JSONExpandedBlock, error) { return client.GetExpandedBlock(blockID) }, + }, + { + name: "Block", + path: "/blocks/" + blockID, + function: func(client *Client) (*blocks.JSONCollapsedBlock, error) { return client.GetBlock(blockID) }, + }, + { + name: "Transaction", + path: "/transactions/" + txID.String(), + function: func(client *Client) (*transactions.Transaction, error) { + return client.GetTransaction(&txID, tccommon.BestRevision, false) + }, + }, + { + name: "Peers", + path: "/node/network/peers", + function: func(client *Client) ([]*node.PeerStats, error) { return client.GetPeers() }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Contains(t, tc.path, r.URL.Path) + + w.WriteHeader(http.StatusInternalServerError) + })) + defer ts.Close() + + client := New(ts.URL) + + fn := reflect.ValueOf(tc.function) + result := fn.Call([]reflect.Value{reflect.ValueOf(client)}) + + if result[len(result)-1].IsNil() { + t.Errorf("expected error for %s, but got nil", tc.name) + return + } + + err := result[len(result)-1].Interface().(error) + assert.Error(t, err) + }) + } +} diff --git a/thorclient/httpclient/http_request.go b/thorclient/httpclient/http_request.go new file mode 100644 index 000000000..cda8a71fd --- /dev/null +++ b/thorclient/httpclient/http_request.go @@ -0,0 +1,75 @@ +// Copyright (c) 2024 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package httpclient + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/vechain/thor/v2/thorclient/common" +) + +func (c *Client) httpRequest(method, url string, payload io.Reader) ([]byte, error) { + body, statusCode, err := c.rawHTTPRequest(method, url, payload) + if err != nil { + return nil, err + } + if !statusCodeIs2xx(statusCode) { + return nil, fmt.Errorf("http error - Status Code %d - %s - %w", statusCode, body, common.ErrNot200Status) + } + return body, nil +} + +func statusCodeIs2xx(statusCode int) bool { + return statusCode >= 200 && statusCode < 300 +} + +func (c *Client) rawHTTPRequest(method, url string, payload io.Reader) ([]byte, int, error) { + req, err := http.NewRequest(method, url, payload) + if err != nil { + return nil, 0, fmt.Errorf("error creating request: %w", err) + } + + if method == "POST" { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := c.c.Do(req) + if err != nil { + return nil, 0, fmt.Errorf("error performing request: %w", err) + } + defer resp.Body.Close() + + responseBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, 0, fmt.Errorf("error reading response body: %w", err) + } + + return responseBody, resp.StatusCode, nil +} + +func (c *Client) httpGET(url string) ([]byte, error) { + return c.httpRequest("GET", url, nil) +} + +func (c *Client) httpPOST(url string, payload interface{}) ([]byte, error) { + var data []byte + + if _, ok := payload.([]byte); ok { + data = payload.([]byte) + } else { + var err error + data, err = json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("unable to marshal payload - %w", err) + } + } + + return c.httpRequest("POST", url, bytes.NewBuffer(data)) +} diff --git a/thorclient/thorclient.go b/thorclient/thorclient.go new file mode 100644 index 000000000..8458a0ae4 --- /dev/null +++ b/thorclient/thorclient.go @@ -0,0 +1,274 @@ +// Copyright (c) 2024 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +// Package thorclient provides a client for interacting with the VeChainThor blockchain. +// It offers a set of methods to interact with accounts, transactions, blocks, events, and other +// features via HTTP and WebSocket connections. + +package thorclient + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/rlp" + "github.com/vechain/thor/v2/api/accounts" + "github.com/vechain/thor/v2/api/blocks" + "github.com/vechain/thor/v2/api/events" + "github.com/vechain/thor/v2/api/node" + "github.com/vechain/thor/v2/api/subscriptions" + "github.com/vechain/thor/v2/api/transactions" + "github.com/vechain/thor/v2/api/transfers" + "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/thorclient/common" + "github.com/vechain/thor/v2/thorclient/httpclient" + "github.com/vechain/thor/v2/thorclient/wsclient" + "github.com/vechain/thor/v2/tx" + + tccommon "github.com/vechain/thor/v2/thorclient/common" +) + +// Client represents the VeChainThor client, allowing communication over HTTP and WebSocket. +type Client struct { + httpConn *httpclient.Client + wsConn *wsclient.Client +} + +// New creates a new Client using the provided HTTP URL. +func New(url string) *Client { + return &Client{ + httpConn: httpclient.New(url), + } +} + +// NewWithWS creates a new Client using the provided HTTP and WebSocket URLs. +// Returns an error if the WebSocket connection fails. +func NewWithWS(url string) (*Client, error) { + wsClient, err := wsclient.NewClient(url) + if err != nil { + return nil, err + } + + return &Client{ + httpConn: httpclient.New(url), + wsConn: wsClient, + }, nil +} + +// Option represents a functional option for customizing client requests. +type Option func(*getOptions) + +// getOptions holds configuration options for client requests. +type getOptions struct { + revision string + pending bool +} + +// applyOptions applies the given functional options to the default options. +func applyOptions(opts []Option) *getOptions { + options := &getOptions{ + revision: tccommon.BestRevision, + pending: false, + } + for _, o := range opts { + o(options) + } + return options +} + +// Revision returns an Option to specify the revision for requests. +func Revision(revision string) Option { + return func(o *getOptions) { + o.revision = revision + } +} + +// Pending returns an Option to specify that the client should fetch pending results. +func Pending() Option { + return func(o *getOptions) { + o.pending = true + } +} + +// RawHTTPClient returns the underlying HTTP client. +func (c *Client) RawHTTPClient() *httpclient.Client { + return c.httpConn +} + +// RawWSClient returns the underlying WebSocket client. +func (c *Client) RawWSClient() *wsclient.Client { + return c.wsConn +} + +// Account retrieves an account from the blockchain based on the provided address and options. +func (c *Client) Account(addr *thor.Address, opts ...Option) (*accounts.Account, error) { + options := applyOptions(opts) + return c.httpConn.GetAccount(addr, options.revision) +} + +// InspectClauses inspects the clauses of a batch call data and returns the call results. +func (c *Client) InspectClauses(calldata *accounts.BatchCallData, opts ...Option) ([]*accounts.CallResult, error) { + options := applyOptions(opts) + return c.httpConn.InspectClauses(calldata, options.revision) +} + +// InspectTxClauses inspects the clauses of a transaction and returns the call results. +// It accepts both signed and unsigned transactions. +func (c *Client) InspectTxClauses(tx *tx.Transaction, senderAddr *thor.Address, opts ...Option) ([]*accounts.CallResult, error) { + clauses := convertToBatchCallData(tx, senderAddr) + return c.InspectClauses(clauses, opts...) +} + +// AccountCode retrieves the account code for a given address. +func (c *Client) AccountCode(addr *thor.Address, opts ...Option) (*accounts.GetCodeResult, error) { + options := applyOptions(opts) + return c.httpConn.GetAccountCode(addr, options.revision) +} + +// AccountStorage retrieves the storage value for a given address and key. +func (c *Client) AccountStorage(addr *thor.Address, key *thor.Bytes32, opts ...Option) (*accounts.GetStorageResult, error) { + options := applyOptions(opts) + return c.httpConn.GetAccountStorage(addr, key, options.revision) +} + +// Transaction retrieves a transaction by its ID. +func (c *Client) Transaction(id *thor.Bytes32, opts ...Option) (*transactions.Transaction, error) { + options := applyOptions(opts) + return c.httpConn.GetTransaction(id, options.revision, options.pending) +} + +// RawTransaction retrieves the raw transaction data by its ID. +func (c *Client) RawTransaction(id *thor.Bytes32, opts ...Option) (*transactions.RawTransaction, error) { + options := applyOptions(opts) + return c.httpConn.GetRawTransaction(id, options.revision, options.pending) +} + +// TransactionReceipt retrieves the receipt for a transaction by its ID. +func (c *Client) TransactionReceipt(id *thor.Bytes32, opts ...Option) (*transactions.Receipt, error) { + options := applyOptions(opts) + return c.httpConn.GetTransactionReceipt(id, options.revision) +} + +// SendTransaction sends a signed transaction to the blockchain. +func (c *Client) SendTransaction(tx *tx.Transaction) (*transactions.SendTxResult, error) { + rlpTx, err := rlp.EncodeToBytes(tx) + if err != nil { + return nil, fmt.Errorf("unable to encode transaction - %w", err) + } + + return c.SendRawTransaction(rlpTx) +} + +// SendRawTransaction sends a raw RLP-encoded transaction to the blockchain. +func (c *Client) SendRawTransaction(rlpTx []byte) (*transactions.SendTxResult, error) { + return c.httpConn.SendTransaction(&transactions.RawTx{Raw: hexutil.Encode(rlpTx)}) +} + +// Block retrieves a block by its revision. +func (c *Client) Block(revision string) (blocks *blocks.JSONCollapsedBlock, err error) { + return c.httpConn.GetBlock(revision) +} + +// ExpandedBlock retrieves an expanded block by its revision. +func (c *Client) ExpandedBlock(revision string) (blocks *blocks.JSONExpandedBlock, err error) { + return c.httpConn.GetExpandedBlock(revision) +} + +// FilterEvents filters events based on the provided filter request. +func (c *Client) FilterEvents(req *events.EventFilter) ([]events.FilteredEvent, error) { + return c.httpConn.FilterEvents(req) +} + +// FilterTransfers filters transfers based on the provided filter request. +func (c *Client) FilterTransfers(req *transfers.TransferFilter) ([]*transfers.FilteredTransfer, error) { + return c.httpConn.FilterTransfers(req) +} + +// Peers retrieves the list of connected peers. +func (c *Client) Peers() ([]*node.PeerStats, error) { + return c.httpConn.GetPeers() +} + +// ChainTag retrieves the chain tag from the genesis block. +func (c *Client) ChainTag() (byte, error) { + genesisBlock, err := c.Block("0") + if err != nil { + return 0, err + } + return genesisBlock.ID[31], nil +} + +// SubscribeBlocks subscribes to block updates over WebSocket. +func (c *Client) SubscribeBlocks(pos string) (*common.Subscription[*blocks.JSONCollapsedBlock], error) { + if c.wsConn == nil { + return nil, fmt.Errorf("not a websocket typed client") + } + return c.wsConn.SubscribeBlocks(pos) +} + +// SubscribeEvents subscribes to event updates over WebSocket. +func (c *Client) SubscribeEvents(pos string, filter *subscriptions.EventFilter) (*common.Subscription[*subscriptions.EventMessage], error) { + if c.wsConn == nil { + return nil, fmt.Errorf("not a websocket typed client") + } + return c.wsConn.SubscribeEvents(pos, filter) +} + +// SubscribeTransfers subscribes to transfer updates over WebSocket. +func (c *Client) SubscribeTransfers(pos string, filter *subscriptions.TransferFilter) (*common.Subscription[*subscriptions.TransferMessage], error) { + if c.wsConn == nil { + return nil, fmt.Errorf("not a websocket typed client") + } + return c.wsConn.SubscribeTransfers(pos, filter) +} + +// SubscribeBeats2 subscribes to Beat2 message updates over WebSocket. +func (c *Client) SubscribeBeats2(pos string) (*common.Subscription[*subscriptions.Beat2Message], error) { + if c.wsConn == nil { + return nil, fmt.Errorf("not a websocket typed client") + } + return c.wsConn.SubscribeBeats2(pos) +} + +// SubscribeTxPool subscribes to pending transaction updates over WebSocket. +func (c *Client) SubscribeTxPool(txID *thor.Bytes32) (*common.Subscription[*subscriptions.PendingTxIDMessage], error) { + if c.wsConn == nil { + return nil, fmt.Errorf("not a websocket typed client") + } + return c.wsConn.SubscribeTxPool(txID) +} + +// convertToBatchCallData converts a transaction and sender address to batch call data format. +func convertToBatchCallData(tx *tx.Transaction, addr *thor.Address) *accounts.BatchCallData { + cls := make(accounts.Clauses, len(tx.Clauses())) + for i, c := range tx.Clauses() { + cls[i] = convertClauseAccounts(c) + } + + blockRef := tx.BlockRef() + encodedBlockRef := hexutil.Encode(blockRef[:]) + + return &accounts.BatchCallData{ + Clauses: cls, + Gas: tx.Gas(), + ProvedWork: nil, // todo hook this field + Caller: addr, + GasPayer: nil, // todo hook this field + GasPrice: nil, // todo hook this field + Expiration: tx.Expiration(), + BlockRef: encodedBlockRef, + } +} + +// convertClauseAccounts converts a transaction clause to accounts.Clause format. +func convertClauseAccounts(c *tx.Clause) accounts.Clause { + value := math.HexOrDecimal256(*c.Value()) + return accounts.Clause{ + To: c.To(), + Value: &value, + Data: hexutil.Encode(c.Data()), + } +} diff --git a/thorclient/thorclient_test.go b/thorclient/thorclient_test.go new file mode 100644 index 000000000..e106fa443 --- /dev/null +++ b/thorclient/thorclient_test.go @@ -0,0 +1,130 @@ +// Copyright (c) 2024 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package thorclient + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vechain/thor/v2/api/accounts" + "github.com/vechain/thor/v2/api/transactions" + "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/tx" + + tccommon "github.com/vechain/thor/v2/thorclient/common" +) + +func TestConvertToBatchCallData(t *testing.T) { + // Test case 1: Empty transaction + tx1 := &tx.Transaction{} + addr1 := &thor.Address{} + expected1 := &accounts.BatchCallData{ + Clauses: make(accounts.Clauses, 0), + Gas: 0, + ProvedWork: nil, + Caller: addr1, + GasPayer: nil, + Expiration: 0, + BlockRef: "0x0000000000000000", + } + assert.Equal(t, expected1, convertToBatchCallData(tx1, addr1)) +} + +func TestRevision(t *testing.T) { + addr := thor.BytesToAddress([]byte("account1")) + revision := "revision1" + + for _, tc := range []struct { + name string + function interface{} + expectedPath string + expectedRevision string + }{ + { + name: "Account", + function: func(client *Client) { client.Account(&addr) }, + expectedPath: "/accounts/" + addr.String(), + expectedRevision: "", + }, + { + name: "GetAccounForRevision", + function: func(client *Client) { client.Account(&addr, Revision(revision)) }, + expectedPath: "/accounts/" + addr.String(), + expectedRevision: "", + }, + { + name: "GetAccountCode", + function: func(client *Client) { client.AccountCode(&addr) }, + expectedPath: "/accounts/" + addr.String() + "/code", + expectedRevision: "", + }, + { + name: "GetAccountCodeForRevision", + function: func(client *Client) { client.AccountCode(&addr, Revision(revision)) }, + expectedPath: "/accounts/" + addr.String() + "/code", + expectedRevision: "", + }, + } { + t.Run(tc.name, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, tc.expectedPath, r.URL.Path) + if tc.expectedRevision != "" { + assert.Equal(t, "revision", r.URL.Query().Get("revision")) + } + + w.Write([]byte{}) + })) + defer ts.Close() + + client := New(ts.URL) + + fn := reflect.ValueOf(tc.function) + fn.Call([]reflect.Value{reflect.ValueOf(client)}) + }) + } +} + +func TestGetTransaction(t *testing.T) { + expectedTx := &transactions.Transaction{ + ID: thor.BytesToBytes32([]byte("txid1")), + } + + for _, tc := range []struct { + name string + function interface{} + isPending bool + }{ + { + name: "Transaction", + function: func(client *Client) { client.Transaction(&expectedTx.ID) }, + isPending: false, + }, + { + name: "GetTransactionPending", + function: func(client *Client) { client.Transaction(&expectedTx.ID, Revision(tccommon.BestRevision), Pending()) }, + isPending: true, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/transactions/"+expectedTx.ID.String(), r.URL.Path) + if tc.isPending { + assert.Equal(t, "true", r.URL.Query().Get("pending")) + } + + w.Write(expectedTx.ID[:]) + })) + defer ts.Close() + + client := New(ts.URL) + fn := reflect.ValueOf(tc.function) + fn.Call([]reflect.Value{reflect.ValueOf(client)}) + }) + } +} diff --git a/thorclient/wsclient/client.go b/thorclient/wsclient/client.go new file mode 100644 index 000000000..057d5aa48 --- /dev/null +++ b/thorclient/wsclient/client.go @@ -0,0 +1,218 @@ +// Copyright (c) 2024 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +// Package wsclient provides a WebSocket client for subscribing to various VeChainThor blockchain events. +// It enables subscriptions to blocks, transfers, events, and other updates via WebSocket. +package wsclient + +import ( + "fmt" + "net/url" + "strings" + "time" + + "github.com/vechain/thor/v2/thor" + + "github.com/gorilla/websocket" + "github.com/vechain/thor/v2/api/blocks" + "github.com/vechain/thor/v2/api/subscriptions" + "github.com/vechain/thor/v2/thorclient/common" +) + +const readTimeout = 60 * time.Second + +// Client represents a WebSocket client that connects to the VeChainThor blockchain via WebSocket +// for subscribing to blockchain events and updates. +type Client struct { + host string + scheme string +} + +// NewClient creates a new WebSocket Client from the provided URL. +// The function parses the URL, determines the appropriate WebSocket scheme (ws or wss), +// and returns the client or an error if the URL is invalid. +func NewClient(url string) (*Client, error) { + var host string + var scheme string + + // Determine the scheme (ws or wss) based on the URL. + if strings.Contains(url, "https://") || strings.Contains(url, "wss://") { + host = strings.TrimPrefix(strings.TrimPrefix(url, "https://"), "wss://") + scheme = "wss" + } else if strings.Contains(url, "http://") || strings.Contains(url, "ws://") { + host = strings.TrimPrefix(strings.TrimPrefix(url, "http://"), "ws://") + scheme = "ws" + } else { + return nil, fmt.Errorf("invalid url") + } + + return &Client{ + host: strings.TrimSuffix(host, "/"), + scheme: scheme, + }, nil +} + +// SubscribeEvents subscribes to blockchain events based on the provided query. +// It returns a Subscription that streams event messages or an error if the connection fails. +func (c *Client) SubscribeEvents(pos string, filter *subscriptions.EventFilter) (*common.Subscription[*subscriptions.EventMessage], error) { + queryValues := &url.Values{} + queryValues.Add("pos", pos) + if filter != nil { + if filter.Address != nil { + queryValues.Add("address", filter.Address.String()) + } + if filter.Topic0 != nil { + queryValues.Add("topic0", filter.Topic0.String()) + } + if filter.Topic1 != nil { + queryValues.Add("topic1", filter.Topic1.String()) + } + if filter.Topic2 != nil { + queryValues.Add("topic2", filter.Topic2.String()) + } + if filter.Topic3 != nil { + queryValues.Add("topic3", filter.Topic3.String()) + } + if filter.Topic4 != nil { + queryValues.Add("topic4", filter.Topic4.String()) + } + } + conn, err := c.connect("/subscriptions/event", queryValues) + if err != nil { + return nil, fmt.Errorf("unable to connect - %w", err) + } + + return subscribe[subscriptions.EventMessage](conn), nil +} + +// SubscribeBlocks subscribes to block updates based on the provided query. +// It returns a Subscription that streams block messages or an error if the connection fails. +func (c *Client) SubscribeBlocks(pos string) (*common.Subscription[*blocks.JSONCollapsedBlock], error) { + queryValues := &url.Values{} + queryValues.Add("pos", pos) + conn, err := c.connect("/subscriptions/block", queryValues) + if err != nil { + return nil, fmt.Errorf("unable to connect - %w", err) + } + + return subscribe[blocks.JSONCollapsedBlock](conn), nil +} + +// SubscribeTransfers subscribes to transfer events based on the provided query. +// It returns a Subscription that streams transfer messages or an error if the connection fails. +func (c *Client) SubscribeTransfers(pos string, filter *subscriptions.TransferFilter) (*common.Subscription[*subscriptions.TransferMessage], error) { + queryValues := &url.Values{} + queryValues.Add("pos", pos) + if filter != nil { + if filter.TxOrigin != nil { + queryValues.Add("txOrigin", filter.TxOrigin.String()) + } + if filter.Sender != nil { + queryValues.Add("sender", filter.Sender.String()) + } + if filter.Recipient != nil { + queryValues.Add("recipient", filter.Recipient.String()) + } + } + conn, err := c.connect("/subscriptions/transfer", queryValues) + if err != nil { + return nil, fmt.Errorf("unable to connect - %w", err) + } + + return subscribe[subscriptions.TransferMessage](conn), nil +} + +// SubscribeTxPool subscribes to pending transaction pool updates based on the provided query. +// It returns a Subscription that streams pending transaction messages or an error if the connection fails. +func (c *Client) SubscribeTxPool(txID *thor.Bytes32) (*common.Subscription[*subscriptions.PendingTxIDMessage], error) { + queryValues := &url.Values{} + if txID != nil { + queryValues.Add("id", txID.String()) + } + + conn, err := c.connect("/subscriptions/txpool", queryValues) + if err != nil { + return nil, fmt.Errorf("unable to connect - %w", err) + } + + return subscribe[subscriptions.PendingTxIDMessage](conn), nil +} + +// SubscribeBeats2 subscribes to Beat2 messages based on the provided query. +// It returns a Subscription that streams Beat2 messages or an error if the connection fails. +func (c *Client) SubscribeBeats2(pos string) (*common.Subscription[*subscriptions.Beat2Message], error) { + queryValues := &url.Values{} + queryValues.Add("pos", pos) + conn, err := c.connect("/subscriptions/beat2", queryValues) + if err != nil { + return nil, fmt.Errorf("unable to connect - %w", err) + } + + return subscribe[subscriptions.Beat2Message](conn), nil +} + +// subscribe starts a new subscription over the given WebSocket connection. +// It returns a read-only channel that streams events of type T. +func subscribe[T any](conn *websocket.Conn) *common.Subscription[*T] { + // Create a new channel for events + eventChan := make(chan common.EventWrapper[*T], 1_000) + var closed bool + + // Start a goroutine to handle receiving messages from the WebSocket connection. + go func() { + defer close(eventChan) + defer conn.Close() + + for { + conn.SetReadDeadline(time.Now().Add(readTimeout)) + var data T + // Read a JSON message from the WebSocket and unmarshal it into the data. + err := conn.ReadJSON(&data) + if err != nil { + if !closed { + // Send an EventWrapper with the error to the channel. + eventChan <- common.EventWrapper[*T]{Error: fmt.Errorf("%w: %w", common.ErrUnexpectedMsg, err)} + } + return + } + + eventChan <- common.EventWrapper[*T]{Data: &data} + } + }() + + return &common.Subscription[*T]{ + EventChan: eventChan, + Unsubscribe: func() { + closed = true + conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")) + conn.Close() + }, + } +} + +// connect establishes a WebSocket connection to the specified endpoint and query. +// It returns the connection or an error if the connection fails. +func (c *Client) connect(endpoint string, queryValues *url.Values) (*websocket.Conn, error) { + u := url.URL{ + Scheme: c.scheme, + Host: c.host, + Path: endpoint, + RawQuery: queryValues.Encode(), + } + + conn, _, err := websocket.DefaultDialer.Dial(u.String(), nil) + if err != nil { + return nil, err + } + + conn.SetPingHandler(func(payload string) error { + // Make a best effort to send the pong message. + _ = conn.WriteControl(websocket.PongMessage, []byte(payload), time.Now().Add(time.Second)) + conn.SetReadDeadline(time.Now().Add(readTimeout)) + return nil + }) + // TODO append to the connection pool + return conn, nil +} diff --git a/thorclient/wsclient/client_test.go b/thorclient/wsclient/client_test.go new file mode 100644 index 000000000..483ae7233 --- /dev/null +++ b/thorclient/wsclient/client_test.go @@ -0,0 +1,449 @@ +// Copyright (c) 2024 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package wsclient + +import ( + "errors" + "net/http" + "net/http/httptest" + "reflect" + "testing" + "time" + + "github.com/vechain/thor/v2/test/datagen" + "github.com/vechain/thor/v2/thor" + + "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" + "github.com/vechain/thor/v2/api/blocks" + "github.com/vechain/thor/v2/api/subscriptions" + "github.com/vechain/thor/v2/thorclient/common" +) + +func TestClient_SubscribeEvents(t *testing.T) { + pos := "best" + expectedEvent := &subscriptions.EventMessage{} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/subscriptions/event", r.URL.Path) + assert.Equal(t, "pos="+pos, r.URL.RawQuery) + + upgrader := websocket.Upgrader{} + + conn, _ := upgrader.Upgrade(w, r, nil) + defer conn.Close() + + conn.WriteJSON(expectedEvent) + })) + defer ts.Close() + + client, err := NewClient(ts.URL) + assert.NoError(t, err) + sub, err := client.SubscribeEvents(pos, nil) + + assert.NoError(t, err) + assert.Equal(t, expectedEvent, (<-sub.EventChan).Data) +} + +func TestClient_SubscribeBlocks(t *testing.T) { + pos := "best" + expectedBlock := &blocks.JSONCollapsedBlock{} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/subscriptions/block", r.URL.Path) + assert.Equal(t, "pos="+pos, r.URL.RawQuery) + + upgrader := websocket.Upgrader{} + + conn, _ := upgrader.Upgrade(w, r, nil) + defer conn.Close() + + conn.WriteJSON(expectedBlock) + })) + defer ts.Close() + + client, err := NewClient(ts.URL) + assert.NoError(t, err) + sub, err := client.SubscribeBlocks(pos) + + assert.NoError(t, err) + assert.Equal(t, expectedBlock, (<-sub.EventChan).Data) +} + +func TestClient_SubscribeTransfers(t *testing.T) { + pos := "best" + expectedTransfer := &subscriptions.TransferMessage{} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/subscriptions/transfer", r.URL.Path) + assert.Equal(t, "pos="+pos, r.URL.RawQuery) + + upgrader := websocket.Upgrader{} + + conn, _ := upgrader.Upgrade(w, r, nil) + defer conn.Close() + + conn.WriteJSON(expectedTransfer) + })) + defer ts.Close() + + client, err := NewClient(ts.URL) + assert.NoError(t, err) + sub, err := client.SubscribeTransfers(pos, nil) + + assert.NoError(t, err) + derp := (<-sub.EventChan).Data + assert.Equal(t, expectedTransfer, derp) +} + +func TestClient_SubscribeTxPool(t *testing.T) { + txID := datagen.RandomHash() + expectedPendingTxID := &subscriptions.PendingTxIDMessage{} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/subscriptions/txpool", r.URL.Path) + assert.Equal(t, "id="+txID.String(), r.URL.RawQuery) + + upgrader := websocket.Upgrader{} + + conn, _ := upgrader.Upgrade(w, r, nil) + defer conn.Close() + + conn.WriteJSON(expectedPendingTxID) + })) + defer ts.Close() + + client, err := NewClient(ts.URL) + assert.NoError(t, err) + sub, err := client.SubscribeTxPool(&txID) + + assert.NoError(t, err) + assert.Equal(t, expectedPendingTxID, (<-sub.EventChan).Data) +} + +func TestClient_SubscribeBeats2(t *testing.T) { + pos := "best" + expectedBeat2 := &subscriptions.Beat2Message{} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/subscriptions/beat2", r.URL.Path) + assert.Equal(t, "pos="+pos, r.URL.RawQuery) + + upgrader := websocket.Upgrader{} + + conn, _ := upgrader.Upgrade(w, r, nil) + defer conn.Close() + + conn.WriteJSON(expectedBeat2) + })) + defer ts.Close() + + client, err := NewClient(ts.URL) + assert.NoError(t, err) + sub, err := client.SubscribeBeats2(pos) + + assert.NoError(t, err) + assert.Equal(t, expectedBeat2, (<-sub.EventChan).Data) +} +func TestNewClient(t *testing.T) { + expectedHost := "example.com" + + for _, tc := range []struct { + name string + url string + expectedSchema string + }{ + { + name: "http", + url: "http://example.com", + expectedSchema: "ws", + }, + { + name: "https", + url: "https://example.com", + expectedSchema: "wss", + }, + { + name: "ws", + url: "ws://example.com", + expectedSchema: "ws", + }, + { + name: "wss", + url: "wss://example.com", + expectedSchema: "wss", + }, + } { + t.Run(tc.name, func(t *testing.T) { + client, err := NewClient(tc.url) + assert.NoError(t, err) + assert.NotNil(t, client) + assert.Equal(t, tc.expectedSchema, client.scheme) + assert.Equal(t, expectedHost, client.host) + }) + } +} + +func TestNewClientError(t *testing.T) { + badURL := "invalid" + client, err := NewClient(badURL) + assert.Error(t, err) + assert.Nil(t, client) +} + +func TestClient_SubscribeError(t *testing.T) { + pos := "examplePos" + badURL := "http://example.com" + client, err := NewClient(badURL) + assert.NoError(t, err) + + for _, tc := range []struct { + name string + subscribeFunc interface{} + args []interface{} + }{ + { + name: "SubscribeEvents", + subscribeFunc: client.SubscribeEvents, + args: []interface{}{pos, (*subscriptions.EventFilter)(nil)}, // pos and a nil EventFilter + }, + { + name: "SubscribeTransfers", + subscribeFunc: client.SubscribeTransfers, + args: []interface{}{pos, (*subscriptions.TransferFilter)(nil)}, // pos and a nil TransferFilter + }, + { + name: "SubscribeTxPool", + subscribeFunc: client.SubscribeTxPool, + args: []interface{}{(*thor.Bytes32)(nil)}, // nil txID + }, + { + name: "SubscribeBeats2", + subscribeFunc: client.SubscribeBeats2, + args: []interface{}{pos}, // only pos + }, + { + name: "SubscribeBlocks", + subscribeFunc: client.SubscribeBlocks, + args: []interface{}{pos}, // only pos + }, + } { + t.Run(tc.name, func(t *testing.T) { + fn := reflect.ValueOf(tc.subscribeFunc) + + // Prepare the arguments for the function call + var reflectArgs []reflect.Value + for _, arg := range tc.args { + reflectArgs = append(reflectArgs, reflect.ValueOf(arg)) + } + + // Call the subscription function + result := fn.Call(reflectArgs) + + // Check if the second returned value is an error and not nil + if result[1].IsNil() { + t.Errorf("expected error for %s, but got nil", tc.name) + return + } + + // Assert that the error is present + err := result[1].Interface().(error) + assert.Error(t, err) + }) + } +} + +func TestClient_SubscribeBlocks_ServerError(t *testing.T) { + pos := "best" + expectedError := "test error" + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/subscriptions/block", r.URL.Path) + assert.Equal(t, "pos="+pos, r.URL.RawQuery) + + upgrader := websocket.Upgrader{} + + conn, _ := upgrader.Upgrade(w, r, nil) + defer conn.Close() + + // Send a message that causes an error on the client side + conn.WriteMessage(websocket.TextMessage, []byte(expectedError)) + })) + defer ts.Close() + + client, err := NewClient(ts.URL) + assert.NoError(t, err) + sub, err := client.SubscribeBlocks(pos) + + assert.NoError(t, err) + + // Read the error from the event channel + event := <-sub.EventChan + assert.Error(t, event.Error) + assert.True(t, errors.Is(event.Error, common.ErrUnexpectedMsg)) +} + +func TestClient_SubscribeBlocks_ServerShutdown(t *testing.T) { + pos := "best" + expectedBlock := &blocks.JSONCollapsedBlock{} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/subscriptions/block", r.URL.Path) + assert.Equal(t, "pos="+pos, r.URL.RawQuery) + + upgrader := websocket.Upgrader{} + + conn, _ := upgrader.Upgrade(w, r, nil) + + // Send a valid block to the client + conn.WriteJSON(expectedBlock) + + // Simulate a server shutdown by closing the WebSocket connection + conn.Close() + })) + defer ts.Close() + + client, err := NewClient(ts.URL) + assert.NoError(t, err) + sub, err := client.SubscribeBlocks(pos) + + assert.NoError(t, err) + + // The first event should be the valid block + event := <-sub.EventChan + assert.NoError(t, event.Error) + assert.Equal(t, expectedBlock, event.Data) + + // The next event should be an error due to the server shutdown + event = <-sub.EventChan + assert.Error(t, event.Error) + assert.Contains(t, event.Error.Error(), "websocket: close") +} + +func TestClient_SubscribeBlocks_ClientShutdown(t *testing.T) { + pos := "best" + expectedBlock := &blocks.JSONCollapsedBlock{} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/subscriptions/block", r.URL.Path) + assert.Equal(t, "pos="+pos, r.URL.RawQuery) + + upgrader := websocket.Upgrader{} + + conn, _ := upgrader.Upgrade(w, r, nil) + + // Send a valid block to the client + + for { + err := conn.WriteJSON(expectedBlock) + if err != nil { + break + } + time.Sleep(10 * time.Millisecond) + } + })) + defer ts.Close() + + client, err := NewClient(ts.URL) + assert.NoError(t, err) + sub, err := client.SubscribeBlocks(pos) + + assert.NoError(t, err) + + // The first 50 events should be the valid block + // the server is producing events at high speed + for i := 0; i < 50; i++ { + event := <-sub.EventChan + assert.NoError(t, event.Error) + assert.Equal(t, expectedBlock, event.Data) + } + + // unsubscribe should close the connection forcing a connection error in the eventChan + sub.Unsubscribe() + + // Ensure no more events are received after unsubscribe + select { + case _, ok := <-sub.EventChan: + if ok { + t.Error("Expected the event channel to be closed after unsubscribe, but it was still open") + } + case <-time.After(200 * time.Second): + // Timeout here is expected since the channel should be closed and not sending events + } +} + +func TestClient_SubscribeBlocks_ClientShutdown_LongBlocks(t *testing.T) { + pos := "best" + expectedBlock := &blocks.JSONCollapsedBlock{} + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "/subscriptions/block", r.URL.Path) + assert.Equal(t, "pos="+pos, r.URL.RawQuery) + + upgrader := websocket.Upgrader{} + + conn, _ := upgrader.Upgrade(w, r, nil) + + // Send a valid block to the client + + for { + err := conn.WriteJSON(expectedBlock) + if err != nil { + break + } + time.Sleep(1 * time.Second) + } + })) + defer ts.Close() + + client, err := NewClient(ts.URL) + assert.NoError(t, err) + sub, err := client.SubscribeBlocks(pos) + + assert.NoError(t, err) + + assert.NoError(t, (<-sub.EventChan).Error) + assert.NotNil(t, (<-sub.EventChan).Data) + + // unsubscribe should close the connection forcing a connection error in the eventChan + sub.Unsubscribe() + + // Ensure no more events are received after unsubscribe + select { + case _, ok := <-sub.EventChan: + if ok { + t.Error("Expected the event channel to be closed after unsubscribe, but it was still open") + } + case <-time.After(200 * time.Millisecond): + // Timeout here is expected since the channel should be closed and not sending events + } +} + +// go test -timeout 80s -run ^TestSubscribeBeats2WithServer$ github.com/vechain/thor/v2/thorclient/wsclient -v +func TestSubscribeBeats2WithServer(t *testing.T) { + t.Skip("manual test") + client, err := NewClient("https://mainnet.vechain.org") + if err != nil { + t.Fatal(err) + } + + sub, err := client.SubscribeBeats2("") + if err != nil { + t.Fatal(err) + } + + go func() { + <-time.After(60 * time.Second) + sub.Unsubscribe() + }() + + for ev := range sub.EventChan { + if ev.Error != nil { + t.Fatal(ev.Error) + } + t.Log(ev.Data) + } +} From af887573cad4da211be98b2883989dce444d2c54 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Wed, 30 Oct 2024 11:36:54 +0000 Subject: [PATCH 02/25] Show all issues on lint (#869) * Show all issues on lint * fix lint --- .github/workflows/lint-go.yaml | 2 +- api/accounts/accounts_test.go | 44 ---------------------------------- api/blocks/blocks_test.go | 14 ----------- api/events/events_test.go | 19 --------------- api/node/node_test.go | 15 ------------ 5 files changed, 1 insertion(+), 93 deletions(-) diff --git a/.github/workflows/lint-go.yaml b/.github/workflows/lint-go.yaml index f2497950c..ea4d2195a 100644 --- a/.github/workflows/lint-go.yaml +++ b/.github/workflows/lint-go.yaml @@ -23,5 +23,5 @@ jobs: version: v1.60.3 # use the default if on main branch, otherwise use the pull request config args: --timeout=30m --config=.golangci.yml - only-new-issues: true + only-new-issues: false skip-cache: true diff --git a/api/accounts/accounts_test.go b/api/accounts/accounts_test.go index cd4aac3cb..e126960b2 100644 --- a/api/accounts/accounts_test.go +++ b/api/accounts/accounts_test.go @@ -6,10 +6,8 @@ package accounts_test import ( - "bytes" "encoding/json" "fmt" - "io" "math/big" "net/http" "net/http/httptest" @@ -578,45 +576,3 @@ func batchCallWithNonExistingRevision(t *testing.T) { assert.Equal(t, http.StatusBadRequest, statusCode, "bad revision") assert.Equal(t, "revision: leveldb: not found\n", string(res), "revision not found") } - -func httpPost(t *testing.T, url string, body interface{}) ([]byte, int) { - data, err := json.Marshal(body) - if err != nil { - t.Fatal(err) - } - res, err := http.Post(url, "application/x-www-form-urlencoded", bytes.NewReader(data)) //#nosec G107 - if err != nil { - t.Fatal(err) - } - r, err := io.ReadAll(res.Body) - res.Body.Close() - if err != nil { - t.Fatal(err) - } - return r, res.StatusCode -} - -func httpGet(t *testing.T, url string) ([]byte, int) { - res, err := http.Get(url) //#nosec G107 - if err != nil { - t.Fatal(err) - } - r, err := io.ReadAll(res.Body) - res.Body.Close() - if err != nil { - t.Fatal(err) - } - return r, res.StatusCode -} - -func httpGetAccount(t *testing.T, path string) *accounts.Account { - res, statusCode := httpGet(t, ts.URL+"/accounts/"+path) - var acc accounts.Account - if err := json.Unmarshal(res, &acc); err != nil { - t.Fatal(err) - } - - assert.Equal(t, http.StatusOK, statusCode, "get account failed") - - return &acc -} diff --git a/api/blocks/blocks_test.go b/api/blocks/blocks_test.go index 60b96b299..a203cadd6 100644 --- a/api/blocks/blocks_test.go +++ b/api/blocks/blocks_test.go @@ -7,7 +7,6 @@ package blocks_test import ( "encoding/json" - "io" "math" "math/big" "net/http" @@ -267,16 +266,3 @@ func checkExpandedBlock(t *testing.T, expBl *block.Block, actBl *blocks.JSONExpa assert.Equal(t, tx.ID(), actBl.Transactions[i].ID, "txid should be equal") } } - -func httpGet(t *testing.T, url string) ([]byte, int) { - res, err := http.Get(url) //#nosec G107 - if err != nil { - t.Fatal(err) - } - r, err := io.ReadAll(res.Body) - res.Body.Close() - if err != nil { - t.Fatal(err) - } - return r, res.StatusCode -} diff --git a/api/events/events_test.go b/api/events/events_test.go index 9f859780b..3d28e41dc 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -6,9 +6,7 @@ package events_test import ( - "bytes" "encoding/json" - "io" "net/http" "net/http/httptest" "strings" @@ -212,23 +210,6 @@ func createDb(t *testing.T) *logdb.LogDB { } // Utilities functions -func httpPost(t *testing.T, url string, body interface{}) ([]byte, int) { - data, err := json.Marshal(body) - if err != nil { - t.Fatal(err) - } - res, err := http.Post(url, "application/x-www-form-urlencoded", bytes.NewReader(data)) //#nosec G107 - if err != nil { - t.Fatal(err) - } - r, err := io.ReadAll(res.Body) - res.Body.Close() - if err != nil { - t.Fatal(err) - } - return r, res.StatusCode -} - func insertBlocks(t *testing.T, db *logdb.LogDB, n int) { b := new(block.Builder).Build() for i := 0; i < n; i++ { diff --git a/api/node/node_test.go b/api/node/node_test.go index fd74326d3..e9bbe95fe 100644 --- a/api/node/node_test.go +++ b/api/node/node_test.go @@ -5,8 +5,6 @@ package node_test import ( - "io" - "net/http" "net/http/httptest" "testing" "time" @@ -54,16 +52,3 @@ func initCommServer(t *testing.T) { node.New(comm).Mount(router, "/node") ts = httptest.NewServer(router) } - -func httpGet(t *testing.T, url string) []byte { - res, err := http.Get(url) //#nosec G107 - if err != nil { - t.Fatal(err) - } - r, err := io.ReadAll(res.Body) - res.Body.Close() - if err != nil { - t.Fatal(err) - } - return r -} From ad5bfbddcabddb4f9b4e3b09e54314ab491adf22 Mon Sep 17 00:00:00 2001 From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com> Date: Mon, 4 Nov 2024 18:08:56 +0000 Subject: [PATCH 03/25] fix(docker): using AWS docker repo for trivy (#872) * fix(docker): using AWS docker repo for trivy * fix(docker): using AWS docker repo for trivy --- .github/workflows/publish-docker-images.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish-docker-images.yaml b/.github/workflows/publish-docker-images.yaml index 49eeb7e58..fd97eb27a 100644 --- a/.github/workflows/publish-docker-images.yaml +++ b/.github/workflows/publish-docker-images.yaml @@ -95,6 +95,9 @@ jobs: annotations: true severity: LOW dockerfile: ./Dockerfile + env: + # See https://github.com/aquasecurity/trivy/discussions/7538 + TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2 - name: Internal CI uses: peter-evans/repository-dispatch@v3 From 9d5b5154498197f60a39c7cc15aa4f2d5513f297 Mon Sep 17 00:00:00 2001 From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com> Date: Tue, 5 Nov 2024 17:31:14 +0000 Subject: [PATCH 04/25] Darren/feat/add subscription cache (#866) * ehancement: create a cache for block based subscriptions * minor: change function names for subscriptions * test: add unit test for message cache * chore: add license headers * refactor: fix up error handling * fix: remove bad test * fix: PR comments * fix: PR comments - remove block cache * refactor(subscriptions): store structs in cache, not bytes * fix(license): add license header * chore(subscriptions): revert unit test changes * enhancement: resolve pr comments to use simplelru * enhancement: resolve pr comments - use id as key --- .github/workflows/on-pull-request.yaml | 2 +- api/subscriptions/beat2_reader.go | 40 ++++++++++----- api/subscriptions/beat2_reader_test.go | 10 ++-- api/subscriptions/beat_reader.go | 46 +++++++++++------ api/subscriptions/beat_reader_test.go | 8 +-- api/subscriptions/block_reader.go | 2 - api/subscriptions/message_cache.go | 65 +++++++++++++++++++++++++ api/subscriptions/message_cache_test.go | 61 +++++++++++++++++++++++ api/subscriptions/subscriptions.go | 12 +++-- 9 files changed, 202 insertions(+), 44 deletions(-) create mode 100644 api/subscriptions/message_cache.go create mode 100644 api/subscriptions/message_cache_test.go diff --git a/.github/workflows/on-pull-request.yaml b/.github/workflows/on-pull-request.yaml index 57cee70ee..6c6300109 100644 --- a/.github/workflows/on-pull-request.yaml +++ b/.github/workflows/on-pull-request.yaml @@ -3,7 +3,7 @@ name: Pull Request CI on: pull_request: branches: - - master + - '*' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} diff --git a/api/subscriptions/beat2_reader.go b/api/subscriptions/beat2_reader.go index 9d1940538..267ba9d00 100644 --- a/api/subscriptions/beat2_reader.go +++ b/api/subscriptions/beat2_reader.go @@ -17,12 +17,14 @@ import ( type beat2Reader struct { repo *chain.Repository blockReader chain.BlockReader + cache *messageCache[Beat2Message] } -func newBeat2Reader(repo *chain.Repository, position thor.Bytes32) *beat2Reader { +func newBeat2Reader(repo *chain.Repository, position thor.Bytes32, cache *messageCache[Beat2Message]) *beat2Reader { return &beat2Reader{ repo: repo, blockReader: repo.NewBlockReader(position), + cache: cache, } } @@ -33,21 +35,32 @@ func (br *beat2Reader) Read() ([]interface{}, bool, error) { } var msgs []interface{} - bloomGenerator := &bloom.Generator{} - - bloomAdd := func(key []byte) { - key = bytes.TrimLeft(key, "\x00") - // exclude non-address key - if len(key) <= thor.AddressLength { - bloomGenerator.Add(key) + for _, block := range blocks { + msg, _, err := br.cache.GetOrAdd(block.Header().ID(), br.generateBeat2Message(block)) + if err != nil { + return nil, false, err } + msgs = append(msgs, msg) } + return msgs, len(blocks) > 0, nil +} + +func (br *beat2Reader) generateBeat2Message(block *chain.ExtendedBlock) func() (Beat2Message, error) { + return func() (Beat2Message, error) { + bloomGenerator := &bloom.Generator{} + + bloomAdd := func(key []byte) { + key = bytes.TrimLeft(key, "\x00") + // exclude non-address key + if len(key) <= thor.AddressLength { + bloomGenerator.Add(key) + } + } - for _, block := range blocks { header := block.Header() receipts, err := br.repo.GetBlockReceipts(header.ID()) if err != nil { - return nil, false, err + return Beat2Message{}, err } txs := block.Transactions() for i, receipt := range receipts { @@ -74,7 +87,7 @@ func (br *beat2Reader) Read() ([]interface{}, bool, error) { const bitsPerKey = 20 filter := bloomGenerator.Generate(bitsPerKey, bloom.K(bitsPerKey)) - msgs = append(msgs, &Beat2Message{ + beat2 := Beat2Message{ Number: header.Number(), ID: header.ID(), ParentID: header.ParentID(), @@ -84,7 +97,8 @@ func (br *beat2Reader) Read() ([]interface{}, bool, error) { Bloom: hexutil.Encode(filter.Bits), K: filter.K, Obsolete: block.Obsolete, - }) + } + + return beat2, nil } - return msgs, len(blocks) > 0, nil } diff --git a/api/subscriptions/beat2_reader_test.go b/api/subscriptions/beat2_reader_test.go index dbeecbd7f..4a292f485 100644 --- a/api/subscriptions/beat2_reader_test.go +++ b/api/subscriptions/beat2_reader_test.go @@ -19,13 +19,13 @@ func TestBeat2Reader_Read(t *testing.T) { newBlock := generatedBlocks[1] // Act - beatReader := newBeat2Reader(repo, genesisBlk.Header().ID()) + beatReader := newBeat2Reader(repo, genesisBlk.Header().ID(), newMessageCache[Beat2Message](10)) res, ok, err := beatReader.Read() // Assert assert.NoError(t, err) assert.True(t, ok) - if beatMsg, ok := res[0].(*Beat2Message); !ok { + if beatMsg, ok := res[0].(Beat2Message); !ok { t.Fatal("unexpected type") } else { assert.Equal(t, newBlock.Header().Number(), beatMsg.Number) @@ -33,6 +33,8 @@ func TestBeat2Reader_Read(t *testing.T) { assert.Equal(t, newBlock.Header().ParentID(), beatMsg.ParentID) assert.Equal(t, newBlock.Header().Timestamp(), beatMsg.Timestamp) assert.Equal(t, uint32(newBlock.Header().TxsFeatures()), beatMsg.TxsFeatures) + // GasLimit is not part of the deprecated BeatMessage + assert.Equal(t, newBlock.Header().GasLimit(), beatMsg.GasLimit) } } @@ -42,7 +44,7 @@ func TestBeat2Reader_Read_NoNewBlocksToRead(t *testing.T) { newBlock := generatedBlocks[1] // Act - beatReader := newBeat2Reader(repo, newBlock.Header().ID()) + beatReader := newBeat2Reader(repo, newBlock.Header().ID(), newMessageCache[Beat2Message](10)) res, ok, err := beatReader.Read() // Assert @@ -56,7 +58,7 @@ func TestBeat2Reader_Read_ErrorWhenReadingBlocks(t *testing.T) { repo, _, _ := initChain(t) // Act - beatReader := newBeat2Reader(repo, thor.MustParseBytes32("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) + beatReader := newBeat2Reader(repo, thor.MustParseBytes32("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), newMessageCache[Beat2Message](10)) res, ok, err := beatReader.Read() // Assert diff --git a/api/subscriptions/beat_reader.go b/api/subscriptions/beat_reader.go index ed4e77147..c315f1dc2 100644 --- a/api/subscriptions/beat_reader.go +++ b/api/subscriptions/beat_reader.go @@ -17,12 +17,14 @@ import ( type beatReader struct { repo *chain.Repository blockReader chain.BlockReader + cache *messageCache[BeatMessage] } -func newBeatReader(repo *chain.Repository, position thor.Bytes32) *beatReader { +func newBeatReader(repo *chain.Repository, position thor.Bytes32, cache *messageCache[BeatMessage]) *beatReader { return &beatReader{ repo: repo, blockReader: repo.NewBlockReader(position), + cache: cache, } } @@ -33,40 +35,51 @@ func (br *beatReader) Read() ([]interface{}, bool, error) { } var msgs []interface{} for _, block := range blocks { + msg, _, err := br.cache.GetOrAdd(block.Header().ID(), br.generateBeatMessage(block)) + if err != nil { + return nil, false, err + } + msgs = append(msgs, msg) + } + return msgs, len(blocks) > 0, nil +} + +func (br *beatReader) generateBeatMessage(block *chain.ExtendedBlock) func() (BeatMessage, error) { + return func() (BeatMessage, error) { header := block.Header() receipts, err := br.repo.GetBlockReceipts(header.ID()) if err != nil { - return nil, false, err + return BeatMessage{}, err } txs := block.Transactions() - bloomContent := &bloomContent{} + content := &bloomContent{} for i, receipt := range receipts { - bloomContent.add(receipt.GasPayer.Bytes()) + content.add(receipt.GasPayer.Bytes()) for _, output := range receipt.Outputs { for _, event := range output.Events { - bloomContent.add(event.Address.Bytes()) + content.add(event.Address.Bytes()) for _, topic := range event.Topics { - bloomContent.add(topic.Bytes()) + content.add(topic.Bytes()) } } for _, transfer := range output.Transfers { - bloomContent.add(transfer.Sender.Bytes()) - bloomContent.add(transfer.Recipient.Bytes()) + content.add(transfer.Sender.Bytes()) + content.add(transfer.Recipient.Bytes()) } } origin, _ := txs[i].Origin() - bloomContent.add(origin.Bytes()) + content.add(origin.Bytes()) } signer, _ := header.Signer() - bloomContent.add(signer.Bytes()) - bloomContent.add(header.Beneficiary().Bytes()) + content.add(signer.Bytes()) + content.add(header.Beneficiary().Bytes()) - k := bloom.LegacyEstimateBloomK(bloomContent.len()) + k := bloom.LegacyEstimateBloomK(content.len()) bloom := bloom.NewLegacyBloom(k) - for _, item := range bloomContent.items { + for _, item := range content.items { bloom.Add(item) } - msgs = append(msgs, &BeatMessage{ + beat := BeatMessage{ Number: header.Number(), ID: header.ID(), ParentID: header.ParentID(), @@ -75,9 +88,10 @@ func (br *beatReader) Read() ([]interface{}, bool, error) { Bloom: hexutil.Encode(bloom.Bits[:]), K: uint32(k), Obsolete: block.Obsolete, - }) + } + + return beat, nil } - return msgs, len(blocks) > 0, nil } type bloomContent struct { diff --git a/api/subscriptions/beat_reader_test.go b/api/subscriptions/beat_reader_test.go index 6e6974af9..07d020a7b 100644 --- a/api/subscriptions/beat_reader_test.go +++ b/api/subscriptions/beat_reader_test.go @@ -19,13 +19,13 @@ func TestBeatReader_Read(t *testing.T) { newBlock := generatedBlocks[1] // Act - beatReader := newBeatReader(repo, genesisBlk.Header().ID()) + beatReader := newBeatReader(repo, genesisBlk.Header().ID(), newMessageCache[BeatMessage](10)) res, ok, err := beatReader.Read() // Assert assert.NoError(t, err) assert.True(t, ok) - if beatMsg, ok := res[0].(*BeatMessage); !ok { + if beatMsg, ok := res[0].(BeatMessage); !ok { t.Fatal("unexpected type") } else { assert.Equal(t, newBlock.Header().Number(), beatMsg.Number) @@ -42,7 +42,7 @@ func TestBeatReader_Read_NoNewBlocksToRead(t *testing.T) { newBlock := generatedBlocks[1] // Act - beatReader := newBeatReader(repo, newBlock.Header().ID()) + beatReader := newBeatReader(repo, newBlock.Header().ID(), newMessageCache[BeatMessage](10)) res, ok, err := beatReader.Read() // Assert @@ -56,7 +56,7 @@ func TestBeatReader_Read_ErrorWhenReadingBlocks(t *testing.T) { repo, _, _ := initChain(t) // Act - beatReader := newBeatReader(repo, thor.MustParseBytes32("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) + beatReader := newBeatReader(repo, thor.MustParseBytes32("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), newMessageCache[BeatMessage](10)) res, ok, err := beatReader.Read() // Assert diff --git a/api/subscriptions/block_reader.go b/api/subscriptions/block_reader.go index 8c1d3ba50..817fb1c1a 100644 --- a/api/subscriptions/block_reader.go +++ b/api/subscriptions/block_reader.go @@ -11,13 +11,11 @@ import ( ) type blockReader struct { - repo *chain.Repository blockReader chain.BlockReader } func newBlockReader(repo *chain.Repository, position thor.Bytes32) *blockReader { return &blockReader{ - repo: repo, blockReader: repo.NewBlockReader(position), } } diff --git a/api/subscriptions/message_cache.go b/api/subscriptions/message_cache.go new file mode 100644 index 000000000..0ef85bf94 --- /dev/null +++ b/api/subscriptions/message_cache.go @@ -0,0 +1,65 @@ +// Copyright (c) 2024 The VeChainThor developers +// +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package subscriptions + +import ( + "fmt" + "sync" + + "github.com/hashicorp/golang-lru/simplelru" + "github.com/vechain/thor/v2/thor" +) + +// messageCache is a generic cache that stores messages of any type. +type messageCache[T any] struct { + cache *simplelru.LRU + mu sync.RWMutex +} + +// newMessageCache creates a new messageCache with the specified cache size. +func newMessageCache[T any](cacheSize uint32) *messageCache[T] { + if cacheSize > 1000 { + cacheSize = 1000 + } + if cacheSize == 0 { + cacheSize = 1 + } + cache, err := simplelru.NewLRU(int(cacheSize), nil) + if err != nil { + // lru.New only throws an error if the number is less than 1 + panic(fmt.Errorf("failed to create message cache: %v", err)) + } + return &messageCache[T]{ + cache: cache, + } +} + +// GetOrAdd returns the message of the block. If the message is not in the cache, +// it will generate the message and add it to the cache. The second return value +// indicates whether the message is newly generated. +func (mc *messageCache[T]) GetOrAdd(id thor.Bytes32, createMessage func() (T, error)) (T, bool, error) { + mc.mu.RLock() + msg, ok := mc.cache.Get(id) + mc.mu.RUnlock() + if ok { + return msg.(T), false, nil + } + + mc.mu.Lock() + defer mc.mu.Unlock() + msg, ok = mc.cache.Get(id) + if ok { + return msg.(T), false, nil + } + + newMsg, err := createMessage() + if err != nil { + var zero T + return zero, false, err + } + mc.cache.Add(id, newMsg) + return newMsg, true, nil +} diff --git a/api/subscriptions/message_cache_test.go b/api/subscriptions/message_cache_test.go new file mode 100644 index 000000000..6bf65c600 --- /dev/null +++ b/api/subscriptions/message_cache_test.go @@ -0,0 +1,61 @@ +// Copyright (c) 2024 The VeChainThor developers +// +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package subscriptions + +import ( + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/vechain/thor/v2/block" +) + +type message struct { + id string +} + +func handler(blk *block.Block) func() (message, error) { + return func() (message, error) { + msg := message{ + id: blk.Header().ID().String(), + } + return msg, nil + } +} + +func TestMessageCache_GetOrAdd(t *testing.T) { + _, generatedBlocks, _ := initChain(t) + + blk0 := generatedBlocks[0] + blk1 := generatedBlocks[1] + + cache := newMessageCache[message](10) + + counter := atomic.Int32{} + wg := sync.WaitGroup{} + for i := 0; i < 100; i++ { + wg.Add(1) + start := time.Now().Add(20 * time.Millisecond) + go func() { + defer wg.Done() + time.Sleep(time.Until(start)) + _, added, err := cache.GetOrAdd(blk0.Header().ID(), handler(blk0)) + assert.NoError(t, err) + if added { + counter.Add(1) + } + }() + } + wg.Wait() + assert.Equal(t, counter.Load(), int32(1)) + + _, added, err := cache.GetOrAdd(blk1.Header().ID(), handler(blk1)) + assert.NoError(t, err) + assert.True(t, added) + assert.Equal(t, cache.cache.Len(), 2) +} diff --git a/api/subscriptions/subscriptions.go b/api/subscriptions/subscriptions.go index 73f895bf3..7582da5bb 100644 --- a/api/subscriptions/subscriptions.go +++ b/api/subscriptions/subscriptions.go @@ -31,6 +31,8 @@ type Subscriptions struct { pendingTx *pendingTx done chan struct{} wg sync.WaitGroup + beat2Cache *messageCache[Beat2Message] + beatCache *messageCache[BeatMessage] } type msgReader interface { @@ -67,8 +69,10 @@ func New(repo *chain.Repository, allowedOrigins []string, backtraceLimit uint32, return false }, }, - pendingTx: newPendingTx(txpool), - done: make(chan struct{}), + pendingTx: newPendingTx(txpool), + done: make(chan struct{}), + beat2Cache: newMessageCache[Beat2Message](backtraceLimit), + beatCache: newMessageCache[BeatMessage](backtraceLimit), } sub.wg.Add(1) @@ -158,7 +162,7 @@ func (s *Subscriptions) handleBeatReader(w http.ResponseWriter, req *http.Reques if err != nil { return nil, err } - return newBeatReader(s.repo, position), nil + return newBeatReader(s.repo, position, s.beatCache), nil } func (s *Subscriptions) handleBeat2Reader(w http.ResponseWriter, req *http.Request) (*beat2Reader, error) { @@ -166,7 +170,7 @@ func (s *Subscriptions) handleBeat2Reader(w http.ResponseWriter, req *http.Reque if err != nil { return nil, err } - return newBeat2Reader(s.repo, position), nil + return newBeat2Reader(s.repo, position, s.beat2Cache), nil } func (s *Subscriptions) handleSubject(w http.ResponseWriter, req *http.Request) error { From 7a89780525e18ba9ab1b40fac63728514b04b3c1 Mon Sep 17 00:00:00 2001 From: Makis Christou Date: Wed, 6 Nov 2024 11:27:39 +0200 Subject: [PATCH 05/25] Add additional block tests (#863) --- consensus/consensus_test.go | 70 +++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/consensus/consensus_test.go b/consensus/consensus_test.go index 14bf7ec43..5bac09763 100644 --- a/consensus/consensus_test.go +++ b/consensus/consensus_test.go @@ -10,6 +10,7 @@ import ( "crypto/rand" "fmt" "math/big" + "strings" "testing" "github.com/ethereum/go-ethereum/crypto" @@ -508,6 +509,54 @@ func TestVerifyBlock(t *testing.T) { assert.Equal(t, err, expected) }, }, + { + "InvalidStateRoot", func(t *testing.T) { + header := tc.original.Header() + builder := new(block.Builder). + ParentID(header.ParentID()). + Timestamp(header.Timestamp()). + TotalScore(header.TotalScore()). + GasLimit(header.GasLimit()). + GasUsed(header.GasUsed()). + Beneficiary(header.Beneficiary()). + StateRoot(thor.Bytes32{123}). + ReceiptsRoot(header.ReceiptsRoot()) + builder.TransactionFeatures(header.TxsFeatures()) + + blk, err := tc.sign(builder) + if err != nil { + t.Fatal(err) + } + expectedPrefix := "block state root mismatch" + err = tc.consent(blk) + + assert.True(t, strings.HasPrefix(err.Error(), expectedPrefix)) + }, + }, + { + "InvalidReceiptsRoot", func(t *testing.T) { + header := tc.original.Header() + builder := new(block.Builder). + ParentID(header.ParentID()). + Timestamp(header.Timestamp()). + TotalScore(header.TotalScore()). + GasLimit(header.GasLimit()). + GasUsed(header.GasUsed()). + Beneficiary(header.Beneficiary()). + StateRoot(header.StateRoot()). + ReceiptsRoot(thor.Bytes32{123}) + builder.TransactionFeatures(header.TxsFeatures()) + + blk, err := tc.sign(builder) + if err != nil { + t.Fatal(err) + } + expectedPrefix := "block receipts root mismatch" + err = tc.consent(blk) + + assert.True(t, strings.HasPrefix(err.Error(), expectedPrefix)) + }, + }, } for _, tt := range tests { @@ -649,6 +698,27 @@ func TestValidateBlockBody(t *testing.T) { assert.Equal(t, err, expected) }, }, + { + "ZeroGasTx", func(t *testing.T) { + txBuilder := new(tx.Builder). + GasPriceCoef(0). + Gas(0). + Expiration(100). + Clause(tx.NewClause(&thor.Address{}).WithValue(big.NewInt(0)).WithData(nil)). + Nonce(0). + ChainTag(30) + + tx := txSign(txBuilder) + + blk, err := tc.sign(tc.builder(tc.original.Header()).Transaction(tx).Transaction(tx)) + if err != nil { + t.Fatal(err) + } + + err = tc.consent(blk) + assert.Equal(t, "intrinsic gas exceeds provided gas", err.Error()) + }, + }, } for _, tt := range tests { From 8be8574a6a0e2b0f5448992b4099489849f6086b Mon Sep 17 00:00:00 2001 From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com> Date: Thu, 7 Nov 2024 08:49:28 +0000 Subject: [PATCH 06/25] enhancement(logging): leverage trace level (#873) --- cmd/thor/node/node.go | 4 ++-- cmd/thor/node/packer_loop.go | 2 +- comm/handle_rpc.go | 2 +- comm/sync.go | 2 +- p2psrv/server.go | 4 ++-- txpool/tx_pool.go | 14 +++++++------- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/cmd/thor/node/node.go b/cmd/thor/node/node.go index e4b53a3c0..d103f227a 100644 --- a/cmd/thor/node/node.go +++ b/cmd/thor/node/node.go @@ -266,7 +266,7 @@ func (n *Node) txStashLoop(ctx context.Context) { if err := stash.Save(txEv.Tx); err != nil { logger.Warn("stash tx", "id", txEv.Tx.ID(), "err", err) } else { - logger.Debug("stashed tx", "id", txEv.Tx.ID()) + logger.Trace("stashed tx", "id", txEv.Tx.ID()) } } } @@ -392,7 +392,7 @@ func (n *Node) processBlock(newBlock *block.Block, stats *blockStats) (bool, err commitElapsed := mclock.Now() - startTime - execElapsed if v, updated := n.bandwidth.Update(newBlock.Header(), time.Duration(realElapsed)); updated { - logger.Debug("bandwidth updated", "gps", v) + logger.Trace("bandwidth updated", "gps", v) } stats.UpdateProcessed(1, len(receipts), execElapsed, commitElapsed, realElapsed, newBlock.Header().GasUsed()) diff --git a/cmd/thor/node/packer_loop.go b/cmd/thor/node/packer_loop.go index 44e7f1ded..675ab041b 100644 --- a/cmd/thor/node/packer_loop.go +++ b/cmd/thor/node/packer_loop.go @@ -205,7 +205,7 @@ func (n *Node) pack(flow *packer.Flow) (err error) { ) if v, updated := n.bandwidth.Update(newBlock.Header(), time.Duration(realElapsed)); updated { - logger.Debug("bandwidth updated", "gps", v) + logger.Trace("bandwidth updated", "gps", v) } metricBlockProcessedTxs().SetWithLabel(int64(len(receipts)), map[string]string{"type": "proposed"}) diff --git a/comm/handle_rpc.go b/comm/handle_rpc.go index 0dbddc764..50ae93cf4 100644 --- a/comm/handle_rpc.go +++ b/comm/handle_rpc.go @@ -21,7 +21,7 @@ import ( // peer will be disconnected if error returned func (c *Communicator) handleRPC(peer *Peer, msg *p2p.Msg, write func(interface{}), txsToSync *txsToSync) (err error) { log := peer.logger.New("msg", proto.MsgName(msg.Code)) - log.Debug("received RPC call") + log.Trace("received RPC call") defer func() { if err != nil { log.Debug("failed to handle RPC call", "err", err) diff --git a/comm/sync.go b/comm/sync.go index 75347098e..74fc9e1a8 100644 --- a/comm/sync.go +++ b/comm/sync.go @@ -196,7 +196,7 @@ func findCommonAncestor(ctx context.Context, repo *chain.Repository, peer *Peer, func (c *Communicator) syncTxs(peer *Peer) { for i := 0; ; i++ { - peer.logger.Debug(fmt.Sprintf("sync txs loop %v", i)) + peer.logger.Trace(fmt.Sprintf("sync txs loop %v", i)) result, err := proto.GetTxs(c.ctx, peer) if err != nil { peer.logger.Debug("failed to request txs", "err", err) diff --git a/p2psrv/server.go b/p2psrv/server.go index c17240bb1..3bdf50ea6 100644 --- a/p2psrv/server.go +++ b/p2psrv/server.go @@ -87,7 +87,7 @@ func (s *Server) Start(protocols []*p2p.Protocol, topic discv5.Topic) error { } log := logger.New("peer", peer, "dir", dir) - log.Debug("peer connected") + log.Trace("peer connected") metricConnectedPeers().Add(1) startTime := mclock.Now() @@ -252,7 +252,7 @@ func (s *Server) discoverLoop(topic discv5.Topic) { if _, found := s.discoveredNodes.Get(node.ID); !found { metricDiscoveredNodes().Add(1) s.discoveredNodes.Set(node.ID, node) - logger.Debug("discovered node", "node", node) + logger.Trace("discovered node", "node", node) } case <-s.done: close(setPeriod) diff --git a/txpool/tx_pool.go b/txpool/tx_pool.go index 8493a8003..928751676 100644 --- a/txpool/tx_pool.go +++ b/txpool/tx_pool.go @@ -135,7 +135,7 @@ func (p *TxPool) housekeeping() { } metricTxPoolGauge().AddWithLabel(0-int64(removed), map[string]string{"source": "washed", "total": "true"}) - logger.Debug("wash done", ctx...) + logger.Trace("wash done", ctx...) } } } @@ -275,7 +275,7 @@ func (p *TxPool) add(newTx *tx.Transaction, rejectNonExecutable bool, localSubmi p.goes.Go(func() { p.txFeed.Send(&TxEvent{newTx, &executable}) }) - logger.Debug("tx added", "id", newTx.ID(), "executable", executable) + logger.Trace("tx added", "id", newTx.ID(), "executable", executable) } else { // we skip steps that rely on head block when chain is not synced, // but check the pool's limit @@ -287,7 +287,7 @@ func (p *TxPool) add(newTx *tx.Transaction, rejectNonExecutable bool, localSubmi if err := p.all.Add(txObj, p.options.LimitPerAccount, func(_ thor.Address, _ *big.Int) error { return nil }); err != nil { return txRejectedError{err.Error()} } - logger.Debug("tx added", "id", newTx.ID()) + logger.Trace("tx added", "id", newTx.ID()) p.goes.Go(func() { p.txFeed.Send(&TxEvent{newTx, nil}) }) @@ -408,21 +408,21 @@ func (p *TxPool) wash(headSummary *chain.BlockSummary) (executables tx.Transacti for _, txObj := range all { if thor.IsOriginBlocked(txObj.Origin()) || p.blocklist.Contains(txObj.Origin()) { toRemove = append(toRemove, txObj) - logger.Debug("tx washed out", "id", txObj.ID(), "err", "blocked") + logger.Trace("tx washed out", "id", txObj.ID(), "err", "blocked") continue } // out of lifetime if !txObj.localSubmitted && now > txObj.timeAdded+int64(p.options.MaxLifetime) { toRemove = append(toRemove, txObj) - logger.Debug("tx washed out", "id", txObj.ID(), "err", "out of lifetime") + logger.Trace("tx washed out", "id", txObj.ID(), "err", "out of lifetime") continue } // settled, out of energy or dep broken executable, err := txObj.Executable(chain, newState(), headSummary.Header) if err != nil { toRemove = append(toRemove, txObj) - logger.Debug("tx washed out", "id", txObj.ID(), "err", err) + logger.Trace("tx washed out", "id", txObj.ID(), "err", err) continue } @@ -430,7 +430,7 @@ func (p *TxPool) wash(headSummary *chain.BlockSummary) (executables tx.Transacti provedWork, err := txObj.ProvedWork(headSummary.Header.Number(), chain.GetBlockID) if err != nil { toRemove = append(toRemove, txObj) - logger.Debug("tx washed out", "id", txObj.ID(), "err", err) + logger.Trace("tx washed out", "id", txObj.ID(), "err", err) continue } txObj.overallGasPrice = txObj.OverallGasPrice(baseGasPrice, provedWork) From 784604debefb42ae8624ac5a27a1373d91c71c7e Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Thu, 7 Nov 2024 10:57:50 +0000 Subject: [PATCH 07/25] Add testchain package (#844) * Refactor thor node * thorchain allows insertion of blocks * remove thorNode, added testchain * clean up + comments * adding license headers * adding templating tests for thorclient * Remove test event hacks * remove types * removed chain_builder + added logdb to testchain * pr comments * Update test/testchain/chain.go Co-authored-by: libotony --------- Co-authored-by: libotony --- api/accounts/accounts_test.go | 72 ++-- api/blocks/blocks_test.go | 56 +--- api/debug/debug_test.go | 58 +--- api/events/events_test.go | 63 ++-- api/metrics_test.go | 37 +-- api/node/node_test.go | 33 +- api/subscriptions/beat2_reader_test.go | 24 +- api/subscriptions/beat_reader_test.go | 23 +- api/subscriptions/block_reader_test.go | 110 ++----- api/subscriptions/event_reader_test.go | 17 +- api/subscriptions/message_cache_test.go | 10 +- api/subscriptions/pending_tx_test.go | 17 +- api/subscriptions/subscriptions_test.go | 171 ++++++---- api/subscriptions/transfer_reader_test.go | 32 +- api/transactions/transactions_test.go | 57 +--- api/transfers/transfers_test.go | 35 +- api/utils/revisions_test.go | 37 +-- cmd/thor/solo/types.go | 3 +- test/eventcontract/event_contract.go | 77 +++++ test/testchain/chain.go | 222 +++++++++++++ thorclient/api_test.go | 381 ++++++++++++++++++++++ 21 files changed, 1030 insertions(+), 505 deletions(-) create mode 100644 test/eventcontract/event_contract.go create mode 100644 test/testchain/chain.go create mode 100644 thorclient/api_test.go diff --git a/api/accounts/accounts_test.go b/api/accounts/accounts_test.go index e126960b2..9294723eb 100644 --- a/api/accounts/accounts_test.go +++ b/api/accounts/accounts_test.go @@ -12,7 +12,6 @@ import ( "net/http" "net/http/httptest" "testing" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -20,19 +19,16 @@ import ( "github.com/gorilla/mux" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - ABI "github.com/vechain/thor/v2/abi" "github.com/vechain/thor/v2/api/accounts" "github.com/vechain/thor/v2/block" - "github.com/vechain/thor/v2/chain" - "github.com/vechain/thor/v2/cmd/thor/solo" "github.com/vechain/thor/v2/genesis" - "github.com/vechain/thor/v2/muxdb" - "github.com/vechain/thor/v2/packer" - "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/test/testchain" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/thorclient" - tccommon "github.com/vechain/thor/v2/thorclient/common" "github.com/vechain/thor/v2/tx" + + ABI "github.com/vechain/thor/v2/abi" + tccommon "github.com/vechain/thor/v2/thorclient/common" ) // pragma solidity ^0.4.18; @@ -94,7 +90,7 @@ const ( ) var ( - gasLimit uint64 + gasLimit = math.MaxUint32 addr = thor.BytesToAddress([]byte("to")) value = big.NewInt(10000) storageKey = thor.Bytes32{} @@ -102,7 +98,6 @@ var ( contractAddr thor.Address bytecode = common.Hex2Bytes("608060405234801561001057600080fd5b50610125806100206000396000f3006080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806324b8ba5f14604e578063bb4e3f4d14607b575b600080fd5b348015605957600080fd5b506079600480360381019080803560ff16906020019092919050505060cf565b005b348015608657600080fd5b5060b3600480360381019080803560ff169060200190929190803560ff16906020019092919050505060ec565b604051808260ff1660ff16815260200191505060405180910390f35b806000806101000a81548160ff021916908360ff16021790555050565b60008183019050929150505600a165627a7a723058201584add23e31d36c569b468097fe01033525686b59bbb263fb3ab82e9553dae50029") runtimeBytecode = common.Hex2Bytes("6080604052600436106049576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806324b8ba5f14604e578063bb4e3f4d14607b575b600080fd5b348015605957600080fd5b506079600480360381019080803560ff16906020019092919050505060cf565b005b348015608657600080fd5b5060b3600480360381019080803560ff169060200190929190803560ff16906020019092919050505060ec565b604051808260ff1660ff16815260200191505060405180910390f35b806000806101000a81548160ff021916908360ff16021790555050565b60008183019050929150505600a165627a7a723058201584add23e31d36c569b468097fe01033525686b59bbb263fb3ab82e9553dae50029") - acc *accounts.Accounts ts *httptest.Server tclient *thorclient.Client ) @@ -270,22 +265,14 @@ func getStorageWithNonExistingRevision(t *testing.T) { } func initAccountServer(t *testing.T) { - db := muxdb.NewMem() - stater := state.NewStater(db) - gene := genesis.NewDevnet() + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } - genesisBlock = b - repo, _ := chain.NewRepository(db, b) + genesisBlock = thorChain.GenesisBlock() claTransfer := tx.NewClause(&addr).WithValue(value) claDeploy := tx.NewClause(nil).WithData(bytecode) - transaction := buildTxWithClauses(repo.ChainTag(), claTransfer, claDeploy) + transaction := buildTxWithClauses(thorChain.Repo().ChainTag(), claTransfer, claDeploy) contractAddr = thor.CreateContractAddress(transaction.ID(), 1, 0) - packTx(repo, stater, transaction, t) - method := "set" abi, _ := ABI.New([]byte(abiJSON)) m, _ := abi.MethodByName(method) @@ -294,13 +281,19 @@ func initAccountServer(t *testing.T) { t.Fatal(err) } claCall := tx.NewClause(&contractAddr).WithData(input) - transactionCall := buildTxWithClauses(repo.ChainTag(), claCall) - packTx(repo, stater, transactionCall, t) + transactionCall := buildTxWithClauses(thorChain.Repo().ChainTag(), claCall) + require.NoError(t, + thorChain.MintTransactions( + genesis.DevAccounts()[0], + transaction, + transactionCall, + ), + ) router := mux.NewRouter() - gasLimit = math.MaxUint32 - acc = accounts.New(repo, stater, gasLimit, thor.NoFork, solo.NewBFTEngine(repo)) - acc.Mount(router, "/accounts") + accounts.New(thorChain.Repo(), thorChain.Stater(), uint64(gasLimit), thor.NoFork, thorChain.Engine()). + Mount(router, "/accounts") + ts = httptest.NewServer(router) } @@ -318,31 +311,6 @@ func buildTxWithClauses(chaiTag byte, clauses ...*tx.Clause) *tx.Transaction { return tx.MustSign(trx, genesis.DevAccounts()[0].PrivateKey) } -func packTx(repo *chain.Repository, stater *state.Stater, transaction *tx.Transaction, t *testing.T) { - packer := packer.New(repo, stater, genesis.DevAccounts()[0].Address, &genesis.DevAccounts()[0].Address, thor.NoFork) - flow, err := packer.Schedule(repo.BestBlockSummary(), uint64(time.Now().Unix())) - if err != nil { - t.Fatal(err) - } - err = flow.Adopt(transaction) - if err != nil { - t.Fatal(err) - } - b, stage, receipts, err := flow.Pack(genesis.DevAccounts()[0].PrivateKey, 0, false) - if err != nil { - t.Fatal(err) - } - if _, err := stage.Commit(); err != nil { - t.Fatal(err) - } - if err := repo.AddBlock(b, receipts, 0); err != nil { - t.Fatal(err) - } - if err := repo.SetBestBlockID(b.Header().ID()); err != nil { - t.Fatal(err) - } -} - func deployContractWithCall(t *testing.T) { badBody := &accounts.CallData{ Gas: 10000000, diff --git a/api/blocks/blocks_test.go b/api/blocks/blocks_test.go index a203cadd6..dcb6c4e94 100644 --- a/api/blocks/blocks_test.go +++ b/api/blocks/blocks_test.go @@ -14,19 +14,14 @@ import ( "strconv" "strings" "testing" - "time" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/api/blocks" "github.com/vechain/thor/v2/block" - "github.com/vechain/thor/v2/chain" - "github.com/vechain/thor/v2/cmd/thor/solo" "github.com/vechain/thor/v2/genesis" - "github.com/vechain/thor/v2/muxdb" - "github.com/vechain/thor/v2/packer" - "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/test/testchain" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/thorclient" "github.com/vechain/thor/v2/tx" @@ -175,22 +170,14 @@ func testGetBlockWithRevisionNumberTooHigh(t *testing.T) { } func initBlockServer(t *testing.T) { - db := muxdb.NewMem() - stater := state.NewStater(db) - gene := genesis.NewDevnet() - - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } - genesisBlock = b + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) - repo, _ := chain.NewRepository(db, b) addr := thor.BytesToAddress([]byte("to")) cla := tx.NewClause(&addr).WithValue(big.NewInt(10000)) trx := tx.MustSign( new(tx.Builder). - ChainTag(repo.ChainTag()). + ChainTag(thorChain.Repo().ChainTag()). GasPriceCoef(1). Expiration(10). Gas(21000). @@ -201,34 +188,17 @@ func initBlockServer(t *testing.T) { genesis.DevAccounts()[0].PrivateKey, ) - packer := packer.New(repo, stater, genesis.DevAccounts()[0].Address, &genesis.DevAccounts()[0].Address, thor.NoFork) - sum, _ := repo.GetBlockSummary(b.Header().ID()) - flow, err := packer.Schedule(sum, uint64(time.Now().Unix())) - if err != nil { - t.Fatal(err) - } - err = flow.Adopt(trx) - if err != nil { - t.Fatal(err) - } - block, stage, receipts, err := flow.Pack(genesis.DevAccounts()[0].PrivateKey, 0, false) - if err != nil { - t.Fatal(err) - } - if _, err := stage.Commit(); err != nil { - t.Fatal(err) - } - if err := repo.AddBlock(block, receipts, 0); err != nil { - t.Fatal(err) - } - if err := repo.SetBestBlockID(block.Header().ID()); err != nil { - t.Fatal(err) - } + require.NoError(t, thorChain.MintTransactions(genesis.DevAccounts()[0], trx)) + + allBlocks, err := thorChain.GetAllBlocks() + require.NoError(t, err) + + genesisBlock = allBlocks[0] + blk = allBlocks[1] + router := mux.NewRouter() - bftEngine := solo.NewBFTEngine(repo) - blocks.New(repo, bftEngine).Mount(router, "/blocks") + blocks.New(thorChain.Repo(), thorChain.Engine()).Mount(router, "/blocks") ts = httptest.NewServer(router) - blk = block } func checkCollapsedBlock(t *testing.T, expBl *block.Block, actBl *blocks.JSONCollapsedBlock) { diff --git a/api/debug/debug_test.go b/api/debug/debug_test.go index 36f26c075..1275a9030 100644 --- a/api/debug/debug_test.go +++ b/api/debug/debug_test.go @@ -13,7 +13,6 @@ import ( "net/http/httptest" "strings" "testing" - "time" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" @@ -22,13 +21,11 @@ import ( "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/block" "github.com/vechain/thor/v2/builtin" - "github.com/vechain/thor/v2/chain" - "github.com/vechain/thor/v2/cmd/thor/solo" "github.com/vechain/thor/v2/genesis" "github.com/vechain/thor/v2/muxdb" - "github.com/vechain/thor/v2/packer" "github.com/vechain/thor/v2/state" "github.com/vechain/thor/v2/test/datagen" + "github.com/vechain/thor/v2/test/testchain" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/thorclient" "github.com/vechain/thor/v2/tracers/logger" @@ -515,22 +512,15 @@ func testStorageRangeDefaultOption(t *testing.T) { } func initDebugServer(t *testing.T) { - db := muxdb.NewMem() - stater := state.NewStater(db) - gene := genesis.NewDevnet() - - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } - repo, _ := chain.NewRepository(db, b) + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) addr := thor.BytesToAddress([]byte("to")) // Adding an empty clause transaction to the block to cover the case of // scanning multiple txs before getting the right one noClausesTx := new(tx.Builder). - ChainTag(repo.ChainTag()). + ChainTag(thorChain.Repo().ChainTag()). Expiration(10). Gas(21000). Build() @@ -539,7 +529,7 @@ func initDebugServer(t *testing.T) { cla := tx.NewClause(&addr).WithValue(big.NewInt(10000)) cla2 := tx.NewClause(&addr).WithValue(big.NewInt(10000)) transaction = new(tx.Builder). - ChainTag(repo.ChainTag()). + ChainTag(thorChain.Repo().ChainTag()). GasPriceCoef(1). Expiration(10). Gas(37000). @@ -550,38 +540,16 @@ func initDebugServer(t *testing.T) { Build() transaction = tx.MustSign(transaction, genesis.DevAccounts()[0].PrivateKey) - packer := packer.New(repo, stater, genesis.DevAccounts()[0].Address, &genesis.DevAccounts()[0].Address, thor.NoFork) - sum, _ := repo.GetBlockSummary(b.Header().ID()) - flow, err := packer.Schedule(sum, uint64(time.Now().Unix())) - if err != nil { - t.Fatal(err) - } - err = flow.Adopt(noClausesTx) - if err != nil { - t.Fatal(err) - } - err = flow.Adopt(transaction) - if err != nil { - t.Fatal(err) - } - b, stage, receipts, err := flow.Pack(genesis.DevAccounts()[0].PrivateKey, 0, false) - blk = b - if err != nil { - t.Fatal(err) - } - if _, err := stage.Commit(); err != nil { - t.Fatal(err) - } - if err := repo.AddBlock(b, receipts, 0); err != nil { - t.Fatal(err) - } - if err := repo.SetBestBlockID(b.Header().ID()); err != nil { - t.Fatal(err) - } + require.NoError(t, thorChain.MintTransactions(genesis.DevAccounts()[0], transaction, noClausesTx)) + require.NoError(t, thorChain.MintTransactions(genesis.DevAccounts()[0])) + + allBlocks, err := thorChain.GetAllBlocks() + require.NoError(t, err) + blk = allBlocks[1] - forkConfig := thor.GetForkConfig(b.Header().ID()) + forkConfig := thor.GetForkConfig(blk.Header().ID()) router := mux.NewRouter() - debug = New(repo, stater, forkConfig, 21000, true, solo.NewBFTEngine(repo), []string{"all"}, false) + debug = New(thorChain.Repo(), thorChain.Stater(), forkConfig, 21000, true, thorChain.Engine(), []string{"all"}, false) debug.Mount(router, "/debug") ts = httptest.NewServer(router) } diff --git a/api/events/events_test.go b/api/events/events_test.go index 3d28e41dc..b1268d378 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -17,11 +17,8 @@ import ( "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/api/events" "github.com/vechain/thor/v2/block" - "github.com/vechain/thor/v2/chain" - "github.com/vechain/thor/v2/genesis" "github.com/vechain/thor/v2/logdb" - "github.com/vechain/thor/v2/muxdb" - "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/test/testchain" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/thorclient" "github.com/vechain/thor/v2/tx" @@ -37,8 +34,7 @@ var ( ) func TestEmptyEvents(t *testing.T) { - db := createDb(t) - initEventServer(t, db, defaultLogLimit) + initEventServer(t, defaultLogLimit) defer ts.Close() tclient = thorclient.New(ts.URL) @@ -51,21 +47,19 @@ func TestEmptyEvents(t *testing.T) { } func TestEvents(t *testing.T) { - db := createDb(t) - initEventServer(t, db, defaultLogLimit) + thorChain := initEventServer(t, defaultLogLimit) defer ts.Close() blocksToInsert := 5 tclient = thorclient.New(ts.URL) - insertBlocks(t, db, blocksToInsert) + insertBlocks(t, thorChain.LogDB(), blocksToInsert) testEventWithBlocks(t, blocksToInsert) } func TestOption(t *testing.T) { - db := createDb(t) - initEventServer(t, db, 5) + thorChain := initEventServer(t, 5) defer ts.Close() - insertBlocks(t, db, 5) + insertBlocks(t, thorChain.LogDB(), 5) tclient = thorclient.New(ts.URL) filter := events.EventFilter{ @@ -75,20 +69,20 @@ func TestOption(t *testing.T) { Order: logdb.DESC, } - res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/events", filter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/event", filter) require.NoError(t, err) assert.Equal(t, "options.limit exceeds the maximum allowed value of 5", strings.Trim(string(res), "\n")) assert.Equal(t, http.StatusForbidden, statusCode) filter.Options.Limit = 5 - _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/events", filter) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/logs/event", filter) require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode) // with nil options, should use default limit, when the filtered lower // or equal to the limit, should return the filtered events filter.Options = nil - res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/events", filter) + res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/logs/event", filter) require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode) var tLogs []*events.FilteredEvent @@ -99,8 +93,8 @@ func TestOption(t *testing.T) { assert.Equal(t, 5, len(tLogs)) // when the filtered events exceed the limit, should return the forbidden - insertBlocks(t, db, 6) - res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/events", filter) + insertBlocks(t, thorChain.LogDB(), 6) + res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/logs/event", filter) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, statusCode) assert.Equal(t, "the number of filtered logs exceeds the maximum allowed value of 5, please use pagination", strings.Trim(string(res), "\n")) @@ -110,7 +104,7 @@ func TestOption(t *testing.T) { func testEventsBadRequest(t *testing.T) { badBody := []byte{0x00, 0x01, 0x02} - _, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/events", badBody) + _, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/event", badBody) assert.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode) } @@ -123,7 +117,7 @@ func testEventWithEmptyDb(t *testing.T) { Order: logdb.DESC, } - res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/events", emptyFilter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/event", emptyFilter) require.NoError(t, err) var tLogs []*events.FilteredEvent if err := json.Unmarshal(res, &tLogs); err != nil { @@ -142,7 +136,7 @@ func testEventWithBlocks(t *testing.T, expectedBlocks int) { Order: logdb.DESC, } - res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/events", emptyFilter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/event", emptyFilter) require.NoError(t, err) var tLogs []*events.FilteredEvent if err := json.Unmarshal(res, &tLogs); err != nil { @@ -169,7 +163,7 @@ func testEventWithBlocks(t *testing.T, expectedBlocks int) { }}, } - res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/events", matchingFilter) + res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/logs/event", matchingFilter) require.NoError(t, err) if err := json.Unmarshal(res, &tLogs); err != nil { t.Fatal(err) @@ -183,30 +177,15 @@ func testEventWithBlocks(t *testing.T, expectedBlocks int) { } // Init functions -func initEventServer(t *testing.T, logDb *logdb.LogDB, limit uint64) { - router := mux.NewRouter() - - muxDb := muxdb.NewMem() - stater := state.NewStater(muxDb) - gene := genesis.NewDevnet() - - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } - - repo, _ := chain.NewRepository(muxDb, b) +func initEventServer(t *testing.T, limit uint64) *testchain.Chain { + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) - events.New(repo, logDb, limit).Mount(router, "/events") + router := mux.NewRouter() + events.New(thorChain.Repo(), thorChain.LogDB(), limit).Mount(router, "/logs/event") ts = httptest.NewServer(router) -} -func createDb(t *testing.T) *logdb.LogDB { - logDb, err := logdb.NewMem() - if err != nil { - t.Fatal(err) - } - return logDb + return thorChain } // Utilities functions diff --git a/api/metrics_test.go b/api/metrics_test.go index 59e86c9c8..7cb1794e4 100644 --- a/api/metrics_test.go +++ b/api/metrics_test.go @@ -20,14 +20,11 @@ import ( "github.com/gorilla/websocket" "github.com/prometheus/common/expfmt" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/api/accounts" "github.com/vechain/thor/v2/api/subscriptions" - "github.com/vechain/thor/v2/chain" - "github.com/vechain/thor/v2/cmd/thor/solo" - "github.com/vechain/thor/v2/genesis" "github.com/vechain/thor/v2/metrics" - "github.com/vechain/thor/v2/muxdb" - "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/test/testchain" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/txpool" ) @@ -37,28 +34,21 @@ func init() { } func TestMetricsMiddleware(t *testing.T) { - db := muxdb.NewMem() - stater := state.NewStater(db) - gene := genesis.NewDevnet() - - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } - repo, _ := chain.NewRepository(db, b) + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) // inject some invalid data to db - data := db.NewStore("chain.data") + data := thorChain.Database().NewStore("chain.data") var blkID thor.Bytes32 rand.Read(blkID[:]) data.Put(blkID[:], []byte("invalid data")) // get summary should fail since the block data is not rlp encoded - _, err = repo.GetBlockSummary(blkID) + _, err = thorChain.Repo().GetBlockSummary(blkID) assert.NotNil(t, err) router := mux.NewRouter() - acc := accounts.New(repo, stater, math.MaxUint64, thor.NoFork, solo.NewBFTEngine(repo)) + acc := accounts.New(thorChain.Repo(), thorChain.Stater(), math.MaxUint64, thor.NoFork, thorChain.Engine()) acc.Mount(router, "/accounts") router.PathPrefix("/metrics").Handler(metrics.HTTPHandler()) router.Use(metricsMiddleware) @@ -109,18 +99,11 @@ func TestMetricsMiddleware(t *testing.T) { } func TestWebsocketMetrics(t *testing.T) { - db := muxdb.NewMem() - stater := state.NewStater(db) - gene := genesis.NewDevnet() - - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } - repo, _ := chain.NewRepository(db, b) + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) router := mux.NewRouter() - sub := subscriptions.New(repo, []string{"*"}, 10, txpool.New(repo, stater, txpool.Options{})) + sub := subscriptions.New(thorChain.Repo(), []string{"*"}, 10, txpool.New(thorChain.Repo(), thorChain.Stater(), txpool.Options{})) sub.Mount(router, "/subscriptions") router.PathPrefix("/metrics").Handler(metrics.HTTPHandler()) router.Use(metricsMiddleware) diff --git a/api/node/node_test.go b/api/node/node_test.go index e9bbe95fe..3dd2e96ee 100644 --- a/api/node/node_test.go +++ b/api/node/node_test.go @@ -13,11 +13,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/api/node" - "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/comm" - "github.com/vechain/thor/v2/genesis" - "github.com/vechain/thor/v2/muxdb" - "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/test/testchain" "github.com/vechain/thor/v2/thorclient" "github.com/vechain/thor/v2/txpool" ) @@ -34,21 +31,19 @@ func TestNode(t *testing.T) { } func initCommServer(t *testing.T) { - db := muxdb.NewMem() - stater := state.NewStater(db) - gene := genesis.NewDevnet() - - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } - repo, _ := chain.NewRepository(db, b) - comm := comm.New(repo, txpool.New(repo, stater, txpool.Options{ - Limit: 10000, - LimitPerAccount: 16, - MaxLifetime: 10 * time.Minute, - })) + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) + + communicator := comm.New( + thorChain.Repo(), + txpool.New(thorChain.Repo(), thorChain.Stater(), txpool.Options{ + Limit: 10000, + LimitPerAccount: 16, + MaxLifetime: 10 * time.Minute, + })) + router := mux.NewRouter() - node.New(comm).Mount(router, "/node") + node.New(communicator).Mount(router, "/node") + ts = httptest.NewServer(router) } diff --git a/api/subscriptions/beat2_reader_test.go b/api/subscriptions/beat2_reader_test.go index 4a292f485..a31884a0b 100644 --- a/api/subscriptions/beat2_reader_test.go +++ b/api/subscriptions/beat2_reader_test.go @@ -9,17 +9,21 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/thor" ) func TestBeat2Reader_Read(t *testing.T) { // Arrange - repo, generatedBlocks, _ := initChain(t) - genesisBlk := generatedBlocks[0] - newBlock := generatedBlocks[1] + thorChain := initChain(t) + allBlocks, err := thorChain.GetAllBlocks() + require.NoError(t, err) + + genesisBlk := allBlocks[0] + newBlock := allBlocks[1] // Act - beatReader := newBeat2Reader(repo, genesisBlk.Header().ID(), newMessageCache[Beat2Message](10)) + beatReader := newBeat2Reader(thorChain.Repo(), genesisBlk.Header().ID(), newMessageCache[Beat2Message](10)) res, ok, err := beatReader.Read() // Assert @@ -40,11 +44,13 @@ func TestBeat2Reader_Read(t *testing.T) { func TestBeat2Reader_Read_NoNewBlocksToRead(t *testing.T) { // Arrange - repo, generatedBlocks, _ := initChain(t) - newBlock := generatedBlocks[1] + thorChain := initChain(t) + allBlocks, err := thorChain.GetAllBlocks() + require.NoError(t, err) + newBlock := allBlocks[1] // Act - beatReader := newBeat2Reader(repo, newBlock.Header().ID(), newMessageCache[Beat2Message](10)) + beatReader := newBeat2Reader(thorChain.Repo(), newBlock.Header().ID(), newMessageCache[Beat2Message](10)) res, ok, err := beatReader.Read() // Assert @@ -55,10 +61,10 @@ func TestBeat2Reader_Read_NoNewBlocksToRead(t *testing.T) { func TestBeat2Reader_Read_ErrorWhenReadingBlocks(t *testing.T) { // Arrange - repo, _, _ := initChain(t) + thorChain := initChain(t) // Act - beatReader := newBeat2Reader(repo, thor.MustParseBytes32("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), newMessageCache[Beat2Message](10)) + beatReader := newBeat2Reader(thorChain.Repo(), thor.MustParseBytes32("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), newMessageCache[Beat2Message](10)) res, ok, err := beatReader.Read() // Assert diff --git a/api/subscriptions/beat_reader_test.go b/api/subscriptions/beat_reader_test.go index 07d020a7b..508913eea 100644 --- a/api/subscriptions/beat_reader_test.go +++ b/api/subscriptions/beat_reader_test.go @@ -9,17 +9,20 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/thor" ) func TestBeatReader_Read(t *testing.T) { // Arrange - repo, generatedBlocks, _ := initChain(t) - genesisBlk := generatedBlocks[0] - newBlock := generatedBlocks[1] + thorChain := initChain(t) + allBlocks, err := thorChain.GetAllBlocks() + require.NoError(t, err) + genesisBlk := allBlocks[0] + newBlock := allBlocks[1] // Act - beatReader := newBeatReader(repo, genesisBlk.Header().ID(), newMessageCache[BeatMessage](10)) + beatReader := newBeatReader(thorChain.Repo(), genesisBlk.Header().ID(), newMessageCache[BeatMessage](10)) res, ok, err := beatReader.Read() // Assert @@ -38,11 +41,13 @@ func TestBeatReader_Read(t *testing.T) { func TestBeatReader_Read_NoNewBlocksToRead(t *testing.T) { // Arrange - repo, generatedBlocks, _ := initChain(t) - newBlock := generatedBlocks[1] + thorChain := initChain(t) + allBlocks, err := thorChain.GetAllBlocks() + require.NoError(t, err) + newBlock := allBlocks[1] // Act - beatReader := newBeatReader(repo, newBlock.Header().ID(), newMessageCache[BeatMessage](10)) + beatReader := newBeatReader(thorChain.Repo(), newBlock.Header().ID(), newMessageCache[BeatMessage](10)) res, ok, err := beatReader.Read() // Assert @@ -53,10 +58,10 @@ func TestBeatReader_Read_NoNewBlocksToRead(t *testing.T) { func TestBeatReader_Read_ErrorWhenReadingBlocks(t *testing.T) { // Arrange - repo, _, _ := initChain(t) + thorChain := initChain(t) // Act - beatReader := newBeatReader(repo, thor.MustParseBytes32("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), newMessageCache[BeatMessage](10)) + beatReader := newBeatReader(thorChain.Repo(), thor.MustParseBytes32("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), newMessageCache[BeatMessage](10)) res, ok, err := beatReader.Read() // Assert diff --git a/api/subscriptions/block_reader_test.go b/api/subscriptions/block_reader_test.go index f772ba389..29a866963 100644 --- a/api/subscriptions/block_reader_test.go +++ b/api/subscriptions/block_reader_test.go @@ -8,27 +8,28 @@ package subscriptions import ( "math/big" "testing" - "time" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/stretchr/testify/assert" - "github.com/vechain/thor/v2/block" - "github.com/vechain/thor/v2/chain" + "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/genesis" - "github.com/vechain/thor/v2/muxdb" - "github.com/vechain/thor/v2/packer" - "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/test/eventcontract" + "github.com/vechain/thor/v2/test/testchain" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/tx" - "github.com/vechain/thor/v2/txpool" ) func TestBlockReader_Read(t *testing.T) { - repo, generatedBlocks, _ := initChain(t) - genesisBlk := generatedBlocks[0] - newBlock := generatedBlocks[1] + // Arrange + thorChain := initChain(t) + allBlocks, err := thorChain.GetAllBlocks() + require.NoError(t, err) + genesisBlk := allBlocks[0] + newBlock := allBlocks[1] // Test case 1: Successful read next blocks - br := newBlockReader(repo, genesisBlk.Header().ID()) + br := newBlockReader(thorChain.Repo(), genesisBlk.Header().ID()) res, ok, err := br.Read() assert.NoError(t, err) @@ -41,7 +42,7 @@ func TestBlockReader_Read(t *testing.T) { } // Test case 2: There is no new block - br = newBlockReader(repo, newBlock.Header().ID()) + br = newBlockReader(thorChain.Repo(), newBlock.Header().ID()) res, ok, err = br.Read() assert.NoError(t, err) @@ -49,7 +50,7 @@ func TestBlockReader_Read(t *testing.T) { assert.Empty(t, res) // Test case 3: Error when reading blocks - br = newBlockReader(repo, thor.MustParseBytes32("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) + br = newBlockReader(thorChain.Repo(), thor.MustParseBytes32("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) res, ok, err = br.Read() assert.Error(t, err) @@ -57,27 +58,14 @@ func TestBlockReader_Read(t *testing.T) { assert.Empty(t, res) } -func initChain(t *testing.T) (*chain.Repository, []*block.Block, *txpool.TxPool) { - db := muxdb.NewMem() - stater := state.NewStater(db) - gene := genesis.NewDevnet() - - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } - repo, _ := chain.NewRepository(db, b) - - txPool := txpool.New(repo, stater, txpool.Options{ - Limit: 100, - LimitPerAccount: 16, - MaxLifetime: time.Hour, - }) +func initChain(t *testing.T) *testchain.Chain { + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) addr := thor.BytesToAddress([]byte("to")) cla := tx.NewClause(&addr).WithValue(big.NewInt(10000)) tr := new(tx.Builder). - ChainTag(repo.ChainTag()). + ChainTag(thorChain.Repo().ChainTag()). GasPriceCoef(1). Expiration(10). Gas(21000). @@ -87,52 +75,20 @@ func initChain(t *testing.T) (*chain.Repository, []*block.Block, *txpool.TxPool) Build() tr = tx.MustSign(tr, genesis.DevAccounts()[0].PrivateKey) - packer := packer.New(repo, stater, genesis.DevAccounts()[0].Address, &genesis.DevAccounts()[0].Address, thor.NoFork) - sum, _ := repo.GetBlockSummary(b.Header().ID()) - flow, err := packer.Schedule(sum, uint64(time.Now().Unix())) - if err != nil { - t.Fatal(err) - } - err = flow.Adopt(tr) - if err != nil { - t.Fatal(err) - } - blk, stage, receipts, err := flow.Pack(genesis.DevAccounts()[0].PrivateKey, 0, false) - if err != nil { - t.Fatal(err) - } - if _, err := stage.Commit(); err != nil { - t.Fatal(err) - } - insertMockOutputEvent(receipts) - if err := repo.AddBlock(blk, receipts, 0); err != nil { - t.Fatal(err) - } - if err := repo.SetBestBlockID(blk.Header().ID()); err != nil { - t.Fatal(err) - } - return repo, []*block.Block{b, blk}, txPool -} + txDeploy := new(tx.Builder). + ChainTag(thorChain.Repo().ChainTag()). + GasPriceCoef(1). + Expiration(100). + Gas(1_000_000). + Nonce(3). + Clause(tx.NewClause(nil).WithData(common.Hex2Bytes(eventcontract.HexBytecode))). + BlockRef(tx.NewBlockRef(0)). + Build() + sigTxDeploy, err := crypto.Sign(txDeploy.SigningHash().Bytes(), genesis.DevAccounts()[1].PrivateKey) + require.NoError(t, err) + txDeploy = txDeploy.WithSignature(sigTxDeploy) -// This is a helper function to forcly insert an event into the output receipts -func insertMockOutputEvent(receipts tx.Receipts) { - oldReceipt := receipts[0] - events := make(tx.Events, 0) - events = append(events, &tx.Event{ - Address: thor.BytesToAddress([]byte("to")), - Topics: []thor.Bytes32{thor.BytesToBytes32([]byte("topic"))}, - Data: []byte("data"), - }) - outputs := &tx.Output{ - Transfers: oldReceipt.Outputs[0].Transfers, - Events: events, - } - receipts[0] = &tx.Receipt{ - Reverted: oldReceipt.Reverted, - GasUsed: oldReceipt.GasUsed, - Outputs: []*tx.Output{outputs}, - GasPayer: oldReceipt.GasPayer, - Paid: oldReceipt.Paid, - Reward: oldReceipt.Reward, - } + require.NoError(t, thorChain.MintTransactions(genesis.DevAccounts()[0], tr, txDeploy)) + + return thorChain } diff --git a/api/subscriptions/event_reader_test.go b/api/subscriptions/event_reader_test.go index b1fe280ba..71122f40f 100644 --- a/api/subscriptions/event_reader_test.go +++ b/api/subscriptions/event_reader_test.go @@ -8,17 +8,22 @@ package subscriptions import ( "testing" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" "github.com/vechain/thor/v2/chain" ) func TestEventReader_Read(t *testing.T) { - repo, generatedBlocks, _ := initChain(t) - genesisBlk := generatedBlocks[0] - newBlock := generatedBlocks[1] + // Arrange + thorChain := initChain(t) + allBlocks, err := thorChain.GetAllBlocks() + require.NoError(t, err) + genesisBlk := allBlocks[0] + newBlock := allBlocks[1] er := &eventReader{ - repo: repo, + repo: thorChain.Repo(), filter: &EventFilter{}, blockReader: &mockBlockReaderWithError{}, } @@ -30,7 +35,7 @@ func TestEventReader_Read(t *testing.T) { assert.False(t, ok) // Test case 2: Events are available to read - er = newEventReader(repo, genesisBlk.Header().ID(), &EventFilter{}) + er = newEventReader(thorChain.Repo(), genesisBlk.Header().ID(), &EventFilter{}) events, ok, err = er.Read() @@ -44,7 +49,7 @@ func TestEventReader_Read(t *testing.T) { t.Fatal("unexpected type") } } - assert.Equal(t, 1, len(eventMessages)) + assert.Equal(t, 2, len(eventMessages)) eventMsg := eventMessages[0] assert.Equal(t, newBlock.Header().ID(), eventMsg.Meta.BlockID) assert.Equal(t, newBlock.Header().Number(), eventMsg.Meta.BlockNumber) diff --git a/api/subscriptions/message_cache_test.go b/api/subscriptions/message_cache_test.go index 6bf65c600..29f40de89 100644 --- a/api/subscriptions/message_cache_test.go +++ b/api/subscriptions/message_cache_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/block" ) @@ -29,10 +30,13 @@ func handler(blk *block.Block) func() (message, error) { } func TestMessageCache_GetOrAdd(t *testing.T) { - _, generatedBlocks, _ := initChain(t) + thorChain := initChain(t) - blk0 := generatedBlocks[0] - blk1 := generatedBlocks[1] + allBlocks, err := thorChain.GetAllBlocks() + require.NoError(t, err) + + blk0 := allBlocks[0] + blk1 := allBlocks[1] cache := newMessageCache[message](10) diff --git a/api/subscriptions/pending_tx_test.go b/api/subscriptions/pending_tx_test.go index a4deb2c89..00e6a0140 100644 --- a/api/subscriptions/pending_tx_test.go +++ b/api/subscriptions/pending_tx_test.go @@ -23,7 +23,14 @@ import ( ) func TestPendingTx_Subscribe(t *testing.T) { - _, _, txPool := initChain(t) + // Arrange + thorChain := initChain(t) + txPool := txpool.New(thorChain.Repo(), thorChain.Stater(), txpool.Options{ + Limit: 100, + LimitPerAccount: 16, + MaxLifetime: time.Hour, + }) + p := newPendingTx(txPool) // When initialized, there should be no listeners @@ -36,7 +43,13 @@ func TestPendingTx_Subscribe(t *testing.T) { } func TestPendingTx_Unsubscribe(t *testing.T) { - _, _, txPool := initChain(t) + // Arrange + thorChain := initChain(t) + txPool := txpool.New(thorChain.Repo(), thorChain.Stater(), txpool.Options{ + Limit: 100, + LimitPerAccount: 16, + MaxLifetime: time.Hour, + }) p := newPendingTx(txPool) ch := make(chan *tx.Transaction) diff --git a/api/subscriptions/subscriptions_test.go b/api/subscriptions/subscriptions_test.go index cce2fd8f2..0c0bffe3a 100644 --- a/api/subscriptions/subscriptions_test.go +++ b/api/subscriptions/subscriptions_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "fmt" "io" + "math/big" "net/http" "net/http/httptest" "net/url" @@ -16,23 +17,22 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/gorilla/mux" "github.com/gorilla/websocket" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/block" - "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/genesis" - "github.com/vechain/thor/v2/muxdb" - "github.com/vechain/thor/v2/packer" - "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/test/eventcontract" + "github.com/vechain/thor/v2/test/testchain" "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/tx" "github.com/vechain/thor/v2/txpool" ) var ts *httptest.Server -var sub *Subscriptions -var txPool *txpool.TxPool -var repo *chain.Repository var blocks []*block.Block func TestSubscriptions(t *testing.T) { @@ -217,25 +217,111 @@ func TestParseAddress(t *testing.T) { } func initSubscriptionsServer(t *testing.T) { - r, generatedBlocks, pool := initChain(t) - repo = r - txPool = pool - blocks = generatedBlocks + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) + + txPool := txpool.New(thorChain.Repo(), thorChain.Stater(), txpool.Options{ + Limit: 100, + LimitPerAccount: 16, + MaxLifetime: time.Hour, + }) + + addr := thor.BytesToAddress([]byte("to")) + cla := tx.NewClause(&addr).WithValue(big.NewInt(10000)) + tr := new(tx.Builder). + ChainTag(thorChain.Repo().ChainTag()). + GasPriceCoef(1). + Expiration(10). + Gas(21000). + Nonce(1). + Clause(cla). + BlockRef(tx.NewBlockRef(0)). + Build() + + sig, err := crypto.Sign(tr.SigningHash().Bytes(), genesis.DevAccounts()[0].PrivateKey) + if err != nil { + t.Fatal(err) + } + tr = tr.WithSignature(sig) + + txDeploy := new(tx.Builder). + ChainTag(thorChain.Repo().ChainTag()). + GasPriceCoef(1). + Expiration(100). + Gas(1_000_000). + Nonce(3). + Clause(tx.NewClause(nil).WithData(common.Hex2Bytes(eventcontract.HexBytecode))). + BlockRef(tx.NewBlockRef(0)). + Build() + sigTxDeploy, err := crypto.Sign(txDeploy.SigningHash().Bytes(), genesis.DevAccounts()[1].PrivateKey) + require.NoError(t, err) + txDeploy = txDeploy.WithSignature(sigTxDeploy) + + require.NoError(t, thorChain.MintTransactions(genesis.DevAccounts()[0], tr, txDeploy)) + + blocks, err = thorChain.GetAllBlocks() + require.NoError(t, err) + router := mux.NewRouter() - sub = New(repo, []string{}, 5, txPool) - sub.Mount(router, "/subscriptions") + New(thorChain.Repo(), []string{}, 5, txPool). + Mount(router, "/subscriptions") ts = httptest.NewServer(router) } func TestSubscriptionsBacktrace(t *testing.T) { - r, generatedBlocks, pool := initChainMultipleBlocks(t, 10) - repo = r - txPool = pool - blocks = generatedBlocks + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) + + txPool := txpool.New(thorChain.Repo(), thorChain.Stater(), txpool.Options{ + Limit: 100, + LimitPerAccount: 16, + MaxLifetime: time.Hour, + }) + + addr := thor.BytesToAddress([]byte("to")) + cla := tx.NewClause(&addr).WithValue(big.NewInt(10000)) + tr := new(tx.Builder). + ChainTag(thorChain.Repo().ChainTag()). + GasPriceCoef(1). + Expiration(10). + Gas(21000). + Nonce(1). + Clause(cla). + BlockRef(tx.NewBlockRef(0)). + Build() + + sig, err := crypto.Sign(tr.SigningHash().Bytes(), genesis.DevAccounts()[0].PrivateKey) + if err != nil { + t.Fatal(err) + } + tr = tr.WithSignature(sig) + + txDeploy := new(tx.Builder). + ChainTag(thorChain.Repo().ChainTag()). + GasPriceCoef(1). + Expiration(100). + Gas(1_000_000). + Nonce(3). + Clause(tx.NewClause(nil).WithData(common.Hex2Bytes(eventcontract.HexBytecode))). + BlockRef(tx.NewBlockRef(0)). + Build() + sigTxDeploy, err := crypto.Sign(txDeploy.SigningHash().Bytes(), genesis.DevAccounts()[1].PrivateKey) + require.NoError(t, err) + txDeploy = txDeploy.WithSignature(sigTxDeploy) + + require.NoError(t, thorChain.MintTransactions(genesis.DevAccounts()[0], tr, txDeploy)) + + for i := 0; i < 10; i++ { + require.NoError(t, thorChain.MintTransactions(genesis.DevAccounts()[0])) + } + + blocks, err = thorChain.GetAllBlocks() + require.NoError(t, err) + router := mux.NewRouter() - sub = New(repo, []string{}, 5, txPool) - sub.Mount(router, "/subscriptions") + New(thorChain.Repo(), []string{}, 5, txPool).Mount(router, "/subscriptions") ts = httptest.NewServer(router) + defer ts.Close() t.Run("testHandleSubjectWithTransferBacktraceLimit", testHandleSubjectWithTransferBacktraceLimit) @@ -258,50 +344,3 @@ func testHandleSubjectWithTransferBacktraceLimit(t *testing.T) { assert.Equal(t, body, []byte("pos: backtrace limit exceeded\n")) assert.Nil(t, conn) } - -func initChainMultipleBlocks(t *testing.T, blockCount int) (*chain.Repository, []*block.Block, *txpool.TxPool) { - db := muxdb.NewMem() - stater := state.NewStater(db) - gene := genesis.NewDevnet() - - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } - repo, _ := chain.NewRepository(db, b) - - txPool := txpool.New(repo, stater, txpool.Options{ - Limit: 100, - LimitPerAccount: 16, - MaxLifetime: time.Hour, - }) - - packer := packer.New(repo, stater, genesis.DevAccounts()[0].Address, &genesis.DevAccounts()[0].Address, thor.NoFork) - - tmpBlock := b - createdBlocks := []*block.Block{b} - for i := 0; i < blockCount; i++ { - sum, _ := repo.GetBlockSummary(tmpBlock.Header().ID()) - flow, err := packer.Schedule(sum, uint64(time.Now().Unix())) - if err != nil { - t.Fatal(err) - } - blk, stage, receipts, err := flow.Pack(genesis.DevAccounts()[0].PrivateKey, 0, false) - if err != nil { - t.Fatal(err) - } - if _, err := stage.Commit(); err != nil { - t.Fatal(err) - } - if err := repo.AddBlock(blk, receipts, 0); err != nil { - t.Fatal(err) - } - if err := repo.SetBestBlockID(blk.Header().ID()); err != nil { - t.Fatal(err) - } - createdBlocks = append(createdBlocks, blk) - tmpBlock = blk - } - - return repo, createdBlocks, txPool -} diff --git a/api/subscriptions/transfer_reader_test.go b/api/subscriptions/transfer_reader_test.go index 8bef2175c..de7d0bf66 100644 --- a/api/subscriptions/transfer_reader_test.go +++ b/api/subscriptions/transfer_reader_test.go @@ -8,19 +8,23 @@ package subscriptions import ( "testing" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/assert" "github.com/vechain/thor/v2/thor" ) func TestTransferReader_Read(t *testing.T) { // Arrange - repo, generatedBlocks, _ := initChain(t) - genesisBlk := generatedBlocks[0] - newBlock := generatedBlocks[1] + thorChain := initChain(t) + allBlocks, err := thorChain.GetAllBlocks() + require.NoError(t, err) + genesisBlk := allBlocks[0] + newBlock := allBlocks[1] filter := &TransferFilter{} // Act - br := newTransferReader(repo, genesisBlk.Header().ID(), filter) + br := newTransferReader(thorChain.Repo(), genesisBlk.Header().ID(), filter) res, ok, err := br.Read() // Assert @@ -38,12 +42,14 @@ func TestTransferReader_Read(t *testing.T) { func TestTransferReader_Read_NoNewBlocksToRead(t *testing.T) { // Arrange - repo, generatedBlocks, _ := initChain(t) - newBlock := generatedBlocks[1] + thorChain := initChain(t) + allBlocks, err := thorChain.GetAllBlocks() + require.NoError(t, err) + newBlock := allBlocks[1] filter := &TransferFilter{} // Act - br := newTransferReader(repo, newBlock.Header().ID(), filter) + br := newTransferReader(thorChain.Repo(), newBlock.Header().ID(), filter) res, ok, err := br.Read() // Assert @@ -54,11 +60,11 @@ func TestTransferReader_Read_NoNewBlocksToRead(t *testing.T) { func TestTransferReader_Read_ErrorWhenReadingBlocks(t *testing.T) { // Arrange - repo, _, _ := initChain(t) + thorChain := initChain(t) filter := &TransferFilter{} // Act - br := newTransferReader(repo, thor.MustParseBytes32("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), filter) + br := newTransferReader(thorChain.Repo(), thor.MustParseBytes32("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), filter) res, ok, err := br.Read() // Assert @@ -69,8 +75,10 @@ func TestTransferReader_Read_ErrorWhenReadingBlocks(t *testing.T) { func TestTransferReader_Read_NoTransferMatchingTheFilter(t *testing.T) { // Arrange - repo, generatedBlocks, _ := initChain(t) - genesisBlk := generatedBlocks[0] + thorChain := initChain(t) + allBlocks, err := thorChain.GetAllBlocks() + require.NoError(t, err) + genesisBlk := allBlocks[0] nonExistingAddress := thor.MustParseAddress("0xffffffffffffffffffffffffffffffffffffffff") badFilter := &TransferFilter{ @@ -78,7 +86,7 @@ func TestTransferReader_Read_NoTransferMatchingTheFilter(t *testing.T) { } // Act - br := newTransferReader(repo, genesisBlk.Header().ID(), badFilter) + br := newTransferReader(thorChain.Repo(), genesisBlk.Header().ID(), badFilter) res, ok, err := br.Read() // Assert diff --git a/api/transactions/transactions_test.go b/api/transactions/transactions_test.go index cef22e191..68c9d535d 100644 --- a/api/transactions/transactions_test.go +++ b/api/transactions/transactions_test.go @@ -20,11 +20,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/api/transactions" - "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/genesis" - "github.com/vechain/thor/v2/muxdb" - "github.com/vechain/thor/v2/packer" - "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/test/testchain" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/thorclient" "github.com/vechain/thor/v2/tx" @@ -32,11 +29,11 @@ import ( ) var ( - repo *chain.Repository ts *httptest.Server transaction *tx.Transaction mempoolTx *tx.Transaction tclient *thorclient.Client + chainTag byte ) func TestTransaction(t *testing.T) { @@ -110,7 +107,6 @@ func getTxReceipt(t *testing.T) { func sendTx(t *testing.T) { var blockRef = tx.NewBlockRef(0) - var chainTag = repo.ChainTag() var expiration = uint32(10) var gas = uint64(21000) @@ -270,19 +266,15 @@ func httpPostAndCheckResponseStatus(t *testing.T, url string, obj interface{}, r } func initTransactionServer(t *testing.T) { - db := muxdb.NewMem() - stater := state.NewStater(db) - gene := genesis.NewDevnet() + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) + + chainTag = thorChain.Repo().ChainTag() - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } - repo, _ = chain.NewRepository(db, b) addr := thor.BytesToAddress([]byte("to")) cla := tx.NewClause(&addr).WithValue(big.NewInt(10000)) transaction = new(tx.Builder). - ChainTag(repo.ChainTag()). + ChainTag(chainTag). GasPriceCoef(1). Expiration(10). Gas(21000). @@ -292,47 +284,26 @@ func initTransactionServer(t *testing.T) { Build() transaction = tx.MustSign(transaction, genesis.DevAccounts()[0].PrivateKey) + require.NoError(t, thorChain.MintTransactions(genesis.DevAccounts()[0], transaction)) + + mempool := txpool.New(thorChain.Repo(), thorChain.Stater(), txpool.Options{Limit: 10000, LimitPerAccount: 16, MaxLifetime: 10 * time.Minute}) + mempoolTx = new(tx.Builder). - ChainTag(repo.ChainTag()). + ChainTag(chainTag). Expiration(10). Gas(21000). Nonce(1). Build() mempoolTx = tx.MustSign(mempoolTx, genesis.DevAccounts()[0].PrivateKey) - packer := packer.New(repo, stater, genesis.DevAccounts()[0].Address, &genesis.DevAccounts()[0].Address, thor.NoFork) - sum, _ := repo.GetBlockSummary(b.Header().ID()) - flow, err := packer.Schedule(sum, uint64(time.Now().Unix())) - if err != nil { - t.Fatal(err) - } - err = flow.Adopt(transaction) - if err != nil { - t.Fatal(err) - } - b, stage, receipts, err := flow.Pack(genesis.DevAccounts()[0].PrivateKey, 0, false) - if err != nil { - t.Fatal(err) - } - if _, err := stage.Commit(); err != nil { - t.Fatal(err) - } - if err := repo.AddBlock(b, receipts, 0); err != nil { - t.Fatal(err) - } - if err := repo.SetBestBlockID(b.Header().ID()); err != nil { - t.Fatal(err) - } - router := mux.NewRouter() - // Add a tx to the mempool to have both pending and non-pending transactions - mempool := txpool.New(repo, stater, txpool.Options{Limit: 10000, LimitPerAccount: 16, MaxLifetime: 10 * time.Minute}) e := mempool.Add(mempoolTx) if e != nil { t.Fatal(e) } - transactions.New(repo, mempool).Mount(router, "/transactions") + router := mux.NewRouter() + transactions.New(thorChain.Repo(), mempool).Mount(router, "/transactions") ts = httptest.NewServer(router) } diff --git a/api/transfers/transfers_test.go b/api/transfers/transfers_test.go index b0764c85f..04a8c7b42 100644 --- a/api/transfers/transfers_test.go +++ b/api/transfers/transfers_test.go @@ -19,12 +19,9 @@ import ( "github.com/vechain/thor/v2/api/events" "github.com/vechain/thor/v2/api/transfers" "github.com/vechain/thor/v2/block" - "github.com/vechain/thor/v2/chain" - "github.com/vechain/thor/v2/genesis" "github.com/vechain/thor/v2/logdb" - "github.com/vechain/thor/v2/muxdb" - "github.com/vechain/thor/v2/state" "github.com/vechain/thor/v2/test/datagen" + "github.com/vechain/thor/v2/test/testchain" "github.com/vechain/thor/v2/thorclient" "github.com/vechain/thor/v2/tx" ) @@ -72,20 +69,20 @@ func TestOption(t *testing.T) { Order: logdb.DESC, } - res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/transfers", filter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/transfers", filter) require.NoError(t, err) assert.Equal(t, "options.limit exceeds the maximum allowed value of 5", strings.Trim(string(res), "\n")) assert.Equal(t, http.StatusForbidden, statusCode) filter.Options.Limit = 5 - _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/transfers", filter) + _, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/logs/transfers", filter) require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode) // with nil options, should use default limit, when the filtered lower // or equal to the limit, should return the filtered transfers filter.Options = nil - res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/transfers", filter) + res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/logs/transfers", filter) require.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode) var tLogs []*events.FilteredEvent @@ -97,7 +94,7 @@ func TestOption(t *testing.T) { // when the filtered transfers exceed the limit, should return the forbidden insertBlocks(t, db, 6) - res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/transfers", filter) + res, statusCode, err = tclient.RawHTTPClient().RawHTTPPost("/logs/transfers", filter) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, statusCode) assert.Equal(t, "the number of filtered logs exceeds the maximum allowed value of 5, please use pagination", strings.Trim(string(res), "\n")) @@ -107,7 +104,7 @@ func TestOption(t *testing.T) { func testTransferBadRequest(t *testing.T) { badBody := []byte{0x00, 0x01, 0x02} - _, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/transfers", badBody) + _, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/transfers", badBody) require.NoError(t, err) assert.Equal(t, http.StatusBadRequest, statusCode) } @@ -120,7 +117,7 @@ func testTransferWithEmptyDb(t *testing.T) { Order: logdb.DESC, } - res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/transfers", emptyFilter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/transfers", emptyFilter) require.NoError(t, err) var tLogs []*transfers.FilteredTransfer if err := json.Unmarshal(res, &tLogs); err != nil { @@ -139,7 +136,7 @@ func testTransferWithBlocks(t *testing.T, expectedBlocks int) { Order: logdb.DESC, } - res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/transfers", emptyFilter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/transfers", emptyFilter) require.NoError(t, err) var tLogs []*transfers.FilteredTransfer if err := json.Unmarshal(res, &tLogs); err != nil { @@ -174,20 +171,12 @@ func insertBlocks(t *testing.T, db *logdb.LogDB, n int) { } func initTransferServer(t *testing.T, logDb *logdb.LogDB, limit uint64) { - router := mux.NewRouter() - - muxDb := muxdb.NewMem() - stater := state.NewStater(muxDb) - gene := genesis.NewDevnet() - - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) - repo, _ := chain.NewRepository(muxDb, b) + router := mux.NewRouter() + transfers.New(thorChain.Repo(), logDb, limit).Mount(router, "/logs/transfers") - transfers.New(repo, logDb, limit).Mount(router, "/transfers") ts = httptest.NewServer(router) } diff --git a/api/utils/revisions_test.go b/api/utils/revisions_test.go index 76a4bbfdc..12926eaa1 100644 --- a/api/utils/revisions_test.go +++ b/api/utils/revisions_test.go @@ -12,11 +12,8 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" - "github.com/vechain/thor/v2/chain" - "github.com/vechain/thor/v2/cmd/thor/solo" - "github.com/vechain/thor/v2/genesis" - "github.com/vechain/thor/v2/muxdb" - "github.com/vechain/thor/v2/state" + "github.com/stretchr/testify/require" + "github.com/vechain/thor/v2/test/testchain" "github.com/vechain/thor/v2/thor" ) @@ -101,15 +98,8 @@ func TestAllowNext(t *testing.T) { } func TestGetSummary(t *testing.T) { - db := muxdb.NewMem() - stater := state.NewStater(db) - gene := genesis.NewDevnet() - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } - repo, _ := chain.NewRepository(db, b) - bft := solo.NewBFTEngine(repo) + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) // Test cases testCases := []struct { @@ -151,7 +141,7 @@ func TestGetSummary(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - summary, err := GetSummary(tc.revision, repo, bft) + summary, err := GetSummary(tc.revision, thorChain.Repo(), thorChain.Engine()) if tc.err != nil { assert.Equal(t, tc.err.Error(), err.Error()) } else { @@ -163,22 +153,17 @@ func TestGetSummary(t *testing.T) { } func TestGetSummaryAndState(t *testing.T) { - db := muxdb.NewMem() - stater := state.NewStater(db) - gene := genesis.NewDevnet() - b, _, _, err := gene.Build(stater) - if err != nil { - t.Fatal(err) - } - repo, _ := chain.NewRepository(db, b) - bft := solo.NewBFTEngine(repo) + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) + + b := thorChain.GenesisBlock() - summary, _, err := GetSummaryAndState(&Revision{revBest}, repo, bft, stater) + summary, _, err := GetSummaryAndState(&Revision{revBest}, thorChain.Repo(), thorChain.Engine(), thorChain.Stater()) assert.Nil(t, err) assert.Equal(t, summary.Header.Number(), b.Header().Number()) assert.Equal(t, summary.Header.Timestamp(), b.Header().Timestamp()) - summary, _, err = GetSummaryAndState(&Revision{revNext}, repo, bft, stater) + summary, _, err = GetSummaryAndState(&Revision{revNext}, thorChain.Repo(), thorChain.Engine(), thorChain.Stater()) assert.Nil(t, err) assert.Equal(t, summary.Header.Number(), b.Header().Number()+1) assert.Equal(t, summary.Header.Timestamp(), b.Header().Timestamp()+thor.BlockInterval) diff --git a/cmd/thor/solo/types.go b/cmd/thor/solo/types.go index c431ece0d..b436419a8 100644 --- a/cmd/thor/solo/types.go +++ b/cmd/thor/solo/types.go @@ -6,6 +6,7 @@ package solo import ( + "github.com/vechain/thor/v2/bft" "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/comm" "github.com/vechain/thor/v2/thor" @@ -34,7 +35,7 @@ func (engine *BFTEngine) Justified() (thor.Bytes32, error) { return engine.justified, nil } -func NewBFTEngine(repo *chain.Repository) *BFTEngine { +func NewBFTEngine(repo *chain.Repository) bft.Committer { return &BFTEngine{ finalized: repo.GenesisBlock().Header().ID(), justified: repo.GenesisBlock().Header().ID(), diff --git a/test/eventcontract/event_contract.go b/test/eventcontract/event_contract.go new file mode 100644 index 000000000..cb239c919 --- /dev/null +++ b/test/eventcontract/event_contract.go @@ -0,0 +1,77 @@ +// Copyright (c) 2024 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package eventcontract + +const Code = `// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +contract EventContract { + + // Event that is triggered on deployment + event Deployed(string message); + + // Event that is triggered when triggerEvent method is called + event Triggered(string message); + + // Constructor is executed upon contract deployment + constructor() { + emit Deployed("it's deployed"); + } + + // Function that triggers the Triggered event + function triggerEvent(string memory message) public { + emit Triggered(message); + } +}` + +const ABI = `[ + { + "inputs": [], + "stateMutability": "nonpayable", + "type": "constructor" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "string", + "name": "message", + "type": "string" + } + ], + "name": "Deployed", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "string", + "name": "message", + "type": "string" + } + ], + "name": "Triggered", + "type": "event" + }, + { + "inputs": [ + { + "internalType": "string", + "name": "message", + "type": "string" + } + ], + "name": "triggerEvent", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + } +]` + +const HexBytecode = `608060405234801561001057600080fd5b507f90e3596779ac8b6be4c38e80c57aadc0ee3d1b6c8c20a9633d1c860890855f8860405161003e906100a8565b60405180910390a16100c8565b600082825260208201905092915050565b7f69742773206465706c6f79656400000000000000000000000000000000000000600082015250565b6000610092600d8361004b565b915061009d8261005c565b602082019050919050565b600060208201905081810360008301526100c181610085565b9050919050565b610309806100d76000396000f3fe608060405234801561001057600080fd5b506004361061002b5760003560e01c8063e6c75c6b14610030575b600080fd5b61004a600480360381019061004591906101e0565b61004c565b005b7f2b22cb97612862145333bc8f03d3ae7b4ea194fc038c00d4994dcd794bbf6f558160405161007b91906102b1565b60405180910390a150565b6000604051905090565b600080fd5b600080fd5b600080fd5b600080fd5b6000601f19601f8301169050919050565b7f4e487b7100000000000000000000000000000000000000000000000000000000600052604160045260246000fd5b6100ed826100a4565b810181811067ffffffffffffffff8211171561010c5761010b6100b5565b5b80604052505050565b600061011f610086565b905061012b82826100e4565b919050565b600067ffffffffffffffff82111561014b5761014a6100b5565b5b610154826100a4565b9050602081019050919050565b82818337600083830152505050565b600061018361017e84610130565b610115565b90508281526020810184848401111561019f5761019e61009f565b5b6101aa848285610161565b509392505050565b600082601f8301126101c7576101c661009a565b5b81356101d7848260208601610170565b91505092915050565b6000602082840312156101f6576101f5610090565b5b600082013567ffffffffffffffff81111561021457610213610095565b5b610220848285016101b2565b91505092915050565b600081519050919050565b600082825260208201905092915050565b60005b83811015610263578082015181840152602081019050610248565b83811115610272576000848401525b50505050565b600061028382610229565b61028d8185610234565b935061029d818560208601610245565b6102a6816100a4565b840191505092915050565b600060208201905081810360008301526102cb8184610278565b90509291505056fea2646970667358221220c8faa509c384033da1201cb3e75b6fe60090e6722ae8c11af2a827446254d80664736f6c634300080a0033` diff --git a/test/testchain/chain.go b/test/testchain/chain.go new file mode 100644 index 000000000..b35687d14 --- /dev/null +++ b/test/testchain/chain.go @@ -0,0 +1,222 @@ +// Copyright (c) 2024 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package testchain + +import ( + "errors" + "fmt" + "slices" + "time" + + "github.com/vechain/thor/v2/bft" + "github.com/vechain/thor/v2/block" + "github.com/vechain/thor/v2/chain" + "github.com/vechain/thor/v2/cmd/thor/solo" + "github.com/vechain/thor/v2/genesis" + "github.com/vechain/thor/v2/logdb" + "github.com/vechain/thor/v2/muxdb" + "github.com/vechain/thor/v2/packer" + "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/tx" +) + +// Chain represents the blockchain structure. +// It includes database (db), genesis information (genesis), consensus engine (engine), +// repository for blocks and state (repo), state manager (stater), and the genesis block (genesisBlock). +type Chain struct { + db *muxdb.MuxDB + genesis *genesis.Genesis + engine bft.Committer + repo *chain.Repository + stater *state.Stater + genesisBlock *block.Block + logDB *logdb.LogDB + forkConfig thor.ForkConfig +} + +func New( + db *muxdb.MuxDB, + genesis *genesis.Genesis, + engine bft.Committer, + repo *chain.Repository, + stater *state.Stater, + genesisBlock *block.Block, + logDB *logdb.LogDB, +) *Chain { + return &Chain{ + db: db, + genesis: genesis, + engine: engine, + repo: repo, + stater: stater, + genesisBlock: genesisBlock, + logDB: logDB, + forkConfig: thor.GetForkConfig(genesisBlock.Header().ID()), + } +} + +// NewIntegrationTestChain is a convenience function that creates a Chain for testing. +// It uses an in-memory database, development network genesis, and a solo BFT engine. +func NewIntegrationTestChain() (*Chain, error) { + // Initialize the database + db := muxdb.NewMem() + + // Create the state manager (Stater) with the initialized database. + stater := state.NewStater(db) + + // Initialize the genesis and retrieve the genesis block + gene := genesis.NewDevnet() + geneBlk, _, _, err := gene.Build(stater) + if err != nil { + return nil, err + } + + // Create the repository which manages chain data, using the database and genesis block. + repo, err := chain.NewRepository(db, geneBlk) + if err != nil { + return nil, err + } + + // Create an inMemory logdb + logDb, err := logdb.NewMem() + if err != nil { + return nil, err + } + + return New( + db, + gene, + solo.NewBFTEngine(repo), + repo, + stater, + geneBlk, + logDb, + ), nil +} + +// Repo returns the blockchain's repository, which stores blocks and other data. +func (c *Chain) Repo() *chain.Repository { + return c.repo +} + +// Stater returns the current state manager of the chain, which is responsible for managing the state of accounts and other elements. +func (c *Chain) Stater() *state.Stater { + return c.stater +} + +// Engine returns the consensus engine responsible for the blockchain's consensus mechanism. +func (c *Chain) Engine() bft.Committer { + return c.engine +} + +// GenesisBlock returns the genesis block of the chain, which is the first block in the blockchain. +func (c *Chain) GenesisBlock() *block.Block { + return c.genesisBlock +} + +// MintTransactions creates a block with the provided transactions and adds it to the blockchain. +// It wraps the transactions with receipts and passes them to MintTransactionsWithReceiptFunc. +func (c *Chain) MintTransactions(account genesis.DevAccount, transactions ...*tx.Transaction) error { + return c.MintBlock(account, transactions...) +} + +// MintBlock creates and finalizes a new block with the given transactions. +// It schedules a new block, adopts transactions, packs them into a block, and commits it to the chain. +func (c *Chain) MintBlock(account genesis.DevAccount, transactions ...*tx.Transaction) error { + // Create a new block packer with the current chain state and account information. + blkPacker := packer.New(c.Repo(), c.Stater(), account.Address, &genesis.DevAccounts()[0].Address, c.forkConfig) + + // Create a new block + blkFlow, err := blkPacker.Mock( + c.Repo().BestBlockSummary(), + c.Repo().BestBlockSummary().Header.Timestamp()+thor.BlockInterval, + c.Repo().BestBlockSummary().Header.GasLimit(), + ) + if err != nil { + return fmt.Errorf("unable to mock a new block: %w", err) + } + + // Adopt the provided transactions into the block. + for _, trx := range transactions { + if err = blkFlow.Adopt(trx); err != nil { + return fmt.Errorf("unable to adopt tx into block: %w", err) + } + } + + // Pack the adopted transactions into a block. + newBlk, stage, receipts, err := blkFlow.Pack(account.PrivateKey, 0, false) + if err != nil { + return fmt.Errorf("unable to pack tx: %w", err) + } + + // Commit the new block to the chain's state. + if _, err := stage.Commit(); err != nil { + return fmt.Errorf("unable to commit tx: %w", err) + } + + // Add the block to the repository. + if err := c.Repo().AddBlock(newBlk, receipts, 0); err != nil { + return fmt.Errorf("unable to add tx to repo: %w", err) + } + + // Set the new block as the best (latest) block in the repository. + if err := c.Repo().SetBestBlockID(newBlk.Header().ID()); err != nil { + return fmt.Errorf("unable to set best block: %w", err) + } + + return nil +} + +// GetAllBlocks retrieves all blocks from the blockchain, starting from the best block and moving backward to the genesis block. +// It limits the retrieval time to 5 seconds to avoid excessive delays. +func (c *Chain) GetAllBlocks() ([]*block.Block, error) { + bestBlkSummary := c.Repo().BestBlockSummary() + var blks []*block.Block + currBlockID := bestBlkSummary.Header.ID() + startTime := time.Now() + + // Traverse the chain backwards until the genesis block is reached or timeout occurs. + for { + blk, err := c.repo.GetBlock(currBlockID) + if err != nil { + return nil, err + } + blks = append(blks, blk) + + // Stop when the genesis block is reached and reverse the slice to have genesis at position 0. + if blk.Header().Number() == c.genesisBlock.Header().Number() { + slices.Reverse(blks) // make sure genesis is at position 0 + return blks, err + } + currBlockID = blk.Header().ParentID() + + // Check if the retrieval process is taking too long (more than 5 seconds). + if time.Since(startTime) > 5*time.Second { + return nil, errors.New("taking more than 5 seconds to retrieve all blocks") + } + } +} + +// BestBlock returns the current best (latest) block in the chain. +func (c *Chain) BestBlock() (*block.Block, error) { + return c.Repo().GetBlock(c.Repo().BestBlockSummary().Header.ID()) +} + +// GetForkConfig returns the current fork configuration based on the ID of the genesis block. +func (c *Chain) GetForkConfig() thor.ForkConfig { + return c.forkConfig +} + +// Database returns the current database. +func (c *Chain) Database() *muxdb.MuxDB { + return c.db +} + +// LogDB returns the current logdb. +func (c *Chain) LogDB() *logdb.LogDB { + return c.logDB +} diff --git a/thorclient/api_test.go b/thorclient/api_test.go new file mode 100644 index 000000000..e6a0e43be --- /dev/null +++ b/thorclient/api_test.go @@ -0,0 +1,381 @@ +// Copyright (c) 2024 The VeChainThor developers + +// Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying +// file LICENSE or + +package thorclient + +import ( + "math/big" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/gorilla/mux" + "github.com/stretchr/testify/require" + "github.com/vechain/thor/v2/api/accounts" + "github.com/vechain/thor/v2/api/blocks" + "github.com/vechain/thor/v2/api/debug" + "github.com/vechain/thor/v2/api/events" + "github.com/vechain/thor/v2/api/node" + "github.com/vechain/thor/v2/comm" + "github.com/vechain/thor/v2/genesis" + "github.com/vechain/thor/v2/logdb" + "github.com/vechain/thor/v2/test/datagen" + "github.com/vechain/thor/v2/test/testchain" + "github.com/vechain/thor/v2/thor" + "github.com/vechain/thor/v2/tx" + "github.com/vechain/thor/v2/txpool" + + // Force-load the tracer native engines to trigger registration + _ "github.com/vechain/thor/v2/tracers/js" + _ "github.com/vechain/thor/v2/tracers/logger" +) + +const ( + gasLimit = 30_000_000 + logDBLimit = 1_000 +) + +func initAPIServer(t *testing.T) (*testchain.Chain, *httptest.Server) { + thorChain, err := testchain.NewIntegrationTestChain() + require.NoError(t, err) + + // mint some transactions to be used in the endpoints + mintTransactions(t, thorChain) + + router := mux.NewRouter() + + accounts.New(thorChain.Repo(), thorChain.Stater(), uint64(gasLimit), thor.NoFork, thorChain.Engine()). + Mount(router, "/accounts") + + blocks.New(thorChain.Repo(), thorChain.Engine()).Mount(router, "/blocks") + + debug.New(thorChain.Repo(), thorChain.Stater(), thorChain.GetForkConfig(), gasLimit, true, thorChain.Engine(), []string{"all"}, false). + Mount(router, "/debug") + + logDb, err := logdb.NewMem() + require.NoError(t, err) + events.New(thorChain.Repo(), logDb, logDBLimit).Mount(router, "/logs/event") + + communicator := comm.New( + thorChain.Repo(), + txpool.New(thorChain.Repo(), thorChain.Stater(), txpool.Options{ + Limit: 10000, + LimitPerAccount: 16, + MaxLifetime: 10 * time.Minute, + }), + ) + node.New(communicator).Mount(router, "/node") + + return thorChain, httptest.NewServer(router) +} + +func mintTransactions(t *testing.T, thorChain *testchain.Chain) { + toAddr := datagen.RandAddress() + + noClausesTx := new(tx.Builder). + ChainTag(thorChain.Repo().ChainTag()). + Expiration(10). + Gas(21000). + Build() + sig, err := crypto.Sign(noClausesTx.SigningHash().Bytes(), genesis.DevAccounts()[0].PrivateKey) + if err != nil { + t.Fatal(err) + } + noClausesTx = noClausesTx.WithSignature(sig) + + cla := tx.NewClause(&toAddr).WithValue(big.NewInt(10000)) + cla2 := tx.NewClause(&toAddr).WithValue(big.NewInt(10000)) + transaction := new(tx.Builder). + ChainTag(thorChain.Repo().ChainTag()). + GasPriceCoef(1). + Expiration(10). + Gas(37000). + Nonce(1). + Clause(cla). + Clause(cla2). + BlockRef(tx.NewBlockRef(0)). + Build() + + sig, err = crypto.Sign(transaction.SigningHash().Bytes(), genesis.DevAccounts()[0].PrivateKey) + if err != nil { + t.Fatal(err) + } + transaction = transaction.WithSignature(sig) + + require.NoError(t, thorChain.MintTransactions(genesis.DevAccounts()[0], transaction, noClausesTx)) +} + +func TestAPIs(t *testing.T) { + thorChain, ts := initAPIServer(t) + defer ts.Close() + + for name, tt := range map[string]func(*testing.T, *testchain.Chain, *httptest.Server){ + "testAccountEndpoint": testAccountEndpoint, + "testBlocksEndpoint": testBlocksEndpoint, + "testDebugEndpoint": testDebugEndpoint, + "testEventsEndpoint": testEventsEndpoint, + "testNodeEndpoint": testNodeEndpoint, + } { + t.Run(name, func(t *testing.T) { + tt(t, thorChain, ts) + }) + } +} + +func testAccountEndpoint(t *testing.T, _ *testchain.Chain, ts *httptest.Server) { + // Example storage key + storageKey := "0x0000000000000000000000000000000000000000000000000000000000000000" + + // Example addresses + address1 := "0x0123456789abcdef0123456789abcdef01234567" + address2 := "0xabcdef0123456789abcdef0123456789abcdef01" + + // 1. Test GET /accounts/{address} + t.Run("GetAccount", func(t *testing.T) { + resp, err := ts.Client().Get(ts.URL + "/accounts/" + address1) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, 200, resp.StatusCode) + // Optionally, you can unmarshal and validate the response body here + }) + + // 2. Test GET /accounts/{address}/code + t.Run("GetCode", func(t *testing.T) { + resp, err := ts.Client().Get(ts.URL + "/accounts/" + address1 + "/code") + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, 200, resp.StatusCode) + // Optionally, you can unmarshal and validate the response body here + }) + + // 3. Test GET /accounts/{address}/storage/{key} + t.Run("GetStorage", func(t *testing.T) { + resp, err := ts.Client().Get(ts.URL + "/accounts/" + address1 + "/storage/" + storageKey) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, 200, resp.StatusCode) + // Optionally, you can unmarshal and validate the response body here + }) + + // 4. Test POST /accounts/* + t.Run("InspectClauses", func(t *testing.T) { + // Define the payload for the batch call + payload := `{ + "clauses": [ + { + "to": "` + address1 + `", + "value": "0x0", + "data": "0x" + }, + { + "to": "` + address2 + `", + "value": "0x1", + "data": "0x" + } + ], + "gas": 1000000, + "gasPrice": "0x0", + "caller": "` + address1 + `" + }` + req, err := http.NewRequest("POST", ts.URL+"/accounts/*", strings.NewReader(payload)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + + // Simulate sending request with revision query parameter + query := req.URL.Query() + query.Add("revision", "best") // Add any revision parameter as expected + req.URL.RawQuery = query.Encode() + + // Perform the request + resp, err := ts.Client().Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + // Ensure the response code is 200 OK + require.Equal(t, 200, resp.StatusCode) + }) +} + +func testBlocksEndpoint(t *testing.T, _ *testchain.Chain, ts *httptest.Server) { + // Example revision (this could be a block number or block ID) + revision := "best" // You can adjust this to a real block number or ID for integration testing + + // 1. Test GET /blocks/{revision} + t.Run("GetBlock", func(t *testing.T) { + // Send request to get block information by revision + resp, err := ts.Client().Get(ts.URL + "/blocks/" + revision) + require.NoError(t, err) + defer resp.Body.Close() + + // Ensure the response code is 200 OK + require.Equal(t, 200, resp.StatusCode) + // Optionally, you can unmarshal and validate the response body here + }) + + // 2. Test GET /blocks/{revision}?expanded=true + t.Run("GetBlockExpanded", func(t *testing.T) { + // Send request to get expanded block information (includes transactions and receipts) + resp, err := ts.Client().Get(ts.URL + "/blocks/" + revision + "?expanded=true") + require.NoError(t, err) + defer resp.Body.Close() + + // Ensure the response code is 200 OK + require.Equal(t, 200, resp.StatusCode) + // Optionally, you can unmarshal and validate the response body here + }) + + // 3. Test GET /blocks/{revision}?expanded=invalid (should return bad request) + t.Run("GetBlockInvalidExpanded", func(t *testing.T) { + // Send request with an invalid 'expanded' parameter + resp, err := ts.Client().Get(ts.URL + "/blocks/" + revision + "?expanded=invalid") + require.NoError(t, err) + defer resp.Body.Close() + + // Ensure the response code is 400 Bad Request + require.Equal(t, 400, resp.StatusCode) + // Optionally, you can unmarshal and validate the response body here + }) +} + +func testDebugEndpoint(t *testing.T, thorChain *testchain.Chain, ts *httptest.Server) { + // Example block ID, transaction index, and clause index + bestBlock, _ := thorChain.BestBlock() + blockID := bestBlock.Header().ID().String() + txIndex := uint64(0) + clauseIndex := uint32(0) + + // Example contract address + contractAddress := "0xabcdef0123456789abcdef0123456789abcdef01" + + // 1. Test POST /debug/tracers (Trace an existing clause) + t.Run("TraceClause", func(t *testing.T) { + payload := `{ + "name": "structLoggerTracer", + "target": "` + blockID + `/` + strconv.FormatUint(txIndex, 10) + `/` + strconv.FormatUint(uint64(clauseIndex), 10) + `" + }` + + req, err := http.NewRequest("POST", ts.URL+"/debug/tracers", strings.NewReader(payload)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + + // Perform the request + resp, err := ts.Client().Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + // Ensure the response code is 200 OK + require.Equal(t, 200, resp.StatusCode) + // Optionally, you can unmarshal and validate the response body here + }) + + // 2. Test POST /debug/tracers/call (Trace a contract call) + t.Run("TraceCall", func(t *testing.T) { + payload := `{ + "name": "structLoggerTracer", + "to": "` + contractAddress + `", + "value": "0x0", + "data": "0x", + "gas": 1000000, + "gasPrice": "0x0", + "caller": "` + contractAddress + `" + }` + + req, err := http.NewRequest("POST", ts.URL+"/debug/tracers/call", strings.NewReader(payload)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + + // Perform the request + resp, err := ts.Client().Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + // Ensure the response code is 200 OK + require.Equal(t, 200, resp.StatusCode) + // Optionally, you can unmarshal and validate the response body here + }) + + // 3. Test POST /debug/storage-range (Debug storage for a contract) + t.Run("DebugStorage", func(t *testing.T) { + payload := `{ + "address": "` + contractAddress + `", + "target": "` + blockID + `/` + strconv.FormatUint(txIndex, 10) + `/` + strconv.FormatUint(uint64(clauseIndex), 10) + `", + "keyStart": "", + "maxResult": 100 + }` + + req, err := http.NewRequest("POST", ts.URL+"/debug/storage-range", strings.NewReader(payload)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + + // Perform the request + resp, err := ts.Client().Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + // Ensure the response code is 200 OK + require.Equal(t, 200, resp.StatusCode) + // Optionally, you can unmarshal and validate the response body here + }) +} + +func testEventsEndpoint(t *testing.T, _ *testchain.Chain, ts *httptest.Server) { + // Example address and topic for filtering events + address := "0x0123456789abcdef0123456789abcdef01234567" + topic := thor.BytesToBytes32([]byte("topic")).String() + + // 1. Test POST /events (Filter events) + t.Run("FilterEvents", func(t *testing.T) { + // Define the payload for filtering events + payload := `{ + "criteriaSet": [ + { + "address": "` + address + `", + "topic0": "` + topic + `" + } + ], + "options": { + "limit": 10, + "offset": 0 + } + }` + + req, err := http.NewRequest("POST", ts.URL+"/logs/event", strings.NewReader(payload)) + require.NoError(t, err) + req.Header.Set("Content-Type", "application/json") + + // Perform the request + resp, err := ts.Client().Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + // Ensure the response code is 200 OK + require.Equal(t, 200, resp.StatusCode) + // Optionally, you can unmarshal and validate the response body here + // body, err := ioutil.ReadAll(resp.Body) + // require.NoError(t, err) + // fmt.Println(string(body)) + }) +} + +func testNodeEndpoint(t *testing.T, _ *testchain.Chain, ts *httptest.Server) { + // 1. Test GET /node/network/peers + t.Run("GetPeersStats", func(t *testing.T) { + // Send request to get peers statistics + resp, err := ts.Client().Get(ts.URL + "/node/network/peers") + require.NoError(t, err) + defer resp.Body.Close() + + // Ensure the response code is 200 OK + require.Equal(t, 200, resp.StatusCode) + // Optionally, you can unmarshal and validate the response body here + // body, err := ioutil.ReadAll(resp.Body) + // require.NoError(t, err) + // fmt.Println(string(body)) + }) +} From 49d97048d319c961252d7d45aa25b710c02550a1 Mon Sep 17 00:00:00 2001 From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:26:58 +0000 Subject: [PATCH 08/25] chore(docs): update spec for validator nodes (#875) * chore(docs): update spec for validator nodes * chore(docs): update cores * chore(docs): remove public node stuff --- docs/hosting-a-node.md | 49 +++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/docs/hosting-a-node.md b/docs/hosting-a-node.md index 8defd58dc..1dc1e12ce 100644 --- a/docs/hosting-a-node.md +++ b/docs/hosting-a-node.md @@ -1,17 +1,20 @@ ## Hosting a Node -_**Please note**: The recommendations and information below are based on the main network as of 22nd April 2024. The +_**Please note**: The recommendations and information below are based on the main network as of 6th November 2024. The requirements may change as the network evolves._ +[VeChain Stats](https://vechainstats.com/charts/#thor-size) provides an up-to-date graphic of the network's current +state, including the disk space required for various node types. + ### Table of Contents - [System Requirements](#system-requirements) - - [Authority Nodes](#authority-nodes) - - [Public Nodes](#public-nodes) + - [Authority Nodes](#authority-nodes) + - [Full Archive Nodes](#full-archive-nodes) - [Node Types](#node-types) - - [Full Archive Node](#full-archive-node) - - [Full Node](#full-node) - - [Full Node without Logs](#full-node-without-logs) + - [Full Archive Node](#full-archive-node) + - [Full Node](#full-node) + - [Full Node without Logs](#full-node-without-logs) - [Metrics](#metrics) --- @@ -27,26 +30,25 @@ available options. #### Authority Nodes -Below spec is the node configured in full node without logs. +- Pruner enabled +- Skip logs | Resource | Minimum Specification | Recommended Specification | |-----------|-----------------------|---------------------------| | CPU | 2 Core | 4 Core | | RAM | 8 GB | 16 GB | | Bandwidth | 10 Mbit | 20 Mbit | -| Disk | 300 GB NVMe SSD | 500 GB NVMe SSD | - -#### Public Nodes +| Disk | 200 GB NVMe SSD | 300 GB NVMe SSD | -**Note**: For public nodes, it is essential to configure them with a robust and secure setup, including protection -against DDoS attacks and intrusion detection systems (IDS). +### Full Archive Nodes -Below spec is the node configured in full archive node. +- Disabled pruner +- Enabled logs | Resource | Minimum Specification | Recommended Specification | |-----------|-----------------------|---------------------------| -| CPU | 8 Core | 16 Core | -| RAM | 16 GB | 64 GB | +| CPU | 2 Core | 4 Core | +| RAM | 16 GB | 32 GB | | Bandwidth | 10 Mbit | 20 Mbit | | Disk | 600 GB SSD | 1 TB SSD | @@ -95,26 +97,29 @@ bin/thor --network main --skip-logs _As of 22nd April 2024, a full node without logs uses **~100 GB** of disk space._ - ### Metrics -Telemetry plays a critical role in monitoring and managing blockchain nodes efficiently. +Telemetry plays a critical role in monitoring and managing blockchain nodes efficiently. Below is an overview of how metrics is integrated and utilized within our node systems. Metrics is enabled in nodes by default. It's possible to disable it by setting `--enable-metrics=false`. -By default, a [prometheus](https://prometheus.io/docs/introduction/overview/) server is available at `localhost:2112/metrics` with the metrics. +By default, a [prometheus](https://prometheus.io/docs/introduction/overview/) server is available at +`localhost:2112/metrics` with the metrics. ```shell curl localhost:2112/metrics ``` -Instrumentation is in a beta phase at this stage. You can read more about the metric types [here](https://prometheus.io/docs/concepts/metric_types/). +Instrumentation is in a beta phase at this stage. You can read more about the metric +types [here](https://prometheus.io/docs/concepts/metric_types/). ### Admin -Admin is used to allow privileged actions to the node by the administrator. Currently it supports changing the logger's verbosity at runtime. +Admin is used to allow privileged actions to the node by the administrator. Currently it supports changing the logger's +verbosity at runtime. -Admin is not enabled in nodes by default. It's possible to enable it by setting `--enable-admin`. Once enabled, an Admin server is available at `localhost:2113/admin` with the following capabilities: +Admin is not enabled in nodes by default. It's possible to enable it by setting `--enable-admin`. Once enabled, an +Admin server is available at `localhost:2113/admin` with the following capabilities: Retrieve the current log level via a GET request to /admin/loglevel. @@ -126,4 +131,4 @@ Change the log level via a POST request to /admin/loglevel. ```shell curl -X POST -H "Content-Type: application/json" -d '{"level": "trace"}' http://localhost:2113/admin/loglevel -``` \ No newline at end of file +``` From ca8b38b8715bc7b06209648f1609fa9a8694adc0 Mon Sep 17 00:00:00 2001 From: Darren Kelly <107671032+darrenvechain@users.noreply.github.com> Date: Tue, 29 Oct 2024 08:17:08 +0000 Subject: [PATCH 09/25] Darren/logdb remove leading zeros (#865) --- logdb/logdb.go | 17 ++++++- logdb/logdb_bench_test.go | 45 +++++++++--------- logdb/logdb_test.go | 97 ++++++++++++++++++++++++++------------- logdb/types.go | 2 +- 4 files changed, 102 insertions(+), 59 deletions(-) diff --git a/logdb/logdb.go b/logdb/logdb.go index bcd793e94..b1979813f 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -398,11 +398,23 @@ func (db *LogDB) NewWriterSyncOff() *Writer { func topicValue(topics []thor.Bytes32, i int) []byte { if i < len(topics) { - return topics[i][:] + return removeLeadingZeros(topics[i][:]) } return nil } +func removeLeadingZeros(bytes []byte) []byte { + i := 0 + // increase i until it reaches the first non-zero byte + for ; i < len(bytes) && bytes[i] == 0; i++ { + } + // ensure at least 1 byte exists + if i == len(bytes) { + return []byte{0} + } + return bytes[i:] +} + // Writer is the transactional log writer. type Writer struct { conn *sql.Conn @@ -481,7 +493,8 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { topicValue(ev.Topics, 1), topicValue(ev.Topics, 2), topicValue(ev.Topics, 3), - topicValue(ev.Topics, 4)); err != nil { + topicValue(ev.Topics, 4), + ); err != nil { return err } diff --git a/logdb/logdb_bench_test.go b/logdb/logdb_bench_test.go index e421ffce3..9e667999b 100644 --- a/logdb/logdb_bench_test.go +++ b/logdb/logdb_bench_test.go @@ -3,7 +3,7 @@ // Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying // file LICENSE or -package logdb_test +package logdb import ( "context" @@ -15,7 +15,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/vechain/thor/v2/block" - "github.com/vechain/thor/v2/logdb" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/tx" ) @@ -39,7 +38,7 @@ func init() { flag.StringVar(&dbPath, "dbPath", "", "Path to the database file") } -// TestLogDB_NewestBlockID performs a series of read/write benchmarks on the NewestBlockID functionality of the LogDB. +// TestLogDB_NewestBlockID performs a series of read/write benchmarks on the NewestBlockID functionality of LogDB. // It benchmarks the creating, writing, committing a new block, followed by fetching this new block as the NewestBlockID func BenchmarkFakeDB_NewestBlockID(t *testing.B) { db, err := createTempDB() @@ -155,7 +154,7 @@ func BenchmarkTestDB_HasBlockID(b *testing.B) { defer db.Close() // find the first 500k blocks with events - events, err := db.FilterEvents(context.Background(), &logdb.EventFilter{Options: &logdb.Options{Offset: 0, Limit: 500_000}}) + events, err := db.FilterEvents(context.Background(), &EventFilter{Options: &Options{Offset: 0, Limit: 500_000}}) require.NoError(b, err) require.GreaterOrEqual(b, len(events), 500_000, "there should be more than 500k events in the db") @@ -178,12 +177,12 @@ func BenchmarkTestDB_FilterEvents(b *testing.B) { vthoAddress := thor.MustParseAddress(VTHO_ADDRESS) topic := thor.MustParseBytes32(VTHO_TOPIC) - addressFilterCriteria := []*logdb.EventCriteria{ + addressFilterCriteria := []*EventCriteria{ { Address: &vthoAddress, }, } - topicFilterCriteria := []*logdb.EventCriteria{ + topicFilterCriteria := []*EventCriteria{ { Topics: [5]*thor.Bytes32{&topic, nil, nil, nil, nil}, }, @@ -191,14 +190,14 @@ func BenchmarkTestDB_FilterEvents(b *testing.B) { tests := []struct { name string - arg *logdb.EventFilter + arg *EventFilter }{ - {"AddressCriteriaFilter", &logdb.EventFilter{CriteriaSet: addressFilterCriteria, Options: &logdb.Options{Offset: 0, Limit: 500000}}}, - {"TopicCriteriaFilter", &logdb.EventFilter{CriteriaSet: topicFilterCriteria, Options: &logdb.Options{Offset: 0, Limit: 500000}}}, - {"EventLimit", &logdb.EventFilter{Order: logdb.ASC, Options: &logdb.Options{Offset: 0, Limit: 500000}}}, - {"EventLimitDesc", &logdb.EventFilter{Order: logdb.DESC, Options: &logdb.Options{Offset: 0, Limit: 500000}}}, - {"EventRange", &logdb.EventFilter{Range: &logdb.Range{From: 500000, To: 1_000_000}}}, - {"EventRangeDesc", &logdb.EventFilter{Range: &logdb.Range{From: 500000, To: 1_000_000}, Order: logdb.DESC}}, + {"AddressCriteriaFilter", &EventFilter{CriteriaSet: addressFilterCriteria, Options: &Options{Offset: 0, Limit: 500000}}}, + {"TopicCriteriaFilter", &EventFilter{CriteriaSet: topicFilterCriteria, Options: &Options{Offset: 0, Limit: 500000}}}, + {"EventLimit", &EventFilter{Order: ASC, Options: &Options{Offset: 0, Limit: 500000}}}, + {"EventLimitDesc", &EventFilter{Order: DESC, Options: &Options{Offset: 0, Limit: 500000}}}, + {"EventRange", &EventFilter{Range: &Range{From: 500000, To: 1_000_000}}}, + {"EventRangeDesc", &EventFilter{Range: &Range{From: 500000, To: 1_000_000}, Order: DESC}}, } for _, tt := range tests { @@ -222,7 +221,7 @@ func BenchmarkTestDB_FilterTransfers(b *testing.B) { defer db.Close() txOrigin := thor.MustParseAddress(TEST_ADDRESS) - transferCriteria := []*logdb.TransferCriteria{ + transferCriteria := []*TransferCriteria{ { TxOrigin: &txOrigin, Sender: nil, @@ -232,12 +231,12 @@ func BenchmarkTestDB_FilterTransfers(b *testing.B) { tests := []struct { name string - arg *logdb.TransferFilter + arg *TransferFilter }{ - {"TransferCriteria", &logdb.TransferFilter{CriteriaSet: transferCriteria, Options: &logdb.Options{Offset: 0, Limit: 500_000}}}, - {"TransferCriteriaDesc", &logdb.TransferFilter{Order: logdb.DESC, CriteriaSet: transferCriteria, Options: &logdb.Options{Offset: 0, Limit: 500_000}}}, - {"Ranged500K", &logdb.TransferFilter{Range: &logdb.Range{From: 500_000, To: 1_000_000}}}, - {"Ranged500KDesc", &logdb.TransferFilter{Range: &logdb.Range{From: 500_000, To: 1_000_000}, Order: logdb.DESC}}, + {"TransferCriteria", &TransferFilter{CriteriaSet: transferCriteria, Options: &Options{Offset: 0, Limit: 500_000}}}, + {"TransferCriteriaDesc", &TransferFilter{Order: DESC, CriteriaSet: transferCriteria, Options: &Options{Offset: 0, Limit: 500_000}}}, + {"Ranged500K", &TransferFilter{Range: &Range{From: 500_000, To: 1_000_000}}}, + {"Ranged500KDesc", &TransferFilter{Range: &Range{From: 500_000, To: 1_000_000}, Order: DESC}}, } for _, tt := range tests { @@ -253,7 +252,7 @@ func BenchmarkTestDB_FilterTransfers(b *testing.B) { } } -func createTempDB() (*logdb.LogDB, error) { +func createTempDB() (*LogDB, error) { dir, err := os.MkdirTemp("", "tempdir-") if err != nil { return nil, fmt.Errorf("failed to create temp directory: %w", err) @@ -268,7 +267,7 @@ func createTempDB() (*logdb.LogDB, error) { return nil, fmt.Errorf("failed to close temp file: %w", err) } - db, err := logdb.New(tmpFile.Name()) + db, err := New(tmpFile.Name()) if err != nil { return nil, fmt.Errorf("unable to load logdb: %w", err) } @@ -276,10 +275,10 @@ func createTempDB() (*logdb.LogDB, error) { return db, nil } -func loadDBFromDisk(b *testing.B) (*logdb.LogDB, error) { +func loadDBFromDisk(b *testing.B) (*LogDB, error) { if dbPath == "" { b.Fatal("Please provide a dbPath") } - return logdb.New(dbPath) + return New(dbPath) } diff --git a/logdb/logdb_test.go b/logdb/logdb_test.go index 7ffdd59b1..fc7c6af56 100644 --- a/logdb/logdb_test.go +++ b/logdb/logdb_test.go @@ -3,7 +3,7 @@ // Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying // file LICENSE or -package logdb_test +package logdb import ( "context" @@ -11,10 +11,10 @@ import ( "math/big" "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/stretchr/testify/assert" "github.com/vechain/thor/v2/block" - logdb "github.com/vechain/thor/v2/logdb" "github.com/vechain/thor/v2/thor" "github.com/vechain/thor/v2/tx" ) @@ -84,9 +84,9 @@ func newTransferOnlyReceipt() *tx.Receipt { } } -type eventLogs []*logdb.Event +type eventLogs []*Event -func (logs eventLogs) Filter(f func(ev *logdb.Event) bool) (ret eventLogs) { +func (logs eventLogs) Filter(f func(ev *Event) bool) (ret eventLogs) { for _, ev := range logs { if f(ev) { ret = append(ret, ev) @@ -102,9 +102,9 @@ func (logs eventLogs) Reverse() (ret eventLogs) { return } -type transferLogs []*logdb.Transfer +type transferLogs []*Transfer -func (logs transferLogs) Filter(f func(tr *logdb.Transfer) bool) (ret transferLogs) { +func (logs transferLogs) Filter(f func(tr *Transfer) bool) (ret transferLogs) { for _, tr := range logs { if f(tr) { ret = append(ret, tr) @@ -121,7 +121,7 @@ func (logs transferLogs) Reverse() (ret transferLogs) { } func TestEvents(t *testing.T) { - db, err := logdb.NewMem() + db, err := NewMem() if err != nil { t.Fatal(err) } @@ -144,7 +144,7 @@ func TestEvents(t *testing.T) { tx := b.Transactions()[j] receipt := receipts[j] origin, _ := tx.Origin() - allEvents = append(allEvents, &logdb.Event{ + allEvents = append(allEvents, &Event{ BlockNumber: b.Header().Number(), Index: uint32(j), BlockID: b.Header().ID(), @@ -157,7 +157,7 @@ func TestEvents(t *testing.T) { Data: receipt.Outputs[0].Events[0].Data, }) - allTransfers = append(allTransfers, &logdb.Transfer{ + allTransfers = append(allTransfers, &Transfer{ BlockNumber: b.Header().Number(), Index: uint32(j), BlockID: b.Header().ID(), @@ -184,21 +184,21 @@ func TestEvents(t *testing.T) { { tests := []struct { name string - arg *logdb.EventFilter + arg *EventFilter want eventLogs }{ - {"query all events", &logdb.EventFilter{}, allEvents}, + {"query all events", &EventFilter{}, allEvents}, {"query all events with nil option", nil, allEvents}, - {"query all events asc", &logdb.EventFilter{Order: logdb.ASC}, allEvents}, - {"query all events desc", &logdb.EventFilter{Order: logdb.DESC}, allEvents.Reverse()}, - {"query all events limit offset", &logdb.EventFilter{Options: &logdb.Options{Offset: 1, Limit: 10}}, allEvents[1:11]}, - {"query all events range", &logdb.EventFilter{Range: &logdb.Range{From: 10, To: 20}}, allEvents.Filter(func(ev *logdb.Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 })}, - {"query events with range and desc", &logdb.EventFilter{Range: &logdb.Range{From: 10, To: 20}, Order: logdb.DESC}, allEvents.Filter(func(ev *logdb.Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 }).Reverse()}, - {"query events with limit with desc", &logdb.EventFilter{Order: logdb.DESC, Options: &logdb.Options{Limit: 10}}, allEvents.Reverse()[0:10]}, - {"query all events with criteria", &logdb.EventFilter{CriteriaSet: []*logdb.EventCriteria{{Address: &allEvents[1].Address}}}, allEvents.Filter(func(ev *logdb.Event) bool { + {"query all events asc", &EventFilter{Order: ASC}, allEvents}, + {"query all events desc", &EventFilter{Order: DESC}, allEvents.Reverse()}, + {"query all events limit offset", &EventFilter{Options: &Options{Offset: 1, Limit: 10}}, allEvents[1:11]}, + {"query all events range", &EventFilter{Range: &Range{From: 10, To: 20}}, allEvents.Filter(func(ev *Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 })}, + {"query events with range and desc", &EventFilter{Range: &Range{From: 10, To: 20}, Order: DESC}, allEvents.Filter(func(ev *Event) bool { return ev.BlockNumber >= 10 && ev.BlockNumber <= 20 }).Reverse()}, + {"query events with limit with desc", &EventFilter{Order: DESC, Options: &Options{Limit: 10}}, allEvents.Reverse()[0:10]}, + {"query all events with criteria", &EventFilter{CriteriaSet: []*EventCriteria{{Address: &allEvents[1].Address}}}, allEvents.Filter(func(ev *Event) bool { return ev.Address == allEvents[1].Address })}, - {"query all events with multi-criteria", &logdb.EventFilter{CriteriaSet: []*logdb.EventCriteria{{Address: &allEvents[1].Address}, {Topics: [5]*thor.Bytes32{allEvents[2].Topics[0]}}, {Topics: [5]*thor.Bytes32{allEvents[3].Topics[0]}}}}, allEvents.Filter(func(ev *logdb.Event) bool { + {"query all events with multi-criteria", &EventFilter{CriteriaSet: []*EventCriteria{{Address: &allEvents[1].Address}, {Topics: [5]*thor.Bytes32{allEvents[2].Topics[0]}}, {Topics: [5]*thor.Bytes32{allEvents[3].Topics[0]}}}}, allEvents.Filter(func(ev *Event) bool { return ev.Address == allEvents[1].Address || *ev.Topics[0] == *allEvents[2].Topics[0] || *ev.Topics[0] == *allEvents[3].Topics[0] })}, } @@ -215,21 +215,21 @@ func TestEvents(t *testing.T) { { tests := []struct { name string - arg *logdb.TransferFilter + arg *TransferFilter want transferLogs }{ - {"query all transfers", &logdb.TransferFilter{}, allTransfers}, + {"query all transfers", &TransferFilter{}, allTransfers}, {"query all transfers with nil option", nil, allTransfers}, - {"query all transfers asc", &logdb.TransferFilter{Order: logdb.ASC}, allTransfers}, - {"query all transfers desc", &logdb.TransferFilter{Order: logdb.DESC}, allTransfers.Reverse()}, - {"query all transfers limit offset", &logdb.TransferFilter{Options: &logdb.Options{Offset: 1, Limit: 10}}, allTransfers[1:11]}, - {"query all transfers range", &logdb.TransferFilter{Range: &logdb.Range{From: 10, To: 20}}, allTransfers.Filter(func(tr *logdb.Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 })}, - {"query transfers with range and desc", &logdb.TransferFilter{Range: &logdb.Range{From: 10, To: 20}, Order: logdb.DESC}, allTransfers.Filter(func(tr *logdb.Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 }).Reverse()}, - {"query transfers with limit with desc", &logdb.TransferFilter{Order: logdb.DESC, Options: &logdb.Options{Limit: 10}}, allTransfers.Reverse()[0:10]}, - {"query all transfers with criteria", &logdb.TransferFilter{CriteriaSet: []*logdb.TransferCriteria{{Sender: &allTransfers[1].Sender}}}, allTransfers.Filter(func(tr *logdb.Transfer) bool { + {"query all transfers asc", &TransferFilter{Order: ASC}, allTransfers}, + {"query all transfers desc", &TransferFilter{Order: DESC}, allTransfers.Reverse()}, + {"query all transfers limit offset", &TransferFilter{Options: &Options{Offset: 1, Limit: 10}}, allTransfers[1:11]}, + {"query all transfers range", &TransferFilter{Range: &Range{From: 10, To: 20}}, allTransfers.Filter(func(tr *Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 })}, + {"query transfers with range and desc", &TransferFilter{Range: &Range{From: 10, To: 20}, Order: DESC}, allTransfers.Filter(func(tr *Transfer) bool { return tr.BlockNumber >= 10 && tr.BlockNumber <= 20 }).Reverse()}, + {"query transfers with limit with desc", &TransferFilter{Order: DESC, Options: &Options{Limit: 10}}, allTransfers.Reverse()[0:10]}, + {"query all transfers with criteria", &TransferFilter{CriteriaSet: []*TransferCriteria{{Sender: &allTransfers[1].Sender}}}, allTransfers.Filter(func(tr *Transfer) bool { return tr.Sender == allTransfers[1].Sender })}, - {"query all transfers with multi-criteria", &logdb.TransferFilter{CriteriaSet: []*logdb.TransferCriteria{{Sender: &allTransfers[1].Sender}, {Recipient: &allTransfers[2].Recipient}}}, allTransfers.Filter(func(tr *logdb.Transfer) bool { + {"query all transfers with multi-criteria", &TransferFilter{CriteriaSet: []*TransferCriteria{{Sender: &allTransfers[1].Sender}, {Recipient: &allTransfers[2].Recipient}}}, allTransfers.Filter(func(tr *Transfer) bool { return tr.Sender == allTransfers[1].Sender || tr.Recipient == allTransfers[2].Recipient })}, } @@ -244,10 +244,10 @@ func TestEvents(t *testing.T) { } } -// TestLogDB_NewestBlockID performs a series of read/write tests on the NewestBlockID functionality of the LogDB. +// TestLogDB_NewestBlockID performs a series of read/write tests on the NewestBlockID functionality of the // It validates the correctness of the NewestBlockID method under various scenarios. func TestLogDB_NewestBlockID(t *testing.T) { - db, err := logdb.NewMem() + db, err := NewMem() if err != nil { t.Fatal(err) } @@ -368,9 +368,9 @@ func TestLogDB_NewestBlockID(t *testing.T) { } } -// TestLogDB_HasBlockID performs a series of tests on the HasBlockID functionality of the LogDB. +// TestLogDB_HasBlockID performs a series of tests on the HasBlockID functionality of the func TestLogDB_HasBlockID(t *testing.T) { - db, err := logdb.NewMem() + db, err := NewMem() if err != nil { t.Fatal(err) } @@ -431,3 +431,34 @@ func TestLogDB_HasBlockID(t *testing.T) { } assert.True(t, has) } + +func TestRemoveLeadingZeros(t *testing.T) { + tests := []struct { + name string + input []byte + expected []byte + }{ + { + "should remove leading zeros", + common.Hex2Bytes("0000000000000000000000006d95e6dca01d109882fe1726a2fb9865fa41e7aa"), + common.Hex2Bytes("6d95e6dca01d109882fe1726a2fb9865fa41e7aa"), + }, + { + "should not remove any bytes", + common.Hex2Bytes("ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"), + common.Hex2Bytes("ddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef"), + }, + { + "should have at least 1 byte", + common.Hex2Bytes("00000000000000000"), + []byte{0}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := removeLeadingZeros(tt.input) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/logdb/types.go b/logdb/types.go index e4ebb1be4..7aa5ce990 100644 --- a/logdb/types.go +++ b/logdb/types.go @@ -71,7 +71,7 @@ func (c *EventCriteria) toWhereCondition() (cond string, args []interface{}) { for i, topic := range c.Topics { if topic != nil { cond += fmt.Sprintf(" AND topic%v = ", i) + refIDQuery - args = append(args, topic.Bytes()) + args = append(args, removeLeadingZeros(topic.Bytes())) } } return From f9173e422ee102c3306457d25098c0a83f7837fa Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Tue, 24 Sep 2024 09:57:36 +0200 Subject: [PATCH 10/25] feat: add new txIndex column to event meta response --- api/events/types.go | 4 ++++ logdb/logdb.go | 18 +++++++++++++++--- logdb/logdb_test.go | 1 + logdb/schema.go | 1 + logdb/types.go | 1 + 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/api/events/types.go b/api/events/types.go index 0dce06aa4..8cd03590e 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -21,6 +21,7 @@ type LogMeta struct { BlockNumber uint32 `json:"blockNumber"` BlockTimestamp uint64 `json:"blockTimestamp"` TxID thor.Bytes32 `json:"txID"` + TxIndex uint32 `json:"txIndex"` TxOrigin thor.Address `json:"txOrigin"` ClauseIndex uint32 `json:"clauseIndex"` } @@ -51,6 +52,7 @@ func convertEvent(event *logdb.Event) *FilteredEvent { BlockNumber: event.BlockNumber, BlockTimestamp: event.BlockTime, TxID: event.TxID, + TxIndex: event.TxIndex, TxOrigin: event.TxOrigin, ClauseIndex: event.ClauseIndex, }, @@ -74,6 +76,7 @@ func (e *FilteredEvent) String() string { blockNumber %v, blockTimestamp %v), txID %v, + txIndex %v, txOrigin %v, clauseIndex %v) )`, @@ -84,6 +87,7 @@ func (e *FilteredEvent) String() string { e.Meta.BlockNumber, e.Meta.BlockTimestamp, e.Meta.TxID, + e.Meta.TxIndex, e.Meta.TxOrigin, e.Meta.ClauseIndex, ) diff --git a/logdb/logdb.go b/logdb/logdb.go index b1979813f..67f6f5c90 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -95,7 +95,7 @@ func (db *LogDB) Path() string { } func (db *LogDB) FilterEvents(ctx context.Context, filter *EventFilter) ([]*Event, error) { - const query = `SELECT e.seq, r0.data, e.blockTime, r1.data, r2.data, e.clauseIndex, r3.data, r4.data, r5.data, r6.data, r7.data, r8.data, e.data + const query = `SELECT e.seq, e.txIndex, r0.data, e.blockTime, r1.data, r2.data, e.clauseIndex, r3.data, r4.data, r5.data, r6.data, r7.data, r8.data, e.data FROM (%v) e LEFT JOIN ref r0 ON e.blockID = r0.id LEFT JOIN ref r1 ON e.txID = r1.id @@ -244,6 +244,7 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac } var ( seq sequence + txIndex uint32 blockID []byte blockTime uint64 txID []byte @@ -255,6 +256,7 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac ) if err := rows.Scan( &seq, + &txIndex, &blockID, &blockTime, &txID, @@ -276,6 +278,7 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac BlockID: thor.BytesToBytes32(blockID), BlockTime: blockTime, TxID: thor.BytesToBytes32(txID), + TxIndex: txIndex, TxOrigin: thor.BytesToAddress(txOrigin), ClauseIndex: clauseIndex, Address: thor.BytesToAddress(address), @@ -455,6 +458,11 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { } ) + indexes := make(map[thor.Bytes32]int, len(txs)) + for i, tx := range txs { + indexes[tx.ID()] = i + } + for i, r := range receipts { if isReceiptEmpty(r) { continue @@ -478,6 +486,9 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { txID = tx.ID() txOrigin, _ = tx.Origin() } + + txIndex := indexes[txID] + if err := w.exec( "INSERT OR IGNORE INTO ref(data) VALUES(?),(?)", txID[:], txOrigin[:]); err != nil { @@ -498,8 +509,8 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { return err } - const query = "INSERT OR IGNORE INTO event(seq, blockTime, clauseIndex, data, blockID, txID, txOrigin, address, topic0, topic1, topic2, topic3, topic4) " + - "VALUES(?,?,?,?," + + const query = "INSERT OR IGNORE INTO event(seq, txIndex, blockTime, clauseIndex, data, blockID, txID, txOrigin, address, topic0, topic1, topic2, topic3, topic4) " + + "VALUES(?,?,?,?,?," + refIDQuery + "," + refIDQuery + "," + refIDQuery + "," + @@ -518,6 +529,7 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { if err := w.exec( query, newSequence(blockNum, eventCount), + txIndex, blockTimestamp, clauseIndex, eventData, diff --git a/logdb/logdb_test.go b/logdb/logdb_test.go index fc7c6af56..aa1cb8df4 100644 --- a/logdb/logdb_test.go +++ b/logdb/logdb_test.go @@ -147,6 +147,7 @@ func TestEvents(t *testing.T) { allEvents = append(allEvents, &Event{ BlockNumber: b.Header().Number(), Index: uint32(j), + TxIndex: uint32(j), BlockID: b.Header().ID(), BlockTime: b.Header().Timestamp(), TxID: tx.ID(), diff --git a/logdb/schema.go b/logdb/schema.go index dccb33d35..1c60513e8 100644 --- a/logdb/schema.go +++ b/logdb/schema.go @@ -14,6 +14,7 @@ const ( // creates events table eventTableSchema = `CREATE TABLE IF NOT EXISTS event ( seq INTEGER PRIMARY KEY NOT NULL, + txIndex INTEGER NOT NULL, blockID INTEGER NOT NULL, blockTime INTEGER NOT NULL, txID INTEGER NOT NULL, diff --git a/logdb/types.go b/logdb/types.go index 7aa5ce990..697385d03 100644 --- a/logdb/types.go +++ b/logdb/types.go @@ -19,6 +19,7 @@ type Event struct { BlockID thor.Bytes32 BlockTime uint64 TxID thor.Bytes32 + TxIndex uint32 TxOrigin thor.Address //contract caller ClauseIndex uint32 Address thor.Address // always a contract address From f3bd272785a13f92c6e57967683badb29091b728 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Wed, 25 Sep 2024 09:57:16 +0200 Subject: [PATCH 11/25] test: add convert event test --- api/events/types.go | 4 +++ api/events/types_test.go | 71 ++++++++++++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 14 deletions(-) diff --git a/api/events/types.go b/api/events/types.go index 8cd03590e..841c7c78b 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -22,6 +22,7 @@ type LogMeta struct { BlockTimestamp uint64 `json:"blockTimestamp"` TxID thor.Bytes32 `json:"txID"` TxIndex uint32 `json:"txIndex"` + LogIndex uint32 `json:"logIndex"` TxOrigin thor.Address `json:"txOrigin"` ClauseIndex uint32 `json:"clauseIndex"` } @@ -53,6 +54,7 @@ func convertEvent(event *logdb.Event) *FilteredEvent { BlockTimestamp: event.BlockTime, TxID: event.TxID, TxIndex: event.TxIndex, + LogIndex: event.Index, TxOrigin: event.TxOrigin, ClauseIndex: event.ClauseIndex, }, @@ -77,6 +79,7 @@ func (e *FilteredEvent) String() string { blockTimestamp %v), txID %v, txIndex %v, + logIndex %v, txOrigin %v, clauseIndex %v) )`, @@ -88,6 +91,7 @@ func (e *FilteredEvent) String() string { e.Meta.BlockTimestamp, e.Meta.TxID, e.Meta.TxIndex, + e.Meta.LogIndex, e.Meta.TxOrigin, e.Meta.ClauseIndex, ) diff --git a/api/events/types_test.go b/api/events/types_test.go index a02f441c5..ec418b7b7 100644 --- a/api/events/types_test.go +++ b/api/events/types_test.go @@ -3,19 +3,20 @@ // Distributed under the GNU Lesser General Public License v3.0 software license, see the accompanying // file LICENSE or -package events_test +package events import ( "math" "testing" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/stretchr/testify/assert" - "github.com/vechain/thor/v2/api/events" "github.com/vechain/thor/v2/chain" "github.com/vechain/thor/v2/genesis" "github.com/vechain/thor/v2/logdb" "github.com/vechain/thor/v2/muxdb" "github.com/vechain/thor/v2/state" + "github.com/vechain/thor/v2/thor" ) func TestEventsTypes(t *testing.T) { @@ -33,13 +34,13 @@ func TestEventsTypes(t *testing.T) { } func testConvertRangeWithBlockRangeType(t *testing.T, chain *chain.Chain) { - rng := &events.Range{ - Unit: events.BlockRangeType, + rng := &Range{ + Unit: BlockRangeType, From: 1, To: 2, } - convertedRng, err := events.ConvertRange(chain, rng) + convertedRng, err := ConvertRange(chain, rng) assert.NoError(t, err) assert.Equal(t, uint32(rng.From), convertedRng.From) @@ -47,8 +48,8 @@ func testConvertRangeWithBlockRangeType(t *testing.T, chain *chain.Chain) { } func testConvertRangeWithTimeRangeTypeLessThenGenesis(t *testing.T, chain *chain.Chain) { - rng := &events.Range{ - Unit: events.TimeRangeType, + rng := &Range{ + Unit: TimeRangeType, From: 1, To: 2, } @@ -57,7 +58,7 @@ func testConvertRangeWithTimeRangeTypeLessThenGenesis(t *testing.T, chain *chain To: math.MaxUint32, } - convRng, err := events.ConvertRange(chain, rng) + convRng, err := ConvertRange(chain, rng) assert.NoError(t, err) assert.Equal(t, expectedEmptyRange, convRng) @@ -68,8 +69,8 @@ func testConvertRangeWithTimeRangeType(t *testing.T, chain *chain.Chain) { if err != nil { t.Fatal(err) } - rng := &events.Range{ - Unit: events.TimeRangeType, + rng := &Range{ + Unit: TimeRangeType, From: 1, To: genesis.Timestamp(), } @@ -78,7 +79,7 @@ func testConvertRangeWithTimeRangeType(t *testing.T, chain *chain.Chain) { To: 0, } - convRng, err := events.ConvertRange(chain, rng) + convRng, err := ConvertRange(chain, rng) assert.NoError(t, err) assert.Equal(t, expectedZeroRange, convRng) @@ -89,8 +90,8 @@ func testConvertRangeWithFromGreaterThanGenesis(t *testing.T, chain *chain.Chain if err != nil { t.Fatal(err) } - rng := &events.Range{ - Unit: events.TimeRangeType, + rng := &Range{ + Unit: TimeRangeType, From: genesis.Timestamp() + 1_000, To: genesis.Timestamp() + 10_000, } @@ -99,7 +100,7 @@ func testConvertRangeWithFromGreaterThanGenesis(t *testing.T, chain *chain.Chain To: math.MaxUint32, } - convRng, err := events.ConvertRange(chain, rng) + convRng, err := ConvertRange(chain, rng) assert.NoError(t, err) assert.Equal(t, expectedEmptyRange, convRng) @@ -123,3 +124,45 @@ func initChain(t *testing.T) *chain.Chain { return repo.NewBestChain() } + +func TestConvertEvent(t *testing.T) { + event := &logdb.Event{ + Address: thor.Address{0x01}, + Data: []byte{0x02, 0x03}, + BlockID: thor.Bytes32{0x04}, + BlockNumber: 5, + BlockTime: 6, + TxID: thor.Bytes32{0x07}, + TxIndex: 8, + Index: 9, + TxOrigin: thor.Address{0x0A}, + ClauseIndex: 10, + Topics: [5]*thor.Bytes32{ + {0x0B}, + {0x0C}, + nil, + nil, + nil, + }, + } + + expectedTopics := []*thor.Bytes32{ + {0x0B}, + {0x0C}, + } + expectedData := hexutil.Encode(event.Data) + + result := convertEvent(event) + + assert.Equal(t, event.Address, result.Address) + assert.Equal(t, expectedData, result.Data) + assert.Equal(t, event.BlockID, result.Meta.BlockID) + assert.Equal(t, event.BlockNumber, result.Meta.BlockNumber) + assert.Equal(t, event.BlockTime, result.Meta.BlockTimestamp) + assert.Equal(t, event.TxID, result.Meta.TxID) + assert.Equal(t, event.TxIndex, result.Meta.TxIndex) + assert.Equal(t, event.Index, result.Meta.LogIndex) + assert.Equal(t, event.TxOrigin, result.Meta.TxOrigin) + assert.Equal(t, event.ClauseIndex, result.Meta.ClauseIndex) + assert.Equal(t, expectedTopics, result.Topics) +} From c5b3c632a68c1ca5d76be1fda4aaa4cc29de7ab3 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 26 Sep 2024 09:33:43 +0200 Subject: [PATCH 12/25] feat: make txLog and txIndex as optional return params --- api/events/events.go | 2 +- api/events/events_test.go | 73 +++++++++++++++++++++++++++++++++ api/events/types.go | 85 +++++++++++++++++++++++++++++---------- api/events/types_test.go | 41 +++++++++++++++++-- 4 files changed, 175 insertions(+), 26 deletions(-) diff --git a/api/events/events.go b/api/events/events.go index 40dff7b09..62bdec355 100644 --- a/api/events/events.go +++ b/api/events/events.go @@ -44,7 +44,7 @@ func (e *Events) filter(ctx context.Context, ef *EventFilter) ([]*FilteredEvent, } fes := make([]*FilteredEvent, len(events)) for i, e := range events { - fes[i] = convertEvent(e) + fes[i] = convertEvent(e, ef.OptionalData) } return fes, nil } diff --git a/api/events/events_test.go b/api/events/events_test.go index b1268d378..513745d58 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -56,6 +56,79 @@ func TestEvents(t *testing.T) { testEventWithBlocks(t, blocksToInsert) } +func TestOptionalData(t *testing.T) { + db := createDb(t) + initEventServer(t, db, defaultLogLimit) + defer ts.Close() + insertBlocks(t, db, 5) + + testCases := []struct { + name string + optData *events.EventOptionalData + expected *events.LogOptionalData + }{ + { + name: "empty optional data", + optData: &events.EventOptionalData{}, + expected: nil, + }, + { + name: "optional data with txIndex", + optData: &events.EventOptionalData{ + TxIndex: true, + }, + expected: &events.LogOptionalData{ + TxIndex: new(uint32), + }, + }, + { + name: "optional data with logIndex", + optData: &events.EventOptionalData{ + LogIndex: true, + }, + expected: &events.LogOptionalData{ + LogIndex: new(uint32), + }, + }, + { + name: "optional data with txIndex and logIndex", + optData: &events.EventOptionalData{ + TxIndex: true, + LogIndex: true, + }, + expected: &events.LogOptionalData{ + TxIndex: new(uint32), + LogIndex: new(uint32), + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + filter := events.EventFilter{ + CriteriaSet: make([]*events.EventCriteria, 0), + Range: nil, + Options: &logdb.Options{Limit: 6}, + Order: logdb.DESC, + OptionalData: tc.optData, + } + + res, statusCode := httpPost(t, ts.URL+"/events", filter) + assert.Equal(t, http.StatusOK, statusCode) + var tLogs []*events.FilteredEvent + if err := json.Unmarshal(res, &tLogs); err != nil { + t.Fatal(err) + } + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, 5, len(tLogs)) + + for _, tLog := range tLogs { + assert.Equal(t, tc.expected, tLog.Meta.OptionalData) + } + }) + } +} + func TestOption(t *testing.T) { thorChain := initEventServer(t, 5) defer ts.Close() diff --git a/api/events/types.go b/api/events/types.go index 841c7c78b..a80ef81d4 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -17,14 +17,33 @@ import ( ) type LogMeta struct { - BlockID thor.Bytes32 `json:"blockID"` - BlockNumber uint32 `json:"blockNumber"` - BlockTimestamp uint64 `json:"blockTimestamp"` - TxID thor.Bytes32 `json:"txID"` - TxIndex uint32 `json:"txIndex"` - LogIndex uint32 `json:"logIndex"` - TxOrigin thor.Address `json:"txOrigin"` - ClauseIndex uint32 `json:"clauseIndex"` + BlockID thor.Bytes32 `json:"blockID"` + BlockNumber uint32 `json:"blockNumber"` + BlockTimestamp uint64 `json:"blockTimestamp"` + TxID thor.Bytes32 `json:"txID"` + TxOrigin thor.Address `json:"txOrigin"` + ClauseIndex uint32 `json:"clauseIndex"` + OptionalData *LogOptionalData `json:"optionalData,omitempty"` +} + +type LogOptionalData struct { + TxIndex *uint32 `json:"txIndex,omitempty"` + LogIndex *uint32 `json:"logIndex,omitempty"` +} + +func (opt *LogOptionalData) Empty() bool { + return opt == nil || (opt.TxIndex == nil && opt.LogIndex == nil) +} + +func (opt *LogOptionalData) String() string { + var parts []string + if opt.TxIndex != nil { + parts = append(parts, fmt.Sprintf("txIndex: %v", *opt.TxIndex)) + } + if opt.LogIndex != nil { + parts = append(parts, fmt.Sprintf("logIndex: %v", *opt.LogIndex)) + } + return fmt.Sprintf("%v", parts) } type TopicSet struct { @@ -44,8 +63,8 @@ type FilteredEvent struct { } // convert a logdb.Event into a json format Event -func convertEvent(event *logdb.Event) *FilteredEvent { - fe := FilteredEvent{ +func convertEvent(event *logdb.Event, eventOptionalData *EventOptionalData) *FilteredEvent { + fe := &FilteredEvent{ Address: event.Address, Data: hexutil.Encode(event.Data), Meta: LogMeta{ @@ -53,19 +72,37 @@ func convertEvent(event *logdb.Event) *FilteredEvent { BlockNumber: event.BlockNumber, BlockTimestamp: event.BlockTime, TxID: event.TxID, - TxIndex: event.TxIndex, - LogIndex: event.Index, TxOrigin: event.TxOrigin, ClauseIndex: event.ClauseIndex, }, } + fe = addOptionalData(fe, event, eventOptionalData) + fe.Topics = make([]*thor.Bytes32, 0) for i := 0; i < 5; i++ { if event.Topics[i] != nil { fe.Topics = append(fe.Topics, event.Topics[i]) } } - return &fe + return fe +} + +func addOptionalData(fe *FilteredEvent, event *logdb.Event, eventOptionalData *EventOptionalData) *FilteredEvent { + if eventOptionalData != nil { + opt := &LogOptionalData{} + + if eventOptionalData.LogIndex { + opt.LogIndex = &event.Index + } + if eventOptionalData.TxIndex { + opt.TxIndex = &event.TxIndex + } + + if !opt.Empty() { + fe.Meta.OptionalData = opt + } + } + return fe } func (e *FilteredEvent) String() string { @@ -78,10 +115,9 @@ func (e *FilteredEvent) String() string { blockNumber %v, blockTimestamp %v), txID %v, - txIndex %v, - logIndex %v, txOrigin %v, - clauseIndex %v) + clauseIndex %v, + optionalData (%v)) )`, e.Address, e.Topics, @@ -90,10 +126,9 @@ func (e *FilteredEvent) String() string { e.Meta.BlockNumber, e.Meta.BlockTimestamp, e.Meta.TxID, - e.Meta.TxIndex, - e.Meta.LogIndex, e.Meta.TxOrigin, e.Meta.ClauseIndex, + e.Meta.OptionalData, ) } @@ -103,10 +138,16 @@ type EventCriteria struct { } type EventFilter struct { - CriteriaSet []*EventCriteria `json:"criteriaSet"` - Range *Range `json:"range"` - Options *logdb.Options `json:"options"` - Order logdb.Order `json:"order"` + CriteriaSet []*EventCriteria `json:"criteriaSet"` + Range *Range `json:"range"` + Options *logdb.Options `json:"options"` + Order logdb.Order `json:"order"` + OptionalData *EventOptionalData `json:"optionalData,omitempty"` +} + +type EventOptionalData struct { + LogIndex bool `json:"logIndex,omitempty"` + TxIndex bool `json:"txIndex,omitempty"` } func convertEventFilter(chain *chain.Chain, filter *EventFilter) (*logdb.EventFilter, error) { diff --git a/api/events/types_test.go b/api/events/types_test.go index ec418b7b7..850e16f06 100644 --- a/api/events/types_test.go +++ b/api/events/types_test.go @@ -145,6 +145,10 @@ func TestConvertEvent(t *testing.T) { nil, }, } + eventOptData := &EventOptionalData{ + LogIndex: true, + TxIndex: true, + } expectedTopics := []*thor.Bytes32{ {0x0B}, @@ -152,7 +156,7 @@ func TestConvertEvent(t *testing.T) { } expectedData := hexutil.Encode(event.Data) - result := convertEvent(event) + result := convertEvent(event, eventOptData) assert.Equal(t, event.Address, result.Address) assert.Equal(t, expectedData, result.Data) @@ -160,9 +164,40 @@ func TestConvertEvent(t *testing.T) { assert.Equal(t, event.BlockNumber, result.Meta.BlockNumber) assert.Equal(t, event.BlockTime, result.Meta.BlockTimestamp) assert.Equal(t, event.TxID, result.Meta.TxID) - assert.Equal(t, event.TxIndex, result.Meta.TxIndex) - assert.Equal(t, event.Index, result.Meta.LogIndex) + assert.Equal(t, event.TxIndex, *result.Meta.OptionalData.TxIndex) + assert.Equal(t, event.Index, *result.Meta.OptionalData.LogIndex) assert.Equal(t, event.TxOrigin, result.Meta.TxOrigin) assert.Equal(t, event.ClauseIndex, result.Meta.ClauseIndex) assert.Equal(t, expectedTopics, result.Topics) } + +func TestIsEmpty(t *testing.T) { + // Empty cases + var nilCase *LogOptionalData + assert.True(t, nilCase.Empty()) + + emptyCase := &LogOptionalData{} + assert.True(t, emptyCase.Empty()) + + emptyCase = &LogOptionalData{ + LogIndex: nil, + } + assert.True(t, emptyCase.Empty()) + + emptyCase = &LogOptionalData{ + TxIndex: nil, + } + assert.True(t, emptyCase.Empty()) + + // Not empty cases + val := uint32(1) + notEmptyCase := &LogOptionalData{ + LogIndex: &val, + } + assert.False(t, notEmptyCase.Empty()) + + notEmptyCase = &LogOptionalData{ + TxIndex: &val, + } + assert.False(t, notEmptyCase.Empty()) +} From 76a38d90eca98fe7a1087125acf16fa4e5853433 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 26 Sep 2024 10:27:56 +0200 Subject: [PATCH 13/25] chore: update swagger with new event optional data --- api/doc/thor.yaml | 84 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 83 insertions(+), 1 deletion(-) diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml index 732a5f1a3..9ba9735bc 100644 --- a/api/doc/thor.yaml +++ b/api/doc/thor.yaml @@ -1009,6 +1009,8 @@ components: enum: - asc - desc + optionalData: + $ref: '#/components/schemas/EventOptionalData' EventLogsResponse: type: array @@ -1020,7 +1022,7 @@ components: - $ref: '#/components/schemas/Event' - properties: meta: - $ref: '#/components/schemas/LogMeta' + $ref: '#/components/schemas/EventLogMeta' TransferLogFilterRequest: type: object @@ -1325,6 +1327,66 @@ components: description: The index of the clause in the transaction, from which the log was generated. example: 0 nullable: false + + EventLogMeta: + title: EventLogMeta + type: object + description: The event or transfer log metadata such as block number, block timestamp, etc. + properties: + blockID: + type: string + format: hex + description: The block identifier in which the log was included. + example: '0x0004f6cc88bb4626a92907718e82f255b8fa511453a78e8797eb8cea3393b215' + nullable: false + pattern: '^0x[0-9a-f]{64}$' + blockNumber: + type: integer + format: uint32 + description: The block number (height) of the block in which the log was included. + example: 325324 + nullable: false + blockTimestamp: + type: integer + format: uint64 + description: The UNIX timestamp of the block in which the log was included. + example: 1533267900 + nullable: false + txID: + type: string + format: hex + description: The transaction identifier, from which the log was generated. + example: '0x284bba50ef777889ff1a367ed0b38d5e5626714477c40de38d71cedd6f9fa477' + nullable: false + pattern: '^0x[0-9a-f]{64}$' + txOrigin: + type: string + description: The account from which the transaction was sent. + example: '0xdb4027477b2a8fe4c83c6dafe7f86678bb1b8a8d' + nullable: false + pattern: '^0x[0-9a-f]{40}$' + clauseIndex: + type: integer + format: uint32 + description: The index of the clause in the transaction, from which the log was generated. + example: 0 + nullable: false + optionalData: + $ref: '#/components/schemas/LogOptionalData' + + LogOptionalData: + title: optionalData + type: object + nullable: true + properties: + txIndex: + type: integer + nullable: true + example: 1 + logIndex: + type: integer + nullable: true + example: 1 Block: title: Block @@ -1916,6 +1978,26 @@ components: } ``` This refers to the range from block 10 to block 1000. + + EventOptionalData: + nullable: true + type: object + title: EventOptionalData + properties: + txIndex: + type: boolean + example: true + nullable: true + description: | + Specifies whether to include in the response the event transaction index. + loglIndex: + type: boolean + example: true + nullable: true + description: | + Specifies whether to include in the response the event log index. + description: | + Specifies all the optional data that can be included in the response. EventCriteria: type: object From ad6204e451fd5c010e19e10470f717eb7c65952a Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Mon, 7 Oct 2024 10:22:26 +0200 Subject: [PATCH 14/25] feat: save logIndex in sequence --- logdb/logdb.go | 32 ++++++++++++++------------------ logdb/schema.go | 1 - logdb/sequence.go | 41 ++++++++++++++++++++++++++++++++--------- logdb/sequence_test.go | 36 +++++++++++++++++++++++------------- 4 files changed, 69 insertions(+), 41 deletions(-) diff --git a/logdb/logdb.go b/logdb/logdb.go index 67f6f5c90..457a34395 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -9,7 +9,6 @@ import ( "context" "database/sql" "fmt" - "math" "math/big" sqlite3 "github.com/mattn/go-sqlite3" @@ -95,7 +94,7 @@ func (db *LogDB) Path() string { } func (db *LogDB) FilterEvents(ctx context.Context, filter *EventFilter) ([]*Event, error) { - const query = `SELECT e.seq, e.txIndex, r0.data, e.blockTime, r1.data, r2.data, e.clauseIndex, r3.data, r4.data, r5.data, r6.data, r7.data, r8.data, e.data + const query = `SELECT e.seq, r0.data, e.blockTime, r1.data, r2.data, e.clauseIndex, r3.data, r4.data, r5.data, r6.data, r7.data, r8.data, e.data FROM (%v) e LEFT JOIN ref r0 ON e.blockID = r0.id LEFT JOIN ref r1 ON e.txID = r1.id @@ -118,10 +117,10 @@ FROM (%v) e if filter.Range != nil { subQuery += " AND seq >= ?" - args = append(args, newSequence(filter.Range.From, 0)) + args = append(args, newSequence(filter.Range.From, 0, 0)) if filter.Range.To >= filter.Range.From { subQuery += " AND seq <= ?" - args = append(args, newSequence(filter.Range.To, uint32(math.MaxInt32))) + args = append(args, newSequence(filter.Range.To, txIndexMask, logIndexMask)) } } @@ -184,10 +183,10 @@ FROM (%v) t if filter.Range != nil { subQuery += " AND seq >= ?" - args = append(args, newSequence(filter.Range.From, 0)) + args = append(args, newSequence(filter.Range.From, 0, 0)) if filter.Range.To >= filter.Range.From { subQuery += " AND seq <= ?" - args = append(args, newSequence(filter.Range.To, uint32(math.MaxInt32))) + args = append(args, newSequence(filter.Range.To, txIndexMask, logIndexMask)) } } @@ -244,7 +243,6 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac } var ( seq sequence - txIndex uint32 blockID []byte blockTime uint64 txID []byte @@ -256,7 +254,6 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac ) if err := rows.Scan( &seq, - &txIndex, &blockID, &blockTime, &txID, @@ -274,11 +271,11 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac } event := &Event{ BlockNumber: seq.BlockNumber(), - Index: seq.Index(), + Index: seq.LogIndex(), BlockID: thor.BytesToBytes32(blockID), BlockTime: blockTime, TxID: thor.BytesToBytes32(txID), - TxIndex: txIndex, + TxIndex: seq.TxIndex(), TxOrigin: thor.BytesToAddress(txOrigin), ClauseIndex: clauseIndex, Address: thor.BytesToAddress(address), @@ -337,7 +334,7 @@ func (db *LogDB) queryTransfers(ctx context.Context, query string, args ...inter } trans := &Transfer{ BlockNumber: seq.BlockNumber(), - Index: seq.Index(), + Index: seq.LogIndex(), BlockID: thor.BytesToBytes32(blockID), BlockTime: blockTime, TxID: thor.BytesToBytes32(txID), @@ -379,7 +376,7 @@ func (db *LogDB) HasBlockID(id thor.Bytes32) (bool, error) { UNION SELECT * FROM (SELECT seq FROM event WHERE seq=? AND blockID=` + refIDQuery + ` LIMIT 1))` - seq := newSequence(block.Number(id), 0) + seq := newSequence(block.Number(id), 0, 0) row := db.stmtCache.MustPrepare(query).QueryRow(seq, id[:], seq, id[:]) var count int if err := row.Scan(&count); err != nil { @@ -429,7 +426,7 @@ type Writer struct { // Truncate truncates the database by deleting logs after blockNum (included). func (w *Writer) Truncate(blockNum uint32) error { - seq := newSequence(blockNum, 0) + seq := newSequence(blockNum, 0, 0) if err := w.exec("DELETE FROM event WHERE seq >= ?", seq); err != nil { return err } @@ -509,8 +506,8 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { return err } - const query = "INSERT OR IGNORE INTO event(seq, txIndex, blockTime, clauseIndex, data, blockID, txID, txOrigin, address, topic0, topic1, topic2, topic3, topic4) " + - "VALUES(?,?,?,?,?," + + const query = "INSERT OR IGNORE INTO event(seq, blockTime, clauseIndex, data, blockID, txID, txOrigin, address, topic0, topic1, topic2, topic3, topic4) " + + "VALUES(?,?,?,?," + refIDQuery + "," + refIDQuery + "," + refIDQuery + "," + @@ -528,8 +525,7 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { if err := w.exec( query, - newSequence(blockNum, eventCount), - txIndex, + newSequence(blockNum, uint32(txIndex), eventCount), blockTimestamp, clauseIndex, eventData, @@ -564,7 +560,7 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { if err := w.exec( query, - newSequence(blockNum, transferCount), + newSequence(blockNum, uint32(txIndex), transferCount), blockTimestamp, clauseIndex, tr.Amount.Bytes(), diff --git a/logdb/schema.go b/logdb/schema.go index 1c60513e8..dccb33d35 100644 --- a/logdb/schema.go +++ b/logdb/schema.go @@ -14,7 +14,6 @@ const ( // creates events table eventTableSchema = `CREATE TABLE IF NOT EXISTS event ( seq INTEGER PRIMARY KEY NOT NULL, - txIndex INTEGER NOT NULL, blockID INTEGER NOT NULL, blockTime INTEGER NOT NULL, txID INTEGER NOT NULL, diff --git a/logdb/sequence.go b/logdb/sequence.go index 52909ffe4..9b5c29f0c 100644 --- a/logdb/sequence.go +++ b/logdb/sequence.go @@ -5,21 +5,44 @@ package logdb -import "math" - type sequence int64 -func newSequence(blockNum uint32, index uint32) sequence { - if (index & math.MaxInt32) != index { - panic("index too large") +// Adjust these constants based on your bit allocation requirements +const ( + blockNumBits = 31 + txIndexBits = 12 + logIndexBits = 21 + // Max = 2^31 - 1 = 2,147,483,647 + blockNumMask = (1 << blockNumBits) - 1 + // Max = 2^12 - 1 = 4,095 + txIndexMask = (1 << txIndexBits) - 1 + // Max = 2^21 - 1 = 2,097,151 + logIndexMask = (1 << logIndexBits) - 1 +) + +func newSequence(blockNum uint32, txIndex uint32, logIndex uint32) sequence { + if blockNum > blockNumMask { + panic("block number too large") + } + if txIndex > txIndexMask { + panic("transaction index too large") } - return (sequence(blockNum) << 31) | sequence(index) + if logIndex > logIndexMask { + panic("log index too large") + } + return (sequence(blockNum) << (txIndexBits + logIndexBits)) | + (sequence(txIndex) << logIndexBits) | + sequence(logIndex) } func (s sequence) BlockNumber() uint32 { - return uint32(s >> 31) + return uint32(s>>(txIndexBits+logIndexBits)) & blockNumMask +} + +func (s sequence) TxIndex() uint32 { + return uint32((s >> logIndexBits) & txIndexMask) } -func (s sequence) Index() uint32 { - return uint32(s & math.MaxInt32) +func (s sequence) LogIndex() uint32 { + return uint32(s & logIndexMask) } diff --git a/logdb/sequence_test.go b/logdb/sequence_test.go index 9fa19fff0..b16e2d0da 100644 --- a/logdb/sequence_test.go +++ b/logdb/sequence_test.go @@ -6,33 +6,36 @@ package logdb import ( - "math" "testing" ) func TestSequence(t *testing.T) { type args struct { blockNum uint32 - index uint32 + txIndex uint32 + logIndex uint32 } tests := []struct { name string args args - want args }{ - {"regular", args{1, 2}, args{1, 2}}, - {"max bn", args{math.MaxUint32, 1}, args{math.MaxUint32, 1}}, - {"max index", args{5, math.MaxInt32}, args{5, math.MaxInt32}}, - {"both max", args{math.MaxUint32, math.MaxInt32}, args{math.MaxUint32, math.MaxInt32}}, + {"regular", args{1, 2, 3}}, + {"max bn", args{blockNumMask, 1, 2}}, + {"max tx index", args{5, txIndexMask, 4}}, + {"max log index", args{5, 4, logIndexMask}}, + {"both max", args{blockNumMask, txIndexMask, logIndexMask}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := newSequence(tt.args.blockNum, tt.args.index) - if bn := got.BlockNumber(); bn != tt.want.blockNum { - t.Errorf("seq.blockNum() = %v, want %v", bn, tt.want.blockNum) + got := newSequence(tt.args.blockNum, tt.args.txIndex, tt.args.logIndex) + if bn := got.BlockNumber(); bn != tt.args.blockNum { + t.Errorf("seq.blockNum() = %v, want %v", bn, tt.args.blockNum) } - if i := got.Index(); i != tt.want.index { - t.Errorf("seq.index() = %v, want %v", i, tt.want.index) + if ti := got.TxIndex(); ti != tt.args.txIndex { + t.Errorf("seq.txIndex() = %v, want %v", ti, tt.args.txIndex) + } + if i := got.LogIndex(); i != tt.args.logIndex { + t.Errorf("seq.index() = %v, want %v", i, tt.args.logIndex) } }) } @@ -42,5 +45,12 @@ func TestSequence(t *testing.T) { t.Errorf("newSequence should panic on 2nd arg > math.MaxInt32") } }() - newSequence(1, math.MaxInt32+1) + newSequence(1, txIndexMask+1, 5) + + defer func() { + if e := recover(); e == nil { + t.Errorf("newSequence should panic on 3rd arg > math.MaxInt32") + } + }() + newSequence(1, 5, logIndexMask+1) } From 1415b40cb62a8a04e66aef9cd935121a21e60d93 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Fri, 11 Oct 2024 09:49:17 +0200 Subject: [PATCH 15/25] feat: tweaked bits in sequence --- logdb/sequence.go | 8 ++++---- thor/params.go | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/logdb/sequence.go b/logdb/sequence.go index 9b5c29f0c..b76ad4821 100644 --- a/logdb/sequence.go +++ b/logdb/sequence.go @@ -9,12 +9,12 @@ type sequence int64 // Adjust these constants based on your bit allocation requirements const ( - blockNumBits = 31 - txIndexBits = 12 + blockNumBits = 28 + txIndexBits = 15 logIndexBits = 21 - // Max = 2^31 - 1 = 2,147,483,647 + // Max = 2^28 - 1 = 268,435,455 blockNumMask = (1 << blockNumBits) - 1 - // Max = 2^12 - 1 = 4,095 + // Max = 2^15 - 1 = 32,767 txIndexMask = (1 << txIndexBits) - 1 // Max = 2^21 - 1 = 2,097,151 logIndexMask = (1 << logIndexBits) - 1 diff --git a/thor/params.go b/thor/params.go index 5912c46c9..6750f2577 100644 --- a/thor/params.go +++ b/thor/params.go @@ -12,6 +12,11 @@ import ( "github.com/ethereum/go-ethereum/params" ) +/* + NOTE: any changes to gas limit or block interval may affect how the txIndex and blockNumber are stored in sequence.go: + - an increase in gas limit may require more bits for txIndex; + - if block frequency is increased, blockNumber will increment faster, potentially exhausting the allocated bits sooner than expected. +*/ // Constants of block chain. const ( BlockInterval uint64 = 10 // time interval between two consecutive blocks. From 94f4070428729dce7fd3e02413f7c9a038e1c3f5 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Fri, 11 Oct 2024 10:03:15 +0200 Subject: [PATCH 16/25] refactor: rename optional log meta field --- api/doc/thor.yaml | 8 ++++---- api/events/events_test.go | 10 +++++----- api/events/types.go | 26 +++++++++++++------------- api/events/types_test.go | 16 ++++++++-------- 4 files changed, 30 insertions(+), 30 deletions(-) diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml index 9ba9735bc..66ff3eba7 100644 --- a/api/doc/thor.yaml +++ b/api/doc/thor.yaml @@ -1371,11 +1371,11 @@ components: description: The index of the clause in the transaction, from which the log was generated. example: 0 nullable: false - optionalData: - $ref: '#/components/schemas/LogOptionalData' + extendedLogMeta: + $ref: '#/components/schemas/ExtendedLogMeta' - LogOptionalData: - title: optionalData + ExtendedLogMeta: + title: ExtendedLogMeta type: object nullable: true properties: diff --git a/api/events/events_test.go b/api/events/events_test.go index 513745d58..2a82a9e20 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -65,7 +65,7 @@ func TestOptionalData(t *testing.T) { testCases := []struct { name string optData *events.EventOptionalData - expected *events.LogOptionalData + expected *events.ExtendedLogMeta }{ { name: "empty optional data", @@ -77,7 +77,7 @@ func TestOptionalData(t *testing.T) { optData: &events.EventOptionalData{ TxIndex: true, }, - expected: &events.LogOptionalData{ + expected: &events.ExtendedLogMeta{ TxIndex: new(uint32), }, }, @@ -86,7 +86,7 @@ func TestOptionalData(t *testing.T) { optData: &events.EventOptionalData{ LogIndex: true, }, - expected: &events.LogOptionalData{ + expected: &events.ExtendedLogMeta{ LogIndex: new(uint32), }, }, @@ -96,7 +96,7 @@ func TestOptionalData(t *testing.T) { TxIndex: true, LogIndex: true, }, - expected: &events.LogOptionalData{ + expected: &events.ExtendedLogMeta{ TxIndex: new(uint32), LogIndex: new(uint32), }, @@ -123,7 +123,7 @@ func TestOptionalData(t *testing.T) { assert.Equal(t, 5, len(tLogs)) for _, tLog := range tLogs { - assert.Equal(t, tc.expected, tLog.Meta.OptionalData) + assert.Equal(t, tc.expected, tLog.Meta.ExtendedLogMeta) } }) } diff --git a/api/events/types.go b/api/events/types.go index a80ef81d4..f9c2f612a 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -17,25 +17,25 @@ import ( ) type LogMeta struct { - BlockID thor.Bytes32 `json:"blockID"` - BlockNumber uint32 `json:"blockNumber"` - BlockTimestamp uint64 `json:"blockTimestamp"` - TxID thor.Bytes32 `json:"txID"` - TxOrigin thor.Address `json:"txOrigin"` - ClauseIndex uint32 `json:"clauseIndex"` - OptionalData *LogOptionalData `json:"optionalData,omitempty"` + BlockID thor.Bytes32 `json:"blockID"` + BlockNumber uint32 `json:"blockNumber"` + BlockTimestamp uint64 `json:"blockTimestamp"` + TxID thor.Bytes32 `json:"txID"` + TxOrigin thor.Address `json:"txOrigin"` + ClauseIndex uint32 `json:"clauseIndex"` + ExtendedLogMeta *ExtendedLogMeta `json:"extendedLogMeta,omitempty"` } -type LogOptionalData struct { +type ExtendedLogMeta struct { TxIndex *uint32 `json:"txIndex,omitempty"` LogIndex *uint32 `json:"logIndex,omitempty"` } -func (opt *LogOptionalData) Empty() bool { +func (opt *ExtendedLogMeta) Empty() bool { return opt == nil || (opt.TxIndex == nil && opt.LogIndex == nil) } -func (opt *LogOptionalData) String() string { +func (opt *ExtendedLogMeta) String() string { var parts []string if opt.TxIndex != nil { parts = append(parts, fmt.Sprintf("txIndex: %v", *opt.TxIndex)) @@ -89,7 +89,7 @@ func convertEvent(event *logdb.Event, eventOptionalData *EventOptionalData) *Fil func addOptionalData(fe *FilteredEvent, event *logdb.Event, eventOptionalData *EventOptionalData) *FilteredEvent { if eventOptionalData != nil { - opt := &LogOptionalData{} + opt := &ExtendedLogMeta{} if eventOptionalData.LogIndex { opt.LogIndex = &event.Index @@ -99,7 +99,7 @@ func addOptionalData(fe *FilteredEvent, event *logdb.Event, eventOptionalData *E } if !opt.Empty() { - fe.Meta.OptionalData = opt + fe.Meta.ExtendedLogMeta = opt } } return fe @@ -128,7 +128,7 @@ func (e *FilteredEvent) String() string { e.Meta.TxID, e.Meta.TxOrigin, e.Meta.ClauseIndex, - e.Meta.OptionalData, + e.Meta.ExtendedLogMeta, ) } diff --git a/api/events/types_test.go b/api/events/types_test.go index 850e16f06..e6216094d 100644 --- a/api/events/types_test.go +++ b/api/events/types_test.go @@ -164,8 +164,8 @@ func TestConvertEvent(t *testing.T) { assert.Equal(t, event.BlockNumber, result.Meta.BlockNumber) assert.Equal(t, event.BlockTime, result.Meta.BlockTimestamp) assert.Equal(t, event.TxID, result.Meta.TxID) - assert.Equal(t, event.TxIndex, *result.Meta.OptionalData.TxIndex) - assert.Equal(t, event.Index, *result.Meta.OptionalData.LogIndex) + assert.Equal(t, event.TxIndex, *result.Meta.ExtendedLogMeta.TxIndex) + assert.Equal(t, event.Index, *result.Meta.ExtendedLogMeta.LogIndex) assert.Equal(t, event.TxOrigin, result.Meta.TxOrigin) assert.Equal(t, event.ClauseIndex, result.Meta.ClauseIndex) assert.Equal(t, expectedTopics, result.Topics) @@ -173,30 +173,30 @@ func TestConvertEvent(t *testing.T) { func TestIsEmpty(t *testing.T) { // Empty cases - var nilCase *LogOptionalData + var nilCase *ExtendedLogMeta assert.True(t, nilCase.Empty()) - emptyCase := &LogOptionalData{} + emptyCase := &ExtendedLogMeta{} assert.True(t, emptyCase.Empty()) - emptyCase = &LogOptionalData{ + emptyCase = &ExtendedLogMeta{ LogIndex: nil, } assert.True(t, emptyCase.Empty()) - emptyCase = &LogOptionalData{ + emptyCase = &ExtendedLogMeta{ TxIndex: nil, } assert.True(t, emptyCase.Empty()) // Not empty cases val := uint32(1) - notEmptyCase := &LogOptionalData{ + notEmptyCase := &ExtendedLogMeta{ LogIndex: &val, } assert.False(t, notEmptyCase.Empty()) - notEmptyCase = &LogOptionalData{ + notEmptyCase = &ExtendedLogMeta{ TxIndex: &val, } assert.False(t, notEmptyCase.Empty()) From 464f530dbf11190e6cfb33a9a63cb8b90f362f8b Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Tue, 5 Nov 2024 09:37:55 +0100 Subject: [PATCH 17/25] refactor: comments, yaml and txIndex counts --- api/doc/thor.yaml | 2 ++ logdb/logdb.go | 7 +------ thor/params.go | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml index 66ff3eba7..fccae4437 100644 --- a/api/doc/thor.yaml +++ b/api/doc/thor.yaml @@ -1380,10 +1380,12 @@ components: nullable: true properties: txIndex: + description: The index of the transaction in the block, from which the log was generated. type: integer nullable: true example: 1 logIndex: + descrption: The index of the log in the receipt's outputs. type: integer nullable: true example: 1 diff --git a/logdb/logdb.go b/logdb/logdb.go index 457a34395..817683286 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -455,11 +455,6 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { } ) - indexes := make(map[thor.Bytes32]int, len(txs)) - for i, tx := range txs { - indexes[tx.ID()] = i - } - for i, r := range receipts { if isReceiptEmpty(r) { continue @@ -484,7 +479,7 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { txOrigin, _ = tx.Origin() } - txIndex := indexes[txID] + txIndex := i if err := w.exec( "INSERT OR IGNORE INTO ref(data) VALUES(?),(?)", diff --git a/thor/params.go b/thor/params.go index 6750f2577..3ec8462f8 100644 --- a/thor/params.go +++ b/thor/params.go @@ -13,7 +13,7 @@ import ( ) /* - NOTE: any changes to gas limit or block interval may affect how the txIndex and blockNumber are stored in sequence.go: + NOTE: any changes to gas limit or block interval may affect how the txIndex and blockNumber are stored in logdb/sequence.go: - an increase in gas limit may require more bits for txIndex; - if block frequency is increased, blockNumber will increment faster, potentially exhausting the allocated bits sooner than expected. */ From 9114c0b9de5e603e4489dd9f79289d4c827451ad Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 7 Nov 2024 14:36:13 +0100 Subject: [PATCH 18/25] rebase to master --- api/doc/thor.yaml | 66 ++++------------------------ api/events/events.go | 2 +- api/events/events_test.go | 65 +++++++++------------------- api/events/types.go | 76 ++++++++++----------------------- api/events/types_test.go | 41 ++---------------- api/transfers/transfers.go | 2 +- api/transfers/transfers_test.go | 51 ++++++++++++++++++++++ api/transfers/types.go | 13 +++++- logdb/types.go | 6 ++- 9 files changed, 123 insertions(+), 199 deletions(-) diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml index fccae4437..075dedb8f 100644 --- a/api/doc/thor.yaml +++ b/api/doc/thor.yaml @@ -1009,8 +1009,6 @@ components: enum: - asc - desc - optionalData: - $ref: '#/components/schemas/EventOptionalData' EventLogsResponse: type: array @@ -1022,7 +1020,7 @@ components: - $ref: '#/components/schemas/Event' - properties: meta: - $ref: '#/components/schemas/EventLogMeta' + $ref: '#/components/schemas/LogMeta' TransferLogFilterRequest: type: object @@ -1327,65 +1325,13 @@ components: description: The index of the clause in the transaction, from which the log was generated. example: 0 nullable: false - - EventLogMeta: - title: EventLogMeta - type: object - description: The event or transfer log metadata such as block number, block timestamp, etc. - properties: - blockID: - type: string - format: hex - description: The block identifier in which the log was included. - example: '0x0004f6cc88bb4626a92907718e82f255b8fa511453a78e8797eb8cea3393b215' - nullable: false - pattern: '^0x[0-9a-f]{64}$' - blockNumber: - type: integer - format: uint32 - description: The block number (height) of the block in which the log was included. - example: 325324 - nullable: false - blockTimestamp: - type: integer - format: uint64 - description: The UNIX timestamp of the block in which the log was included. - example: 1533267900 - nullable: false - txID: - type: string - format: hex - description: The transaction identifier, from which the log was generated. - example: '0x284bba50ef777889ff1a367ed0b38d5e5626714477c40de38d71cedd6f9fa477' - nullable: false - pattern: '^0x[0-9a-f]{64}$' - txOrigin: - type: string - description: The account from which the transaction was sent. - example: '0xdb4027477b2a8fe4c83c6dafe7f86678bb1b8a8d' - nullable: false - pattern: '^0x[0-9a-f]{40}$' - clauseIndex: - type: integer - format: uint32 - description: The index of the clause in the transaction, from which the log was generated. - example: 0 - nullable: false - extendedLogMeta: - $ref: '#/components/schemas/ExtendedLogMeta' - - ExtendedLogMeta: - title: ExtendedLogMeta - type: object - nullable: true - properties: txIndex: description: The index of the transaction in the block, from which the log was generated. type: integer nullable: true example: 1 logIndex: - descrption: The index of the log in the receipt's outputs. + description: The index of the log in the receipt's outputs. type: integer nullable: true example: 1 @@ -1919,6 +1865,11 @@ components: The limit of records to be included in the output. Use this parameter for pagination. Default's to all results. + includeIndexes: + type: boolean + example: true + nullable: true + description: Include both transaction and log index in the response. description: | Include these parameters to receive filtered results in a paged format. @@ -1929,7 +1880,8 @@ components: { "options": { "offset": 0, - "limit": 10 + "limit": 10, + "includeIndexes": true } } ``` diff --git a/api/events/events.go b/api/events/events.go index 62bdec355..0001280df 100644 --- a/api/events/events.go +++ b/api/events/events.go @@ -44,7 +44,7 @@ func (e *Events) filter(ctx context.Context, ef *EventFilter) ([]*FilteredEvent, } fes := make([]*FilteredEvent, len(events)) for i, e := range events { - fes[i] = convertEvent(e, ef.OptionalData) + fes[i] = convertEvent(e, ef.Options.IncludeIndexes) } return fes, nil } diff --git a/api/events/events_test.go b/api/events/events_test.go index 2a82a9e20..0f924e8af 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -56,64 +56,40 @@ func TestEvents(t *testing.T) { testEventWithBlocks(t, blocksToInsert) } -func TestOptionalData(t *testing.T) { - db := createDb(t) - initEventServer(t, db, defaultLogLimit) +func TestOptionalIndexes(t *testing.T) { + thorChain := initEventServer(t, defaultLogLimit) defer ts.Close() - insertBlocks(t, db, 5) + insertBlocks(t, thorChain.LogDB(), 5) + tclient = thorclient.New(ts.URL) testCases := []struct { - name string - optData *events.EventOptionalData - expected *events.ExtendedLogMeta + name string + includeIndexes bool + expected *uint32 }{ { - name: "empty optional data", - optData: &events.EventOptionalData{}, - expected: nil, - }, - { - name: "optional data with txIndex", - optData: &events.EventOptionalData{ - TxIndex: true, - }, - expected: &events.ExtendedLogMeta{ - TxIndex: new(uint32), - }, + name: "do not include indexes", + includeIndexes: false, + expected: nil, }, { - name: "optional data with logIndex", - optData: &events.EventOptionalData{ - LogIndex: true, - }, - expected: &events.ExtendedLogMeta{ - LogIndex: new(uint32), - }, - }, - { - name: "optional data with txIndex and logIndex", - optData: &events.EventOptionalData{ - TxIndex: true, - LogIndex: true, - }, - expected: &events.ExtendedLogMeta{ - TxIndex: new(uint32), - LogIndex: new(uint32), - }, + name: "include indexes", + includeIndexes: true, + expected: new(uint32), }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { filter := events.EventFilter{ - CriteriaSet: make([]*events.EventCriteria, 0), - Range: nil, - Options: &logdb.Options{Limit: 6}, - Order: logdb.DESC, - OptionalData: tc.optData, + CriteriaSet: make([]*events.EventCriteria, 0), + Range: nil, + Options: &logdb.Options{Limit: 6, IncludeIndexes: tc.includeIndexes}, + Order: logdb.DESC, } - res, statusCode := httpPost(t, ts.URL+"/events", filter) + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/event", filter) + assert.NoError(t, err) assert.Equal(t, http.StatusOK, statusCode) var tLogs []*events.FilteredEvent if err := json.Unmarshal(res, &tLogs); err != nil { @@ -123,7 +99,8 @@ func TestOptionalData(t *testing.T) { assert.Equal(t, 5, len(tLogs)) for _, tLog := range tLogs { - assert.Equal(t, tc.expected, tLog.Meta.ExtendedLogMeta) + assert.Equal(t, tc.expected, tLog.Meta.TxIndex) + assert.Equal(t, tc.expected, tLog.Meta.LogIndex) } }) } diff --git a/api/events/types.go b/api/events/types.go index f9c2f612a..278b66f76 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -17,33 +17,14 @@ import ( ) type LogMeta struct { - BlockID thor.Bytes32 `json:"blockID"` - BlockNumber uint32 `json:"blockNumber"` - BlockTimestamp uint64 `json:"blockTimestamp"` - TxID thor.Bytes32 `json:"txID"` - TxOrigin thor.Address `json:"txOrigin"` - ClauseIndex uint32 `json:"clauseIndex"` - ExtendedLogMeta *ExtendedLogMeta `json:"extendedLogMeta,omitempty"` -} - -type ExtendedLogMeta struct { - TxIndex *uint32 `json:"txIndex,omitempty"` - LogIndex *uint32 `json:"logIndex,omitempty"` -} - -func (opt *ExtendedLogMeta) Empty() bool { - return opt == nil || (opt.TxIndex == nil && opt.LogIndex == nil) -} - -func (opt *ExtendedLogMeta) String() string { - var parts []string - if opt.TxIndex != nil { - parts = append(parts, fmt.Sprintf("txIndex: %v", *opt.TxIndex)) - } - if opt.LogIndex != nil { - parts = append(parts, fmt.Sprintf("logIndex: %v", *opt.LogIndex)) - } - return fmt.Sprintf("%v", parts) + BlockID thor.Bytes32 `json:"blockID"` + BlockNumber uint32 `json:"blockNumber"` + BlockTimestamp uint64 `json:"blockTimestamp"` + TxID thor.Bytes32 `json:"txID"` + TxOrigin thor.Address `json:"txOrigin"` + ClauseIndex uint32 `json:"clauseIndex"` + TxIndex *uint32 `json:"txIndex,omitempty"` + LogIndex *uint32 `json:"logIndex,omitempty"` } type TopicSet struct { @@ -63,7 +44,7 @@ type FilteredEvent struct { } // convert a logdb.Event into a json format Event -func convertEvent(event *logdb.Event, eventOptionalData *EventOptionalData) *FilteredEvent { +func convertEvent(event *logdb.Event, addIndexes bool) *FilteredEvent { fe := &FilteredEvent{ Address: event.Address, Data: hexutil.Encode(event.Data), @@ -76,7 +57,11 @@ func convertEvent(event *logdb.Event, eventOptionalData *EventOptionalData) *Fil ClauseIndex: event.ClauseIndex, }, } - fe = addOptionalData(fe, event, eventOptionalData) + + if addIndexes { + fe.Meta.TxIndex = &event.TxIndex + fe.Meta.LogIndex = &event.Index + } fe.Topics = make([]*thor.Bytes32, 0) for i := 0; i < 5; i++ { @@ -87,24 +72,6 @@ func convertEvent(event *logdb.Event, eventOptionalData *EventOptionalData) *Fil return fe } -func addOptionalData(fe *FilteredEvent, event *logdb.Event, eventOptionalData *EventOptionalData) *FilteredEvent { - if eventOptionalData != nil { - opt := &ExtendedLogMeta{} - - if eventOptionalData.LogIndex { - opt.LogIndex = &event.Index - } - if eventOptionalData.TxIndex { - opt.TxIndex = &event.TxIndex - } - - if !opt.Empty() { - fe.Meta.ExtendedLogMeta = opt - } - } - return fe -} - func (e *FilteredEvent) String() string { return fmt.Sprintf(` Event( @@ -117,7 +84,8 @@ func (e *FilteredEvent) String() string { txID %v, txOrigin %v, clauseIndex %v, - optionalData (%v)) + txIndex: %v, + logIndex: %v) )`, e.Address, e.Topics, @@ -128,7 +96,8 @@ func (e *FilteredEvent) String() string { e.Meta.TxID, e.Meta.TxOrigin, e.Meta.ClauseIndex, - e.Meta.ExtendedLogMeta, + e.Meta.TxIndex, + e.Meta.LogIndex, ) } @@ -138,11 +107,10 @@ type EventCriteria struct { } type EventFilter struct { - CriteriaSet []*EventCriteria `json:"criteriaSet"` - Range *Range `json:"range"` - Options *logdb.Options `json:"options"` - Order logdb.Order `json:"order"` - OptionalData *EventOptionalData `json:"optionalData,omitempty"` + CriteriaSet []*EventCriteria `json:"criteriaSet"` + Range *Range `json:"range"` + Options *logdb.Options `json:"options"` + Order logdb.Order `json:"order"` } type EventOptionalData struct { diff --git a/api/events/types_test.go b/api/events/types_test.go index e6216094d..cefc56768 100644 --- a/api/events/types_test.go +++ b/api/events/types_test.go @@ -145,10 +145,6 @@ func TestConvertEvent(t *testing.T) { nil, }, } - eventOptData := &EventOptionalData{ - LogIndex: true, - TxIndex: true, - } expectedTopics := []*thor.Bytes32{ {0x0B}, @@ -156,7 +152,7 @@ func TestConvertEvent(t *testing.T) { } expectedData := hexutil.Encode(event.Data) - result := convertEvent(event, eventOptData) + result := convertEvent(event, true) assert.Equal(t, event.Address, result.Address) assert.Equal(t, expectedData, result.Data) @@ -164,40 +160,9 @@ func TestConvertEvent(t *testing.T) { assert.Equal(t, event.BlockNumber, result.Meta.BlockNumber) assert.Equal(t, event.BlockTime, result.Meta.BlockTimestamp) assert.Equal(t, event.TxID, result.Meta.TxID) - assert.Equal(t, event.TxIndex, *result.Meta.ExtendedLogMeta.TxIndex) - assert.Equal(t, event.Index, *result.Meta.ExtendedLogMeta.LogIndex) + assert.Equal(t, event.TxIndex, *result.Meta.TxIndex) + assert.Equal(t, event.Index, *result.Meta.LogIndex) assert.Equal(t, event.TxOrigin, result.Meta.TxOrigin) assert.Equal(t, event.ClauseIndex, result.Meta.ClauseIndex) assert.Equal(t, expectedTopics, result.Topics) } - -func TestIsEmpty(t *testing.T) { - // Empty cases - var nilCase *ExtendedLogMeta - assert.True(t, nilCase.Empty()) - - emptyCase := &ExtendedLogMeta{} - assert.True(t, emptyCase.Empty()) - - emptyCase = &ExtendedLogMeta{ - LogIndex: nil, - } - assert.True(t, emptyCase.Empty()) - - emptyCase = &ExtendedLogMeta{ - TxIndex: nil, - } - assert.True(t, emptyCase.Empty()) - - // Not empty cases - val := uint32(1) - notEmptyCase := &ExtendedLogMeta{ - LogIndex: &val, - } - assert.False(t, notEmptyCase.Empty()) - - notEmptyCase = &ExtendedLogMeta{ - TxIndex: &val, - } - assert.False(t, notEmptyCase.Empty()) -} diff --git a/api/transfers/transfers.go b/api/transfers/transfers.go index cad4ee6b3..7d548b29d 100644 --- a/api/transfers/transfers.go +++ b/api/transfers/transfers.go @@ -50,7 +50,7 @@ func (t *Transfers) filter(ctx context.Context, filter *TransferFilter) ([]*Filt } tLogs := make([]*FilteredTransfer, len(transfers)) for i, trans := range transfers { - tLogs[i] = convertTransfer(trans) + tLogs[i] = convertTransfer(trans, filter.Options.IncludeIndexes) } return tLogs, nil } diff --git a/api/transfers/transfers_test.go b/api/transfers/transfers_test.go index 04a8c7b42..a41e0ca08 100644 --- a/api/transfers/transfers_test.go +++ b/api/transfers/transfers_test.go @@ -100,6 +100,57 @@ func TestOption(t *testing.T) { assert.Equal(t, "the number of filtered logs exceeds the maximum allowed value of 5, please use pagination", strings.Trim(string(res), "\n")) } +func TestOptionalData(t *testing.T) { + db := createDb(t) + initTransferServer(t, db, defaultLogLimit) + defer ts.Close() + insertBlocks(t, db, 5) + tclient = thorclient.New(ts.URL) + + testCases := []struct { + name string + includeIndexes bool + expected *uint32 + }{ + { + name: "do not include indexes", + includeIndexes: false, + expected: nil, + }, + { + name: "include indexes", + includeIndexes: true, + expected: new(uint32), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + filter := transfers.TransferFilter{ + CriteriaSet: make([]*logdb.TransferCriteria, 0), + Range: nil, + Options: &logdb.Options{Limit: 5, IncludeIndexes: tc.includeIndexes}, + Order: logdb.DESC, + } + + res, statusCode, err := tclient.RawHTTPClient().RawHTTPPost("/logs/transfers", filter) + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, statusCode) + var tLogs []*transfers.FilteredTransfer + if err := json.Unmarshal(res, &tLogs); err != nil { + t.Fatal(err) + } + assert.Equal(t, http.StatusOK, statusCode) + assert.Equal(t, 5, len(tLogs)) + + for _, tLog := range tLogs { + assert.Equal(t, tc.expected, tLog.Meta.TxIndex) + assert.Equal(t, tc.expected, tLog.Meta.LogIndex) + } + }) + } +} + // Test functions func testTransferBadRequest(t *testing.T) { badBody := []byte{0x00, 0x01, 0x02} diff --git a/api/transfers/types.go b/api/transfers/types.go index 29ad9b328..440c89c5d 100644 --- a/api/transfers/types.go +++ b/api/transfers/types.go @@ -19,6 +19,8 @@ type LogMeta struct { TxID thor.Bytes32 `json:"txID"` TxOrigin thor.Address `json:"txOrigin"` ClauseIndex uint32 `json:"clauseIndex"` + TxIndex *uint32 `json:"txIndex,omitempty"` + LogIndex *uint32 `json:"logIndex,omitempty"` } type FilteredTransfer struct { @@ -28,9 +30,9 @@ type FilteredTransfer struct { Meta LogMeta `json:"meta"` } -func convertTransfer(transfer *logdb.Transfer) *FilteredTransfer { +func convertTransfer(transfer *logdb.Transfer, addIndexes bool) *FilteredTransfer { v := math.HexOrDecimal256(*transfer.Amount) - return &FilteredTransfer{ + ft := &FilteredTransfer{ Sender: transfer.Sender, Recipient: transfer.Recipient, Amount: &v, @@ -43,6 +45,13 @@ func convertTransfer(transfer *logdb.Transfer) *FilteredTransfer { ClauseIndex: transfer.ClauseIndex, }, } + + if addIndexes { + ft.Meta.TxIndex = &transfer.TxIndex + ft.Meta.LogIndex = &transfer.Index + } + + return ft } type TransferFilter struct { diff --git a/logdb/types.go b/logdb/types.go index 697385d03..298f20680 100644 --- a/logdb/types.go +++ b/logdb/types.go @@ -34,6 +34,7 @@ type Transfer struct { BlockID thor.Bytes32 BlockTime uint64 TxID thor.Bytes32 + TxIndex uint32 TxOrigin thor.Address ClauseIndex uint32 Sender thor.Address @@ -54,8 +55,9 @@ type Range struct { } type Options struct { - Offset uint64 - Limit uint64 + Offset uint64 + Limit uint64 + IncludeIndexes bool } type EventCriteria struct { From a80a62a432e134b1cbee8d7b4f9c578aeac47aef Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Wed, 6 Nov 2024 16:31:48 +0100 Subject: [PATCH 19/25] fix: remove stale struct --- api/events/types.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/api/events/types.go b/api/events/types.go index 278b66f76..65fca444f 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -113,11 +113,6 @@ type EventFilter struct { Order logdb.Order `json:"order"` } -type EventOptionalData struct { - LogIndex bool `json:"logIndex,omitempty"` - TxIndex bool `json:"txIndex,omitempty"` -} - func convertEventFilter(chain *chain.Chain, filter *EventFilter) (*logdb.EventFilter, error) { rng, err := ConvertRange(chain, filter.Range) if err != nil { From 9217741a09ceb30563c956033916abb0df6a49a0 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 7 Nov 2024 11:10:05 +0100 Subject: [PATCH 20/25] add txIndex to returned logdb query --- logdb/logdb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/logdb/logdb.go b/logdb/logdb.go index 817683286..a94afd4a6 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -338,6 +338,7 @@ func (db *LogDB) queryTransfers(ctx context.Context, query string, args ...inter BlockID: thor.BytesToBytes32(blockID), BlockTime: blockTime, TxID: thor.BytesToBytes32(txID), + TxIndex: seq.TxIndex(), TxOrigin: thor.BytesToAddress(txOrigin), ClauseIndex: clauseIndex, Sender: thor.BytesToAddress(sender), From 297ffd529719def60f5470f8ef6fdaba5427991e Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 7 Nov 2024 13:11:29 +0100 Subject: [PATCH 21/25] reset to 0 eventCount and transferCount each receipt and write blockId only once --- logdb/logdb.go | 9 ++++++--- logdb/logdb_test.go | 5 +++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/logdb/logdb.go b/logdb/logdb.go index a94afd4a6..b986dae5c 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -444,8 +444,6 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { blockNum = b.Header().Number() blockTimestamp = b.Header().Timestamp() txs = b.Transactions() - eventCount, - transferCount uint32 isReceiptEmpty = func(r *tx.Receipt) bool { for _, o := range r.Outputs { if len(o.Events) > 0 || len(o.Transfers) > 0 { @@ -456,18 +454,23 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { } ) + writeBlockId := true + for i, r := range receipts { + eventCount, transferCount := uint32(0), uint32(0) + if isReceiptEmpty(r) { continue } - if eventCount == 0 && transferCount == 0 { + if writeBlockId { // block id is not yet inserted if err := w.exec( "INSERT OR IGNORE INTO ref(data) VALUES(?)", blockID[:]); err != nil { return err } + writeBlockId = false } var ( diff --git a/logdb/logdb_test.go b/logdb/logdb_test.go index aa1cb8df4..f40b23a4a 100644 --- a/logdb/logdb_test.go +++ b/logdb/logdb_test.go @@ -146,7 +146,7 @@ func TestEvents(t *testing.T) { origin, _ := tx.Origin() allEvents = append(allEvents, &Event{ BlockNumber: b.Header().Number(), - Index: uint32(j), + Index: uint32(0), TxIndex: uint32(j), BlockID: b.Header().ID(), BlockTime: b.Header().Timestamp(), @@ -160,7 +160,8 @@ func TestEvents(t *testing.T) { allTransfers = append(allTransfers, &Transfer{ BlockNumber: b.Header().Number(), - Index: uint32(j), + Index: uint32(0), + TxIndex: uint32(j), BlockID: b.Header().ID(), BlockTime: b.Header().Timestamp(), TxID: tx.ID(), From 9ff47cb887baa8084f8c5edca45e5359d7e0b65a Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 7 Nov 2024 14:40:01 +0100 Subject: [PATCH 22/25] fix lint --- logdb/logdb.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/logdb/logdb.go b/logdb/logdb.go index b986dae5c..e49f869c9 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -454,7 +454,7 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { } ) - writeBlockId := true + writeBlockID := true for i, r := range receipts { eventCount, transferCount := uint32(0), uint32(0) @@ -463,14 +463,14 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { continue } - if writeBlockId { + if writeBlockID { // block id is not yet inserted if err := w.exec( "INSERT OR IGNORE INTO ref(data) VALUES(?)", blockID[:]); err != nil { return err } - writeBlockId = false + writeBlockID = false } var ( From 6f4c9eff3e4c5c0fedd9ab2c18b2b13d49d81b07 Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 7 Nov 2024 14:54:46 +0100 Subject: [PATCH 23/25] rephrase logIndex description in yaml file --- api/doc/thor.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/doc/thor.yaml b/api/doc/thor.yaml index 075dedb8f..2a0d30b9e 100644 --- a/api/doc/thor.yaml +++ b/api/doc/thor.yaml @@ -1331,7 +1331,7 @@ components: nullable: true example: 1 logIndex: - description: The index of the log in the receipt's outputs. + description: The index of the log in the receipt's outputs. This is an overall index among all clauses. type: integer nullable: true example: 1 From f81bc85fd1c8a2852aaaea9fb5f6f99d0fb3e00e Mon Sep 17 00:00:00 2001 From: Paolo Galli Date: Thu, 7 Nov 2024 17:35:00 +0100 Subject: [PATCH 24/25] refactor: use filter.Option instead of eventFilter.Option --- api/events/events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/events/events.go b/api/events/events.go index 0001280df..8c1550471 100644 --- a/api/events/events.go +++ b/api/events/events.go @@ -44,7 +44,7 @@ func (e *Events) filter(ctx context.Context, ef *EventFilter) ([]*FilteredEvent, } fes := make([]*FilteredEvent, len(events)) for i, e := range events { - fes[i] = convertEvent(e, ef.Options.IncludeIndexes) + fes[i] = convertEvent(e, filter.Options.IncludeIndexes) } return fes, nil } From a2a74f828f50703532e87db02ec27b7a2c5c078e Mon Sep 17 00:00:00 2001 From: tony Date: Fri, 8 Nov 2024 18:00:55 +0800 Subject: [PATCH 25/25] move includeIndexes to api --- api/events/events.go | 9 +++--- api/events/events_test.go | 4 +-- api/events/types.go | 55 ++++++++++----------------------- api/events/types_test.go | 4 +-- api/transfers/transfers.go | 14 ++++++--- api/transfers/transfers_test.go | 4 +-- api/transfers/types.go | 4 +-- cmd/thor/sync_logdb.go | 12 +++++-- logdb/logdb.go | 12 +++---- logdb/logdb_test.go | 4 +-- logdb/types.go | 9 +++--- 11 files changed, 60 insertions(+), 71 deletions(-) diff --git a/api/events/events.go b/api/events/events.go index 8c1550471..b4c93fadc 100644 --- a/api/events/events.go +++ b/api/events/events.go @@ -44,7 +44,7 @@ func (e *Events) filter(ctx context.Context, ef *EventFilter) ([]*FilteredEvent, } fes := make([]*FilteredEvent, len(events)) for i, e := range events { - fes[i] = convertEvent(e, filter.Options.IncludeIndexes) + fes[i] = convertEvent(e, ef.Options.IncludeIndexes) } return fes, nil } @@ -60,9 +60,10 @@ func (e *Events) handleFilter(w http.ResponseWriter, req *http.Request) error { if filter.Options == nil { // if filter.Options is nil, set to the default limit +1 // to detect whether there are more logs than the default limit - filter.Options = &logdb.Options{ - Offset: 0, - Limit: e.limit + 1, + filter.Options = &Options{ + Offset: 0, + Limit: e.limit + 1, + IncludeIndexes: false, } } diff --git a/api/events/events_test.go b/api/events/events_test.go index 0f924e8af..89aafd36f 100644 --- a/api/events/events_test.go +++ b/api/events/events_test.go @@ -84,7 +84,7 @@ func TestOptionalIndexes(t *testing.T) { filter := events.EventFilter{ CriteriaSet: make([]*events.EventCriteria, 0), Range: nil, - Options: &logdb.Options{Limit: 6, IncludeIndexes: tc.includeIndexes}, + Options: &events.Options{Limit: 6, IncludeIndexes: tc.includeIndexes}, Order: logdb.DESC, } @@ -115,7 +115,7 @@ func TestOption(t *testing.T) { filter := events.EventFilter{ CriteriaSet: make([]*events.EventCriteria, 0), Range: nil, - Options: &logdb.Options{Limit: 6}, + Options: &events.Options{Limit: 6}, Order: logdb.DESC, } diff --git a/api/events/types.go b/api/events/types.go index 65fca444f..575f8d855 100644 --- a/api/events/types.go +++ b/api/events/types.go @@ -6,7 +6,6 @@ package events import ( - "fmt" "math" "github.com/ethereum/go-ethereum/common/hexutil" @@ -60,7 +59,7 @@ func convertEvent(event *logdb.Event, addIndexes bool) *FilteredEvent { if addIndexes { fe.Meta.TxIndex = &event.TxIndex - fe.Meta.LogIndex = &event.Index + fe.Meta.LogIndex = &event.LogIndex } fe.Topics = make([]*thor.Bytes32, 0) @@ -72,45 +71,22 @@ func convertEvent(event *logdb.Event, addIndexes bool) *FilteredEvent { return fe } -func (e *FilteredEvent) String() string { - return fmt.Sprintf(` - Event( - address: %v, - topics: %v, - data: %v, - meta: (blockID %v, - blockNumber %v, - blockTimestamp %v), - txID %v, - txOrigin %v, - clauseIndex %v, - txIndex: %v, - logIndex: %v) - )`, - e.Address, - e.Topics, - e.Data, - e.Meta.BlockID, - e.Meta.BlockNumber, - e.Meta.BlockTimestamp, - e.Meta.TxID, - e.Meta.TxOrigin, - e.Meta.ClauseIndex, - e.Meta.TxIndex, - e.Meta.LogIndex, - ) -} - type EventCriteria struct { Address *thor.Address `json:"address"` TopicSet } +type Options struct { + Offset uint64 + Limit uint64 + IncludeIndexes bool +} + type EventFilter struct { - CriteriaSet []*EventCriteria `json:"criteriaSet"` - Range *Range `json:"range"` - Options *logdb.Options `json:"options"` - Order logdb.Order `json:"order"` + CriteriaSet []*EventCriteria + Range *Range + Options *Options + Order logdb.Order // default asc } func convertEventFilter(chain *chain.Chain, filter *EventFilter) (*logdb.EventFilter, error) { @@ -119,9 +95,12 @@ func convertEventFilter(chain *chain.Chain, filter *EventFilter) (*logdb.EventFi return nil, err } f := &logdb.EventFilter{ - Range: rng, - Options: filter.Options, - Order: filter.Order, + Range: rng, + Options: &logdb.Options{ + Offset: filter.Options.Offset, + Limit: filter.Options.Limit, + }, + Order: filter.Order, } if len(filter.CriteriaSet) > 0 { f.CriteriaSet = make([]*logdb.EventCriteria, len(filter.CriteriaSet)) diff --git a/api/events/types_test.go b/api/events/types_test.go index cefc56768..75eafe3a7 100644 --- a/api/events/types_test.go +++ b/api/events/types_test.go @@ -134,7 +134,7 @@ func TestConvertEvent(t *testing.T) { BlockTime: 6, TxID: thor.Bytes32{0x07}, TxIndex: 8, - Index: 9, + LogIndex: 9, TxOrigin: thor.Address{0x0A}, ClauseIndex: 10, Topics: [5]*thor.Bytes32{ @@ -161,7 +161,7 @@ func TestConvertEvent(t *testing.T) { assert.Equal(t, event.BlockTime, result.Meta.BlockTimestamp) assert.Equal(t, event.TxID, result.Meta.TxID) assert.Equal(t, event.TxIndex, *result.Meta.TxIndex) - assert.Equal(t, event.Index, *result.Meta.LogIndex) + assert.Equal(t, event.LogIndex, *result.Meta.LogIndex) assert.Equal(t, event.TxOrigin, result.Meta.TxOrigin) assert.Equal(t, event.ClauseIndex, result.Meta.ClauseIndex) assert.Equal(t, expectedTopics, result.Topics) diff --git a/api/transfers/transfers.go b/api/transfers/transfers.go index 7d548b29d..2a6cbfb9e 100644 --- a/api/transfers/transfers.go +++ b/api/transfers/transfers.go @@ -42,8 +42,11 @@ func (t *Transfers) filter(ctx context.Context, filter *TransferFilter) ([]*Filt transfers, err := t.db.FilterTransfers(ctx, &logdb.TransferFilter{ CriteriaSet: filter.CriteriaSet, Range: rng, - Options: filter.Options, - Order: filter.Order, + Options: &logdb.Options{ + Offset: filter.Options.Offset, + Limit: filter.Options.Limit, + }, + Order: filter.Order, }) if err != nil { return nil, err @@ -66,9 +69,10 @@ func (t *Transfers) handleFilterTransferLogs(w http.ResponseWriter, req *http.Re if filter.Options == nil { // if filter.Options is nil, set to the default limit +1 // to detect whether there are more logs than the default limit - filter.Options = &logdb.Options{ - Offset: 0, - Limit: t.limit + 1, + filter.Options = &events.Options{ + Offset: 0, + Limit: t.limit + 1, + IncludeIndexes: false, } } diff --git a/api/transfers/transfers_test.go b/api/transfers/transfers_test.go index a41e0ca08..eb028414f 100644 --- a/api/transfers/transfers_test.go +++ b/api/transfers/transfers_test.go @@ -65,7 +65,7 @@ func TestOption(t *testing.T) { filter := transfers.TransferFilter{ CriteriaSet: make([]*logdb.TransferCriteria, 0), Range: nil, - Options: &logdb.Options{Limit: 6}, + Options: &events.Options{Limit: 6}, Order: logdb.DESC, } @@ -129,7 +129,7 @@ func TestOptionalData(t *testing.T) { filter := transfers.TransferFilter{ CriteriaSet: make([]*logdb.TransferCriteria, 0), Range: nil, - Options: &logdb.Options{Limit: 5, IncludeIndexes: tc.includeIndexes}, + Options: &events.Options{Limit: 5, IncludeIndexes: tc.includeIndexes}, Order: logdb.DESC, } diff --git a/api/transfers/types.go b/api/transfers/types.go index 440c89c5d..1574acf5a 100644 --- a/api/transfers/types.go +++ b/api/transfers/types.go @@ -48,7 +48,7 @@ func convertTransfer(transfer *logdb.Transfer, addIndexes bool) *FilteredTransfe if addIndexes { ft.Meta.TxIndex = &transfer.TxIndex - ft.Meta.LogIndex = &transfer.Index + ft.Meta.LogIndex = &transfer.LogIndex } return ft @@ -57,6 +57,6 @@ func convertTransfer(transfer *logdb.Transfer, addIndexes bool) *FilteredTransfe type TransferFilter struct { CriteriaSet []*logdb.TransferCriteria Range *events.Range - Options *logdb.Options + Options *events.Options Order logdb.Order //default asc } diff --git a/cmd/thor/sync_logdb.go b/cmd/thor/sync_logdb.go index 9fccf3127..edfb78793 100644 --- a/cmd/thor/sync_logdb.go +++ b/cmd/thor/sync_logdb.go @@ -285,6 +285,8 @@ func verifyLogDBPerBlock( n := block.Header().Number() id := block.Header().ID() ts := block.Header().Timestamp() + evCount := 0 + trCount := 0 var expectedEvLogs []*logdb.Event var expectedTrLogs []*logdb.Transfer @@ -292,6 +294,8 @@ func verifyLogDBPerBlock( for txIndex, r := range receipts { tx := txs[txIndex] origin, _ := tx.Origin() + evCount = 0 + trCount = 0 for clauseIndex, output := range r.Outputs { for _, ev := range output.Events { @@ -301,7 +305,7 @@ func verifyLogDBPerBlock( } expectedEvLogs = append(expectedEvLogs, &logdb.Event{ BlockNumber: n, - Index: uint32(len(expectedEvLogs)), + LogIndex: uint32(evCount), BlockID: id, BlockTime: ts, TxID: tx.ID(), @@ -310,12 +314,14 @@ func verifyLogDBPerBlock( Address: ev.Address, Topics: convertTopics(ev.Topics), Data: data, + TxIndex: uint32(txIndex), }) + evCount++ } for _, tr := range output.Transfers { expectedTrLogs = append(expectedTrLogs, &logdb.Transfer{ BlockNumber: n, - Index: uint32(len(expectedTrLogs)), + LogIndex: uint32(trCount), BlockID: id, BlockTime: ts, TxID: tx.ID(), @@ -324,7 +330,9 @@ func verifyLogDBPerBlock( Sender: tr.Sender, Recipient: tr.Recipient, Amount: tr.Amount, + TxIndex: uint32(txIndex), }) + trCount++ } } } diff --git a/logdb/logdb.go b/logdb/logdb.go index e49f869c9..f172ebf1d 100644 --- a/logdb/logdb.go +++ b/logdb/logdb.go @@ -271,7 +271,7 @@ func (db *LogDB) queryEvents(ctx context.Context, query string, args ...interfac } event := &Event{ BlockNumber: seq.BlockNumber(), - Index: seq.LogIndex(), + LogIndex: seq.LogIndex(), BlockID: thor.BytesToBytes32(blockID), BlockTime: blockTime, TxID: thor.BytesToBytes32(txID), @@ -334,7 +334,7 @@ func (db *LogDB) queryTransfers(ctx context.Context, query string, args ...inter } trans := &Transfer{ BlockNumber: seq.BlockNumber(), - Index: seq.LogIndex(), + LogIndex: seq.LogIndex(), BlockID: thor.BytesToBytes32(blockID), BlockTime: blockTime, TxID: thor.BytesToBytes32(txID), @@ -452,10 +452,9 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { } return true } + blockIDInserted bool ) - writeBlockID := true - for i, r := range receipts { eventCount, transferCount := uint32(0), uint32(0) @@ -463,14 +462,14 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { continue } - if writeBlockID { + if !blockIDInserted { // block id is not yet inserted if err := w.exec( "INSERT OR IGNORE INTO ref(data) VALUES(?)", blockID[:]); err != nil { return err } - writeBlockID = false + blockIDInserted = true } var ( @@ -484,7 +483,6 @@ func (w *Writer) Write(b *block.Block, receipts tx.Receipts) error { } txIndex := i - if err := w.exec( "INSERT OR IGNORE INTO ref(data) VALUES(?),(?)", txID[:], txOrigin[:]); err != nil { diff --git a/logdb/logdb_test.go b/logdb/logdb_test.go index f40b23a4a..454d3a1e8 100644 --- a/logdb/logdb_test.go +++ b/logdb/logdb_test.go @@ -146,7 +146,7 @@ func TestEvents(t *testing.T) { origin, _ := tx.Origin() allEvents = append(allEvents, &Event{ BlockNumber: b.Header().Number(), - Index: uint32(0), + LogIndex: uint32(0), TxIndex: uint32(j), BlockID: b.Header().ID(), BlockTime: b.Header().Timestamp(), @@ -160,7 +160,7 @@ func TestEvents(t *testing.T) { allTransfers = append(allTransfers, &Transfer{ BlockNumber: b.Header().Number(), - Index: uint32(0), + LogIndex: uint32(0), TxIndex: uint32(j), BlockID: b.Header().ID(), BlockTime: b.Header().Timestamp(), diff --git a/logdb/types.go b/logdb/types.go index 298f20680..8e772cc0c 100644 --- a/logdb/types.go +++ b/logdb/types.go @@ -15,7 +15,7 @@ import ( // Event represents tx.Event that can be stored in db. type Event struct { BlockNumber uint32 - Index uint32 + LogIndex uint32 BlockID thor.Bytes32 BlockTime uint64 TxID thor.Bytes32 @@ -30,7 +30,7 @@ type Event struct { // Transfer represents tx.Transfer that can be stored in db. type Transfer struct { BlockNumber uint32 - Index uint32 + LogIndex uint32 BlockID thor.Bytes32 BlockTime uint64 TxID thor.Bytes32 @@ -55,9 +55,8 @@ type Range struct { } type Options struct { - Offset uint64 - Limit uint64 - IncludeIndexes bool + Offset uint64 + Limit uint64 } type EventCriteria struct {