From 7028930bd6a3dcd0985d5c1e49ff7fa17e579cda Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Thu, 13 Apr 2023 12:00:52 +0300 Subject: [PATCH 01/29] network: do not use error channel to start network srv It's obsolete thing, we have looger and it perfectly suits our needs. Signed-off-by: Anna Shaleva --- cli/server/server.go | 2 +- internal/testcli/executor.go | 2 +- pkg/network/server.go | 2 +- pkg/network/server_test.go | 18 +++--------------- pkg/services/rpcsrv/subscription_test.go | 4 ++-- 5 files changed, 8 insertions(+), 20 deletions(-) diff --git a/cli/server/server.go b/cli/server/server.go index 2856c4f573..50ba57d469 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -489,7 +489,7 @@ func startServer(ctx *cli.Context) error { rpcServer := rpcsrv.New(chain, cfg.ApplicationConfiguration.RPC, serv, oracleSrv, log, errChan) serv.AddService(&rpcServer) - go serv.Start(errChan) + go serv.Start() if !cfg.ApplicationConfiguration.RPC.StartWhenSynchronized { rpcServer.Start() } diff --git a/internal/testcli/executor.go b/internal/testcli/executor.go index d658233341..20fa01a646 100644 --- a/internal/testcli/executor.go +++ b/internal/testcli/executor.go @@ -164,7 +164,7 @@ func NewTestChain(t *testing.T, f func(*config.Config), run bool) (*core.Blockch }) require.NoError(t, err) netSrv.AddConsensusService(cons, cons.OnPayload, cons.OnTransaction) - go netSrv.Start(make(chan error, 1)) + go netSrv.Start() errCh := make(chan error, 2) rpcServer := rpcsrv.New(chain, cfg.ApplicationConfiguration.RPC, netSrv, nil, logger, errCh) rpcServer.Start() diff --git a/pkg/network/server.go b/pkg/network/server.go index ca37d774fd..b518bfc09e 100644 --- a/pkg/network/server.go +++ b/pkg/network/server.go @@ -265,7 +265,7 @@ func (s *Server) ID() uint32 { // Start will start the server and its underlying transport. Calling it twice // is an error. -func (s *Server) Start(errChan chan error) { +func (s *Server) Start() { s.log.Info("node started", zap.Uint32("blockHeight", s.chain.BlockHeight()), zap.Uint32("headerHeight", s.chain.HeaderHeight())) diff --git a/pkg/network/server_test.go b/pkg/network/server_test.go index edd1968b13..4b95eee4ac 100644 --- a/pkg/network/server_test.go +++ b/pkg/network/server_test.go @@ -87,20 +87,11 @@ func TestNewServer(t *testing.T) { }) } -func startWithChannel(s *Server) chan error { - ch := make(chan error) - go func() { - s.Start(ch) - close(ch) - }() - return ch -} - func TestServerStartAndShutdown(t *testing.T) { t.Run("no consensus", func(t *testing.T) { s := newTestServer(t, ServerConfig{}) - ch := startWithChannel(s) + go s.Start() p := newLocalPeer(t, s) s.register <- p require.Eventually(t, func() bool { return 1 == s.PeerCount() }, time.Second, time.Millisecond*10) @@ -109,7 +100,6 @@ func TestServerStartAndShutdown(t *testing.T) { assert.Nil(t, s.txCallback) s.Shutdown() - <-ch require.True(t, s.transports[0].(*fakeTransp).closed.Load()) err, ok := p.droppedWith.Load().(error) @@ -121,14 +111,13 @@ func TestServerStartAndShutdown(t *testing.T) { cons := new(fakeConsensus) s.AddConsensusService(cons, cons.OnPayload, cons.OnTransaction) - ch := startWithChannel(s) + go s.Start() p := newLocalPeer(t, s) s.register <- p assert.True(t, s.services["fake"].(*fakeConsensus).started.Load()) s.Shutdown() - <-ch require.True(t, s.services["fake"].(*fakeConsensus).stopped.Load()) }) @@ -401,10 +390,9 @@ func startTestServer(t *testing.T, protocolCfg ...func(*config.Blockchain)) *Ser } func startWithCleanup(t *testing.T, s *Server) { - ch := startWithChannel(s) + go s.Start() t.Cleanup(func() { s.Shutdown() - <-ch }) } diff --git a/pkg/services/rpcsrv/subscription_test.go b/pkg/services/rpcsrv/subscription_test.go index 752ed03fa0..64cedcd2d1 100644 --- a/pkg/services/rpcsrv/subscription_test.go +++ b/pkg/services/rpcsrv/subscription_test.go @@ -99,7 +99,7 @@ func TestSubscriptions(t *testing.T) { defer chain.Close() defer rpcSrv.Shutdown() - go rpcSrv.coreServer.Start(make(chan error)) + go rpcSrv.coreServer.Start() defer rpcSrv.coreServer.Shutdown() for _, feed := range subFeeds { @@ -374,7 +374,7 @@ func TestFilteredNotaryRequestSubscriptions(t *testing.T) { } chain, rpcSrv, c, respMsgs, finishedFlag := initCleanServerAndWSClient(t) - go rpcSrv.coreServer.Start(make(chan error, 1)) + go rpcSrv.coreServer.Start() defer chain.Close() defer rpcSrv.Shutdown() From 56b93d279e67c9022bae100ad86ead01e1cbc2c0 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Thu, 13 Apr 2023 12:43:21 +0300 Subject: [PATCH 02/29] config: use uint32 for validators/committee members count Signed-off-by: Anna Shaleva --- docs/node-configuration.md | 4 +-- pkg/config/protocol_config.go | 14 +++++------ pkg/config/protocol_config_test.go | 40 +++++++++++++++--------------- pkg/core/blockchain_core_test.go | 4 +-- pkg/neorpc/result/version.go | 12 ++++----- pkg/rpcclient/actor/maker.go | 4 +-- pkg/rpcclient/actor/maker_test.go | 2 +- 7 files changed, 40 insertions(+), 40 deletions(-) diff --git a/docs/node-configuration.md b/docs/node-configuration.md index b4b0242056..bbd7d4455c 100644 --- a/docs/node-configuration.md +++ b/docs/node-configuration.md @@ -339,7 +339,7 @@ protocol-related settings described in the table below. | Section | Type | Default value | Description | Notes | | --- | --- | --- | --- | --- | -| CommitteeHistory | map[uint32]int | none | Number of committee members after the given height, for example `{0: 1, 20: 4}` sets up a chain with one committee member since the genesis and then changes the setting to 4 committee members at the height of 20. `StandbyCommittee` committee setting must have the number of keys equal or exceeding the highest value in this option. Blocks numbers where the change happens must be divisible by the old and by the new values simultaneously. If not set, committee size is derived from the `StandbyCommittee` setting and never changes. | +| CommitteeHistory | map[uint32]uint32 | none | Number of committee members after the given height, for example `{0: 1, 20: 4}` sets up a chain with one committee member since the genesis and then changes the setting to 4 committee members at the height of 20. `StandbyCommittee` committee setting must have the number of keys equal or exceeding the highest value in this option. Blocks numbers where the change happens must be divisible by the old and by the new values simultaneously. If not set, committee size is derived from the `StandbyCommittee` setting and never changes. | | GarbageCollectionPeriod | `uint32` | 10000 | Controls MPT garbage collection interval (in blocks) for configurations with `RemoveUntraceableBlocks` enabled and `KeepOnlyLatestState` disabled. In this mode the node stores a number of MPT trees (corresponding to `MaxTraceableBlocks` and `StateSyncInterval`), but the DB needs to be clean from old entries from time to time. Doing it too often will cause too much processing overhead, doing it too rarely will leave more useless data in the DB. This setting is deprecated in favor of the same setting in the ApplicationConfiguration and will be removed in future node versions. If both settings are used, ApplicationConfiguration is prioritized over this one. | | Hardforks | `map[string]uint32` | [] | The set of incompatible changes that affect node behaviour starting from the specified height. The default value is an empty set which should be interpreted as "each known hard-fork is applied from the zero blockchain height". The list of valid hard-fork names:
• `Aspidochelone` represents hard-fork introduced in [#2469](https://github.com/nspcc-dev/neo-go/pull/2469) (ported from the [reference](https://github.com/neo-project/neo/pull/2712)). It adjusts the prices of `System.Contract.CreateStandardAccount` and `System.Contract.CreateMultisigAccount` interops so that the resulting prices are in accordance with `sha256` method of native `CryptoLib` contract. It also includes [#2519](https://github.com/nspcc-dev/neo-go/pull/2519) (ported from the [reference](https://github.com/neo-project/neo/pull/2749)) that adjusts the price of `System.Runtime.GetRandom` interop and fixes its vulnerability. A special NeoGo-specific change is included as well for ContractManagement's update/deploy call flags behaviour to be compatible with pre-0.99.0 behaviour that was changed because of the [3.2.0 protocol change](https://github.com/neo-project/neo/pull/2653). | | KeepOnlyLatestState | `bool` | `false` | Specifies if MPT should only store the latest state (or a set of latest states, see `P2PStateExcangeExtensions` section for details). If true, DB size will be smaller, but older roots won't be accessible. This value should remain the same for the same database. | This setting is deprecated in favor of the same setting in the ApplicationConfiguration and will be removed in future node versions. If both settings are used, setting any of them to true enables the function. | @@ -364,6 +364,6 @@ protocol-related settings described in the table below. | StateSyncInterval | `int` | `40000` | The number of blocks between state heights available for MPT state data synchronization. | `P2PStateExchangeExtensions` should be enabled to use this setting. | | TimePerBlock | `Duration` | `15s` | Minimal (and targeted for) time interval between blocks. Must be an integer number of milliseconds. | | ValidatorsCount | `int` | `0` | Number of validators set for the whole network lifetime, can't be set if `ValidatorsHistory` setting is used. | -| ValidatorsHistory | map[uint32]int | none | Number of consensus nodes to use after given height (see `CommitteeHistory` also). Heights where the change occurs must be divisible by the number of committee members at that height. Can't be used with `ValidatorsCount` not equal to zero. | +| ValidatorsHistory | map[uint32]uint32 | none | Number of consensus nodes to use after given height (see `CommitteeHistory` also). Heights where the change occurs must be divisible by the number of committee members at that height. Can't be used with `ValidatorsCount` not equal to zero. | | VerifyBlocks | `bool` | `false` | This setting is deprecated and no longer works, please use `SkipBlockVerification` in the `ApplicationConfiguration`, it will be removed in future node versions. | | VerifyTransactions | `bool` | `false` | Denotes whether to verify transactions in the received blocks. | diff --git a/pkg/config/protocol_config.go b/pkg/config/protocol_config.go index 653f1a6e26..943ad67918 100644 --- a/pkg/config/protocol_config.go +++ b/pkg/config/protocol_config.go @@ -15,7 +15,7 @@ import ( type ( ProtocolConfiguration struct { // CommitteeHistory stores committee size change history (height: size). - CommitteeHistory map[uint32]int `yaml:"CommitteeHistory"` + CommitteeHistory map[uint32]uint32 `yaml:"CommitteeHistory"` // GarbageCollectionPeriod sets the number of blocks to wait before // starting the next MPT garbage collection cycle when RemoveUntraceableBlocks // option is used. @@ -84,7 +84,7 @@ type ( TimePerBlock time.Duration `yaml:"TimePerBlock"` ValidatorsCount int `yaml:"ValidatorsCount"` // Validators stores history of changes to consensus node number (height: number). - ValidatorsHistory map[uint32]int `yaml:"ValidatorsHistory"` + ValidatorsHistory map[uint32]uint32 `yaml:"ValidatorsHistory"` // Whether to verify received blocks. // // Deprecated: please use the same setting in the ApplicationConfiguration, this field will be removed in future versions. @@ -97,7 +97,7 @@ type ( // heightNumber is an auxiliary structure for configuration checks. type heightNumber struct { h uint32 - n int + n uint32 } // Validate checks ProtocolConfiguration for internal consistency and returns @@ -187,11 +187,11 @@ func (p *ProtocolConfiguration) GetCommitteeSize(height uint32) int { if len(p.CommitteeHistory) == 0 { return len(p.StandbyCommittee) } - return getBestFromMap(p.CommitteeHistory, height) + return int(getBestFromMap(p.CommitteeHistory, height)) } -func getBestFromMap(dict map[uint32]int, height uint32) int { - var res int +func getBestFromMap(dict map[uint32]uint32, height uint32) uint32 { + var res uint32 var bestH = uint32(0) for h, n := range dict { if h >= bestH && h <= height { @@ -208,7 +208,7 @@ func (p *ProtocolConfiguration) GetNumOfCNs(height uint32) int { if len(p.ValidatorsHistory) == 0 { return p.ValidatorsCount } - return getBestFromMap(p.ValidatorsHistory, height) + return int(getBestFromMap(p.ValidatorsHistory, height)) } // ShouldUpdateCommitteeAt answers the question of whether the committee diff --git a/pkg/config/protocol_config_test.go b/pkg/config/protocol_config_test.go index aa6e6d68b5..05731813ba 100644 --- a/pkg/config/protocol_config_test.go +++ b/pkg/config/protocol_config_test.go @@ -52,7 +52,7 @@ func TestProtocolConfigurationValidation(t *testing.T) { "02a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd62", }, ValidatorsCount: 4, - ValidatorsHistory: map[uint32]int{0: 4}, + ValidatorsHistory: map[uint32]uint32{0: 4}, } require.Error(t, p.Validate()) p = &ProtocolConfiguration{ @@ -62,8 +62,8 @@ func TestProtocolConfigurationValidation(t *testing.T) { "03d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699", "02a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd62", }, - CommitteeHistory: map[uint32]int{0: 4}, - ValidatorsHistory: map[uint32]int{0: 4, 1000: 5}, + CommitteeHistory: map[uint32]uint32{0: 4}, + ValidatorsHistory: map[uint32]uint32{0: 4, 1000: 5}, } require.Error(t, p.Validate()) p = &ProtocolConfiguration{ @@ -73,8 +73,8 @@ func TestProtocolConfigurationValidation(t *testing.T) { "03d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699", "02a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd62", }, - CommitteeHistory: map[uint32]int{0: 4, 1000: 5}, - ValidatorsHistory: map[uint32]int{0: 4}, + CommitteeHistory: map[uint32]uint32{0: 4, 1000: 5}, + ValidatorsHistory: map[uint32]uint32{0: 4}, } require.Error(t, p.Validate()) p = &ProtocolConfiguration{ @@ -84,8 +84,8 @@ func TestProtocolConfigurationValidation(t *testing.T) { "03d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699", "02a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd62", }, - CommitteeHistory: map[uint32]int{0: 1, 999: 4}, - ValidatorsHistory: map[uint32]int{0: 1}, + CommitteeHistory: map[uint32]uint32{0: 1, 999: 4}, + ValidatorsHistory: map[uint32]uint32{0: 1}, } require.Error(t, p.Validate()) p = &ProtocolConfiguration{ @@ -95,8 +95,8 @@ func TestProtocolConfigurationValidation(t *testing.T) { "03d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699", "02a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd62", }, - CommitteeHistory: map[uint32]int{0: 1, 1000: 4}, - ValidatorsHistory: map[uint32]int{0: 1, 999: 4}, + CommitteeHistory: map[uint32]uint32{0: 1, 1000: 4}, + ValidatorsHistory: map[uint32]uint32{0: 1, 999: 4}, } require.Error(t, p.Validate()) p = &ProtocolConfiguration{ @@ -106,8 +106,8 @@ func TestProtocolConfigurationValidation(t *testing.T) { "03d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699", "02a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd62", }, - CommitteeHistory: map[uint32]int{0: 1, 100: 4}, - ValidatorsHistory: map[uint32]int{0: 4, 100: 4}, + CommitteeHistory: map[uint32]uint32{0: 1, 100: 4}, + ValidatorsHistory: map[uint32]uint32{0: 4, 100: 4}, } require.Error(t, p.Validate()) p = &ProtocolConfiguration{ @@ -123,8 +123,8 @@ func TestProtocolConfigurationValidation(t *testing.T) { "03d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699", "02a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd62", }, - CommitteeHistory: map[uint32]int{0: 1, 100: 4}, - ValidatorsHistory: map[uint32]int{0: 1, 100: 4}, + CommitteeHistory: map[uint32]uint32{0: 1, 100: 4}, + ValidatorsHistory: map[uint32]uint32{0: 1, 100: 4}, } require.NoError(t, p.Validate()) } @@ -137,8 +137,8 @@ func TestGetCommitteeAndCNs(t *testing.T) { "03d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699", "02a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd62", }, - CommitteeHistory: map[uint32]int{0: 1, 100: 4}, - ValidatorsHistory: map[uint32]int{0: 1, 200: 4}, + CommitteeHistory: map[uint32]uint32{0: 1, 100: 4}, + ValidatorsHistory: map[uint32]uint32{0: 1, 200: 4}, } require.Equal(t, 1, p.GetCommitteeSize(0)) require.Equal(t, 1, p.GetCommitteeSize(99)) @@ -173,8 +173,8 @@ func TestProtocolConfigurationEquals(t *testing.T) { o = &cfg2.ProtocolConfiguration require.True(t, p.Equals(o)) - o.CommitteeHistory = map[uint32]int{111: 7} - p.CommitteeHistory = map[uint32]int{111: 7} + o.CommitteeHistory = map[uint32]uint32{111: 7} + p.CommitteeHistory = map[uint32]uint32{111: 7} require.True(t, p.Equals(o)) p.CommitteeHistory[111] = 8 require.False(t, p.Equals(o)) @@ -220,9 +220,9 @@ func TestProtocolConfigurationEquals(t *testing.T) { p.StandbyCommittee = nil o.StandbyCommittee = nil - o.ValidatorsHistory = map[uint32]int{111: 0} - p.ValidatorsHistory = map[uint32]int{111: 0} + o.ValidatorsHistory = map[uint32]uint32{111: 0} + p.ValidatorsHistory = map[uint32]uint32{111: 0} require.True(t, p.Equals(o)) - p.ValidatorsHistory = map[uint32]int{112: 0} + p.ValidatorsHistory = map[uint32]uint32{112: 0} require.False(t, p.Equals(o)) } diff --git a/pkg/core/blockchain_core_test.go b/pkg/core/blockchain_core_test.go index eadb0b1835..5b1944c523 100644 --- a/pkg/core/blockchain_core_test.go +++ b/pkg/core/blockchain_core_test.go @@ -247,12 +247,12 @@ func TestBlockchain_InitWithIncompleteStateJump(t *testing.T) { func TestChainWithVolatileNumOfValidators(t *testing.T) { bc := newTestChainWithCustomCfg(t, func(c *config.Config) { c.ProtocolConfiguration.ValidatorsCount = 0 - c.ProtocolConfiguration.CommitteeHistory = map[uint32]int{ + c.ProtocolConfiguration.CommitteeHistory = map[uint32]uint32{ 0: 1, 4: 4, 24: 6, } - c.ProtocolConfiguration.ValidatorsHistory = map[uint32]int{ + c.ProtocolConfiguration.ValidatorsHistory = map[uint32]uint32{ 0: 1, 4: 4, } diff --git a/pkg/neorpc/result/version.go b/pkg/neorpc/result/version.go index aea901da49..deb4f33d40 100644 --- a/pkg/neorpc/result/version.go +++ b/pkg/neorpc/result/version.go @@ -34,13 +34,13 @@ type ( // returned by the server in case they're enabled. // CommitteeHistory stores height:size map of the committee size. - CommitteeHistory map[uint32]int + CommitteeHistory map[uint32]uint32 // P2PSigExtensions is true when Notary subsystem is enabled on the network. P2PSigExtensions bool // StateRootInHeader is true if state root is contained in block header. StateRootInHeader bool // ValidatorsHistory stores height:size map of the validators count. - ValidatorsHistory map[uint32]int + ValidatorsHistory map[uint32]uint32 } // protocolMarshallerAux is an auxiliary struct used for Protocol JSON marshalling. @@ -55,10 +55,10 @@ type ( ValidatorsCount byte `json:"validatorscount"` InitialGasDistribution int64 `json:"initialgasdistribution"` - CommitteeHistory map[uint32]int `json:"committeehistory,omitempty"` - P2PSigExtensions bool `json:"p2psigextensions,omitempty"` - StateRootInHeader bool `json:"staterootinheader,omitempty"` - ValidatorsHistory map[uint32]int `json:"validatorshistory,omitempty"` + CommitteeHistory map[uint32]uint32 `json:"committeehistory,omitempty"` + P2PSigExtensions bool `json:"p2psigextensions,omitempty"` + StateRootInHeader bool `json:"staterootinheader,omitempty"` + ValidatorsHistory map[uint32]uint32 `json:"validatorshistory,omitempty"` } ) diff --git a/pkg/rpcclient/actor/maker.go b/pkg/rpcclient/actor/maker.go index d185dd1fd1..f58547cb03 100644 --- a/pkg/rpcclient/actor/maker.go +++ b/pkg/rpcclient/actor/maker.go @@ -214,7 +214,7 @@ func (a *Actor) CalculateValidUntilBlock() (uint32, error) { if err != nil { return 0, fmt.Errorf("can't get block count: %w", err) } - var vc = int(a.version.Protocol.ValidatorsCount) + var vc = uint32(a.version.Protocol.ValidatorsCount) var bestH = uint32(0) for h, n := range a.version.Protocol.ValidatorsHistory { // In case it's enabled. if h >= bestH && h <= blockCount { @@ -223,5 +223,5 @@ func (a *Actor) CalculateValidUntilBlock() (uint32, error) { } } - return blockCount + uint32(vc+1), nil + return blockCount + vc + 1, nil } diff --git a/pkg/rpcclient/actor/maker_test.go b/pkg/rpcclient/actor/maker_test.go index 1b17510e8e..83fa42a5b8 100644 --- a/pkg/rpcclient/actor/maker_test.go +++ b/pkg/rpcclient/actor/maker_test.go @@ -25,7 +25,7 @@ func TestCalculateValidUntilBlock(t *testing.T) { require.NoError(t, err) require.Equal(t, uint32(42+7+1), vub) - client.version.Protocol.ValidatorsHistory = map[uint32]int{ + client.version.Protocol.ValidatorsHistory = map[uint32]uint32{ 0: 7, 40: 4, 80: 10, From 6cd0f786498f5a3ea8dd2f0380618e6799454ecb Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Thu, 13 Apr 2023 13:02:36 +0300 Subject: [PATCH 03/29] config: do not allow zero numbers for validators/committee history Signed-off-by: Anna Shaleva --- pkg/config/protocol_config.go | 6 ++++++ pkg/config/protocol_config_test.go | 22 ++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/pkg/config/protocol_config.go b/pkg/config/protocol_config.go index 943ad67918..6dc7455049 100644 --- a/pkg/config/protocol_config.go +++ b/pkg/config/protocol_config.go @@ -130,6 +130,9 @@ func (p *ProtocolConfiguration) Validate() error { } var arr = make([]heightNumber, 0, len(p.CommitteeHistory)) for h, n := range p.CommitteeHistory { + if n == 0 { + return fmt.Errorf("invalid CommitteeHistory: bad members count (%d) for height %d", n, h) + } if int(n) > len(p.StandbyCommittee) { return fmt.Errorf("too small StandbyCommittee for required number of committee members at %d", h) } @@ -148,6 +151,9 @@ func (p *ProtocolConfiguration) Validate() error { } arr = arr[:0] for h, n := range p.ValidatorsHistory { + if n == 0 { + return fmt.Errorf("invalid ValidatorsHistory: bad members count (%d) for height %d", n, h) + } if int(n) > len(p.StandbyCommittee) { return fmt.Errorf("too small StandbyCommittee for required number of validators at %d", h) } diff --git a/pkg/config/protocol_config_test.go b/pkg/config/protocol_config_test.go index 05731813ba..c95a2fc1a9 100644 --- a/pkg/config/protocol_config_test.go +++ b/pkg/config/protocol_config_test.go @@ -110,6 +110,28 @@ func TestProtocolConfigurationValidation(t *testing.T) { ValidatorsHistory: map[uint32]uint32{0: 4, 100: 4}, } require.Error(t, p.Validate()) + p = &ProtocolConfiguration{ + StandbyCommittee: []string{ + "02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2", + "02103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e", + "03d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699", + "02a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd62", + }, + CommitteeHistory: map[uint32]uint32{0: 0, 100: 4}, + ValidatorsHistory: map[uint32]uint32{0: 1, 100: 4}, + } + require.Error(t, p.Validate()) + p = &ProtocolConfiguration{ + StandbyCommittee: []string{ + "02b3622bf4017bdfe317c58aed5f4c753f206b7db896046fa7d774bbc4bf7f8dc2", + "02103a7f7dd016558597f7960d27c516a4394fd968b9e65155eb4b013e4040406e", + "03d90c07df63e690ce77912e10ab51acc944b66860237b608c4f8f8309e71ee699", + "02a7bc55fe8684e0119768d104ba30795bdcc86619e864add26156723ed185cd62", + }, + CommitteeHistory: map[uint32]uint32{0: 1, 100: 4}, + ValidatorsHistory: map[uint32]uint32{0: 0, 100: 4}, + } + require.Error(t, p.Validate()) p = &ProtocolConfiguration{ Hardforks: map[string]uint32{ "Unknown": 123, // Unknown hard-fork. From d5f964f1819509a541de5ef4355832b1f1834da9 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Thu, 13 Apr 2023 13:10:01 +0300 Subject: [PATCH 04/29] config: do not allow negative validators count Signed-off-by: Anna Shaleva --- docs/node-configuration.md | 2 +- pkg/config/protocol_config.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/node-configuration.md b/docs/node-configuration.md index bbd7d4455c..95a389b7c6 100644 --- a/docs/node-configuration.md +++ b/docs/node-configuration.md @@ -363,7 +363,7 @@ protocol-related settings described in the table below. | StateRootInHeader | `bool` | `false` | Enables storing state root in block header. | Experimental protocol extension! | | StateSyncInterval | `int` | `40000` | The number of blocks between state heights available for MPT state data synchronization. | `P2PStateExchangeExtensions` should be enabled to use this setting. | | TimePerBlock | `Duration` | `15s` | Minimal (and targeted for) time interval between blocks. Must be an integer number of milliseconds. | -| ValidatorsCount | `int` | `0` | Number of validators set for the whole network lifetime, can't be set if `ValidatorsHistory` setting is used. | +| ValidatorsCount | `uint32` | `0` | Number of validators set for the whole network lifetime, can't be set if `ValidatorsHistory` setting is used. | | ValidatorsHistory | map[uint32]uint32 | none | Number of consensus nodes to use after given height (see `CommitteeHistory` also). Heights where the change occurs must be divisible by the number of committee members at that height. Can't be used with `ValidatorsCount` not equal to zero. | | VerifyBlocks | `bool` | `false` | This setting is deprecated and no longer works, please use `SkipBlockVerification` in the `ApplicationConfiguration`, it will be removed in future node versions. | | VerifyTransactions | `bool` | `false` | Denotes whether to verify transactions in the received blocks. | diff --git a/pkg/config/protocol_config.go b/pkg/config/protocol_config.go index 6dc7455049..240d82288d 100644 --- a/pkg/config/protocol_config.go +++ b/pkg/config/protocol_config.go @@ -82,7 +82,7 @@ type ( // TimePerBlock is the time interval between blocks that consensus nodes work with. // It must be an integer number of milliseconds. TimePerBlock time.Duration `yaml:"TimePerBlock"` - ValidatorsCount int `yaml:"ValidatorsCount"` + ValidatorsCount uint32 `yaml:"ValidatorsCount"` // Validators stores history of changes to consensus node number (height: number). ValidatorsHistory map[uint32]uint32 `yaml:"ValidatorsHistory"` // Whether to verify received blocks. @@ -125,7 +125,7 @@ func (p *ProtocolConfiguration) Validate() error { if p.ValidatorsCount != 0 && len(p.ValidatorsHistory) != 0 { return errors.New("configuration should either have ValidatorsCount or ValidatorsHistory, not both") } - if len(p.StandbyCommittee) < p.ValidatorsCount { + if len(p.StandbyCommittee) < int(p.ValidatorsCount) { return errors.New("validators count can't exceed the size of StandbyCommittee") } var arr = make([]heightNumber, 0, len(p.CommitteeHistory)) @@ -212,7 +212,7 @@ func getBestFromMap(dict map[uint32]uint32, height uint32) uint32 { // It implies valid configuration file. func (p *ProtocolConfiguration) GetNumOfCNs(height uint32) int { if len(p.ValidatorsHistory) == 0 { - return p.ValidatorsCount + return int(p.ValidatorsCount) } return int(getBestFromMap(p.ValidatorsHistory, height)) } From c8808731054f0e82c4cfb8d6bb968b709ea4fbfd Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Thu, 13 Apr 2023 18:20:52 +0300 Subject: [PATCH 05/29] core: improve documentation to SetOracle/SetNotary I've carefully checked the way how new service can be added to the Blockchain instance or to be removed from it. Current implemention of SetNotary and SetOracle methods doesn't contain dangerous code, and native contracts have atomic values everywhere where service is stored. Current implementation of Notary, Oracle and StateRoot services' reload/disabling/enabling on SIGUSR1 is safe and doesn't require any adjustment. This commit closes #2944, it's not a bug in the code, it's just stale documentation. Signed-off-by: Anna Shaleva --- pkg/core/blockchain.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 2c28c06a35..6112442287 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -328,8 +328,8 @@ func NewBlockchain(s storage.Store, cfg config.Blockchain, log *zap.Logger) (*Bl return bc, nil } -// SetOracle sets oracle module. It doesn't protected by mutex and -// must be called before `bc.Run()` to avoid data race. +// SetOracle sets oracle module. It can safely be called on the running blockchain. +// To unregister Oracle service use SetOracle(nil). func (bc *Blockchain) SetOracle(mod native.OracleService) { orc := bc.contracts.Oracle if mod != nil { @@ -356,8 +356,8 @@ func (bc *Blockchain) SetOracle(mod native.OracleService) { bc.contracts.Designate.OracleService.Store(&mod) } -// SetNotary sets notary module. It doesn't protected by mutex and -// must be called before `bc.Run()` to avoid data race. +// SetNotary sets notary module. It may safely be called on the running blockchain. +// To unregister Notary service use SetNotary(nil). func (bc *Blockchain) SetNotary(mod native.NotaryService) { if mod != nil { keys, _, err := bc.contracts.Designate.GetDesignatedByRole(bc.dao, noderoles.P2PNotary, bc.BlockHeight()) From 758c455c7e0e033ddc2c1ea43f3f08a31650272c Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Thu, 13 Apr 2023 11:48:18 +0400 Subject: [PATCH 06/29] services/rpcsrv: Test `Server` shutdown with failed precondition There is an existing problem with RPC server shutdown freeze after start failure due to some init actions (at least HTTP listen) described in #2896. Add dedicated unit test which checks that `Shutdown` returns within 5s after `Start` method encounters internal problems. Signed-off-by: Leonard Lyubich --- pkg/services/rpcsrv/server_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/services/rpcsrv/server_test.go b/pkg/services/rpcsrv/server_test.go index 08d636d077..98d8324ef2 100644 --- a/pkg/services/rpcsrv/server_test.go +++ b/pkg/services/rpcsrv/server_test.go @@ -50,6 +50,7 @@ import ( "github.com/nspcc-dev/neo-go/pkg/wallet" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/atomic" "go.uber.org/zap/zapcore" ) @@ -3331,3 +3332,21 @@ func BenchmarkHandleIn(b *testing.B) { {"type": "Integer", "value": "42"}, {"type": "Boolean", "value": false}]]}`)) }) } + +func TestFailedPreconditionShutdown(t *testing.T) { + _, srv, _ := initClearServerWithCustomConfig(t, func(c *config.Config) { + c.ApplicationConfiguration.RPC.Addresses = []string{"not an address"} + }) + + srv.Start() + require.Positive(t, len(srv.errChan)) // this is how Start reports internal failures + + var stopped atomic.Bool + + go func() { + srv.Shutdown() + stopped.Store(true) + }() + + require.Eventually(t, stopped.Load, 5*time.Second, 100*time.Millisecond, "Shutdown should return") +} From f8227aa5f7b02f4b6e0922c59674e05a7beb8ec9 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Thu, 13 Apr 2023 12:03:18 +0400 Subject: [PATCH 07/29] services/rpcsrv: Fix potential shutdown deadlock of RPC server Previously RPC server could never be shut down completely due to some start precondition failure (in particular, inability to serve HTTP on any configured endpoint). The problem was caused by next facts: * start method ran subscription routine after HTTP init succeeded only * stop method blocked waiting for the subscription routine to return Run `handleSubEvents` routine on fresh `Start` unconditionally. With this change, `Shutdown` method won't produce deadlock since `handleSubEvents` closes wait channel. Refs #2896. Signed-off-by: Leonard Lyubich --- pkg/services/rpcsrv/server.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/services/rpcsrv/server.go b/pkg/services/rpcsrv/server.go index 6531ccaf80..c0c78ab76a 100644 --- a/pkg/services/rpcsrv/server.go +++ b/pkg/services/rpcsrv/server.go @@ -344,6 +344,9 @@ func (s *Server) Start() { s.log.Info("RPC server already started") return } + + go s.handleSubEvents() + for _, srv := range s.http { srv.Handler = http.HandlerFunc(s.handleHTTPRequest) s.log.Info("starting rpc-server", zap.String("endpoint", srv.Addr)) @@ -363,7 +366,6 @@ func (s *Server) Start() { }(srv) } - go s.handleSubEvents() if cfg := s.config.TLSConfig; cfg.Enabled { for _, srv := range s.https { srv.Handler = http.HandlerFunc(s.handleHTTPRequest) From 465d3f43d2a52c72bfc128bfbd6e9b34c2ac7eed Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Thu, 13 Apr 2023 12:36:39 +0400 Subject: [PATCH 08/29] services/rpcsrv: Wait for subscription process to complete when stopped Previously RPC server shutdown procedure listened to the execution channel and stopped at the first element that arrived in the queue. This could lead to the following problems: * stopper could steal the execution result from subscriber * stopper didn't wait for other subscription actions to complete Add dedicated channel to `Server` for subscription routine. Close the channel on `handleSubEvents` return and wait for signal in `Shutdown`. Signed-off-by: Leonard Lyubich --- pkg/services/rpcsrv/server.go | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pkg/services/rpcsrv/server.go b/pkg/services/rpcsrv/server.go index c0c78ab76a..0ba5f87693 100644 --- a/pkg/services/rpcsrv/server.go +++ b/pkg/services/rpcsrv/server.go @@ -148,11 +148,12 @@ type ( transactionSubs int notaryRequestSubs int - blockCh chan *block.Block - executionCh chan *state.AppExecResult - notificationCh chan *state.ContainedNotificationEvent - transactionCh chan *transaction.Transaction - notaryRequestCh chan mempoolevent.Event + blockCh chan *block.Block + executionCh chan *state.AppExecResult + notificationCh chan *state.ContainedNotificationEvent + transactionCh chan *transaction.Transaction + notaryRequestCh chan mempoolevent.Event + subEventsToExitCh chan struct{} } // session holds a set of iterators got after invoke* call with corresponding @@ -319,11 +320,12 @@ func New(chain Ledger, conf config.RPC, coreServer *network.Server, subscribers: make(map[*subscriber]bool), // These are NOT buffered to preserve original order of events. - blockCh: make(chan *block.Block), - executionCh: make(chan *state.AppExecResult), - notificationCh: make(chan *state.ContainedNotificationEvent), - transactionCh: make(chan *transaction.Transaction), - notaryRequestCh: make(chan mempoolevent.Event), + blockCh: make(chan *block.Block), + executionCh: make(chan *state.AppExecResult), + notificationCh: make(chan *state.ContainedNotificationEvent), + transactionCh: make(chan *transaction.Transaction), + notaryRequestCh: make(chan mempoolevent.Event), + subEventsToExitCh: make(chan struct{}), } } @@ -438,7 +440,7 @@ func (s *Server) Shutdown() { } // Wait for handleSubEvents to finish. - <-s.executionCh + <-s.subEventsToExitCh } // SetOracleHandler allows to update oracle handler used by the Server. @@ -2667,6 +2669,8 @@ func (s *Server) unsubscribeFromChannel(event neorpc.EventID) { } } +// handleSubEvents processes Server subscriptions until Shutdown. Upon +// completion signals to subEventCh channel. func (s *Server) handleSubEvents() { var overflowEvent = neorpc.Notification{ JSONRPC: neorpc.JSONRPCVersion, @@ -2780,12 +2784,14 @@ drainloop: } } // It's not required closing these, but since they're drained already - // this is safe and it also allows to give a signal to Shutdown routine. + // this is safe. close(s.blockCh) close(s.transactionCh) close(s.notificationCh) close(s.executionCh) close(s.notaryRequestCh) + // notify Shutdown routine + close(s.subEventsToExitCh) } func (s *Server) blockHeightFromParam(param *params.Param) (uint32, *neorpc.Error) { From 6e990f39de2a8e15636ed9c48ce8139a9322b4d7 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Thu, 13 Apr 2023 12:53:58 +0400 Subject: [PATCH 09/29] cli/node: Fix deadlock produced by instant RPC service start If `StartWhenSynchronized` is unset in config, `node` command runs RPC service instantly. Previously there was a ground for deadlock. Command started RPC server synchronously. According to server implementation, it sends all internal failures to the parameterized error channel. Deadlock occured because main routine didn't scan the channel. Run `rpcsrv.Server.Start` in a separate go-routine in `startServer`. This prevents potential deadlock caused by writing into unread channel. Fixes #2896. Signed-off-by: Leonard Lyubich --- cli/server/server.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cli/server/server.go b/cli/server/server.go index 50ba57d469..6b87e17667 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -491,7 +491,10 @@ func startServer(ctx *cli.Context) error { go serv.Start() if !cfg.ApplicationConfiguration.RPC.StartWhenSynchronized { - rpcServer.Start() + // Run RPC server in a separate routine. This is necessary to avoid a potential + // deadlock: Start() can write errors to errChan which is not yet read in the + // current execution context (see for-loop below). + go rpcServer.Start() } sigCh := make(chan os.Signal, 1) @@ -546,7 +549,8 @@ Main: rpcServer = rpcsrv.New(chain, cfgnew.ApplicationConfiguration.RPC, serv, oracleSrv, log, errChan) serv.AddService(&rpcServer) if !cfgnew.ApplicationConfiguration.RPC.StartWhenSynchronized || serv.IsInSync() { - rpcServer.Start() + // Here similar to the initial run (see above for-loop), so async. + go rpcServer.Start() } pprof.ShutDown() pprof = metrics.NewPprofService(cfgnew.ApplicationConfiguration.Pprof, log) From ab64f7cfe4807b5ea0109e13a6fb53fe32cd4d73 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 18 Apr 2023 18:52:57 +0300 Subject: [PATCH 10/29] docs: fix typo Remove unnecessary word left after documentation refactoring. Signed-off-by: Anna Shaleva --- docs/notary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/notary.md b/docs/notary.md index 8fe10b3d28..7c348940c9 100644 --- a/docs/notary.md +++ b/docs/notary.md @@ -384,7 +384,7 @@ subpackage with an example written in Go doc. `Invocation` script **should be empty**. - A multisignature witness must have regular `Verification` script filled even if `Invocation` script is to be collected from other notary requests. - `Invocation` script either **should be empty**. + `Invocation` script **should be empty**. 8. Calculate network fee for the transaction (that will be `NetworkFee` transaction field). Use [func (*Client) CalculateNetworkFee](https://pkg.go.dev/github.com/nspcc-dev/neo-go@v0.99.2/pkg/rpcclient#Client.CalculateNetworkFee) method with the main transaction given to it. From 08b273266bc1cf2453fc8cc329f36861674eb096 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 18 Apr 2023 19:34:38 +0300 Subject: [PATCH 11/29] rpcclient: correctly handle request channel closure wsReader() closes c.done first and then goes over the list of c.respChannels. Technically this means that any of the two can be taken in this select. Signed-off-by: Roman Khimov --- pkg/rpcclient/wsclient.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/rpcclient/wsclient.go b/pkg/rpcclient/wsclient.go index 2234260fcc..1ac87ff6ab 100644 --- a/pkg/rpcclient/wsclient.go +++ b/pkg/rpcclient/wsclient.go @@ -638,7 +638,10 @@ func (c *WSClient) makeWsRequest(r *neorpc.Request) (*neorpc.Response, error) { select { case <-c.done: return nil, errors.New("connection lost while waiting for the response") - case resp := <-ch: + case resp, ok := <-ch: + if !ok { + return nil, errors.New("connection lost while waiting for the response") + } c.unregisterRespChannel(r.ID) return resp, nil } From 288dee8871187216f8546b3ac8a13708b0f0588f Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Tue, 18 Apr 2023 19:36:57 +0300 Subject: [PATCH 12/29] rpcclient: close subscriber channels on wsReader exit The reader is about to exit and it will close legacy c.Notifications, but it will leave subscription channels at the same time. This is wrong since these channels will no longer receive any new events, game over. Signed-off-by: Roman Khimov --- pkg/rpcclient/wsclient.go | 13 +++++++++- pkg/rpcclient/wsclient_test.go | 43 ++++++++++++++++++---------------- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/pkg/rpcclient/wsclient.go b/pkg/rpcclient/wsclient.go index 1ac87ff6ab..ec3ed4cd70 100644 --- a/pkg/rpcclient/wsclient.go +++ b/pkg/rpcclient/wsclient.go @@ -46,7 +46,8 @@ import ( // subscriptions share the same receiver channel, then matching notification is // only sent once per channel. The receiver channel will be closed by the WSClient // immediately after MissedEvent is received from the server; no unsubscription -// is performed in this case, so it's the user responsibility to unsubscribe. +// is performed in this case, so it's the user responsibility to unsubscribe. It +// will also be closed on disconnection from server. type WSClient struct { Client // Notifications is a channel that is used to send events received from @@ -539,6 +540,16 @@ readloop: } c.respChannels = nil c.respLock.Unlock() + c.subscriptionsLock.Lock() + for rcvrCh, ids := range c.receivers { + rcvr := c.subscriptions[ids[0]] + _, ok := rcvr.(*naiveReceiver) + if !ok { // naiveReceiver uses c.Notifications that is about to be closed below. + c.subscriptions[ids[0]].Close() + } + delete(c.receivers, rcvrCh) + } + c.subscriptionsLock.Unlock() close(c.Notifications) c.Client.ctxCancel() } diff --git a/pkg/rpcclient/wsclient_test.go b/pkg/rpcclient/wsclient_test.go index 1d7f3f7747..c3f30c8947 100644 --- a/pkg/rpcclient/wsclient_test.go +++ b/pkg/rpcclient/wsclient_test.go @@ -30,10 +30,17 @@ import ( ) func TestWSClientClose(t *testing.T) { - srv := initTestServer(t, "") + srv := initTestServer(t, `{"jsonrpc": "2.0", "id": 1, "result": "55aaff00"}`) wsc, err := NewWS(context.TODO(), httpURLtoWS(srv.URL), Options{}) require.NoError(t, err) + wsc.getNextRequestID = getTestRequestID + bCh := make(chan *block.Block) + _, err = wsc.ReceiveBlocks(nil, bCh) + require.NoError(t, err) wsc.Close() + // Subscriber channel must be closed by server. + _, ok := <-bCh + require.False(t, ok) } func TestWSClientSubscription(t *testing.T) { @@ -296,10 +303,6 @@ func TestWSExecutionVMStateCheck(t *testing.T) { } func TestWSFilteredSubscriptions(t *testing.T) { - bCh := make(chan *block.Block) - txCh := make(chan *transaction.Transaction) - aerCh := make(chan *state.AppExecResult) - ntfCh := make(chan *state.ContainedNotificationEvent) var cases = []struct { name string clientCode func(*testing.T, *WSClient) @@ -308,7 +311,7 @@ func TestWSFilteredSubscriptions(t *testing.T) { {"blocks primary", func(t *testing.T, wsc *WSClient) { primary := 3 - _, err := wsc.ReceiveBlocks(&neorpc.BlockFilter{Primary: &primary}, bCh) + _, err := wsc.ReceiveBlocks(&neorpc.BlockFilter{Primary: &primary}, make(chan *block.Block)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { @@ -323,7 +326,7 @@ func TestWSFilteredSubscriptions(t *testing.T) { {"blocks since", func(t *testing.T, wsc *WSClient) { var since uint32 = 3 - _, err := wsc.ReceiveBlocks(&neorpc.BlockFilter{Since: &since}, bCh) + _, err := wsc.ReceiveBlocks(&neorpc.BlockFilter{Since: &since}, make(chan *block.Block)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { @@ -338,7 +341,7 @@ func TestWSFilteredSubscriptions(t *testing.T) { {"blocks till", func(t *testing.T, wsc *WSClient) { var till uint32 = 3 - _, err := wsc.ReceiveBlocks(&neorpc.BlockFilter{Till: &till}, bCh) + _, err := wsc.ReceiveBlocks(&neorpc.BlockFilter{Till: &till}, make(chan *block.Block)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { @@ -361,7 +364,7 @@ func TestWSFilteredSubscriptions(t *testing.T) { Primary: &primary, Since: &since, Till: &till, - }, bCh) + }, make(chan *block.Block)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { @@ -376,7 +379,7 @@ func TestWSFilteredSubscriptions(t *testing.T) { {"transactions sender", func(t *testing.T, wsc *WSClient) { sender := util.Uint160{1, 2, 3, 4, 5} - _, err := wsc.ReceiveTransactions(&neorpc.TxFilter{Sender: &sender}, txCh) + _, err := wsc.ReceiveTransactions(&neorpc.TxFilter{Sender: &sender}, make(chan *transaction.Transaction)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { @@ -390,7 +393,7 @@ func TestWSFilteredSubscriptions(t *testing.T) { {"transactions signer", func(t *testing.T, wsc *WSClient) { signer := util.Uint160{0, 42} - _, err := wsc.ReceiveTransactions(&neorpc.TxFilter{Signer: &signer}, txCh) + _, err := wsc.ReceiveTransactions(&neorpc.TxFilter{Signer: &signer}, make(chan *transaction.Transaction)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { @@ -405,7 +408,7 @@ func TestWSFilteredSubscriptions(t *testing.T) { func(t *testing.T, wsc *WSClient) { sender := util.Uint160{1, 2, 3, 4, 5} signer := util.Uint160{0, 42} - _, err := wsc.ReceiveTransactions(&neorpc.TxFilter{Sender: &sender, Signer: &signer}, txCh) + _, err := wsc.ReceiveTransactions(&neorpc.TxFilter{Sender: &sender, Signer: &signer}, make(chan *transaction.Transaction)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { @@ -419,7 +422,7 @@ func TestWSFilteredSubscriptions(t *testing.T) { {"notifications contract hash", func(t *testing.T, wsc *WSClient) { contract := util.Uint160{1, 2, 3, 4, 5} - _, err := wsc.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract}, ntfCh) + _, err := wsc.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract}, make(chan *state.ContainedNotificationEvent)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { @@ -433,7 +436,7 @@ func TestWSFilteredSubscriptions(t *testing.T) { {"notifications name", func(t *testing.T, wsc *WSClient) { name := "my_pretty_notification" - _, err := wsc.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Name: &name}, ntfCh) + _, err := wsc.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Name: &name}, make(chan *state.ContainedNotificationEvent)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { @@ -448,7 +451,7 @@ func TestWSFilteredSubscriptions(t *testing.T) { func(t *testing.T, wsc *WSClient) { contract := util.Uint160{1, 2, 3, 4, 5} name := "my_pretty_notification" - _, err := wsc.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract, Name: &name}, ntfCh) + _, err := wsc.ReceiveExecutionNotifications(&neorpc.NotificationFilter{Contract: &contract, Name: &name}, make(chan *state.ContainedNotificationEvent)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { @@ -461,8 +464,8 @@ func TestWSFilteredSubscriptions(t *testing.T) { }, {"executions state", func(t *testing.T, wsc *WSClient) { - state := "FAULT" - _, err := wsc.ReceiveExecutions(&neorpc.ExecutionFilter{State: &state}, aerCh) + vmstate := "FAULT" + _, err := wsc.ReceiveExecutions(&neorpc.ExecutionFilter{State: &vmstate}, make(chan *state.AppExecResult)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { @@ -476,7 +479,7 @@ func TestWSFilteredSubscriptions(t *testing.T) { {"executions container", func(t *testing.T, wsc *WSClient) { container := util.Uint256{1, 2, 3} - _, err := wsc.ReceiveExecutions(&neorpc.ExecutionFilter{Container: &container}, aerCh) + _, err := wsc.ReceiveExecutions(&neorpc.ExecutionFilter{Container: &container}, make(chan *state.AppExecResult)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { @@ -489,9 +492,9 @@ func TestWSFilteredSubscriptions(t *testing.T) { }, {"executions state and container", func(t *testing.T, wsc *WSClient) { - state := "FAULT" + vmstate := "FAULT" container := util.Uint256{1, 2, 3} - _, err := wsc.ReceiveExecutions(&neorpc.ExecutionFilter{State: &state, Container: &container}, aerCh) + _, err := wsc.ReceiveExecutions(&neorpc.ExecutionFilter{State: &vmstate, Container: &container}, make(chan *state.AppExecResult)) require.NoError(t, err) }, func(t *testing.T, p *params.Params) { From 97b93c68335138d9875afaebbf7b4b6253b39c1f Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 25 Apr 2023 13:12:18 +0300 Subject: [PATCH 13/29] *: adjust Prometheus metrics initialisation on node start Initialize Prometheus metrics on node start where appropriate and review the usage of the following metrics: ``` anna@kiwi:~/Documents/GitProjects/nspcc-dev/neo-go$ find | grep prometheus.go ./pkg/network/prometheus.go ./pkg/core/stateroot/prometheus.go ./pkg/core/prometheus.go ./pkg/services/rpcsrv/prometheus.go ./pkg/services/metrics/prometheus.go ``` Close #2970. Signed-off-by: Anna Shaleva --- pkg/core/blockchain.go | 6 ++++++ pkg/core/stateroot/module.go | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 6112442287..cf5dc1ea10 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -503,6 +503,10 @@ func (bc *Blockchain) init() error { } } + updateBlockHeightMetric(bHeight) + updatePersistedHeightMetric(bHeight) + updateHeaderHeightMetric(bc.HeaderHeight()) + return bc.updateExtensibleWhitelist(bHeight) } @@ -632,6 +636,8 @@ func (bc *Blockchain) resetRAMState(height uint32, resetHeaders bool) error { } updateBlockHeightMetric(height) + updatePersistedHeightMetric(height) + updateHeaderHeightMetric(bc.HeaderHeight()) return nil } diff --git a/pkg/core/stateroot/module.go b/pkg/core/stateroot/module.go index 751e75d1b3..442164d8b5 100644 --- a/pkg/core/stateroot/module.go +++ b/pkg/core/stateroot/module.go @@ -152,7 +152,9 @@ func (s *Module) CurrentValidatedHeight() uint32 { func (s *Module) Init(height uint32) error { data, err := s.Store.Get([]byte{byte(storage.DataMPTAux), prefixValidated}) if err == nil { - s.validatedHeight.Store(binary.LittleEndian.Uint32(data)) + h := binary.LittleEndian.Uint32(data) + s.validatedHeight.Store(h) + updateStateHeightMetric(h) } if height == 0 { From d0718a680f19577587ca3abd73f44cc5975f0420 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 26 Apr 2023 12:52:59 +0300 Subject: [PATCH 14/29] core: add InitializeCache method to Contract interface Make the contracts cache initialization unified. The order of cache iniitialization is not important and Nottary contract is added to the bc.contracts.Contracts wrt P2PSigExtensions setting, thus no functional changes, just refactoring for future applications. Signed-off-by: Anna Shaleva --- pkg/core/blockchain.go | 23 +++-------------------- pkg/core/interop/context.go | 5 +++++ pkg/core/native/crypto.go | 6 ++++++ pkg/core/native/designate.go | 2 +- pkg/core/native/ledger.go | 5 +++++ pkg/core/native/management.go | 2 +- pkg/core/native/management_test.go | 4 ++-- pkg/core/native/native_gas.go | 5 +++++ pkg/core/native/native_neo.go | 3 ++- pkg/core/native/notary.go | 2 +- pkg/core/native/oracle.go | 3 ++- pkg/core/native/policy.go | 2 +- pkg/core/native/std.go | 6 ++++++ 13 files changed, 40 insertions(+), 28 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index cf5dc1ea10..f083bdcb22 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -888,29 +888,12 @@ func (bc *Blockchain) resetStateInternal(height uint32, stage stateChangeStage) } func (bc *Blockchain) initializeNativeCache(blockHeight uint32, d *dao.Simple) error { - err := bc.contracts.NEO.InitializeCache(blockHeight, d) - if err != nil { - return fmt.Errorf("can't init cache for NEO native contract: %w", err) - } - err = bc.contracts.Management.InitializeCache(d) - if err != nil { - return fmt.Errorf("can't init cache for Management native contract: %w", err) - } - err = bc.contracts.Designate.InitializeCache(d) - if err != nil { - return fmt.Errorf("can't init cache for Designation native contract: %w", err) - } - bc.contracts.Oracle.InitializeCache(d) - if bc.P2PSigExtensionsEnabled() { - err = bc.contracts.Notary.InitializeCache(d) + for _, c := range bc.contracts.Contracts { + err := c.InitializeCache(blockHeight, d) if err != nil { - return fmt.Errorf("can't init cache for Notary native contract: %w", err) + return fmt.Errorf("failed to initialize cache for %s: %w", c.Metadata().Name, err) } } - err = bc.contracts.Policy.InitializeCache(d) - if err != nil { - return fmt.Errorf("can't init cache for Policy native contract: %w", err) - } return nil } diff --git a/pkg/core/interop/context.go b/pkg/core/interop/context.go index 1d1448014f..9a8270aab1 100644 --- a/pkg/core/interop/context.go +++ b/pkg/core/interop/context.go @@ -153,6 +153,11 @@ type MethodAndPrice struct { // Contract is an interface for all native contracts. type Contract interface { Initialize(*Context) error + // InitializeCache aimed to initialize contract's cache when the contract has + // been deployed, but in-memory cached data were lost due to the node reset. + // It should be called each time after node restart iff the contract was + // deployed and no Initialize method was called. + InitializeCache(blockHeight uint32, d *dao.Simple) error Metadata() *ContractMD OnPersist(*Context) error PostPersist(*Context) error diff --git a/pkg/core/native/crypto.go b/pkg/core/native/crypto.go index 0228bc9ccc..0fef3d73af 100644 --- a/pkg/core/native/crypto.go +++ b/pkg/core/native/crypto.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/decred/dcrd/dcrec/secp256k1/v4" + "github.com/nspcc-dev/neo-go/pkg/core/dao" "github.com/nspcc-dev/neo-go/pkg/core/interop" "github.com/nspcc-dev/neo-go/pkg/core/native/nativenames" "github.com/nspcc-dev/neo-go/pkg/crypto/hash" @@ -147,6 +148,11 @@ func (c *Crypto) Initialize(ic *interop.Context) error { return nil } +// InitializeCache implements the Contract interface. +func (c *Crypto) InitializeCache(blockHeight uint32, d *dao.Simple) error { + return nil +} + // OnPersist implements the Contract interface. func (c *Crypto) OnPersist(ic *interop.Context) error { return nil diff --git a/pkg/core/native/designate.go b/pkg/core/native/designate.go index caf587ccf2..12d8fd79a0 100644 --- a/pkg/core/native/designate.go +++ b/pkg/core/native/designate.go @@ -132,7 +132,7 @@ func (s *Designate) Initialize(ic *interop.Context) error { // InitializeCache fills native Designate cache from DAO. It is called at non-zero height, thus // we can fetch the roles data right from the storage. -func (s *Designate) InitializeCache(d *dao.Simple) error { +func (s *Designate) InitializeCache(blockHeight uint32, d *dao.Simple) error { cache := &DesignationCache{} roles := []noderoles.Role{noderoles.Oracle, noderoles.NeoFSAlphabet, noderoles.StateValidator} if s.p2pSigExtensionsEnabled { diff --git a/pkg/core/native/ledger.go b/pkg/core/native/ledger.go index 51734c2260..ba21a34878 100644 --- a/pkg/core/native/ledger.go +++ b/pkg/core/native/ledger.go @@ -85,6 +85,11 @@ func (l *Ledger) Initialize(ic *interop.Context) error { return nil } +// InitializeCache implements the Contract interface. +func (l *Ledger) InitializeCache(blockHeight uint32, d *dao.Simple) error { + return nil +} + // OnPersist implements the Contract interface. func (l *Ledger) OnPersist(ic *interop.Context) error { // Actual block/tx processing is done in Blockchain.storeBlock(). diff --git a/pkg/core/native/management.go b/pkg/core/native/management.go index 889ceab9c1..f7148f8b59 100644 --- a/pkg/core/native/management.go +++ b/pkg/core/native/management.go @@ -608,7 +608,7 @@ func (m *Management) OnPersist(ic *interop.Context) error { // InitializeCache initializes contract cache with the proper values from storage. // Cache initialization should be done apart from Initialize because Initialize is // called only when deploying native contracts. -func (m *Management) InitializeCache(d *dao.Simple) error { +func (m *Management) InitializeCache(blockHeight uint32, d *dao.Simple) error { cache := &ManagementCache{ contracts: make(map[util.Uint160]*state.Contract), nep11: make(map[util.Uint160]struct{}), diff --git a/pkg/core/native/management_test.go b/pkg/core/native/management_test.go index a4cb7bc879..cc8e1e1f84 100644 --- a/pkg/core/native/management_test.go +++ b/pkg/core/native/management_test.go @@ -82,7 +82,7 @@ func TestManagement_Initialize(t *testing.T) { t.Run("good", func(t *testing.T) { d := dao.NewSimple(storage.NewMemoryStore(), false, false) mgmt := newManagement() - require.NoError(t, mgmt.InitializeCache(d)) + require.NoError(t, mgmt.InitializeCache(0, d)) }) /* See #2801 t.Run("invalid contract state", func(t *testing.T) { @@ -101,7 +101,7 @@ func TestManagement_GetNEP17Contracts(t *testing.T) { err := mgmt.Initialize(&interop.Context{DAO: d}) require.NoError(t, err) require.NoError(t, mgmt.Policy.Initialize(&interop.Context{DAO: d})) - err = mgmt.InitializeCache(d) + err = mgmt.InitializeCache(0, d) require.NoError(t, err) require.Empty(t, mgmt.GetNEP17Contracts(d)) diff --git a/pkg/core/native/native_gas.go b/pkg/core/native/native_gas.go index e60f3f9595..7276572b63 100644 --- a/pkg/core/native/native_gas.go +++ b/pkg/core/native/native_gas.go @@ -99,6 +99,11 @@ func (g *GAS) Initialize(ic *interop.Context) error { return nil } +// InitializeCache implements the Contract interface. +func (g *GAS) InitializeCache(blockHeight uint32, d *dao.Simple) error { + return nil +} + // OnPersist implements the Contract interface. func (g *GAS) OnPersist(ic *interop.Context) error { if len(ic.Block.Transactions) == 0 { diff --git a/pkg/core/native/native_neo.go b/pkg/core/native/native_neo.go index 756f308ef1..9014047f54 100644 --- a/pkg/core/native/native_neo.go +++ b/pkg/core/native/native_neo.go @@ -305,7 +305,8 @@ func (n *NEO) Initialize(ic *interop.Context) error { // InitializeCache initializes all NEO cache with the proper values from the storage. // Cache initialization should be done apart from Initialize because Initialize is -// called only when deploying native contracts. +// called only when deploying native contracts. InitializeCache implements the Contract +// interface. func (n *NEO) InitializeCache(blockHeight uint32, d *dao.Simple) error { cache := &NeoCache{ gasPerVoteCache: make(map[string]big.Int), diff --git a/pkg/core/native/notary.go b/pkg/core/native/notary.go index 93f2af7880..f58e628a7c 100644 --- a/pkg/core/native/notary.go +++ b/pkg/core/native/notary.go @@ -152,7 +152,7 @@ func (n *Notary) Initialize(ic *interop.Context) error { return nil } -func (n *Notary) InitializeCache(d *dao.Simple) error { +func (n *Notary) InitializeCache(blockHeight uint32, d *dao.Simple) error { cache := &NotaryCache{ maxNotValidBeforeDelta: uint32(getIntWithKey(n.ID, d, maxNotValidBeforeDeltaKey)), notaryServiceFeePerKey: getIntWithKey(n.ID, d, notaryServiceFeeKey), diff --git a/pkg/core/native/oracle.go b/pkg/core/native/oracle.go index 0189f4fd3e..ef943966e3 100644 --- a/pkg/core/native/oracle.go +++ b/pkg/core/native/oracle.go @@ -251,10 +251,11 @@ func (o *Oracle) Initialize(ic *interop.Context) error { return nil } -func (o *Oracle) InitializeCache(d *dao.Simple) { +func (o *Oracle) InitializeCache(blockHeight uint32, d *dao.Simple) error { cache := &OracleCache{} cache.requestPrice = getIntWithKey(o.ID, d, prefixRequestPrice) d.SetCache(o.ID, cache) + return nil } func getResponse(tx *transaction.Transaction) *transaction.OracleResponse { diff --git a/pkg/core/native/policy.go b/pkg/core/native/policy.go index 570dfc40a6..d45972c733 100644 --- a/pkg/core/native/policy.go +++ b/pkg/core/native/policy.go @@ -153,7 +153,7 @@ func (p *Policy) Initialize(ic *interop.Context) error { return nil } -func (p *Policy) InitializeCache(d *dao.Simple) error { +func (p *Policy) InitializeCache(blockHeight uint32, d *dao.Simple) error { cache := &PolicyCache{} err := p.fillCacheFromDAO(cache, d) if err != nil { diff --git a/pkg/core/native/std.go b/pkg/core/native/std.go index 733fd73382..18dabccc5d 100644 --- a/pkg/core/native/std.go +++ b/pkg/core/native/std.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/mr-tron/base58" + "github.com/nspcc-dev/neo-go/pkg/core/dao" "github.com/nspcc-dev/neo-go/pkg/core/interop" "github.com/nspcc-dev/neo-go/pkg/core/native/nativenames" base58neogo "github.com/nspcc-dev/neo-go/pkg/encoding/base58" @@ -429,6 +430,11 @@ func (s *Std) Initialize(ic *interop.Context) error { return nil } +// InitializeCache implements the Contract interface. +func (s *Std) InitializeCache(blockHeight uint32, d *dao.Simple) error { + return nil +} + // OnPersist implements the Contract interface. func (s *Std) OnPersist(ic *interop.Context) error { return nil From 2c984e2393855a9a5c498426a408c435b2aedc81 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 26 Apr 2023 13:22:13 +0300 Subject: [PATCH 15/29] core: initialize natives cache wrt NativeActivations If the contract was deployed then cache must be initialized after in-memory data reset. If the contract isn't active yet, then no cache will be initialized on deploy (i.e. on call to Initialize() method by native Management). Close #2984. Signed-off-by: Anna Shaleva --- pkg/core/blockchain.go | 11 +++-- pkg/core/blockchain_neotest_test.go | 65 +++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index f083bdcb22..9ac21f6720 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -889,9 +889,14 @@ func (bc *Blockchain) resetStateInternal(height uint32, stage stateChangeStage) func (bc *Blockchain) initializeNativeCache(blockHeight uint32, d *dao.Simple) error { for _, c := range bc.contracts.Contracts { - err := c.InitializeCache(blockHeight, d) - if err != nil { - return fmt.Errorf("failed to initialize cache for %s: %w", c.Metadata().Name, err) + for _, h := range c.Metadata().UpdateHistory { + if blockHeight >= h { // check that contract was deployed. + err := c.InitializeCache(blockHeight, d) + if err != nil { + return fmt.Errorf("failed to initialize cache for %s: %w", c.Metadata().Name, err) + } + break + } } } return nil diff --git a/pkg/core/blockchain_neotest_test.go b/pkg/core/blockchain_neotest_test.go index f71b8890b8..6be7aac8d8 100644 --- a/pkg/core/blockchain_neotest_test.go +++ b/pkg/core/blockchain_neotest_test.go @@ -299,6 +299,71 @@ func TestBlockchain_StartFromExistingDB(t *testing.T) { }) } +// This test enables Notary native contract at non-zero height and checks that no +// Notary cache initialization is performed before that height on node restart. +func TestBlockchain_InitializeNativeCacheWrtNativeActivations(t *testing.T) { + const notaryEnabledHeight = 3 + ps, path := newLevelDBForTestingWithPath(t, "") + customConfig := func(c *config.Blockchain) { + c.P2PSigExtensions = true + c.NativeUpdateHistories = make(map[string][]uint32) + for _, n := range []string{ + nativenames.Neo, + nativenames.Gas, + nativenames.Designation, + nativenames.Management, + nativenames.CryptoLib, + nativenames.Ledger, + nativenames.Management, + nativenames.Oracle, + nativenames.Policy, + nativenames.StdLib, + nativenames.Notary, + } { + if n == nativenames.Notary { + c.NativeUpdateHistories[n] = []uint32{notaryEnabledHeight} + } else { + c.NativeUpdateHistories[n] = []uint32{0} + } + } + } + bc, validators, committee, err := chain.NewMultiWithCustomConfigAndStoreNoCheck(t, customConfig, ps) + require.NoError(t, err) + go bc.Run() + e := neotest.NewExecutor(t, bc, validators, committee) + e.AddNewBlock(t) + bc.Close() // Ensure persist is done and persistent store is properly closed. + + ps, _ = newLevelDBForTestingWithPath(t, path) + + // If NativeActivations are not taken into account during native cache initialization, + // bs.init() will panic on Notary cache initialization as it's not deployed yet. + require.NotPanics(t, func() { + bc, _, _, err = chain.NewMultiWithCustomConfigAndStoreNoCheck(t, customConfig, ps) + require.NoError(t, err) + }) + go bc.Run() + defer bc.Close() + e = neotest.NewExecutor(t, bc, validators, committee) + h := e.Chain.BlockHeight() + + // Notary isn't initialized yet, so accessing Notary cache should panic. + require.Panics(t, func() { + _ = e.Chain.GetMaxNotValidBeforeDelta() + }) + + // Ensure Notary will be properly initialized and accessing Notary cache works + // as expected. + for i := 0; i < notaryEnabledHeight; i++ { + require.NotPanics(t, func() { + e.AddNewBlock(t) + }, h+uint32(i)+1) + } + require.NotPanics(t, func() { + _ = e.Chain.GetMaxNotValidBeforeDelta() + }) +} + func TestBlockchain_AddHeaders(t *testing.T) { bc, acc := chain.NewSingleWithCustomConfig(t, func(c *config.Blockchain) { c.StateRootInHeader = true From d5603959858d0221214f0ba4e093bea414153541 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 26 Apr 2023 14:10:12 +0300 Subject: [PATCH 16/29] core: prevent direct access to Notary contract if not active Otherwise it will cause panic, which isn't expected behaviour. Signed-off-by: Anna Shaleva --- pkg/core/blockchain.go | 14 ++++++++++---- pkg/core/blockchain_neotest_test.go | 20 +++++++++++--------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/pkg/core/blockchain.go b/pkg/core/blockchain.go index 9ac21f6720..51a5f1d2f4 100644 --- a/pkg/core/blockchain.go +++ b/pkg/core/blockchain.go @@ -2447,7 +2447,10 @@ func (bc *Blockchain) verifyTxAttributes(d *dao.Simple, tx *transaction.Transact nvb := tx.Attributes[i].Value.(*transaction.NotValidBefore).Height curHeight := bc.BlockHeight() if isPartialTx { - maxNVBDelta := bc.contracts.Notary.GetMaxNotValidBeforeDelta(bc.dao) + maxNVBDelta, err := bc.GetMaxNotValidBeforeDelta() + if err != nil { + return fmt.Errorf("%w: failed to retrieve MaxNotValidBeforeDelta value from native Notary contract: %v", ErrInvalidAttribute, err) + } if curHeight+maxNVBDelta < nvb { return fmt.Errorf("%w: NotValidBefore (%d) bigger than MaxNVBDelta (%d) allows at height %d", ErrInvalidAttribute, nvb, maxNVBDelta, curHeight) } @@ -2852,11 +2855,14 @@ func (bc *Blockchain) GetMaxVerificationGAS() int64 { } // GetMaxNotValidBeforeDelta returns maximum NotValidBeforeDelta Notary limit. -func (bc *Blockchain) GetMaxNotValidBeforeDelta() uint32 { +func (bc *Blockchain) GetMaxNotValidBeforeDelta() (uint32, error) { if !bc.config.P2PSigExtensions { - panic("disallowed call to Notary") + panic("disallowed call to Notary") // critical error, thus panic. + } + if bc.contracts.Notary.Metadata().UpdateHistory[0] > bc.BlockHeight() { + return 0, fmt.Errorf("native Notary is active starting from %d", bc.contracts.Notary.Metadata().UpdateHistory[0]) } - return bc.contracts.Notary.GetMaxNotValidBeforeDelta(bc.dao) + return bc.contracts.Notary.GetMaxNotValidBeforeDelta(bc.dao), nil } // GetStoragePrice returns current storage price. diff --git a/pkg/core/blockchain_neotest_test.go b/pkg/core/blockchain_neotest_test.go index 6be7aac8d8..9180435d01 100644 --- a/pkg/core/blockchain_neotest_test.go +++ b/pkg/core/blockchain_neotest_test.go @@ -347,10 +347,9 @@ func TestBlockchain_InitializeNativeCacheWrtNativeActivations(t *testing.T) { e = neotest.NewExecutor(t, bc, validators, committee) h := e.Chain.BlockHeight() - // Notary isn't initialized yet, so accessing Notary cache should panic. - require.Panics(t, func() { - _ = e.Chain.GetMaxNotValidBeforeDelta() - }) + // Notary isn't initialized yet, so accessing Notary cache should return error. + _, err = e.Chain.GetMaxNotValidBeforeDelta() + require.Error(t, err) // Ensure Notary will be properly initialized and accessing Notary cache works // as expected. @@ -359,9 +358,8 @@ func TestBlockchain_InitializeNativeCacheWrtNativeActivations(t *testing.T) { e.AddNewBlock(t) }, h+uint32(i)+1) } - require.NotPanics(t, func() { - _ = e.Chain.GetMaxNotValidBeforeDelta() - }) + _, err = e.Chain.GetMaxNotValidBeforeDelta() + require.NoError(t, err) } func TestBlockchain_AddHeaders(t *testing.T) { @@ -1952,11 +1950,15 @@ func TestBlockchain_VerifyTx(t *testing.T) { require.Error(t, bc.PoolTxWithData(tx, 5, mp, bc, verificationF)) }) t.Run("bad NVB: too big", func(t *testing.T) { - tx := getPartiallyFilledTx(bc.BlockHeight()+bc.GetMaxNotValidBeforeDelta()+1, bc.BlockHeight()+1) + maxNVB, err := bc.GetMaxNotValidBeforeDelta() + require.NoError(t, err) + tx := getPartiallyFilledTx(bc.BlockHeight()+maxNVB+1, bc.BlockHeight()+1) require.True(t, errors.Is(bc.PoolTxWithData(tx, 5, mp, bc, verificationF), core.ErrInvalidAttribute)) }) t.Run("bad ValidUntilBlock: too small", func(t *testing.T) { - tx := getPartiallyFilledTx(bc.BlockHeight(), bc.BlockHeight()+bc.GetMaxNotValidBeforeDelta()+1) + maxNVB, err := bc.GetMaxNotValidBeforeDelta() + require.NoError(t, err) + tx := getPartiallyFilledTx(bc.BlockHeight(), bc.BlockHeight()+maxNVB+1) require.True(t, errors.Is(bc.PoolTxWithData(tx, 5, mp, bc, verificationF), core.ErrInvalidAttribute)) }) t.Run("good", func(t *testing.T) { From 6abf5f6388ef1f8727fb33764a99f4ee8bc13957 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Fri, 12 May 2023 16:53:57 +0300 Subject: [PATCH 17/29] cli: allow to provide network-specific node config file Close #2978. Signed-off-by: Anna Shaleva --- cli/options/options.go | 21 ++++++++++++++++----- cli/server/server.go | 10 +++++----- cli/server/server_test.go | 26 +++++++++++++++++++------- cli/vm/vm.go | 2 +- docs/cli.md | 4 ++++ 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/cli/options/options.go b/cli/options/options.go index e5e5f45b37..09bb08572c 100644 --- a/cli/options/options.go +++ b/cli/options/options.go @@ -35,9 +35,9 @@ const RPCEndpointFlag = "rpc-endpoint" // Network is a set of flags for choosing the network to operate on // (privnet/mainnet/testnet). var Network = []cli.Flag{ - cli.BoolFlag{Name: "privnet, p", Usage: "use private network configuration"}, - cli.BoolFlag{Name: "mainnet, m", Usage: "use mainnet network configuration"}, - cli.BoolFlag{Name: "testnet, t", Usage: "use testnet network configuration"}, + cli.BoolFlag{Name: "privnet, p", Usage: "use private network configuration (if --config-file option is not specified)"}, + cli.BoolFlag{Name: "mainnet, m", Usage: "use mainnet network configuration (if --config-file option is not specified)"}, + cli.BoolFlag{Name: "testnet, t", Usage: "use testnet network configuration (if --config-file option is not specified)"}, cli.BoolFlag{Name: "unittest", Hidden: true}, } @@ -63,7 +63,14 @@ var Historic = cli.StringFlag{ // Config is a flag for commands that use node configuration. var Config = cli.StringFlag{ Name: "config-path", - Usage: "path to directory with configuration files", + Usage: "path to directory with per-network configuration files (may be overridden by --config-file option for the configuration file)", +} + +// ConfigFile is a flag for commands that use node configuration and provide +// path to the specific config file instead of config path. +var ConfigFile = cli.StringFlag{ + Name: "config-file", + Usage: "path to the node configuration file (overrides --config-path option)", } // Debug is a flag for commands that allow node in debug mode usage. @@ -152,7 +159,11 @@ func GetRPCWithInvoker(gctx context.Context, ctx *cli.Context, signers []transac // GetConfigFromContext looks at the path and the mode flags in the given config and // returns an appropriate config. func GetConfigFromContext(ctx *cli.Context) (config.Config, error) { - configPath := "./config" + var configFile = ctx.String("config-file") + if len(configFile) != 0 { + return config.LoadFile(configFile) + } + var configPath = "./config" if argCp := ctx.String("config-path"); argCp != "" { configPath = argCp } diff --git a/cli/server/server.go b/cli/server/server.go index 6b87e17667..9d139ae690 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -34,7 +34,7 @@ import ( // NewCommands returns 'node' command. func NewCommands() []cli.Command { - cfgFlags := []cli.Flag{options.Config} + cfgFlags := []cli.Flag{options.Config, options.ConfigFile} cfgFlags = append(cfgFlags, options.Network...) var cfgWithCountFlags = make([]cli.Flag, len(cfgFlags)) copy(cfgWithCountFlags, cfgFlags) @@ -85,7 +85,7 @@ func NewCommands() []cli.Command { { Name: "node", Usage: "start a NeoGo node", - UsageText: "neo-go node [--config-path path] [-d] [-p/-m/-t]", + UsageText: "neo-go node [--config-path path] [-d] [-p/-m/-t] [--config-file file]", Action: startServer, Flags: cfgFlags, }, @@ -96,21 +96,21 @@ func NewCommands() []cli.Command { { Name: "dump", Usage: "dump blocks (starting with block #1) to the file", - UsageText: "neo-go db dump -o file [-s start] [-c count] [--config-path path] [-p/-m/-t]", + UsageText: "neo-go db dump -o file [-s start] [-c count] [--config-path path] [-p/-m/-t] [--config-file file]", Action: dumpDB, Flags: cfgCountOutFlags, }, { Name: "restore", Usage: "restore blocks from the file", - UsageText: "neo-go db restore -i file [--dump] [-n] [-c count] [--config-path path] [-p/-m/-t]", + UsageText: "neo-go db restore -i file [--dump] [-n] [-c count] [--config-path path] [-p/-m/-t] [--config-file file]", Action: restoreDB, Flags: cfgCountInFlags, }, { Name: "reset", Usage: "reset database to the previous state", - UsageText: "neo-go db reset --height height [--config-path path] [-p/-m/-t]", + UsageText: "neo-go db reset --height height [--config-path path] [-p/-m/-t] [--config-file file]", Action: resetDB, Flags: cfgHeightFlags, }, diff --git a/cli/server/server_test.go b/cli/server/server_test.go index 5a7a36948a..62ff9b11ba 100644 --- a/cli/server/server_test.go +++ b/cli/server/server_test.go @@ -30,13 +30,25 @@ func init() { } func TestGetConfigFromContext(t *testing.T) { - set := flag.NewFlagSet("flagSet", flag.ExitOnError) - set.String("config-path", "../../config", "") - set.Bool("testnet", true, "") - ctx := cli.NewContext(cli.NewApp(), set, nil) - cfg, err := options.GetConfigFromContext(ctx) - require.NoError(t, err) - require.Equal(t, netmode.TestNet, cfg.ProtocolConfiguration.Magic) + t.Run("config-path", func(t *testing.T) { + set := flag.NewFlagSet("flagSet", flag.ExitOnError) + set.String("config-path", "../../config", "") + set.Bool("testnet", true, "") + ctx := cli.NewContext(cli.NewApp(), set, nil) + cfg, err := options.GetConfigFromContext(ctx) + require.NoError(t, err) + require.Equal(t, netmode.TestNet, cfg.ProtocolConfiguration.Magic) + }) + t.Run("config-file", func(t *testing.T) { + set := flag.NewFlagSet("flagSet", flag.ExitOnError) + set.String("config-path", "../../config", "") + set.Bool("testnet", true, "") + set.String("config-file", "../../config/protocol.testnet.yml", "") + ctx := cli.NewContext(cli.NewApp(), set, nil) + cfg, err := options.GetConfigFromContext(ctx) + require.NoError(t, err) + require.Equal(t, netmode.TestNet, cfg.ProtocolConfiguration.Magic) + }) } func TestHandleLoggingParams(t *testing.T) { diff --git a/cli/vm/vm.go b/cli/vm/vm.go index 23b9938dee..89dd351bb9 100644 --- a/cli/vm/vm.go +++ b/cli/vm/vm.go @@ -13,7 +13,7 @@ import ( // NewCommands returns 'vm' command. func NewCommands() []cli.Command { - cfgFlags := []cli.Flag{options.Config} + cfgFlags := []cli.Flag{options.Config, options.ConfigFile} cfgFlags = append(cfgFlags, options.Network...) return []cli.Command{{ Name: "vm", diff --git a/docs/cli.md b/docs/cli.md index 310ad95627..1f734ad073 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -35,6 +35,10 @@ If you want to use some non-default configuration directory path, specify The file loaded is chosen automatically depending on network mode flag. +Or just provide path to the configuration file using `--config-file` flag: + +`./bin/neo-go node --config-file /user/yourConfigPath/yourConfigFile.yml` + Refer to the [node configuration documentation](./node-configuration.md) for detailed configuration file description. From 4383832aa2af87b6aa527cac9ca17e7b676b88ec Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Mon, 5 Jun 2023 14:58:17 +0300 Subject: [PATCH 18/29] go.mod: upgrade bolt to v1.3.7 Refs. nspcc-dev/neofs-node#999, it fixes a number of other Windows-related issues as well. Signed-off-by: Roman Khimov --- go.mod | 4 ++-- go.sum | 46 +++------------------------------------------- 2 files changed, 5 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index 5a9e395f9f..92f8ded249 100644 --- a/go.mod +++ b/go.mod @@ -18,11 +18,11 @@ require ( github.com/pierrec/lz4 v2.6.1+incompatible github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.13.0 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 github.com/twmb/murmur3 v1.1.5 github.com/urfave/cli v1.22.5 - go.etcd.io/bbolt v1.3.6 + go.etcd.io/bbolt v1.3.7 go.uber.org/atomic v1.9.0 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.4.0 diff --git a/go.sum b/go.sum index 4ade3e148b..d5c9f78be1 100644 --- a/go.sum +++ b/go.sum @@ -342,6 +342,7 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -350,6 +351,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/syndtr/goleveldb v0.0.0-20180307113352-169b1b37be73/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= @@ -364,13 +366,11 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20190514113301-1cd887cd7036/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yuin/gopher-lua v0.0.0-20191128022950-c6266f4fe8d7/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -379,17 +379,12 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -402,9 +397,6 @@ golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -436,8 +428,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -473,14 +463,9 @@ golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -500,8 +485,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -550,25 +533,15 @@ golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210429154555-c04ba851c2a4/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -577,9 +550,6 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -627,13 +597,10 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -686,7 +653,6 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 h1:PDIOdWxZ8eRizhKa1AAvY53xsvLB1cWorMjslvY3VA8= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -702,7 +668,6 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -717,17 +682,14 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/abiosoft/ishell.v2 v2.0.0/go.mod h1:sFp+cGtH6o4s1FtpVPTMcHq2yue+c4DGOVohJCPUzwY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -736,11 +698,9 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From b560475e67adb168c424cf121b50d451845803c3 Mon Sep 17 00:00:00 2001 From: Roman Khimov Date: Thu, 29 Jun 2023 11:41:04 +0300 Subject: [PATCH 19/29] server: allow to create incremental dumps Anything not starting from 0 is incremental by definition. Signed-off-by: Roman Khimov --- cli/server/server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/server/server.go b/cli/server/server.go index 9d139ae690..39e67739ca 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -198,6 +198,9 @@ func dumpDB(ctx *cli.Context) error { if count == 0 { count = chainCount - start } + if start != 0 { + writer.WriteU32LE(start) + } writer.WriteU32LE(count) err = chaindump.Dump(chain, writer, start, count) if err != nil { From ebe4c4ce2b96810d6d9f53a225ee15bb287c106a Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Mon, 3 Jul 2023 13:24:11 +0300 Subject: [PATCH 20/29] cli: add test for incremental DB dump and restore Signed-off-by: Anna Shaleva --- cli/server/cli_dump_test.go | 49 +++++++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/cli/server/cli_dump_test.go b/cli/server/cli_dump_test.go index b743f90155..f7756add53 100644 --- a/cli/server/cli_dump_test.go +++ b/cli/server/cli_dump_test.go @@ -13,6 +13,9 @@ import ( "gopkg.in/yaml.v3" ) +// generated via `go run ./scripts/gendump/main.go --out ./cli/server/testdata/chain50x2.acc --blocks 50 --txs 2`. +const inDump = "./testdata/chain50x2.acc" + func TestDBRestoreDump(t *testing.T) { tmpDir := t.TempDir() @@ -32,8 +35,6 @@ func TestDBRestoreDump(t *testing.T) { cfgPath := filepath.Join(tmpDir, "protocol.unit_testnet.yml") require.NoError(t, os.WriteFile(cfgPath, out, os.ModePerm)) - // generated via `go run ./scripts/gendump/main.go --out ./cli/server/testdata/chain50x2.acc --blocks 50 --txs 2` - const inDump = "./testdata/chain50x2.acc" e := testcli.NewExecutor(t, false) stateDump := filepath.Join(tmpDir, "neogo.teststate") @@ -111,3 +112,47 @@ func TestDBRestoreDump(t *testing.T) { require.NoError(t, err) require.Equal(t, d1, d2, "dumps differ") } + +func TestDBDumpRestoreIncremental(t *testing.T) { + tmpDir := t.TempDir() + chainPath := filepath.Join(tmpDir, "neogotestchain") + nonincDump := filepath.Join(tmpDir, "nonincDump.acc") + incDump := filepath.Join(tmpDir, "incDump.acc") + + cfg, err := config.LoadFile(filepath.Join("..", "..", "config", "protocol.unit_testnet.yml")) + require.NoError(t, err, "could not load config") + cfg.ApplicationConfiguration.DBConfiguration.Type = dbconfig.LevelDB + cfg.ApplicationConfiguration.DBConfiguration.LevelDBOptions.DataDirectoryPath = chainPath + out, err := yaml.Marshal(cfg) + require.NoError(t, err) + + cfgPath := filepath.Join(tmpDir, "protocol.unit_testnet.yml") + require.NoError(t, os.WriteFile(cfgPath, out, os.ModePerm)) + + e := testcli.NewExecutor(t, false) + + // Create DB from dump. + e.Run(t, "neo-go", "db", "restore", "--unittest", "--config-path", tmpDir, "--in", inDump) + + // Create two dumps: non-incremental and incremental. + dumpBaseArgs := []string{"neo-go", "db", "dump", "--unittest", + "--config-path", tmpDir} + + // Dump first 15 blocks to a non-incremental dump. + e.Run(t, append(dumpBaseArgs, "--out", nonincDump, "--count", "15")...) + + // Dump second 15 blocks to an incremental dump. + e.Run(t, append(dumpBaseArgs, "--out", incDump, "--start", "15", "--count", "15")...) + + // Clean the DB. + require.NoError(t, os.RemoveAll(chainPath)) + + // Restore chain from two dumps. + restoreBaseArgs := []string{"neo-go", "db", "restore", "--unittest", "--config-path", tmpDir} + + // Restore first 15 blocks from non-incremental dump. + e.Run(t, append(restoreBaseArgs, "--in", nonincDump)...) + + // Restore second 15 blocks from incremental dump. + e.Run(t, append(restoreBaseArgs, "--in", incDump, "-n", "--count", "15")...) +} From e30e262e6644b180023f25965e474a84c9f2560a Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Thu, 20 Jul 2023 10:55:11 +0300 Subject: [PATCH 21/29] network: forbid Notary contract to be a sender of main transaction This prevents the possible attack on notary request sender when malicious partie is allowed to send notary request with main transaction being someone else's fallback. Signed-off-by: Anna Shaleva --- pkg/network/server.go | 3 +++ pkg/network/server_test.go | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkg/network/server.go b/pkg/network/server.go index b518bfc09e..90de8d3a6e 100644 --- a/pkg/network/server.go +++ b/pkg/network/server.go @@ -1199,6 +1199,9 @@ func (s *Server) verifyNotaryRequest(_ *transaction.Transaction, data interface{ if r.FallbackTransaction.Sender() != notaryHash { return fmt.Errorf("P2PNotary contract should be a sender of the fallback transaction, got %s", address.Uint160ToString(r.FallbackTransaction.Sender())) } + if r.MainTransaction.Sender() == notaryHash { + return errors.New("P2PNotary contract is not allowed to be the sender of the main transaction") + } depositExpiration := s.chain.GetNotaryDepositExpiration(payer) if r.FallbackTransaction.ValidUntilBlock >= depositExpiration { return fmt.Errorf("fallback transaction is valid after deposit is unlocked: ValidUntilBlock is %d, deposit lock for %s expires at %d", r.FallbackTransaction.ValidUntilBlock, address.Uint160ToString(payer), depositExpiration) diff --git a/pkg/network/server_test.go b/pkg/network/server_test.go index 4b95eee4ac..4d19679881 100644 --- a/pkg/network/server_test.go +++ b/pkg/network/server_test.go @@ -1036,7 +1036,10 @@ func TestVerifyNotaryRequest(t *testing.T) { require.NoError(t, err) newNotaryRequest := func() *payload.P2PNotaryRequest { return &payload.P2PNotaryRequest{ - MainTransaction: &transaction.Transaction{Script: []byte{0, 1, 2}}, + MainTransaction: &transaction.Transaction{ + Script: []byte{0, 1, 2}, + Signers: []transaction.Signer{{Account: random.Uint160()}}, + }, FallbackTransaction: &transaction.Transaction{ ValidUntilBlock: 321, Signers: []transaction.Signer{{Account: bc.NotaryContractScriptHash}, {Account: random.Uint160()}}, @@ -1057,6 +1060,13 @@ func TestVerifyNotaryRequest(t *testing.T) { require.Error(t, s.verifyNotaryRequest(nil, r)) }) + t.Run("bad main sender", func(t *testing.T) { + bc.VerifyWitnessF = func() (int64, error) { return 0, nil } + r := newNotaryRequest() + r.MainTransaction.Signers[0] = transaction.Signer{Account: bc.NotaryContractScriptHash} + require.Error(t, s.verifyNotaryRequest(nil, r)) + }) + t.Run("expired deposit", func(t *testing.T) { r := newNotaryRequest() bc.NotaryDepositExpiration = r.FallbackTransaction.ValidUntilBlock From 8db997c58ab9e43247542309f37502e72f108822 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 12 Jul 2023 08:33:26 +0300 Subject: [PATCH 22/29] neorpc: adjust SignerWithWitness scopes parsing Ensure that Scopes can be properly parsed not only from the string representation, but also from a single byte. transaction.Signer is not affected (checked against the C# implementation), only RPC-related signer scopes are allowed to be unmarshalled from byte. Close #3059. Signed-off-by: Anna Shaleva --- pkg/core/transaction/witness_scope.go | 14 +++ pkg/core/transaction/witness_scope_test.go | 62 +++++++++++++ pkg/neorpc/types.go | 32 ++++++- pkg/services/rpcsrv/params/param_test.go | 102 ++++++++++++++++----- 4 files changed, 185 insertions(+), 25 deletions(-) diff --git a/pkg/core/transaction/witness_scope.go b/pkg/core/transaction/witness_scope.go index aa67eb33a9..c1cf2b7d86 100644 --- a/pkg/core/transaction/witness_scope.go +++ b/pkg/core/transaction/witness_scope.go @@ -3,6 +3,7 @@ package transaction //go:generate stringer -type=WitnessScope -linecomment -output=witness_scope_string.go import ( "encoding/json" + "errors" "fmt" "strings" ) @@ -29,6 +30,19 @@ const ( Global WitnessScope = 0x80 ) +// ScopesFromByte converts byte to a set of WitnessScopes and performs validity +// check. +func ScopesFromByte(b byte) (WitnessScope, error) { + var res = WitnessScope(b) + if (res&Global != 0) && (res&(None|CalledByEntry|CustomContracts|CustomGroups|Rules) != 0) { + return 0, errors.New("Global scope can not be combined with other scopes") + } + if res&^(None|CalledByEntry|CustomContracts|CustomGroups|Rules|Global) != 0 { + return 0, fmt.Errorf("invalid scope %d", res) + } + return res, nil +} + // ScopesFromString converts string of comma-separated scopes to a set of scopes // (case-sensitive). String can combine several scopes, e.g. be any of: 'Global', // 'CalledByEntry,CustomGroups' etc. In case of an empty string an error will be diff --git a/pkg/core/transaction/witness_scope_test.go b/pkg/core/transaction/witness_scope_test.go index 42a672e686..aea706a080 100644 --- a/pkg/core/transaction/witness_scope_test.go +++ b/pkg/core/transaction/witness_scope_test.go @@ -1,6 +1,7 @@ package transaction import ( + "strconv" "testing" "github.com/stretchr/testify/require" @@ -51,3 +52,64 @@ func TestScopesFromString(t *testing.T) { require.NoError(t, err) require.Equal(t, CalledByEntry|CustomGroups|CustomContracts, s) } + +func TestScopesFromByte(t *testing.T) { + testCases := []struct { + in byte + expected WitnessScope + shouldFail bool + }{ + { + in: 0, + expected: None, + }, + { + in: 1, + expected: CalledByEntry, + }, + { + in: 16, + expected: CustomContracts, + }, + { + in: 32, + expected: CustomGroups, + }, + { + in: 64, + expected: Rules, + }, + { + in: 128, + expected: Global, + }, + { + in: 17, + expected: CalledByEntry | CustomContracts, + }, + { + in: 48, + expected: CustomContracts | CustomGroups, + }, + { + in: 128 + 1, // Global can't be combined with others. + shouldFail: true, + }, + { + in: 2, // No such scope. + shouldFail: true, + }, + } + + for _, tc := range testCases { + t.Run(strconv.Itoa(int(tc.in)), func(t *testing.T) { + actual, err := ScopesFromByte(tc.in) + if tc.shouldFail { + require.Error(t, err, tc.in) + } else { + require.NoError(t, err, tc.in) + require.Equal(t, tc.expected, actual, tc.in) + } + }) + } +} diff --git a/pkg/neorpc/types.go b/pkg/neorpc/types.go index 223cd591c0..9c60a7b97d 100644 --- a/pkg/neorpc/types.go +++ b/pkg/neorpc/types.go @@ -82,7 +82,7 @@ type ( // DisallowUnknownFields JSON marshaller setting. type signerWithWitnessAux struct { Account string `json:"account"` - Scopes transaction.WitnessScope `json:"scopes"` + Scopes json.RawMessage `json:"scopes"` AllowedContracts []util.Uint160 `json:"allowedcontracts,omitempty"` AllowedGroups []*keys.PublicKey `json:"allowedgroups,omitempty"` Rules []transaction.WitnessRule `json:"rules,omitempty"` @@ -92,9 +92,13 @@ type signerWithWitnessAux struct { // MarshalJSON implements the json.Marshaler interface. func (s *SignerWithWitness) MarshalJSON() ([]byte, error) { + sc, err := s.Scopes.MarshalJSON() + if err != nil { + return nil, fmt.Errorf("failed to marshal scopes: %w", err) + } signer := &signerWithWitnessAux{ Account: s.Account.StringLE(), - Scopes: s.Scopes, + Scopes: sc, AllowedContracts: s.AllowedContracts, AllowedGroups: s.AllowedGroups, Rules: s.Rules, @@ -118,9 +122,31 @@ func (s *SignerWithWitness) UnmarshalJSON(data []byte) error { if err != nil { return fmt.Errorf("not a signer: %w", err) } + var ( + jStr string + jByte byte + scopes transaction.WitnessScope + ) + if len(aux.Scopes) != 0 { + if err := json.Unmarshal(aux.Scopes, &jStr); err == nil { + scopes, err = transaction.ScopesFromString(jStr) + if err != nil { + return fmt.Errorf("failed to retrieve scopes from string: %w", err) + } + } else { + err := json.Unmarshal(aux.Scopes, &jByte) + if err != nil { + return fmt.Errorf("failed to unmarshal scopes from byte: %w", err) + } + scopes, err = transaction.ScopesFromByte(jByte) + if err != nil { + return fmt.Errorf("failed to retrieve scopes from byte: %w", err) + } + } + } s.Signer = transaction.Signer{ Account: acc, - Scopes: aux.Scopes, + Scopes: scopes, AllowedContracts: aux.AllowedContracts, AllowedGroups: aux.AllowedGroups, Rules: aux.Rules, diff --git a/pkg/services/rpcsrv/params/param_test.go b/pkg/services/rpcsrv/params/param_test.go index af7c0bbc10..dfe0d2f3f1 100644 --- a/pkg/services/rpcsrv/params/param_test.go +++ b/pkg/services/rpcsrv/params/param_test.go @@ -243,38 +243,96 @@ func TestGetWitness(t *testing.T) { require.NoError(t, err) testCases := []struct { - raw string - expected neorpc.SignerWithWitness + raw string + expected neorpc.SignerWithWitness + shouldFail bool }{ - {`{"account": "0xcadb3dc2faa3ef14a13b619c9a43124755aa2569"}`, neorpc.SignerWithWitness{ - Signer: transaction.Signer{ - Account: accountHash, - Scopes: transaction.None, - }}, + { + raw: `{"account": "0xcadb3dc2faa3ef14a13b619c9a43124755aa2569"}`, + expected: neorpc.SignerWithWitness{ + Signer: transaction.Signer{ + Account: accountHash, + Scopes: transaction.None, + }, + }, }, - {`{"account": "NYxb4fSZVKAz8YsgaPK2WkT3KcAE9b3Vag", "scopes": "Global"}`, neorpc.SignerWithWitness{ - Signer: transaction.Signer{ - Account: addrHash, - Scopes: transaction.Global, - }}, + { + raw: `{"account": "NYxb4fSZVKAz8YsgaPK2WkT3KcAE9b3Vag", "scopes": "Global"}`, + expected: neorpc.SignerWithWitness{ + Signer: transaction.Signer{ + Account: addrHash, + Scopes: transaction.Global, + }, + }, }, - {`{"account": "0xcadb3dc2faa3ef14a13b619c9a43124755aa2569", "scopes": "Global"}`, neorpc.SignerWithWitness{ - Signer: transaction.Signer{ - Account: accountHash, - Scopes: transaction.Global, - }}, + { + raw: `{"account": "0xcadb3dc2faa3ef14a13b619c9a43124755aa2569", "scopes": "Global"}`, + expected: neorpc.SignerWithWitness{ + Signer: transaction.Signer{ + Account: accountHash, + Scopes: transaction.Global, + }, + }, + }, + { + raw: `{"account": "0xcadb3dc2faa3ef14a13b619c9a43124755aa2569", "scopes": 128}`, + expected: neorpc.SignerWithWitness{ + Signer: transaction.Signer{ + Account: accountHash, + Scopes: transaction.Global, + }, + }, + }, + { + raw: `{"account": "0xcadb3dc2faa3ef14a13b619c9a43124755aa2569", "scopes": 0}`, + expected: neorpc.SignerWithWitness{ + Signer: transaction.Signer{ + Account: accountHash, + Scopes: transaction.None, + }, + }, + }, + { + raw: `{"account": "0xcadb3dc2faa3ef14a13b619c9a43124755aa2569", "scopes": 1}`, + expected: neorpc.SignerWithWitness{ + Signer: transaction.Signer{ + Account: accountHash, + Scopes: transaction.CalledByEntry, + }, + }, + }, + { + raw: `{"account": "0xcadb3dc2faa3ef14a13b619c9a43124755aa2569", "scopes": 17}`, + expected: neorpc.SignerWithWitness{ + Signer: transaction.Signer{ + Account: accountHash, + Scopes: transaction.CalledByEntry | transaction.CustomContracts, + }, + }, + }, + { + raw: `{"account": "0xcadb3dc2faa3ef14a13b619c9a43124755aa2569", "scopes": 178}`, + shouldFail: true, + }, + { + raw: `{"account": "0xcadb3dc2faa3ef14a13b619c9a43124755aa2569", "scopes": 2}`, + shouldFail: true, }, } for _, tc := range testCases { p := Param{RawMessage: json.RawMessage(tc.raw)} actual, err := p.GetSignerWithWitness() - require.NoError(t, err) - require.Equal(t, tc.expected, actual) + if tc.shouldFail { + require.Error(t, err, tc.raw) + } else { + require.NoError(t, err, tc.raw) + require.Equal(t, tc.expected, actual) - actual, err = p.GetSignerWithWitness() // valid second invocation. - require.NoError(t, err) - require.Equal(t, tc.expected, actual) + actual, err = p.GetSignerWithWitness() // valid second invocation. + require.NoError(t, err, tc.raw) + require.Equal(t, tc.expected, actual) + } } } From 96aa10bb80013bd096d8534fe1d8b38f45ae8404 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 12 Jul 2023 08:39:15 +0300 Subject: [PATCH 23/29] neorpc: adjust the way SignerWithWitness is marshalled Marshal account with `0x` prefix and add a test. Signed-off-by: Anna Shaleva --- pkg/neorpc/types.go | 2 +- pkg/neorpc/types_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 pkg/neorpc/types_test.go diff --git a/pkg/neorpc/types.go b/pkg/neorpc/types.go index 9c60a7b97d..225cd28fc5 100644 --- a/pkg/neorpc/types.go +++ b/pkg/neorpc/types.go @@ -97,7 +97,7 @@ func (s *SignerWithWitness) MarshalJSON() ([]byte, error) { return nil, fmt.Errorf("failed to marshal scopes: %w", err) } signer := &signerWithWitnessAux{ - Account: s.Account.StringLE(), + Account: `0x` + s.Account.StringLE(), Scopes: sc, AllowedContracts: s.AllowedContracts, AllowedGroups: s.AllowedGroups, diff --git a/pkg/neorpc/types_test.go b/pkg/neorpc/types_test.go new file mode 100644 index 0000000000..f0c99ea231 --- /dev/null +++ b/pkg/neorpc/types_test.go @@ -0,0 +1,40 @@ +package neorpc + +import ( + "encoding/json" + "testing" + + "github.com/nspcc-dev/neo-go/internal/testserdes" + "github.com/nspcc-dev/neo-go/pkg/core/transaction" + "github.com/nspcc-dev/neo-go/pkg/util" + "github.com/stretchr/testify/require" +) + +func TestSignerWithWitnessMarshalUnmarshalJSON(t *testing.T) { + s := &SignerWithWitness{ + Signer: transaction.Signer{ + Account: util.Uint160{1, 2, 3}, + Scopes: transaction.CalledByEntry | transaction.CustomContracts, + AllowedContracts: []util.Uint160{{1, 2, 3, 4}}, + }, + Witness: transaction.Witness{ + InvocationScript: []byte{1, 2, 3}, + VerificationScript: []byte{4, 5, 6}, + }, + } + testserdes.MarshalUnmarshalJSON(t, s, &SignerWithWitness{}) + + // Check marshalling separately to ensure Scopes are marshalled OK. + expected := `{"account":"0xcadb3dc2faa3ef14a13b619c9a43124755aa2569","scopes":"CalledByEntry, CustomContracts"}` + acc, err := util.Uint160DecodeStringLE("cadb3dc2faa3ef14a13b619c9a43124755aa2569") + require.NoError(t, err) + s = &SignerWithWitness{ + Signer: transaction.Signer{ + Account: acc, + Scopes: transaction.CalledByEntry | transaction.CustomContracts, + }, + } + actual, err := json.Marshal(s) + require.NoError(t, err) + require.Equal(t, expected, string(actual)) +} From bf871760c6f602c345769a13085baec77370679e Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Wed, 12 Jul 2023 12:31:40 +0300 Subject: [PATCH 24/29] core: do not use formatted error if not needed Signed-off-by: Anna Shaleva --- pkg/core/transaction/witness_scope.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/core/transaction/witness_scope.go b/pkg/core/transaction/witness_scope.go index c1cf2b7d86..5c60e983b7 100644 --- a/pkg/core/transaction/witness_scope.go +++ b/pkg/core/transaction/witness_scope.go @@ -68,7 +68,7 @@ func ScopesFromString(s string) (WitnessScope, error) { return result, fmt.Errorf("invalid witness scope: %v", scopeStr) } if isGlobal && !(scope == Global) { - return result, fmt.Errorf("Global scope can not be combined with other scopes") + return result, errors.New("Global scope can not be combined with other scopes") } result |= scope if scope == Global { From c0abc61613cdf0497bdb32e820995833ca34e5d9 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Fri, 21 Jul 2023 13:18:08 +0300 Subject: [PATCH 25/29] smartcontract: allow to pass nil as parameter to (*Invoker).Call Signed-off-by: Anna Shaleva --- pkg/smartcontract/parameter.go | 2 ++ pkg/smartcontract/parameter_test.go | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/pkg/smartcontract/parameter.go b/pkg/smartcontract/parameter.go index 06556eca19..c591f5f0de 100644 --- a/pkg/smartcontract/parameter.go +++ b/pkg/smartcontract/parameter.go @@ -360,6 +360,8 @@ func NewParameterFromValue(value interface{}) (Parameter, error) { } result.Type = ArrayType result.Value = arr + case nil: + result.Type = AnyType default: return result, fmt.Errorf("unsupported parameter %T", value) } diff --git a/pkg/smartcontract/parameter_test.go b/pkg/smartcontract/parameter_test.go index 295ed8f932..2ec7a01087 100644 --- a/pkg/smartcontract/parameter_test.go +++ b/pkg/smartcontract/parameter_test.go @@ -690,6 +690,11 @@ func TestParameterFromValue(t *testing.T) { expType: PublicKeyType, expVal: pk2.PublicKey().Bytes(), }, + { + value: nil, + expType: AnyType, + expVal: nil, + }, { value: [][]byte{{1, 2, 3}, {3, 2, 1}}, expType: ArrayType, From 6615cce81d317e88053724952c9f3a4f7da20730 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Thu, 27 Jul 2023 16:28:19 +0300 Subject: [PATCH 26/29] rpcclient: adjust `unwrapContract` helper There may be no such contract, then Null stackitem is expected on stack. Signed-off-by: Anna Shaleva --- pkg/rpcclient/management/management.go | 9 ++++++++- pkg/rpcclient/management/management_test.go | 13 ++++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pkg/rpcclient/management/management.go b/pkg/rpcclient/management/management.go index 6c8e375f94..8f13776c26 100644 --- a/pkg/rpcclient/management/management.go +++ b/pkg/rpcclient/management/management.go @@ -105,16 +105,23 @@ func (c *ContractReader) GetContract(hash util.Uint160) (*state.Contract, error) return unwrapContract(c.invoker.Call(Hash, "getContract", hash)) } -// GetContractByID allows to get contract data from its ID. +// GetContractByID allows to get contract data from its ID. In case of missing +// contract it returns nil state.Contract and nil error. func (c *ContractReader) GetContractByID(id int32) (*state.Contract, error) { return unwrapContract(c.invoker.Call(Hash, "getContractById", id)) } +// unwrapContract tries to retrieve state.Contract from the provided result.Invoke. +// If the resulting stack contains stackitem.Null, then nil state and nil error +// will be returned. func unwrapContract(r *result.Invoke, err error) (*state.Contract, error) { itm, err := unwrap.Item(r, err) if err != nil { return nil, err } + if itm.Equals(stackitem.Null{}) { + return nil, nil + } res := new(state.Contract) err = res.FromStackItem(itm) if err != nil { diff --git a/pkg/rpcclient/management/management_test.go b/pkg/rpcclient/management/management_test.go index 31a54484d3..5ed8b89bc4 100644 --- a/pkg/rpcclient/management/management_test.go +++ b/pkg/rpcclient/management/management_test.go @@ -100,6 +100,17 @@ func TestReader(t *testing.T) { require.NoError(t, err) require.False(t, hm) + ta.res = &result.Invoke{ + State: "HALT", + Stack: []stackitem.Item{ + stackitem.Null{}, + }, + } + + cs, err := man.GetContract(util.Uint160{1, 2, 3}) + require.NoError(t, err) + require.Nil(t, cs) + ta.res = &result.Invoke{ State: "HALT", Stack: []stackitem.Item{ @@ -127,7 +138,7 @@ func TestReader(t *testing.T) { }), }, } - cs, err := man.GetContract(util.Uint160{1, 2, 3}) + cs, err = man.GetContract(util.Uint160{1, 2, 3}) require.NoError(t, err) require.Equal(t, int32(1), cs.ID) require.Equal(t, uint16(0), cs.UpdateCounter) From 8d5a41de6e132c71f051b5b1f4e0f1e4a00ce0de Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Thu, 27 Jul 2023 18:12:02 +0300 Subject: [PATCH 27/29] vm: allow parsing scientific JSON numbers 52-bit precision is not enough for our 256-bit VM, but this value matches the reference implementation, see the https://github.com/neo-project/neo/issues/2879. MaxIntegerPrec will be increased (or even removed) as soon as the ref. issue is resolved. Signed-off-by: Anna Shaleva --- pkg/vm/stackitem/json.go | 40 +++++++++++++++++++++++++++-------- pkg/vm/stackitem/json_test.go | 28 +++++++++++++++++++++++- pkg/vm/testdata/neo-vm | 2 +- 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/pkg/vm/stackitem/json.go b/pkg/vm/stackitem/json.go index d7b93f6237..508d91fbda 100644 --- a/pkg/vm/stackitem/json.go +++ b/pkg/vm/stackitem/json.go @@ -26,6 +26,11 @@ const MaxAllowedInteger = 2<<53 - 1 // MaxJSONDepth is the maximum allowed nesting level of an encoded/decoded JSON. const MaxJSONDepth = 10 +// MaxIntegerPrec is the maximum precision allowed for big.Integer parsing. +// It equals to the reference value and doesn't allow to precisely parse big +// numbers, see the https://github.com/neo-project/neo/issues/2879. +const MaxIntegerPrec = 53 + // ErrInvalidValue is returned when an item value doesn't fit some constraints // during serialization or deserialization. var ErrInvalidValue = errors.New("invalid value") @@ -213,18 +218,35 @@ func (d *decoder) decode() (Item, error) { return NewByteArray([]byte(t)), nil case json.Number: ts := t.String() - dot := strings.IndexByte(ts, '.') - if dot != -1 { - // As a special case numbers like 123.000 are allowed (SetString rejects them). - // And yes, that's the way C# code works also. - for _, r := range ts[dot+1:] { - if r != '0' { - return nil, fmt.Errorf("%w (real value for int)", ErrInvalidValue) + var ( + num *big.Int + ok bool + ) + isScientific := strings.Contains(ts, "e+") || strings.Contains(ts, "E+") + if isScientific { + // As a special case numbers like 2.8e+22 are allowed (SetString rejects them). + // That's the way how C# code works. + f, _, err := big.ParseFloat(ts, 10, MaxIntegerPrec, big.ToNearestEven) + if err != nil { + return nil, fmt.Errorf("%w (malformed exp value for int)", ErrInvalidValue) + } + num = new(big.Int) + _, acc := f.Int(num) + ok = acc == big.Exact + } else { + dot := strings.IndexByte(ts, '.') + if dot != -1 { + // As a special case numbers like 123.000 are allowed (SetString rejects them). + // And yes, that's the way C# code works also. + for _, r := range ts[dot+1:] { + if r != '0' { + return nil, fmt.Errorf("%w (real value for int)", ErrInvalidValue) + } } + ts = ts[:dot] } - ts = ts[:dot] + num, ok = new(big.Int).SetString(ts, 10) } - num, ok := new(big.Int).SetString(ts, 10) if !ok { return nil, fmt.Errorf("%w (integer)", ErrInvalidValue) } diff --git a/pkg/vm/stackitem/json_test.go b/pkg/vm/stackitem/json_test.go index c68db70f03..8ef6112fb0 100644 --- a/pkg/vm/stackitem/json_test.go +++ b/pkg/vm/stackitem/json_test.go @@ -10,6 +10,10 @@ import ( ) func getTestDecodeFunc(js string, expected ...interface{}) func(t *testing.T) { + return getTestDecodeEncodeFunc(js, true, expected...) +} + +func getTestDecodeEncodeFunc(js string, needEncode bool, expected ...interface{}) func(t *testing.T) { return func(t *testing.T) { actual, err := FromJSON([]byte(js), 20) if expected[0] == nil { @@ -19,7 +23,7 @@ func getTestDecodeFunc(js string, expected ...interface{}) func(t *testing.T) { require.NoError(t, err) require.Equal(t, Make(expected[0]), actual) - if len(expected) == 1 { + if needEncode && len(expected) == 1 { encoded, err := ToJSON(actual) require.NoError(t, err) require.Equal(t, js, string(encoded)) @@ -28,6 +32,8 @@ func getTestDecodeFunc(js string, expected ...interface{}) func(t *testing.T) { } func TestFromToJSON(t *testing.T) { + bigInt, ok := new(big.Int).SetString("28000000000000000000000", 10) + require.True(t, ok) t.Run("ByteString", func(t *testing.T) { t.Run("Empty", getTestDecodeFunc(`""`, []byte{})) t.Run("Base64", getTestDecodeFunc(`"test"`, "test")) @@ -36,6 +42,8 @@ func TestFromToJSON(t *testing.T) { t.Run("BigInteger", func(t *testing.T) { t.Run("ZeroFloat", getTestDecodeFunc(`12.000`, 12, nil)) t.Run("NonZeroFloat", getTestDecodeFunc(`12.01`, nil)) + t.Run("ExpInteger", getTestDecodeEncodeFunc(`2.8e+22`, false, bigInt)) + t.Run("ExpFloat", getTestDecodeEncodeFunc(`1.2345e+3`, false, nil)) // float value, parsing should fail for it. t.Run("Negative", getTestDecodeFunc(`-4`, -4)) t.Run("Positive", getTestDecodeFunc(`123`, 123)) }) @@ -123,6 +131,24 @@ func TestFromToJSON(t *testing.T) { }) } +// TestFromJSON_CompatBigInt ensures that maximum BigInt parsing precision matches +// the C# one, ref. https://github.com/neo-project/neo/issues/2879. +func TestFromJSON_CompatBigInt(t *testing.T) { + tcs := map[string]string{ + `9.05e+28`: "90499999999999993918259200000", + `1.871e+21`: "1871000000000000000000", + `3.0366e+32`: "303660000000000004445016810323968", + `1e+30`: "1000000000000000019884624838656", + } + for in, expected := range tcs { + t.Run(in, func(t *testing.T) { + actual, err := FromJSON([]byte(in), 5) + require.NoError(t, err) + require.Equal(t, expected, actual.Value().(*big.Int).String()) + }) + } +} + func testToJSON(t *testing.T, expectedErr error, item Item) { data, err := ToJSON(item) if expectedErr != nil { diff --git a/pkg/vm/testdata/neo-vm b/pkg/vm/testdata/neo-vm index 02f2c68e7b..7e5996844a 160000 --- a/pkg/vm/testdata/neo-vm +++ b/pkg/vm/testdata/neo-vm @@ -1 +1 @@ -Subproject commit 02f2c68e7ba2694aff88c143631e7acf158d378a +Subproject commit 7e5996844a90b514739f879bc9f873f9a34c9a67 From e2c6bbb6b146f76cacdd8b159323bdd0a160ae5a Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 1 Aug 2023 15:46:01 +0300 Subject: [PATCH 28/29] rpcsrv: properly set content-type and CORS for all headers Not only for successful ones. Close #3075. Signed-off-by: Anna Shaleva --- pkg/services/rpcsrv/server.go | 8 ++++---- pkg/services/rpcsrv/server_test.go | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/pkg/services/rpcsrv/server.go b/pkg/services/rpcsrv/server.go index 0ba5f87693..3602cad85d 100644 --- a/pkg/services/rpcsrv/server.go +++ b/pkg/services/rpcsrv/server.go @@ -2861,16 +2861,16 @@ func (s *Server) writeHTTPServerResponse(r *params.Request, w http.ResponseWrite resp.RunForErrors(func(jsonErr *neorpc.Error) { s.logRequestError(r, jsonErr) }) + w.Header().Set("Content-Type", "application/json; charset=utf-8") + if s.config.EnableCORSWorkaround { + setCORSOriginHeaders(w.Header()) + } if r.In != nil { resp := resp.(abstract) if resp.Error != nil { w.WriteHeader(getHTTPCodeForError(resp.Error)) } } - w.Header().Set("Content-Type", "application/json; charset=utf-8") - if s.config.EnableCORSWorkaround { - setCORSOriginHeaders(w.Header()) - } encoder := json.NewEncoder(w) err := encoder.Encode(resp) diff --git a/pkg/services/rpcsrv/server_test.go b/pkg/services/rpcsrv/server_test.go index 98d8324ef2..352a454c0c 100644 --- a/pkg/services/rpcsrv/server_test.go +++ b/pkg/services/rpcsrv/server_test.go @@ -3350,3 +3350,21 @@ func TestFailedPreconditionShutdown(t *testing.T) { require.Eventually(t, stopped.Load, 5*time.Second, 100*time.Millisecond, "Shutdown should return") } + +func TestErrorResponseContentType(t *testing.T) { + chain, rpcSrv, httpSrv := initClearServerWithServices(t, true, false, false) + defer chain.Close() + defer rpcSrv.Shutdown() + + const ( + expectedContentType = "application/json; charset=utf-8" + req = `{"jsonrpc":"2.0", "method":"unknown","params":[]}` + ) + + cl := http.Client{Timeout: time.Second} + resp, err := cl.Post(httpSrv.URL, "application/json", strings.NewReader(req)) + require.NoErrorf(t, err, "could not make a POST request") + resp.Body.Close() + contentType := resp.Header.Get("Content-Type") + require.Equal(t, expectedContentType, contentType) +} From 80fdb1c90c2693c6cf7b410c35fd3160e30283d6 Mon Sep 17 00:00:00 2001 From: Anna Shaleva Date: Tue, 1 Aug 2023 18:53:20 +0300 Subject: [PATCH 29/29] CHANGELOG: release 0.101.4 Signed-off-by: Anna Shaleva --- CHANGELOG.md | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e0d49bb20..11f4a489c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,47 @@ This document outlines major changes between releases. +## 0.101.4 "Yarborough" (01 Aug 2023) + +Another one 3.5.0-compatible version that is aimed to fix T5 testnet state +difference that has happened at block 2336911 which leads to inability to process +new blocks since 2418703. The issue is fixed by allowing JSON numbers +unmarshalling from scientific notation to Integer stackitem. Maximum parsing +precision for such numbers is currently restricted by 53 bits. This is a +temporary C#-compatible solution that is likely to change in the future versions +when an appropriate C# node bug is fixed (neo-project/neo#2879). + +A set of minor bug fixes is included as well to flush some of the long-awaited +changes that were blocked by the 0.102.0 release delay (caused by v3.6.0 C# node +release delay). In particular, invalid headers returned by an RPC server for +error responses, invalid format of incremental dumps created by CLI and deadlock +on unhealthy RPC server shutdown. Long-awaited `--config-file` CLI option to +start the node providing a single configuration file is added. + +T5 testnet chain requires a complete resynchronization for this version. Mainnet +chain resynchronization is recommended, but not required. + +New features: + * `--config-file` CLI option allowing to start the node with a single configuration file (#3014) + +Improvements: + * blockchain Notary and Oracle services documentation improvement (#2972) + * BoltDB (`go.etcd.io/bbolt`) dependency upgrade that fixes a number of Windows-related issues (#3034) + +Bugs fixed: + * panic on node start with invalid configuration (#2968) + * deadlock on unhealthy RPC server shutdown (#2966) + * improper WSClient notification channels managing after disconnection (#2980) + * missing Prometheus metric initialisation on node start (#2992) + * invalid initialisation of native contracts cache (#2994) + * incorrect way of incremental DB dumps creation (#3047) + * Notary contract is allowed to be a sender of main Notary request transaction (#3065) + * discrepancy in signer's witness scope parsing on the RPC server side (#3060) + * Invoker calling API isn't allowed to accept nil parameter (#3067) + * contract RPC Client unwrapper helper can't handle missing contract case (#3072) + * JSON numbers can't be unmarshalled to stackitem from scientific notation (#3073) + * invalid content-type header returned by RPC server on error responses (#3075) + ## 0.101.3 "Yuckiness" (08 Jul 2023) Yet another 3.5.0-compatible emergency version that removes scrupulous