diff --git a/common/channelconfig/util.go b/common/channelconfig/util.go index 1966e08298..738d3c268f 100644 --- a/common/channelconfig/util.go +++ b/common/channelconfig/util.go @@ -15,6 +15,7 @@ import ( cb "github.com/hyperledger/fabric-protos-go/common" mspprotos "github.com/hyperledger/fabric-protos-go/msp" ab "github.com/hyperledger/fabric-protos-go/orderer" + "github.com/hyperledger/fabric-protos-go/orderer/bdls" "github.com/hyperledger/fabric-protos-go/orderer/etcdraft" "github.com/hyperledger/fabric-protos-go/orderer/smartbft" pb "github.com/hyperledger/fabric-protos-go/peer" @@ -364,3 +365,36 @@ func MarshalSmartBFTMetadata(md *smartbft.ConfigMetadata) ([]byte, error) { } return proto.Marshal(copyMd) } + +// MarshalBdlsMetadata serializes Bdls metadata. +func MarshalBdlsMetadata(md *bdls.ConfigMetadata) ([]byte, error) { + copyMd := proto.Clone(md).(*bdls.ConfigMetadata) + for _, c := range copyMd.Consenters { + // Expect the user to set the config value for client/server certs to the + // path where they are persisted locally, then load these files to memory. + clientCert, err := ioutil.ReadFile(string(c.GetClientTlsCert())) + if err != nil { + return nil, errors.Errorf("cannot load client cert for consenter %s:%d: %s", c.GetHost(), c.GetPort(), err) + } + c.ClientTlsCert = clientCert + + serverCert, err := ioutil.ReadFile(string(c.GetServerTlsCert())) + if err != nil { + return nil, errors.Errorf("cannot load server cert for consenter %s:%d: %s", c.GetHost(), c.GetPort(), err) + } + c.ServerTlsCert = serverCert + + // Load OSN signing identity certificate + idBytes, err := ioutil.ReadFile(string(c.Identity)) + if err != nil { + return nil, errors.Errorf("cannot load consenter identity certificate %s:%d, %s", c.GetHost(), c.GetPort(), err) + } + + c.Identity, err = msp.NewSerializedIdentity(c.MspId, idBytes) + + if err != nil { + return nil, errors.Errorf("cannot marshal consenter serialized identity %s:%d: %s", c.GetHost(), c.GetPort(), err) + } + } + return proto.Marshal(copyMd) +} diff --git a/core/peer/peer.go b/core/peer/peer.go index 18ce208d7a..1b62e47e23 100644 --- a/core/peer/peer.go +++ b/core/peer/peer.go @@ -12,6 +12,7 @@ import ( "github.com/golang/protobuf/proto" "github.com/hyperledger/fabric-protos-go/common" + "github.com/hyperledger/fabric-protos-go/orderer/bdls" "github.com/hyperledger/fabric-protos-go/orderer/smartbft" pb "github.com/hyperledger/fabric-protos-go/peer" "github.com/hyperledger/fabric/bccsp" @@ -286,6 +287,31 @@ func (p *Peer) SmartBFTId2Identities(cid string) map[uint64][]byte { return res } +// BdlsId2Identities get identities from last known configuration. +func (p *Peer) BdlsId2Identities(cid string) map[uint64][]byte { + c := p.Channel(cid) + if c == nil { + return nil + } + + c.lock.RLock() + defer c.lock.RUnlock() + oc, ok := c.Resources().OrdererConfig() + if !ok { + return nil + } + + m := &bdls.ConfigMetadata{} + proto.Unmarshal(oc.ConsensusMetadata(), m) + + res := make(map[uint64][]byte) + for _, consenter := range m.Consenters { + res[consenter.ConsenterId] = consenter.Identity + } + + return res +} + type IdentityFethcer struct { Adaptee *Peer } @@ -294,6 +320,10 @@ func (i *IdentityFethcer) Id2Identities(cid string) map[uint64][]byte { return i.Adaptee.SmartBFTId2Identities(cid) } +func (i *IdentityFethcer) Id2IdentitiesBdls(cid string) map[uint64][]byte { + return i.Adaptee.BdlsId2Identities(cid) +} + // createChannel creates a new channel object and insert it into the channels slice. func (p *Peer) createChannel( cid string, diff --git a/integration/nwo/configblock.go b/integration/nwo/configblock.go index d8f25414c2..f3bcf70a93 100644 --- a/integration/nwo/configblock.go +++ b/integration/nwo/configblock.go @@ -16,6 +16,7 @@ import ( "github.com/hyperledger/fabric-protos-go/common" "github.com/hyperledger/fabric-protos-go/msp" protosorderer "github.com/hyperledger/fabric-protos-go/orderer" + "github.com/hyperledger/fabric-protos-go/orderer/bdls" "github.com/hyperledger/fabric-protos-go/orderer/smartbft" "github.com/hyperledger/fabric/common/channelconfig" "github.com/hyperledger/fabric/integration/nwo/commands" @@ -347,6 +348,21 @@ func UpdateSmartBFTMetadata(network *Network, peer *Peer, orderer *Orderer, chan }) } +// UpdateBdlsMetadata executes a config update that updates the bdls metadata according to the given function f +func UpdateBdlsMetadata(network *Network, peer *Peer, orderer *Orderer, channel string, f func(md *bdls.ConfigMetadata)) { + UpdateConsensusMetadata(network, peer, orderer, channel, func(originalMetadata []byte) []byte { + metadata := &bdls.ConfigMetadata{} + err := proto.Unmarshal(originalMetadata, metadata) + Expect(err).NotTo(HaveOccurred()) + + f(metadata) + + newMetadata, err := proto.Marshal(metadata) + Expect(err).NotTo(HaveOccurred()) + return newMetadata + }) +} + // UpdateOrdererEndpoints executes a config update that updates the orderer metadata according to the given endpoints func UpdateOrdererEndpoints(network *Network, peer *Peer, orderer *Orderer, channel string, endpoints ...string) { config := GetConfig(network, peer, orderer, channel) diff --git a/integration/nwo/configtx_template.go b/integration/nwo/configtx_template.go index a2a338be73..8472952541 100644 --- a/integration/nwo/configtx_template.go +++ b/integration/nwo/configtx_template.go @@ -181,6 +181,18 @@ Profiles:{{ range .Profiles }} ConsenterId: {{ $w.OrdererIndex . }} {{- end }}{{- end }} {{- end }} + {{- if eq $w.Consensus.Type "bdls" }} + bdls: + Consenters:{{ range .Orderers }}{{ with $w.Orderer . }} + - Host: 127.0.0.1 + Port: {{ $w.OrdererPort . "Cluster" }} + ClientTLSCert: {{ $w.OrdererLocalCryptoDir . "tls" }}/server.crt + ServerTLSCert: {{ $w.OrdererLocalCryptoDir . "tls" }}/server.crt + MSPID: {{ $w.OrdererMSPID . }} + Identity: {{ $w.OrdererCert . }} + ConsenterId: {{ $w.OrdererIndex . }} + {{- end }}{{- end }} + {{- end }} Organizations:{{ range $w.OrgsForOrderers .Orderers }} - *{{ .MSPID }} {{- end }} @@ -198,6 +210,11 @@ Profiles:{{ range .Profiles }} BlockValidation: Type: ImplicitOrderer Rule: SMARTBFT + {{- else }} + {{- if eq $w.Consensus.Type "bdls" }} + BlockValidation: + Type: ImplicitOrderer + Rule: SMARTBFT {{- else }} BlockValidation: Type: ImplicitMeta diff --git a/integration/nwo/standard_networks.go b/integration/nwo/standard_networks.go index 6a06fc9d25..e04321131f 100644 --- a/integration/nwo/standard_networks.go +++ b/integration/nwo/standard_networks.go @@ -185,6 +185,59 @@ func MinimalRaft() *Config { return config } + +/* +BDLS function config BasicBdls & MultiNodeBdls +*/ +func BasicBdls() *Config { + config := BasicSolo() + config.Consensus.Type = "bdls" + config.Profiles = []*Profile{{ + Name: "SampleDevModeBdls", + Orderers: []string{"orderer"}, + }, { + Name: "TwoOrgsChannel", + Consortium: "SampleConsortium", + Organizations: []string{"Org1", "Org2"}, + }} + for _, peer := range config.Peers { + peer.BFTDeliveryClient = true + } + config.SystemChannel.Profile = "SampleDevModeBdls" + return config +} + +func MultiNodeBdls() *Config { + config := BasicBdls() + config.Orderers = []*Orderer{ + {Name: "orderer1", Organization: "OrdererOrg"}, + {Name: "orderer2", Organization: "OrdererOrg"}, + {Name: "orderer3", Organization: "OrdererOrg"}, + {Name: "orderer4", Organization: "OrdererOrg"}, + } + config.Profiles = []*Profile{{ + Name: "SampleDevModeBdls", + Orderers: []string{"orderer1", "orderer2", "orderer3", "orderer4"}, + }, { + Name: "TwoOrgsChannel", + Consortium: "SampleConsortium", + Organizations: []string{"Org1", "Org2"}, + }} + + config.Channels = []*Channel{ + {Name: "testchannel1", Profile: "TwoOrgsChannel"}, + {Name: "testchannel2", Profile: "TwoOrgsChannel"}} + + for _, peer := range config.Peers { + peer.Channels = []*PeerChannel{ + {Name: "testchannel1", Anchor: true}, + {Name: "testchannel2", Anchor: true}, + } + } + return config +} +/*********************BDLS end*********************/ + func BasicSmartBFT() *Config { config := BasicSolo() config.Consensus.Type = "smartbft" diff --git a/integration/ports.go b/integration/ports.go index a6214f101b..595831320a 100644 --- a/integration/ports.go +++ b/integration/ports.go @@ -39,6 +39,7 @@ const ( RaftBasePort SBEBasePort SmartBFTBasePort + BdlsBasePort ) // On linux, the default ephemeral port range is 32768-60999 and can be diff --git a/internal/configtxgen/encoder/encoder.go b/internal/configtxgen/encoder/encoder.go index af0752a81a..678b9cd129 100644 --- a/internal/configtxgen/encoder/encoder.go +++ b/internal/configtxgen/encoder/encoder.go @@ -43,6 +43,8 @@ const ( ConsensusTypeEtcdRaft = "etcdraft" // ConsensusTypeSmartBFT identifies the SmartBFT-based consensus implementation. ConsensusTypeSmartBFT = "smartbft" + // ConsensusTypeBdls identifies the BDLS-based consensus implementation. + ConsensusTypeBdls = "bdls" // BlockValidationPolicyKey TODO BlockValidationPolicyKey = "BlockValidation" @@ -228,6 +230,11 @@ func NewOrdererGroup(conf *genesisconfig.Orderer) (*cb.ConfigGroup, error) { if consensusMetadata, err = channelconfig.MarshalSmartBFTMetadata(conf.SmartBFT); err != nil { return nil, errors.Errorf("cannot marshal metadata for orderer type %s: %s", ConsensusTypeSmartBFT, err) } + //TODO: replace the MarshalSmartBFTMetadata + case ConsensusTypeBdls: + if consensusMetadata, err = channelconfig.MarshalSmartBFTMetadata(conf.SmartBFT); err != nil { + return nil, errors.Errorf("cannot marshal metadata for orderer type %s: %s", ConsensusTypeBdls, err) + } default: return nil, errors.Errorf("unknown orderer type: %s", conf.OrdererType) } diff --git a/internal/configtxgen/encoder/encoder_test.go b/internal/configtxgen/encoder/encoder_test.go index 6a31f68325..992ae5c715 100644 --- a/internal/configtxgen/encoder/encoder_test.go +++ b/internal/configtxgen/encoder/encoder_test.go @@ -17,6 +17,7 @@ import ( ab "github.com/hyperledger/fabric-protos-go/orderer" "github.com/hyperledger/fabric-protos-go/orderer/etcdraft" "github.com/hyperledger/fabric-protos-go/orderer/smartbft" + "github.com/hyperledger/fabric-protos-go/orderer/bdls" "github.com/hyperledger/fabric/common/util" "github.com/hyperledger/fabric/internal/configtxgen/encoder" "github.com/hyperledger/fabric/internal/configtxgen/encoder/fakes" @@ -516,6 +517,74 @@ var _ = Describe("Encoder", func() { }) }) + Context("when the consensus type is bdls", func() { + BeforeEach(func() { + conf.OrdererType = "bdls" + conf.Bdls = &bdls.ConfigMetadata{ + Options: &bdls.Options{ + RequestBatchMaxCount: uint64(100), + RequestBatchMaxBytes: uint64(1000000), + RequestBatchMaxInterval: "50ms", + IncomingMessageBufferSize: uint64(200), + RequestPoolSize: uint64(400), + RequestForwardTimeout: "2s", + RequestComplainTimeout: "10s", + RequestAutoRemoveTimeout: "1m", + ViewChangeResendInterval: "5s", + ViewChangeTimeout: "20s", + LeaderHeartbeatTimeout: "30s", + LeaderHeartbeatCount: uint64(10), + CollectTimeout: "1m", + SyncOnStart: false, + SpeedUpViewChange: false, + }, + } + }) + + It("adds the bdls metadata", func() { + cg, err := encoder.NewOrdererGroup(conf) + Expect(err).NotTo(HaveOccurred()) + Expect(len(cg.Values)).To(Equal(5)) + consensusType := &ab.ConsensusType{} + err = proto.Unmarshal(cg.Values["ConsensusType"].Value, consensusType) + Expect(err).NotTo(HaveOccurred()) + Expect(consensusType.Type).To(Equal("bdls")) + metadata := &bdls.ConfigMetadata{} + err = proto.Unmarshal(consensusType.Metadata, metadata) + Expect(err).NotTo(HaveOccurred()) + Expect(metadata.Options.RequestBatchMaxCount).To(Equal(uint64(100))) + Expect(metadata.Options.RequestBatchMaxBytes).To(Equal(uint64(1000000))) + Expect(metadata.Options.RequestBatchMaxInterval).To(Equal("50ms")) + Expect(metadata.Options.IncomingMessageBufferSize).To(Equal(uint64(200))) + Expect(metadata.Options.RequestPoolSize).To(Equal(uint64(400))) + Expect(metadata.Options.RequestForwardTimeout).To(Equal("2s")) + Expect(metadata.Options.RequestComplainTimeout).To(Equal("10s")) + Expect(metadata.Options.RequestAutoRemoveTimeout).To(Equal("1m")) + Expect(metadata.Options.ViewChangeResendInterval).To(Equal("5s")) + Expect(metadata.Options.ViewChangeTimeout).To(Equal("20s")) + Expect(metadata.Options.LeaderHeartbeatTimeout).To(Equal("30s")) + Expect(metadata.Options.LeaderHeartbeatCount).To(Equal(uint64(10))) + Expect(metadata.Options.CollectTimeout).To(Equal("1m")) + Expect(metadata.Options.SyncOnStart).To(Equal(false)) + Expect(metadata.Options.SpeedUpViewChange).To(Equal(false)) + }) + + Context("when the bdls configuration is bad", func() { + BeforeEach(func() { + conf.Bdls = &bdls.ConfigMetadata{ + Consenters: []*bdls.Consenter{ + {}, + }, + } + }) + + It("wraps and returns the error", func() { + _, err := encoder.NewOrdererGroup(conf) + Expect(err).To(MatchError("cannot marshal metadata for orderer type bdls: cannot load client cert for consenter :0: open : no such file or directory")) + }) + }) + }) + Context("when the consensus type is unknown", func() { BeforeEach(func() { conf.OrdererType = "bad-type" diff --git a/internal/configtxgen/genesisconfig/config.go b/internal/configtxgen/genesisconfig/config.go index 4e8e624c04..bd0a169ffd 100644 --- a/internal/configtxgen/genesisconfig/config.go +++ b/internal/configtxgen/genesisconfig/config.go @@ -14,6 +14,7 @@ import ( "time" "github.com/SmartBFT-Go/consensus/pkg/types" + "github.com/hyperledger/fabric-protos-go/orderer/bdls" "github.com/hyperledger/fabric-protos-go/orderer/etcdraft" "github.com/hyperledger/fabric-protos-go/orderer/smartbft" "github.com/hyperledger/fabric/common/flogging" @@ -61,6 +62,10 @@ const ( // the SmartBFT-based ordering service. SampleDevModeSmartBFTProfile = "SampleDevModeSmartBFT" + // SampleDevModeBdlsProfile references the sample profile used for testing + // the Bdls-based ordering service. + SampleDevModeBdlsProfile = "SampleDevModeBdls" + // SampleAppChannelInsecureSoloProfile references the sample profile which // does not include any MSPs and uses solo for ordering. SampleAppChannelInsecureSoloProfile = "SampleAppChannelInsecureSolo" @@ -166,6 +171,7 @@ type Orderer struct { Kafka Kafka `yaml:"Kafka"` EtcdRaft *etcdraft.ConfigMetadata `yaml:"EtcdRaft"` SmartBFT *smartbft.ConfigMetadata `yaml:"SmartBFT"` + Bdls *bdls.ConfigMetadata `yaml:"Bdls"` Organizations []*Organization `yaml:"Organizations"` MaxChannels uint64 `yaml:"MaxChannels"` Capabilities map[string]bool `yaml:"Capabilities"` @@ -224,6 +230,25 @@ var genesisDefaults = TopLevel{ SpeedUpViewChange: types.DefaultConfig.SpeedUpViewChange, }, }, + Bdls: &bdls.ConfigMetadata{ + Options: &bdls.Options{ + RequestBatchMaxCount: uint64(types.DefaultConfig.RequestBatchMaxCount), + RequestBatchMaxBytes: uint64(types.DefaultConfig.RequestBatchMaxBytes), + RequestBatchMaxInterval: types.DefaultConfig.RequestBatchMaxInterval.String(), + IncomingMessageBufferSize: uint64(types.DefaultConfig.IncomingMessageBufferSize), + RequestPoolSize: uint64(types.DefaultConfig.RequestPoolSize), + RequestForwardTimeout: types.DefaultConfig.RequestForwardTimeout.String(), + RequestComplainTimeout: types.DefaultConfig.RequestComplainTimeout.String(), + RequestAutoRemoveTimeout: types.DefaultConfig.RequestAutoRemoveTimeout.String(), + ViewChangeResendInterval: types.DefaultConfig.ViewChangeResendInterval.String(), + ViewChangeTimeout: types.DefaultConfig.ViewChangeTimeout.String(), + LeaderHeartbeatTimeout: types.DefaultConfig.LeaderHeartbeatTimeout.String(), + LeaderHeartbeatCount: uint64(types.DefaultConfig.LeaderHeartbeatCount), + CollectTimeout: types.DefaultConfig.CollectTimeout.String(), + SyncOnStart: types.DefaultConfig.SyncOnStart, + SpeedUpViewChange: types.DefaultConfig.SpeedUpViewChange, + }, + }, }, } @@ -434,6 +459,9 @@ loop: cf.TranslatePathInPlace(configDir, &serverCertPath) c.ServerTlsCert = []byte(serverCertPath) } + //TODO: add body for the bdls case + case "bdls": + // Do nothing case SmartBFT: if ord.SmartBFT == nil { logger.Panicf("%s configuration missing", SmartBFT) diff --git a/orderer/common/server/main.go b/orderer/common/server/main.go index 38694a5c19..b8be2eb6b5 100644 --- a/orderer/common/server/main.go +++ b/orderer/common/server/main.go @@ -822,19 +822,17 @@ func initializeMultichannelRegistrar( // the orderer can start without channels at all and have an initialized cluster type consenter switch consenterType { - case "etcdraft": - consenters["etcdraft"] = etcdraft.New(clusterDialer, conf, srvConf, srv, registrar, nil, metricsProvider, bccsp) - case "smartbft": - consenters["smartbft"] = smartbft.New(nil, dpmr.Registry(), signer, clusterDialer, conf, srvConf, srv, registrar, metricsProvider, bccsp) + //case "etcdraft": consenters["etcdraft"] = etcdraft.New(clusterDialer, conf, srvConf, srv, registrar, nil, metricsProvider, bccsp) + //case "smartbft": consenters["smartbft"] = smartbft.New(nil, dpmr.Registry(), signer, clusterDialer, conf, srvConf, srv, registrar, metricsProvider, bccsp) default: logger.Panicf("Unknown cluster type consenter '%s'", consenterType) } } } - consenters["solo"] = solo.New() + //consenters["solo"] = solo.New() var kafkaMetrics *kafka.Metrics - consenters["kafka"], kafkaMetrics = kafka.New(conf.Kafka, metricsProvider, healthChecker, icr, registrar.CreateChain) + //consenters["kafka"], kafkaMetrics = kafka.New(conf.Kafka, metricsProvider, healthChecker, icr, registrar.CreateChain) // Note, we pass a 'nil' channel here, we could pass a channel that // closes if we wished to cleanup this routine on exit. diff --git a/orderer/consensus/bdls/.gitignore b/orderer/consensus/bdls/.gitignore new file mode 100644 index 0000000000..cfac083912 --- /dev/null +++ b/orderer/consensus/bdls/.gitignore @@ -0,0 +1,17 @@ +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +*.orig + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ +.idea/ diff --git a/orderer/consensus/bdls/.travis.yml b/orderer/consensus/bdls/.travis.yml new file mode 100644 index 0000000000..80264d46a5 --- /dev/null +++ b/orderer/consensus/bdls/.travis.yml @@ -0,0 +1,17 @@ +language: go +go: + - 1.13.x + +before_install: + - go get -t -v ./... + +install: + - go get github.com/Sperax/bdls/ + +script: + - go test -v -coverprofile=coverage.txt.tmp -covermode=atomic -timeout 12h -run "(Verify)|(Full20Participants)|(Propose)|(Round)|(Commit)|(Lock)|(Stage)" + - cat coverage.txt.tmp | grep -v "pb.go" > coverage.txt + - rm coverage.txt.tmp + +after_success: + - bash <(curl -s https://codecov.io/bash) diff --git a/orderer/consensus/bdls/Dockerfile b/orderer/consensus/bdls/Dockerfile new file mode 100644 index 0000000000..bc8591e071 --- /dev/null +++ b/orderer/consensus/bdls/Dockerfile @@ -0,0 +1,7 @@ +merge the protocol BDLS into the clone directory: + +fabric/orderer/consensus/bdls + +and run make in the orderer folder. + + diff --git a/orderer/consensus/bdls/LICENSE b/orderer/consensus/bdls/LICENSE new file mode 100644 index 0000000000..517c662a9e --- /dev/null +++ b/orderer/consensus/bdls/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2020, Sperax +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/orderer/consensus/bdls/README.md b/orderer/consensus/bdls/README.md new file mode 100644 index 0000000000..e560480f13 --- /dev/null +++ b/orderer/consensus/bdls/README.md @@ -0,0 +1,226 @@ +[![GoDoc][1]][2] [![License][3]][4] [![Build Status][5]][6] [![Go Report Card][7]][8] [![Coverage Statusd][9]][10] [![Sourcegraph][11]][12] + +[1]: https://godoc.org/github.com/Sperax/bdls?status.svg +[2]: https://godoc.org/github.com/Sperax/bdls +[3]: https://img.shields.io/github/license/Sperax/bdls +[4]: LICENSE +[5]: https://travis-ci.org/Sperax/bdls.svg?branch=master +[6]: https://travis-ci.org/Sperax/bdls +[7]: https://goreportcard.com/badge/github.com/Sperax/bdls?bdls +[8]: https://goreportcard.com/report/github.com/Sperax/bdls +[9]: https://codecov.io/gh/Sperax/bdls/branch/master/graph/badge.svg +[10]: https://codecov.io/gh/Sperax/bdls +[11]: https://sourcegraph.com/github.com/Sperax/bdls/-/badge.svg +[12]: https://sourcegraph.com/github.com/Sperax/bdls?badge + +# BDLS Consensus + +## Introduction + +BDLS is an innovative BFT consensus algorithm that features safety and liveness by +presenting a mathematically proven secure BFT protocol that is resilient in open networks such as +the Internet. With BDLS, we invented a new random beacons to ensure verifiable +unpredictability and fairness of validators. More importantly, BDLS overcomes many +problems, such as DoS attacks, as well as the deadlock problem caused by unreliable +p2p/broadcast channels. These problems are all very relevant to existing realistic open +network scenarios, and are the focus of extensive work in improving Internet security, but it +is an area largely ignored by most in mainstream BFT protocol design.(Paper: https://eprint.iacr.org/2019/1460.pdf) + +For this library, to make the runtime behavior of consensus algorithm predictable as function: +y = f(x, t), where 'x' is the message it received, and 't' is the time while being called, + then'y' is the deterministic status of consensus after 'x' and 't' applied to 'f', +it has been designed in a deterministic scheme, without parallel computing, networking, and +the correctness of program implementation can be proven with proper test cases. + +For more information on the BDLS consensus, you could view here https://medium.com/sperax/bdls-protocol-best-efficiency-best-security-best-performance-4cc2770608dd + +## Features + +1. Pure algorithm implementation in deterministic and predictable behavior, easily to be integrated into existing projects, refer to [DFA](https://en.wikipedia.org/wiki/Deterministic_finite_automaton) for more. +2. Well-tested on various platforms with complicated cases. +3. Auto back-off under heavy payload, guaranteed finalization(worst case gurantee). +4. Easy integratation into Blockchain & non-Blockchain consensus, like [WAL replication](https://en.wikipedia.org/wiki/Replication_(computing)#Database_replication) in database. +5. Builtin network emulation for various network latency with comprehensive statistics. + +## Documentation + +For complete documentation, see the associated [Godoc](https://godoc.org/github.com/Sperax/bdls). + +## Performance + +``` +DATE: 2020/03/18 +OS: Linux 4.19.84-microsoft-standard #1 SMP Wed Nov 13 11:44:37 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux +MEM: 64GB +CPU: AMD Ryzen 7 2700X Eight-Core Processor + +TERMINOLOGY: + +DECIDE.AVG = Average finalization time for each height. +DECIDE.ROUNDS = The rounds where decides has made. +PEER.NUM = Actual participantion. +PJ.NUM = Participants(Quorum) +NET.MSGS = Total network number of messages exchanged in all heights. +NET.BYTES = Total network bytes exchanged in all heights. +MSG.AVGSIZE = Average message size.(Tested with 1KB State.) +NET.MSGRATE = Network message rate(messages/second). +PEER.RATE = Peer's average bandwidth. +DELAY.MIN = Actual minimal network latency(network latency is randomized with normal distribution). +DELAY.MAX = Actual maximal network latency. +DELAY.AVG = Actual average latency. +DELAY.EXP = Expected Latency set to consensus algorithm. + +COMMANDS: +$ go test -v -cpuprofile=cpu.out -memprofile=mem.out -timeout 2h + +TESTING CASES: +============= + +Case 1: 20 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 1.47s | 1;1;1;1;1 | 20 | 20 | 9765 | 16.1M | 1.7K | 66.32/s | 110.1K/s | 59.61ms | 135.47ms | 100.02ms | 100ms | +| 2.3s | 1;1;1;1;1 | 20 | 20 | 9756 | 16.2M | 1.7K | 42.31/s | 70.6K/s | 121.64ms | 273.65ms | 200.12ms | 200ms | +| 3.11s | 1;1;1;1;1 | 20 | 20 | 9758 | 15.9M | 1.7K | 31.30/s | 50.9K/s | 177.7ms | 421.95ms | 300.04ms | 300ms | +| 4.76s | 1;1;1;1;1 | 20 | 20 | 9756 | 15.9M | 1.7K | 20.48/s | 33.4K/s | 308.55ms | 674.98ms | 499.04ms | 500ms | +| 8.85s | 1;1;1;1;1 | 20 | 20 | 9753 | 15.8M | 1.7K | 11.02/s | 17.8K/s | 638.02ms | 1.38348s | 999.99ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 2: 30 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 1.53s | 1;1;1;1;1 | 30 | 30 | 22152 | 36.2M | 1.7K | 96.17/s | 156.5K/s | 55.18ms | 141.39ms | 100.09ms | 100ms | +| 2.33s | 1;1;1;1;1 | 30 | 30 | 22152 | 36.3M | 1.7K | 63.23/s | 104.2K/s | 125.74ms | 275.21ms | 199.87ms | 200ms | +| 3.14s | 1;1;1;1;1 | 30 | 30 | 22137 | 36.2M | 1.7K | 46.90/s | 76.7K/s | 176.14ms | 415.37ms | 300.16ms | 300ms | +| 4.75s | 1;1;1;1;1 | 30 | 30 | 22136 | 35.9M | 1.7K | 31.03/s | 50.3K/s | 317.97ms | 695.47ms | 499.76ms | 500ms | +| 8.9s | 1;1;1;1;1 | 30 | 30 | 22135 | 36M | 1.7K | 16.57/s | 26.9K/s | 532.09ms | 1.34651s | 1.00002s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 3: 50 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 1.82s | 1;1;1;1;1 | 50 | 50 | 59819 | 104.3M | 1.8K | 131.20/s | 227.1K/s | 56.7ms | 137.95ms | 99.91ms | 100ms | +| 2.59s | 1;1;1;1;1 | 50 | 50 | 61951 | 102.9M | 1.7K | 95.61/s | 155.7K/s | 115.92ms | 289.78ms | 200.05ms | 200ms | +| 3.32s | 1;1;1;1;1 | 50 | 50 | 61916 | 101.8M | 1.7K | 74.52/s | 122.2K/s | 170.95ms | 421.28ms | 300.02ms | 300ms | +| 4.9s | 1;1;1;1;1 | 50 | 50 | 61905 | 101.6M | 1.7K | 50.50/s | 82.8K/s | 288.33ms | 731.75ms | 500.06ms | 500ms | +| 8.97s | 1;1;1;1;1 | 50 | 50 | 61906 | 101.4M | 1.7K | 27.60/s | 45.2K/s | 570.08ms | 1.42545s | 1.00002s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 4: 80 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 5.6s | 2;2;1;2;2 | 80 | 80 | 267894 | 1.8G | 7.1K | 119.58/s | 834.2K/s | 53.01ms | 150.23ms | 99.9ms | 100ms | +| 3.13s | 1;1;1;1;1 | 80 | 80 | 153622 | 278.2M | 1.9K | 122.58/s | 217.2K/s | 110.94ms | 285.03ms | 199.81ms | 200ms | +| 3.74s | 1;1;1;1;1 | 80 | 80 | 156056 | 261.3M | 1.7K | 104.26/s | 171.5K/s | 164.08ms | 429.22ms | 299.94ms | 300ms | +| 5.24s | 1;1;1;1;1 | 80 | 80 | 158652 | 260.2M | 1.7K | 75.64/s | 122.2K/s | 273.16ms | 718.12ms | 500.22ms | 500ms | +| 9.38s | 1;1;1;1;1 | 80 | 80 | 159054 | 261M | 1.7K | 42.35/s | 68.8K/s | 553.89ms | 1.44921s | 1.00012s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 5: 100 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 9.5s | 2;2;3;2;2 | 100 | 100 | 505578 | 3.5G | 7.3K | 106.42/s | 764.4K/s | 52.3ms | 145.48ms | 99.92ms | 100ms | +| 7.43s | 2;2;2;1;1 | 100 | 100 | 361084 | 2.3G | 6.6K | 97.18/s | 626.3K/s | 100.07ms | 300.16ms | 199.57ms | 200ms | +| 7.66s | 1;2;1;2;1 | 100 | 100 | 330107 | 2G | 6.3K | 86.08/s | 527.8K/s | 167.46ms | 444.85ms | 299.79ms | 300ms | +| 5.78s | 1;1;1;1;1 | 100 | 100 | 241856 | 404.4M | 1.7K | 83.56/s | 136.7K/s | 239.32ms | 736.61ms | 499.73ms | 500ms | +| 9.54s | 1;1;1;1;1 | 100 | 100 | 248825 | 409.1M | 1.7K | 52.12/s | 85.2K/s | 560.41ms | 1.48048s | 1.00002s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+----------- + +Case 6: 20 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 2.59s | 2;1;2;2;1 | 13 | 20 | 5016 | 8.2M | 1.7K | 29.79/s | 49.6K/s | 58.49ms | 140.78ms | 100ms | 100ms | +| 9.98s | 1;2;3;4;2 | 13 | 20 | 6264 | 9.6M | 1.6K | 9.65/s | 15.1K/s | 118.56ms | 269.81ms | 200.2ms | 200ms | +| 7.2s | 1;1;2;3;1 | 13 | 20 | 5016 | 8.2M | 1.7K | 10.72/s | 17.8K/s | 198.85ms | 408.56ms | 300.19ms | 300ms | +| 30.95s | 5;2;1;1;1 | 13 | 20 | 5640 | 8.9M | 1.6K | 2.80/s | 4.5K/s | 304.56ms | 663.24ms | 499.11ms | 500ms | +| 23.43s | 2;2;2;1;2 | 13 | 20 | 5328 | 8.6M | 1.7K | 3.50/s | 5.7K/s | 621.09ms | 1.37182s | 1.00037s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 7: 30 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 24.06s | 1;2;1;7;4 | 20 | 30 | 17381 | 25.7M | 1.5K | 7.22/s | 10.9K/s | 60.17ms | 143.24ms | 99.98ms | 100ms | +| 27.88s | 2;6;2;1;4 | 20 | 30 | 17383 | 25.7M | 1.5K | 6.23/s | 9.4K/s | 123.31ms | 269.15ms | 200.06ms | 200ms | +| 15.14s | 3;1;2;2;4 | 20 | 30 | 15101 | 23.1M | 1.6K | 9.97/s | 15.6K/s | 186.63ms | 422.51ms | 299.87ms | 300ms | +| 11.47s | 3;1;1;2;1 | 20 | 30 | 12060 | 19.7M | 1.7K | 10.51/s | 17.4K/s | 307.34ms | 682.54ms | 499.59ms | 500ms | +| 44.77s | 3;2;4;1;2 | 20 | 30 | 15100 | 23.1M | 1.6K | 3.37/s | 5.3K/s | 622.36ms | 1.34913s | 999.89ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 8: 50 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 6.23s | 4;3;3;1;1 | 33 | 50 | 41680 | 65M | 1.6K | 40.50/s | 64.2K/s | 55.59ms | 139.67ms | 100.03ms | 100ms | +| 10.51s | 2;1;4;4;1 | 33 | 50 | 41664 | 64.4M | 1.6K | 24.02/s | 37.8K/s | 117.5ms | 290.2ms | 199.97ms | 200ms | +| 15.37s | 2;3;3;1;4 | 33 | 50 | 43776 | 66.8M | 1.6K | 17.25/s | 26.8K/s | 179.9ms | 421.71ms | 299.84ms | 300ms | +| 10.91s | 2;1;2;2;1 | 33 | 50 | 33216 | 54.9M | 1.7K | 18.45/s | 30.9K/s | 303.06ms | 713.69ms | 500.5ms | 500ms | +| 38.8s | 3;3;1;2;3 | 33 | 50 | 41664 | 64.4M | 1.6K | 6.51/s | 10.2K/s | 609.65ms | 1.38302s | 1.00107s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 9: 80 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 5.25s | 2;2;3;2;3 | 53 | 80 | 121369 | 520.3M | 4.4K | 87.18/s | 377.5K/s | 59.25ms | 149.4ms | 99.95ms | 100ms | +| 6s | 1;2;2;2;2 | 53 | 80 | 91790 | 152.8M | 1.7K | 57.64/s | 96.5K/s | 120.06ms | 286.27ms | 199.98ms | 200ms | +| 5.87s | 2;1;2;1;1 | 53 | 80 | 80728 | 138.1M | 1.8K | 51.84/s | 88.6K/s | 175.44ms | 423.91ms | 300.17ms | 300ms | +| 11.21s | 2;1;2;1;2 | 53 | 80 | 86217 | 142.9M | 1.7K | 29.02/s | 48.5K/s | 291.37ms | 734.8ms | 500.38ms | 500ms | +| 21.97s | 1;1;2;2;2 | 53 | 80 | 86216 | 142.9M | 1.7K | 14.80/s | 24.9K/s | 533.87ms | 1.43307s | 1.00042s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 10: 100 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 8.92s | 4;2;3;2;4 | 67 | 100 | 248185 | 1.7G | 7K | 82.99/s | 577.1K/s | 50.01ms | 144ms | 99.98ms | 100ms | +| 8.28s | 2;2;3;2;3 | 67 | 100 | 175047 | 286.4M | 1.7K | 63.08/s | 103.9K/s | 116.41ms | 287.7ms | 199.87ms | 200ms | +| 10.21s | 2;3;1;1;3 | 67 | 100 | 156156 | 279M | 1.8K | 45.63/s | 81.9K/s | 157.92ms | 422.21ms | 300.11ms | 300ms | +| 19.24s | 1;4;1;2;1 | 67 | 100 | 146918 | 239M | 1.7K | 22.79/s | 37.5K/s | 289.88ms | 705.65ms | 500.13ms | 500ms | +| 38.13s | 1;2;4;2;1 | 67 | 100 | 155760 | 248.7M | 1.6K | 12.19/s | 19.8K/s | 558.98ms | 1.49546s | 1.00018s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 11: 50 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s incorrectly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 2.9s | 2;2;2;2;2 | 50 | 50 | 116591 | 438.1M | 3.8K | 160.49/s | 605.2K/s | 58.21ms | 145.52ms | 99.98ms | 50ms | +| 2.81s | 1;1;2;1;1 | 50 | 50 | 81768 | 250.3M | 3.1K | 116.32/s | 343.2K/s | 98.55ms | 281.13ms | 199.73ms | 100ms | +| 3.61s | 2;1;1;1;1 | 50 | 50 | 82816 | 219.3M | 2.7K | 91.68/s | 239K/s | 175.29ms | 441.01ms | 299.71ms | 150ms | +| 4.45s | 1;1;1;1;1 | 50 | 50 | 73038 | 114.1M | 1.6K | 65.56/s | 101.8K/s | 287.19ms | 743.13ms | 500ms | 250ms | +| 7.99s | 1;1;1;1;1 | 50 | 50 | 73225 | 114.2M | 1.6K | 36.62/s | 56.9K/s | 606.02ms | 1.41356s | 999.36ms | 500ms | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 12: 50 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s incorrectly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 3.45s | 2;2;3;3;3 | 33 | 50 | 50621 | 80.2M | 1.6K | 88.78/s | 141.4K/s | 58.45ms | 141.37ms | 100.02ms | 50ms | +| 8.49s | 2;3;4;4;2 | 33 | 50 | 55392 | 119M | 2.2K | 39.54/s | 86.3K/s | 119.07ms | 279.23ms | 200.16ms | 100ms | +| 4.57s | 1;1;2;2;2 | 33 | 50 | 42720 | 144.1M | 3.5K | 56.65/s | 192.2K/s | 174.43ms | 422.48ms | 300.09ms | 150ms | +| 8.04s | 2;1;1;3;1 | 33 | 50 | 38497 | 60.9M | 1.6K | 28.99/s | 46.3K/s | 297.08ms | 699.25ms | 499.54ms | 250ms | +| 18.06s | 1;2;3;2;1 | 33 | 50 | 40608 | 63.2M | 1.6K | 13.63/s | 21.4K/s | 595.42ms | 1.45499s | 1.00061s | 500ms | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +``` + +See also overload benchmark: [PI4-OVERLOAD.TXT](benchmarks/PI4-OVERLOAD.TXT) + +## Specification + +1. Consensus messages are specified in [message.proto](message.proto), users of this library can encapsulate this message in a carrier message, like gossip in TCP. +2. Consensus algorithm is **NOT** thread-safe, it **MUST** be protected by some synchronization mechanism, like `sync.Mutex` or `chan` + `goroutine`. + +## Usage + +1. A testing IPC peer -- [ipc_peer.go](ipc_peer.go) +2. A testing TCP node -- [TCP based Consensus Emualtor](cmd/emucon) + +## Status + +GA diff --git a/orderer/consensus/bdls/agent-tcp/doc.go b/orderer/consensus/bdls/agent-tcp/doc.go new file mode 100644 index 0000000000..512d68c5f1 --- /dev/null +++ b/orderer/consensus/bdls/agent-tcp/doc.go @@ -0,0 +1,3 @@ +// Package agent-tcp implements a TCP based agent to participate in consensus +// Challenge-Response scheme has been adopted to do interactive authentication +package agent diff --git a/orderer/consensus/bdls/agent-tcp/ecdh.go b/orderer/consensus/bdls/agent-tcp/ecdh.go new file mode 100644 index 0000000000..ca0cde12ba --- /dev/null +++ b/orderer/consensus/bdls/agent-tcp/ecdh.go @@ -0,0 +1,41 @@ +// BSD 3-Clause License +// +// Copyright (c) 2020, Sperax +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package agent + +import ( + "crypto/ecdsa" + "math/big" +) + +func ECDH(publicKey *ecdsa.PublicKey, key *ecdsa.PrivateKey) *big.Int { + secret, _ := key.Curve.ScalarMult(publicKey.X, publicKey.Y, key.D.Bytes()) + return secret +} diff --git a/orderer/consensus/bdls/agent-tcp/ecdh_test.go b/orderer/consensus/bdls/agent-tcp/ecdh_test.go new file mode 100644 index 0000000000..245f0786b4 --- /dev/null +++ b/orderer/consensus/bdls/agent-tcp/ecdh_test.go @@ -0,0 +1,22 @@ +package agent + +import ( + "crypto/ecdsa" + "crypto/rand" + "testing" + + "github.com/Sperax/bdls" + "github.com/stretchr/testify/assert" +) + +func TestECDH(t *testing.T) { + key1, err := ecdsa.GenerateKey(bdls.S256Curve, rand.Reader) + assert.Nil(t, err) + key2, err := ecdsa.GenerateKey(bdls.S256Curve, rand.Reader) + assert.Nil(t, err) + + s1 := ECDH(&key1.PublicKey, key2) + s2 := ECDH(&key2.PublicKey, key1) + + assert.Equal(t, s1, s2) +} diff --git a/orderer/consensus/bdls/agent-tcp/errors.go b/orderer/consensus/bdls/agent-tcp/errors.go new file mode 100644 index 0000000000..3257cf92e3 --- /dev/null +++ b/orderer/consensus/bdls/agent-tcp/errors.go @@ -0,0 +1,43 @@ +// BSD 3-Clause License +// +// Copyright (c) 2020, Sperax +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package agent + +import "errors" + +var ( + ErrLocalKeyAuthInit = errors.New("incorrect state for local KeyAuthInitmessage") + ErrKeyNotOnCurve = errors.New("the public key is not on curve") + ErrPeerKeyAuthInit = errors.New("incorrect state for peer KeyAuthInit message") + ErrPeerKeyAuthChallenge = errors.New("incorrect state for peer KeyAuthChallenge message") + ErrPeerKeyAuthChallengeResponse = errors.New("incorrect state for peer KeyAuthChallengeResponse message") + ErrPeerAuthenticatedFailed = errors.New("public key authentication failed for peer") + ErrMessageLengthExceed = errors.New("message size exceeded maximum") +) diff --git a/orderer/consensus/bdls/agent-tcp/gossip.pb.go b/orderer/consensus/bdls/agent-tcp/gossip.pb.go new file mode 100644 index 0000000000..8470a867ba --- /dev/null +++ b/orderer/consensus/bdls/agent-tcp/gossip.pb.go @@ -0,0 +1,1130 @@ +// Code generated by protoc-gen-gogo. DO NOT EDIT. +// source: gossip.proto + +package agent + +import ( + fmt "fmt" + proto "github.com/gogo/protobuf/proto" + io "io" + math "math" + math_bits "math/bits" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package + +// CommandType defines supported commands +type CommandType int32 + +const ( + CommandType_NOP CommandType = 0 + CommandType_KEY_AUTH_INIT CommandType = 1 + CommandType_KEY_AUTH_CHALLENGE CommandType = 2 + CommandType_KEY_AUTH_CHALLENGE_REPLY CommandType = 3 + CommandType_CONSENSUS CommandType = 4 +) + +var CommandType_name = map[int32]string{ + 0: "NOP", + 1: "KEY_AUTH_INIT", + 2: "KEY_AUTH_CHALLENGE", + 3: "KEY_AUTH_CHALLENGE_REPLY", + 4: "CONSENSUS", +} + +var CommandType_value = map[string]int32{ + "NOP": 0, + "KEY_AUTH_INIT": 1, + "KEY_AUTH_CHALLENGE": 2, + "KEY_AUTH_CHALLENGE_REPLY": 3, + "CONSENSUS": 4, +} + +func (x CommandType) String() string { + return proto.EnumName(CommandType_name, int32(x)) +} + +func (CommandType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_878fa4887b90140c, []int{0} +} + +// Gossip defines a stream based protocol +type Gossip struct { + Command CommandType `protobuf:"varint,1,opt,name=Command,proto3,enum=agent.CommandType" json:"Command,omitempty"` + Message []byte `protobuf:"bytes,2,opt,name=Message,proto3" json:"Message,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Gossip) Reset() { *m = Gossip{} } +func (m *Gossip) String() string { return proto.CompactTextString(m) } +func (*Gossip) ProtoMessage() {} +func (*Gossip) Descriptor() ([]byte, []int) { + return fileDescriptor_878fa4887b90140c, []int{0} +} +func (m *Gossip) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Gossip) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Gossip.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Gossip) XXX_Merge(src proto.Message) { + xxx_messageInfo_Gossip.Merge(m, src) +} +func (m *Gossip) XXX_Size() int { + return m.Size() +} +func (m *Gossip) XXX_DiscardUnknown() { + xxx_messageInfo_Gossip.DiscardUnknown(m) +} + +var xxx_messageInfo_Gossip proto.InternalMessageInfo + +func (m *Gossip) GetCommand() CommandType { + if m != nil { + return m.Command + } + return CommandType_NOP +} + +func (m *Gossip) GetMessage() []byte { + if m != nil { + return m.Message + } + return nil +} + +type KeyAuthInit struct { + // client public key + X []byte `protobuf:"bytes,1,opt,name=X,proto3" json:"X,omitempty"` + Y []byte `protobuf:"bytes,2,opt,name=Y,proto3" json:"Y,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *KeyAuthInit) Reset() { *m = KeyAuthInit{} } +func (m *KeyAuthInit) String() string { return proto.CompactTextString(m) } +func (*KeyAuthInit) ProtoMessage() {} +func (*KeyAuthInit) Descriptor() ([]byte, []int) { + return fileDescriptor_878fa4887b90140c, []int{1} +} +func (m *KeyAuthInit) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *KeyAuthInit) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_KeyAuthInit.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *KeyAuthInit) XXX_Merge(src proto.Message) { + xxx_messageInfo_KeyAuthInit.Merge(m, src) +} +func (m *KeyAuthInit) XXX_Size() int { + return m.Size() +} +func (m *KeyAuthInit) XXX_DiscardUnknown() { + xxx_messageInfo_KeyAuthInit.DiscardUnknown(m) +} + +var xxx_messageInfo_KeyAuthInit proto.InternalMessageInfo + +func (m *KeyAuthInit) GetX() []byte { + if m != nil { + return m.X + } + return nil +} + +func (m *KeyAuthInit) GetY() []byte { + if m != nil { + return m.Y + } + return nil +} + +type KeyAuthChallenge struct { + // server ephermal publickey for client authentication + X []byte `protobuf:"bytes,1,opt,name=X,proto3" json:"X,omitempty"` + Y []byte `protobuf:"bytes,2,opt,name=Y,proto3" json:"Y,omitempty"` + // the challenge message, the peer can create the correct HMAC with this message + Challenge []byte `protobuf:"bytes,3,opt,name=Challenge,proto3" json:"Challenge,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *KeyAuthChallenge) Reset() { *m = KeyAuthChallenge{} } +func (m *KeyAuthChallenge) String() string { return proto.CompactTextString(m) } +func (*KeyAuthChallenge) ProtoMessage() {} +func (*KeyAuthChallenge) Descriptor() ([]byte, []int) { + return fileDescriptor_878fa4887b90140c, []int{2} +} +func (m *KeyAuthChallenge) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *KeyAuthChallenge) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_KeyAuthChallenge.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *KeyAuthChallenge) XXX_Merge(src proto.Message) { + xxx_messageInfo_KeyAuthChallenge.Merge(m, src) +} +func (m *KeyAuthChallenge) XXX_Size() int { + return m.Size() +} +func (m *KeyAuthChallenge) XXX_DiscardUnknown() { + xxx_messageInfo_KeyAuthChallenge.DiscardUnknown(m) +} + +var xxx_messageInfo_KeyAuthChallenge proto.InternalMessageInfo + +func (m *KeyAuthChallenge) GetX() []byte { + if m != nil { + return m.X + } + return nil +} + +func (m *KeyAuthChallenge) GetY() []byte { + if m != nil { + return m.Y + } + return nil +} + +func (m *KeyAuthChallenge) GetChallenge() []byte { + if m != nil { + return m.Challenge + } + return nil +} + +type KeyAuthChallengeReply struct { + HMAC []byte `protobuf:"bytes,1,opt,name=HMAC,proto3" json:"HMAC,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *KeyAuthChallengeReply) Reset() { *m = KeyAuthChallengeReply{} } +func (m *KeyAuthChallengeReply) String() string { return proto.CompactTextString(m) } +func (*KeyAuthChallengeReply) ProtoMessage() {} +func (*KeyAuthChallengeReply) Descriptor() ([]byte, []int) { + return fileDescriptor_878fa4887b90140c, []int{3} +} +func (m *KeyAuthChallengeReply) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *KeyAuthChallengeReply) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_KeyAuthChallengeReply.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *KeyAuthChallengeReply) XXX_Merge(src proto.Message) { + xxx_messageInfo_KeyAuthChallengeReply.Merge(m, src) +} +func (m *KeyAuthChallengeReply) XXX_Size() int { + return m.Size() +} +func (m *KeyAuthChallengeReply) XXX_DiscardUnknown() { + xxx_messageInfo_KeyAuthChallengeReply.DiscardUnknown(m) +} + +var xxx_messageInfo_KeyAuthChallengeReply proto.InternalMessageInfo + +func (m *KeyAuthChallengeReply) GetHMAC() []byte { + if m != nil { + return m.HMAC + } + return nil +} + +func init() { + proto.RegisterEnum("agent.CommandType", CommandType_name, CommandType_value) + proto.RegisterType((*Gossip)(nil), "agent.Gossip") + proto.RegisterType((*KeyAuthInit)(nil), "agent.KeyAuthInit") + proto.RegisterType((*KeyAuthChallenge)(nil), "agent.KeyAuthChallenge") + proto.RegisterType((*KeyAuthChallengeReply)(nil), "agent.KeyAuthChallengeReply") +} + +func init() { proto.RegisterFile("gossip.proto", fileDescriptor_878fa4887b90140c) } + +var fileDescriptor_878fa4887b90140c = []byte{ + // 285 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x91, 0xcd, 0x6a, 0x83, 0x50, + 0x10, 0x85, 0x7b, 0x63, 0x1a, 0xc9, 0xc4, 0x94, 0xdb, 0x81, 0x16, 0x17, 0x41, 0x82, 0xab, 0xf4, + 0x07, 0x17, 0xed, 0x13, 0x58, 0x91, 0x28, 0x31, 0x46, 0xd4, 0x40, 0x5c, 0x89, 0xa5, 0x17, 0x13, + 0x30, 0x2a, 0xd5, 0x2e, 0x7c, 0xc3, 0x2e, 0xfb, 0x08, 0xc5, 0x27, 0x29, 0xbd, 0x98, 0xb4, 0xb4, + 0x90, 0xdd, 0x9c, 0x8f, 0x6f, 0x0e, 0x0c, 0x03, 0x52, 0x5a, 0x54, 0xd5, 0xae, 0xd4, 0xca, 0xd7, + 0xa2, 0x2e, 0xf0, 0x3c, 0x49, 0x59, 0x5e, 0xab, 0x1e, 0x0c, 0xe6, 0x1c, 0xe3, 0x3d, 0x88, 0x46, + 0xb1, 0xdf, 0x27, 0xf9, 0x8b, 0x4c, 0xa6, 0x64, 0x76, 0xf1, 0x80, 0x1a, 0x57, 0xb4, 0x8e, 0x86, + 0x4d, 0xc9, 0xfc, 0x83, 0x82, 0x32, 0x88, 0x4b, 0x56, 0x55, 0x49, 0xca, 0xe4, 0xde, 0x94, 0xcc, + 0x24, 0xff, 0x10, 0xd5, 0x1b, 0x18, 0x2d, 0x58, 0xa3, 0xbf, 0xd5, 0x5b, 0x3b, 0xdf, 0xd5, 0x28, + 0x01, 0xd9, 0xf0, 0x42, 0xc9, 0x27, 0x9b, 0xef, 0x14, 0x75, 0x0b, 0x24, 0x52, 0x1d, 0xa0, 0x9d, + 0x6a, 0x6c, 0x93, 0x2c, 0x63, 0x79, 0xca, 0x4e, 0xf9, 0x38, 0x81, 0xe1, 0x51, 0x94, 0x05, 0x4e, + 0x7f, 0x80, 0x7a, 0x07, 0x57, 0x7f, 0xdb, 0x7c, 0x56, 0x66, 0x0d, 0x22, 0xf4, 0xad, 0xa5, 0x6e, + 0x74, 0xad, 0x7c, 0xbe, 0xcd, 0x61, 0xf4, 0xeb, 0x2e, 0x14, 0x41, 0x70, 0x57, 0x1e, 0x3d, 0xc3, + 0x4b, 0x18, 0x2f, 0xcc, 0x28, 0xd6, 0xd7, 0xa1, 0x15, 0xdb, 0xae, 0x1d, 0x52, 0x82, 0xd7, 0x80, + 0x47, 0x64, 0x58, 0xba, 0xe3, 0x98, 0xee, 0xdc, 0xa4, 0x3d, 0x9c, 0x80, 0xfc, 0x9f, 0xc7, 0xbe, + 0xe9, 0x39, 0x11, 0x15, 0x70, 0x0c, 0x43, 0x63, 0xe5, 0x06, 0xa6, 0x1b, 0xac, 0x03, 0xda, 0x7f, + 0x92, 0xde, 0x5b, 0x85, 0x7c, 0xb4, 0x0a, 0xf9, 0x6c, 0x15, 0xf2, 0x3c, 0xe0, 0x3f, 0x78, 0xfc, + 0x0a, 0x00, 0x00, 0xff, 0xff, 0x96, 0x66, 0x6e, 0x1f, 0x93, 0x01, 0x00, 0x00, +} + +func (m *Gossip) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Gossip) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Gossip) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Message) > 0 { + i -= len(m.Message) + copy(dAtA[i:], m.Message) + i = encodeVarintGossip(dAtA, i, uint64(len(m.Message))) + i-- + dAtA[i] = 0x12 + } + if m.Command != 0 { + i = encodeVarintGossip(dAtA, i, uint64(m.Command)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *KeyAuthInit) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *KeyAuthInit) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *KeyAuthInit) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Y) > 0 { + i -= len(m.Y) + copy(dAtA[i:], m.Y) + i = encodeVarintGossip(dAtA, i, uint64(len(m.Y))) + i-- + dAtA[i] = 0x12 + } + if len(m.X) > 0 { + i -= len(m.X) + copy(dAtA[i:], m.X) + i = encodeVarintGossip(dAtA, i, uint64(len(m.X))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *KeyAuthChallenge) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *KeyAuthChallenge) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *KeyAuthChallenge) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Challenge) > 0 { + i -= len(m.Challenge) + copy(dAtA[i:], m.Challenge) + i = encodeVarintGossip(dAtA, i, uint64(len(m.Challenge))) + i-- + dAtA[i] = 0x1a + } + if len(m.Y) > 0 { + i -= len(m.Y) + copy(dAtA[i:], m.Y) + i = encodeVarintGossip(dAtA, i, uint64(len(m.Y))) + i-- + dAtA[i] = 0x12 + } + if len(m.X) > 0 { + i -= len(m.X) + copy(dAtA[i:], m.X) + i = encodeVarintGossip(dAtA, i, uint64(len(m.X))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *KeyAuthChallengeReply) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *KeyAuthChallengeReply) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *KeyAuthChallengeReply) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.HMAC) > 0 { + i -= len(m.HMAC) + copy(dAtA[i:], m.HMAC) + i = encodeVarintGossip(dAtA, i, uint64(len(m.HMAC))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func encodeVarintGossip(dAtA []byte, offset int, v uint64) int { + offset -= sovGossip(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *Gossip) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Command != 0 { + n += 1 + sovGossip(uint64(m.Command)) + } + l = len(m.Message) + if l > 0 { + n += 1 + l + sovGossip(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *KeyAuthInit) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.X) + if l > 0 { + n += 1 + l + sovGossip(uint64(l)) + } + l = len(m.Y) + if l > 0 { + n += 1 + l + sovGossip(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *KeyAuthChallenge) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.X) + if l > 0 { + n += 1 + l + sovGossip(uint64(l)) + } + l = len(m.Y) + if l > 0 { + n += 1 + l + sovGossip(uint64(l)) + } + l = len(m.Challenge) + if l > 0 { + n += 1 + l + sovGossip(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *KeyAuthChallengeReply) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.HMAC) + if l > 0 { + n += 1 + l + sovGossip(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovGossip(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozGossip(x uint64) (n int) { + return sovGossip(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *Gossip) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGossip + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Gossip: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Gossip: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Command", wireType) + } + m.Command = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGossip + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Command |= CommandType(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGossip + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthGossip + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthGossip + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Message = append(m.Message[:0], dAtA[iNdEx:postIndex]...) + if m.Message == nil { + m.Message = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGossip(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGossip + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthGossip + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *KeyAuthInit) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGossip + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: KeyAuthInit: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: KeyAuthInit: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field X", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGossip + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthGossip + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthGossip + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.X = append(m.X[:0], dAtA[iNdEx:postIndex]...) + if m.X == nil { + m.X = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Y", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGossip + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthGossip + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthGossip + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Y = append(m.Y[:0], dAtA[iNdEx:postIndex]...) + if m.Y == nil { + m.Y = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGossip(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGossip + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthGossip + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *KeyAuthChallenge) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGossip + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: KeyAuthChallenge: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: KeyAuthChallenge: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field X", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGossip + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthGossip + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthGossip + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.X = append(m.X[:0], dAtA[iNdEx:postIndex]...) + if m.X == nil { + m.X = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Y", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGossip + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthGossip + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthGossip + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Y = append(m.Y[:0], dAtA[iNdEx:postIndex]...) + if m.Y == nil { + m.Y = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Challenge", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGossip + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthGossip + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthGossip + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Challenge = append(m.Challenge[:0], dAtA[iNdEx:postIndex]...) + if m.Challenge == nil { + m.Challenge = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGossip(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGossip + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthGossip + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *KeyAuthChallengeReply) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGossip + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: KeyAuthChallengeReply: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: KeyAuthChallengeReply: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field HMAC", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGossip + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthGossip + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthGossip + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.HMAC = append(m.HMAC[:0], dAtA[iNdEx:postIndex]...) + if m.HMAC == nil { + m.HMAC = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGossip(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthGossip + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthGossip + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipGossip(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowGossip + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowGossip + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowGossip + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthGossip + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupGossip + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthGossip + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthGossip = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowGossip = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupGossip = fmt.Errorf("proto: unexpected end of group") +) diff --git a/orderer/consensus/bdls/agent-tcp/gossip.proto b/orderer/consensus/bdls/agent-tcp/gossip.proto new file mode 100644 index 0000000000..b97846321b --- /dev/null +++ b/orderer/consensus/bdls/agent-tcp/gossip.proto @@ -0,0 +1,65 @@ +// BSD 3-Clause License +// +// Copyright (c) 2020, Sperax +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; +package agent; + +// CommandType defines supported commands +enum CommandType { + NOP = 0; + KEY_AUTH_INIT=1; + KEY_AUTH_CHALLENGE=2; + KEY_AUTH_CHALLENGE_REPLY= 3; + CONSENSUS=4; +} + +// Gossip defines a stream based protocol +message Gossip{ + CommandType Command = 1; + bytes Message=2; +} + +message KeyAuthInit { + // client public key + bytes X = 1; + bytes Y = 2; +} + +message KeyAuthChallenge { + // server ephermal publickey for client authentication + bytes X=1; + bytes Y=2; + // the challenge message, the peer can create the correct HMAC with this message + bytes Challenge=3; +} + +message KeyAuthChallengeReply{ + bytes HMAC=1; +} diff --git a/orderer/consensus/bdls/agent-tcp/tcp_peer.go b/orderer/consensus/bdls/agent-tcp/tcp_peer.go new file mode 100644 index 0000000000..4d4ade85a9 --- /dev/null +++ b/orderer/consensus/bdls/agent-tcp/tcp_peer.go @@ -0,0 +1,668 @@ +// BSD 3-Clause License +// +// Copyright (c) 2020, Sperax +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package agent + +import ( + "crypto/ecdsa" + "crypto/rand" + "crypto/subtle" + "encoding/binary" + fmt "fmt" + io "io" + "log" + "math/big" + "net" + "sync" + "time" + "unsafe" + + "github.com/Sperax/bdls" + "github.com/Sperax/bdls/crypto/blake2b" + "github.com/Sperax/bdls/timer" + proto "github.com/gogo/protobuf/proto" +) + +const ( + // Frame format: + // |MessageLength(4bytes)| Message(MessageLength) ... | + MessageLength = 4 + + // Message max length(32MB) + MaxMessageLength = 32 * 1024 * 1024 + + // timeout for a unresponsive connection + defaultReadTimeout = 60 * time.Second + defaultWriteTimeout = 60 * time.Second + + // challengeSize + challengeSize = 1024 +) + +// authenticationState is the authentication status for both peer +type authenticationState byte + +// peer initated public-key authentication status +const ( + // peerNotAuthenticated: the peer has just connected + peerNotAuthenticated authenticationState = iota + // peerSentAuthkey: the peer begined it's public key authentication, + // and we've sent out our challenge. + peerAuthkeyReceived + // peerAuthenticated: the peer has been authenticated to it's public key + peerAuthenticated + // peer failed to accept our challenge + peerAuthenticatedFailed +) + +// local initated public key authentication status +const ( + localNotAuthenticated authenticationState = iota + // localSentAuthKey: we have sent auth key command to the peer + localAuthKeySent + // localChallengeAccepted: we have received challenge from peer and responded + localChallengeAccepted +) + +// A TCPAgent binds consensus core to a TCPAgent object, which may have multiple TCPPeer +type TCPAgent struct { + consensus *bdls.Consensus // the consensus core + privateKey *ecdsa.PrivateKey // a private key to sign messages + peers []*TCPPeer // connected peers + consensusMessages [][]byte // all consensus message awaiting to be processed + chConsensusMessages chan struct{} // notification of new consensus message + + die chan struct{} // tcp agent closing + dieOnce sync.Once + sync.Mutex // fields lock +} + +// NewTCPAgent initiate a TCPAgent which talks consensus protocol with peers +func NewTCPAgent(consensus *bdls.Consensus, privateKey *ecdsa.PrivateKey) *TCPAgent { + agent := new(TCPAgent) + agent.consensus = consensus + agent.privateKey = privateKey + agent.die = make(chan struct{}) + agent.chConsensusMessages = make(chan struct{}, 1) + go agent.inputConsensusMessage() + return agent +} + +// AddPeer adds a peer to this agent +func (agent *TCPAgent) AddPeer(p *TCPPeer) bool { + agent.Lock() + defer agent.Unlock() + + select { + case <-agent.die: + return false + default: + agent.peers = append(agent.peers, p) + return agent.consensus.Join(p) + } +} + +// RemovePeer removes a TCPPeer from this agent +func (agent *TCPAgent) RemovePeer(p *TCPPeer) bool { + agent.Lock() + defer agent.Unlock() + + peerAddress := p.RemoteAddr().String() + for k := range agent.peers { + if agent.peers[k].RemoteAddr().String() == peerAddress { + copy(agent.peers[k:], agent.peers[k+1:]) + agent.peers = agent.peers[:len(agent.peers)-1] + return agent.consensus.Leave(p.RemoteAddr()) + } + } + return false +} + +// Close stops all activities on this agent +func (agent *TCPAgent) Close() { + agent.Lock() + defer agent.Unlock() + + agent.dieOnce.Do(func() { + close(agent.die) + // close all peers + for k := range agent.peers { + agent.peers[k].Close() + } + }) +} + +// Update is the consensus updater +func (agent *TCPAgent) Update() { + agent.Lock() + defer agent.Unlock() + + select { + case <-agent.die: + default: + // call consensus update + agent.consensus.Update(time.Now()) + timer.SystemTimedSched.Put(agent.Update, time.Now().Add(20*time.Millisecond)) + } +} + +// Propose a state, awaiting to be finalized at next height. +func (agent *TCPAgent) Propose(s bdls.State) { + agent.Lock() + defer agent.Unlock() + agent.consensus.Propose(s) +} + +// GetLatestState returns latest state +func (agent *TCPAgent) GetLatestState() (height uint64, round uint64, data bdls.State) { + agent.Lock() + defer agent.Unlock() + return agent.consensus.CurrentState() +} + +// handleConsensusMessage will be called if TCPPeer received a consensus message +func (agent *TCPAgent) handleConsensusMessage(bts []byte) { + agent.Lock() + defer agent.Unlock() + agent.consensusMessages = append(agent.consensusMessages, bts) + agent.notifyConsensus() +} + +func (agent *TCPAgent) notifyConsensus() { + select { + case agent.chConsensusMessages <- struct{}{}: + default: + } +} + +// consensus message receiver +func (agent *TCPAgent) inputConsensusMessage() { + for { + select { + case <-agent.chConsensusMessages: + agent.Lock() + msgs := agent.consensusMessages + agent.consensusMessages = nil + + for _, msg := range msgs { + agent.consensus.ReceiveMessage(msg, time.Now()) + } + agent.Unlock() + case <-agent.die: + return + } + } +} + +// fake address for Pipe +type fakeAddress string + +func (fakeAddress) Network() string { return "pipe" } +func (f fakeAddress) String() string { return string(f) } + +// TCPPeer represents a peer(endpoint) related to a tcp connection +type TCPPeer struct { + agent *TCPAgent // the agent it belongs to + conn net.Conn // the connection to this peer + peerAuthStatus authenticationState // peer authentication status + // the announced public key of the peer, only becomes valid if peerAuthStatus == peerAuthenticated + peerPublicKey *ecdsa.PublicKey + + // local authentication status + localAuthState authenticationState + + // the HMAC of the challenge text if peer has requested key authentication + hmac []byte + + // message queues and their notifications + consensusMessages [][]byte // all pending outgoing consensus messages to this peer + chConsensusMessage chan struct{} // notification on new consensus data + + // agent messages + agentMessages [][]byte // all pending outgoing agent messages to this peer. + chAgentMessage chan struct{} // notification on new agent exchange messages + + // peer closing signal + die chan struct{} + dieOnce sync.Once + + // mutex for all fields + sync.Mutex +} + +// NewTCPPeer creates a TCPPeer with protocol over this connection +func NewTCPPeer(conn net.Conn, agent *TCPAgent) *TCPPeer { + p := new(TCPPeer) + p.chConsensusMessage = make(chan struct{}, 1) + p.chAgentMessage = make(chan struct{}, 1) + p.conn = conn + p.agent = agent + p.die = make(chan struct{}) + // we start readLoop & sendLoop for each connection + go p.readLoop() + go p.sendLoop() + return p +} + +// RemoteAddr implements PeerInterface, GetPublicKey returns peer's +// public key, returns nil if peer's has not authenticated it's public-key +func (p *TCPPeer) GetPublicKey() *ecdsa.PublicKey { + p.Lock() + defer p.Unlock() + if p.peerAuthStatus == peerAuthenticated { + //log.Println("get public key:", p.peerPublicKey) + return p.peerPublicKey + } + return nil +} + +// RemoteAddr implements PeerInterface, returns peer's address as connection identity +func (p *TCPPeer) RemoteAddr() net.Addr { + if p.conn.RemoteAddr().Network() == "pipe" { + return fakeAddress(fmt.Sprint(unsafe.Pointer(p))) + } + return p.conn.RemoteAddr() +} + +// Send implements PeerInterface, to send message to this peer +func (p *TCPPeer) Send(out []byte) error { + p.Lock() + defer p.Unlock() + p.consensusMessages = append(p.consensusMessages, out) + p.notifyConsensusMessage() + return nil +} + +// notifyConsensusMessage notifies goroutines there're messages pending to send +func (p *TCPPeer) notifyConsensusMessage() { + select { + case p.chConsensusMessage <- struct{}{}: + default: + } +} + +// notifyAgentMessage, notifies goroutines there're agent messages pending to send +func (p *TCPPeer) notifyAgentMessage() { + select { + case p.chAgentMessage <- struct{}{}: + default: + } +} + +// Close terminates connection to this peer +func (p *TCPPeer) Close() { + p.dieOnce.Do(func() { + p.conn.Close() + close(p.die) + }) + go p.agent.RemovePeer(p) +} + +// InitiatePublicKeyAuthentication will initate a procedure to convince +// the other peer to trust my ownership of public key +func (p *TCPPeer) InitiatePublicKeyAuthentication() error { + p.Lock() + defer p.Unlock() + if p.localAuthState == localNotAuthenticated { + auth := KeyAuthInit{} + auth.X = p.agent.privateKey.PublicKey.X.Bytes() + auth.Y = p.agent.privateKey.PublicKey.Y.Bytes() + + // proto marshal + bts, err := proto.Marshal(&auth) + if err != nil { + panic(err) + } + + g := Gossip{Command: CommandType_KEY_AUTH_INIT, Message: bts} + // proto marshal + out, err := proto.Marshal(&g) + if err != nil { + panic(err) + } + + // enqueue + p.agentMessages = append(p.agentMessages, out) + p.notifyAgentMessage() + p.localAuthState = localAuthKeySent + return nil + } else { + return ErrPeerKeyAuthInit + } +} + +// handleGossip will process all messages from this peer based on it's message types +func (p *TCPPeer) handleGossip(msg *Gossip) error { + switch msg.Command { + case CommandType_NOP: // NOP can be used for connection keepalive + case CommandType_KEY_AUTH_INIT: + // this peer initated it's publickey authentication + var m KeyAuthInit + err := proto.Unmarshal(msg.Message, &m) + if err != nil { + return err + } + + err = p.handleKeyAuthInit(&m) + if err != nil { + return err + } + case CommandType_KEY_AUTH_CHALLENGE: + // received a challenge from this peer + var m KeyAuthChallenge + err := proto.Unmarshal(msg.Message, &m) + if err != nil { + return err + } + + err = p.handleKeyAuthChallenge(&m) + if err != nil { + return err + } + + case CommandType_KEY_AUTH_CHALLENGE_REPLY: + // this peer sends back a challenge reply to authenticate it's publickey + var m KeyAuthChallengeReply + err := proto.Unmarshal(msg.Message, &m) + if err != nil { + return err + } + + err = p.handleKeyAuthChallengeReply(&m) + if err != nil { + return err + } + + case CommandType_CONSENSUS: + // received a consensus message from this peer + p.agent.handleConsensusMessage(msg.Message) + default: + panic(msg) + } + return nil +} + +// peer initiated key authentication +func (p *TCPPeer) handleKeyAuthInit(authKey *KeyAuthInit) error { + p.Lock() + defer p.Unlock() + // only when in init status, authentication process cannot rollback + // to prevent from malicious re-authentication DoS + if p.peerAuthStatus == peerNotAuthenticated { + peerPublicKey := &ecdsa.PublicKey{Curve: bdls.S256Curve, X: big.NewInt(0).SetBytes(authKey.X), Y: big.NewInt(0).SetBytes(authKey.Y)} + + // on curve test + if !bdls.S256Curve.IsOnCurve(peerPublicKey.X, peerPublicKey.Y) { + p.peerAuthStatus = peerAuthenticatedFailed + return ErrKeyNotOnCurve + } + // temporarily stored announced key + p.peerPublicKey = peerPublicKey + + // create ephermal key for authentication + ephemeral, err := ecdsa.GenerateKey(bdls.S256Curve, rand.Reader) + if err != nil { + panic(err) + } + // derive secret + secret := ECDH(p.peerPublicKey, ephemeral) + + // generate challenge texts + var challenge KeyAuthChallenge + challenge.X = ephemeral.PublicKey.X.Bytes() + challenge.Y = ephemeral.PublicKey.Y.Bytes() + challenge.Challenge = make([]byte, challengeSize) + _, err = io.ReadFull(rand.Reader, challenge.Challenge) + if err != nil { + panic(err) + } + + // calculates & store HMAC for this random message + hmac, err := blake2b.New256(secret.Bytes()) + if err != nil { + panic(err) + } + hmac.Write(challenge.Challenge) + p.hmac = hmac.Sum(nil) + + // proto marshal + bts, err := proto.Marshal(&challenge) + if err != nil { + panic(err) + } + + g := Gossip{Command: CommandType_KEY_AUTH_CHALLENGE, Message: bts} + // proto marshal + out, err := proto.Marshal(&g) + if err != nil { + panic(err) + } + + // enqueue + p.agentMessages = append(p.agentMessages, out) + p.notifyAgentMessage() + + // state shift + p.peerAuthStatus = peerAuthkeyReceived + return nil + } else { + return ErrPeerKeyAuthInit + } +} + +// handle key authentication challenge +func (p *TCPPeer) handleKeyAuthChallenge(challenge *KeyAuthChallenge) error { + p.Lock() + defer p.Unlock() + if p.localAuthState == localAuthKeySent { + // use ECDH to recover shared-key + pubkey := &ecdsa.PublicKey{Curve: bdls.S256Curve, X: big.NewInt(0).SetBytes(challenge.X), Y: big.NewInt(0).SetBytes(challenge.Y)} + // derive secret with my private key + secret := ECDH(pubkey, p.agent.privateKey) + + // calculates HMAC for the challenge with the key above + var response KeyAuthChallengeReply + hmac, err := blake2b.New256(secret.Bytes()) + if err != nil { + panic(err) + } + hmac.Write(challenge.Challenge) + response.HMAC = hmac.Sum(nil) + + // proto marshal + bts, err := proto.Marshal(&response) + if err != nil { + panic(err) + } + + g := Gossip{Command: CommandType_KEY_AUTH_CHALLENGE_REPLY, Message: bts} + // proto marshal + out, err := proto.Marshal(&g) + if err != nil { + panic(err) + } + + // enqueue + p.agentMessages = append(p.agentMessages, out) + p.notifyAgentMessage() + + // state shift + p.localAuthState = localChallengeAccepted + return nil + } else { + return ErrPeerKeyAuthChallenge + } +} + +// handle key authentication challenge reply +func (p *TCPPeer) handleKeyAuthChallengeReply(response *KeyAuthChallengeReply) error { + p.Lock() + defer p.Unlock() + if p.peerAuthStatus == peerAuthkeyReceived { + if subtle.ConstantTimeCompare(p.hmac, response.HMAC) == 1 { + p.hmac = nil + p.peerAuthStatus = peerAuthenticated + return nil + } else { + p.peerAuthStatus = peerAuthenticatedFailed + return ErrPeerAuthenticatedFailed + } + } else { + return ErrPeerKeyAuthInit + } +} + +// readLoop keeps reading messages from peer +func (p *TCPPeer) readLoop() { + defer p.Close() + msgLength := make([]byte, MessageLength) + + for { + select { + case <-p.die: + return + default: + // read message size + p.conn.SetReadDeadline(time.Now().Add(defaultReadTimeout)) + _, err := io.ReadFull(p.conn, msgLength) + if err != nil { + return + } + + // check length + length := binary.LittleEndian.Uint32(msgLength) + if length > MaxMessageLength { + log.Println(err) + return + } + + if length == 0 { + log.Println("zero length") + return + } + + // read message bytes + p.conn.SetReadDeadline(time.Now().Add(defaultReadTimeout)) + bts := make([]byte, length) + _, err = io.ReadFull(p.conn, bts) + if err != nil { + return + } + + // unmarshal bytes to message + var gossip Gossip + err = proto.Unmarshal(bts, &gossip) + if err != nil { + log.Println(err) + return + } + + err = p.handleGossip(&gossip) + if err != nil { + log.Println(err) + return + } + } + } +} + +// sendLoop keeps sending consensus message to this peer +func (p *TCPPeer) sendLoop() { + defer p.Close() + + var pending [][]byte + var msg Gossip + msg.Command = CommandType_CONSENSUS + msgLength := make([]byte, MessageLength) + + for { + select { + case <-p.chConsensusMessage: + p.Lock() + pending = p.consensusMessages + p.consensusMessages = nil + p.Unlock() + + for _, bts := range pending { + // we need to encapsulate consensus messages + msg.Message = bts + out, err := proto.Marshal(&msg) + if err != nil { + panic(err) + } + + if len(out) > MaxMessageLength { + panic("maximum message size exceeded") + } + + binary.LittleEndian.PutUint32(msgLength, uint32(len(out))) + p.conn.SetWriteDeadline(time.Now().Add(defaultWriteTimeout)) + // write length + _, err = p.conn.Write(msgLength) + if err != nil { + log.Println(err) + return + } + + // write message + _, err = p.conn.Write(out) + if err != nil { + log.Println(err) + return + } + } + case <-p.chAgentMessage: + p.Lock() + pending = p.agentMessages + p.agentMessages = nil + p.Unlock() + + for _, bts := range pending { + binary.LittleEndian.PutUint32(msgLength, uint32(len(bts))) + // write length + _, err := p.conn.Write(msgLength) + if err != nil { + log.Println(err) + return + } + + // write message + _, err = p.conn.Write(bts) + if err != nil { + log.Println(err) + return + } + } + + case <-p.die: + return + } + } +} diff --git a/orderer/consensus/bdls/agent-tcp/tcp_peer_test.go b/orderer/consensus/bdls/agent-tcp/tcp_peer_test.go new file mode 100644 index 0000000000..03295436c6 --- /dev/null +++ b/orderer/consensus/bdls/agent-tcp/tcp_peer_test.go @@ -0,0 +1,218 @@ +package agent + +import ( + "bytes" + "crypto/ecdsa" + "crypto/rand" + "encoding/binary" + "encoding/hex" + io "io" + "log" + "net" + "net/http" + _ "net/http/pprof" + "sync" + "testing" + "time" + + "github.com/Sperax/bdls" + "github.com/Sperax/bdls/crypto/blake2b" + "github.com/davecgh/go-spew/spew" + "github.com/stretchr/testify/assert" +) + +// init will listen for 6060 while debugging +func init() { + go func() { + log.Println(http.ListenAndServe("0.0.0.0:6060", nil)) + }() +} + +type testParam struct { + numPeers int + numParticipants int + stopHeight int + expectedLatency time.Duration +} + +func TestTCPPeer(t *testing.T) { + var params = []testParam{ + { + numPeers: 20, + numParticipants: 20, + stopHeight: 5, + expectedLatency: 100 * time.Millisecond, + }, + { + numPeers: 20, + numParticipants: 20, + stopHeight: 5, + expectedLatency: 200 * time.Millisecond, + }, + { + numPeers: 20, + numParticipants: 20, + stopHeight: 5, + expectedLatency: 300 * time.Millisecond, + }, + { + numPeers: 20, + numParticipants: 20, + stopHeight: 5, + expectedLatency: 500 * time.Millisecond, + }, + { + numPeers: 20, + numParticipants: 20, + stopHeight: 5, + expectedLatency: 1000 * time.Millisecond, + }, + } + for i := 0; i < len(params); i++ { + t.Logf("-=-=- TESTING CASE: [%v/%v] -=-=-", i+1, len(params)) + testConsensus(t, ¶ms[i]) + } +} + +func testConsensus(t *testing.T, param *testParam) { + t.Logf("PARAMETERS: %+v", spew.Sprintf("%+v", param)) + var participants []*ecdsa.PrivateKey + var coords []bdls.Identity + for i := 0; i < param.numParticipants; i++ { + privateKey, err := ecdsa.GenerateKey(bdls.S256Curve, rand.Reader) + if err != nil { + t.Fatal(err) + } + + participants = append(participants, privateKey) + coords = append(coords, bdls.DefaultPubKeyToIdentity(&privateKey.PublicKey)) + } + + // consensus for one height + consensusOneHeight := func(currentHeight uint64) { + // randomize participants, fisher yates shuffle + n := uint32(len(participants)) + for i := n - 1; i > 0; i-- { + var j uint32 + binary.Read(rand.Reader, binary.LittleEndian, &j) + j = j % (i + 1) + participants[i], participants[j] = participants[j], participants[i] + } + + // created a locked consensus object + var all []*bdls.Consensus + + // same epoch + epoch := time.Now() + // create numPeer peers + for i := 0; i < param.numPeers; i++ { + // initiate config + config := new(bdls.Config) + config.Epoch = epoch + config.CurrentHeight = currentHeight + config.PrivateKey = participants[i] // randomized participants + config.Participants = coords // keep all pubkeys + + // should replace with real function + config.StateCompare = func(a bdls.State, b bdls.State) int { return bytes.Compare(a, b) } + config.StateValidate = func(a bdls.State) bool { return true } + + // consensus + consensus, err := bdls.NewConsensus(config) + assert.Nil(t, err) + consensus.SetLatency(param.expectedLatency) + all = append(all, consensus) + } + + // establish full connected mesh with tcp_peer + numConns := 0 + agents := make([]*TCPAgent, len(all)) + for i := 0; i < len(all); i++ { + agents[i] = NewTCPAgent(all[i], participants[i]) + } + + for i := 0; i < len(all); i++ { + for j := 0; j < len(all); j++ { + if i != j { + c1, c2 := net.Pipe() // in memory duplex pipe to connection i & j + p1 := NewTCPPeer(c1, agents[i]) + p2 := NewTCPPeer(c2, agents[j]) + ok := agents[i].AddPeer(p1) + assert.True(t, ok) + ok = agents[j].AddPeer(p2) + assert.True(t, ok) + numConns += 2 + + // auth public key + p1.InitiatePublicKeyAuthentication() + p2.InitiatePublicKeyAuthentication() + } + } + } + + <-time.After(2 * time.Second) + + // make sure authentication completed + for i := 0; i < len(all); i++ { + for _, peer := range agents[i].peers { + peer.Lock() + assert.Equal(t, peer.localAuthState, localChallengeAccepted) + assert.Equal(t, peer.peerAuthStatus, peerAuthenticated) + peer.Unlock() + } + } + + // after all connections have established, start updater, + // this must be done after connection establishement + // to prevent from missing messages + for i := 0; i < len(all); i++ { + agents[i].Update() + } + + var wg sync.WaitGroup + wg.Add(param.numPeers) + + // selected random peers + for k := range agents { + go func(i int) { + agent := agents[i] + defer wg.Done() + + data := make([]byte, 1024) + io.ReadFull(rand.Reader, data) + agent.Propose(data) + + for { + newHeight, newRound, newState := agent.GetLatestState() + if newHeight > currentHeight { + now := time.Now() + // only one peer print the decide + if i == 0 { + h := blake2b.Sum256(newState) + t.Logf("%v at height:%v round:%v hash:%v", now.Format("15:04:05"), newHeight, newRound, hex.EncodeToString(h[:])) + } + + return + } + + // wait + <-time.After(20 * time.Millisecond) + } + }(k) + } + + // wait for all peers exit + wg.Wait() + // close all peers when waitgroup exit + for k := range agents { + agents[k].Close() + } + } + + // loop to stopHeight + for i := 0; i < param.stopHeight; i++ { + consensusOneHeight(uint64(i)) + } + + t.Logf("consensus stopped at height:%v for %v peers %v participants", param.stopHeight, param.numPeers, param.numParticipants) +} diff --git a/orderer/consensus/bdls/benchmarks/AMD-NORMAL.TXT b/orderer/consensus/bdls/benchmarks/AMD-NORMAL.TXT new file mode 100644 index 0000000000..50f734b9ab --- /dev/null +++ b/orderer/consensus/bdls/benchmarks/AMD-NORMAL.TXT @@ -0,0 +1,155 @@ +DATE: 2020/03/21 +OS: Linux 4.19.84-microsoft-standard #1 SMP Wed Nov 13 11:44:37 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux +MEM: 64GB +CPU: AMD Ryzen 7 2700X Eight-Core Processor + +TERMINOLOGY: + +DECIDE.AVG = Average finalization time for each height. +DECIDE.ROUNDS = The rounds where decides has made. +PEER.NUM = Actual participantion. +PJ.NUM = Participants(Quorum) +NET.MSGS = Total network number of messages exchanged in all heights. +NET.BYTES = Total network bytes exchanged in all heights. +MSG.AVGSIZE = Average message size.(Tested with 1KB State.) +NET.MSGRATE = Network message rate(messages/second). +PEER.RATE = Peer's average bandwidth. +DELAY.MIN = Actual minimal network latency(network latency is randomized with normal distribution). +DELAY.MAX = Actual maximal network latency. +DELAY.AVG = Actual average latency. +DELAY.EXP = Expected Latency set to consensus algorithm. + +TESTING CASES: +============= + +Case 1: 20 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 1.48s | 1;1;1;1;1 | 20 | 20 | 7963 | 14.2M | 1.8K | 53.68/s | 96.9K/s | 64.46ms | 140.57ms | 100.02ms | 100ms | +| 2.28s | 1;1;1;1;1 | 20 | 20 | 7958 | 13.9M | 1.8K | 34.76/s | 61K/s | 118.89ms | 268.65ms | 199.85ms | 200ms | +| 3.12s | 1;1;1;1;1 | 20 | 20 | 7951 | 14.1M | 1.8K | 25.44/s | 45.2K/s | 193.54ms | 419.38ms | 299.92ms | 300ms | +| 4.73s | 1;1;1;1;1 | 20 | 20 | 7950 | 13.9M | 1.8K | 16.79/s | 29.2K/s | 323.98ms | 677.95ms | 499.45ms | 500ms | +| 8.86s | 1;1;1;1;1 | 20 | 20 | 7948 | 13.8M | 1.8K | 8.96/s | 15.6K/s | 623.92ms | 1.36059s | 998.96ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 2: 30 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 1.54s | 1;1;1;1;1 | 30 | 30 | 17943 | 31.7M | 1.8K | 77.57/s | 136.8K/s | 57.05ms | 141.33ms | 99.93ms | 100ms | +| 2.34s | 1;1;1;1;1 | 30 | 30 | 17940 | 31.8M | 1.8K | 50.91/s | 90.5K/s | 122.94ms | 275.47ms | 199.93ms | 200ms | +| 3.14s | 1;1;1;1;1 | 30 | 30 | 17936 | 31.4M | 1.8K | 38.05/s | 66.8K/s | 189.2ms | 413.06ms | 299.87ms | 300ms | +| 4.82s | 1;1;1;1;1 | 30 | 30 | 17934 | 31.2M | 1.8K | 24.78/s | 43.2K/s | 300.65ms | 695.86ms | 499.62ms | 500ms | +| 8.88s | 1;1;1;1;1 | 30 | 30 | 17929 | 31.3M | 1.8K | 13.46/s | 23.4K/s | 599.88ms | 1.36947s | 999.69ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 3: 50 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 1.87s | 1;1;1;1;1 | 50 | 50 | 48219 | 92.8M | 2K | 102.67/s | 195.6K/s | 55ms | 138.57ms | 99.96ms | 100ms | +| 2.58s | 1;1;1;1;1 | 50 | 50 | 49785 | 89M | 1.8K | 77.18/s | 136K/s | 116.73ms | 278.85ms | 199.98ms | 200ms | +| 3.39s | 1;1;1;1;1 | 50 | 50 | 49902 | 88.1M | 1.8K | 58.84/s | 102.3K/s | 178.59ms | 421.13ms | 299.82ms | 300ms | +| 4.94s | 1;1;1;1;1 | 50 | 50 | 49903 | 88M | 1.8K | 40.40/s | 70.8K/s | 283.57ms | 719.42ms | 499.86ms | 500ms | +| 9.05s | 1;1;1;1;1 | 50 | 50 | 49904 | 87.9M | 1.8K | 22.06/s | 38.6K/s | 517.2ms | 1.40479s | 999.96ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 4: 80 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 5.47s | 2;2;2;2;2 | 80 | 80 | 226779 | 1.6G | 7.4K | 103.64/s | 758.4K/s | 55.91ms | 145.08ms | 100ms | 100ms | +| 3.56s | 1;1;1;2;1 | 80 | 80 | 142067 | 253.5M | 1.8K | 99.57/s | 176K/s | 109.34ms | 291.31ms | 199.93ms | 200ms | +| 4.02s | 1;1;1;1;1 | 80 | 80 | 126898 | 229.3M | 1.8K | 78.77/s | 140.2K/s | 175.69ms | 439.64ms | 300.13ms | 300ms | +| 5.38s | 1;1;1;1;1 | 80 | 80 | 127408 | 227.1M | 1.8K | 59.18/s | 103.3K/s | 270.44ms | 729.47ms | 499.96ms | 500ms | +| 9.32s | 1;1;1;1;1 | 80 | 80 | 127853 | 226.1M | 1.8K | 34.27/s | 59.9K/s | 562.83ms | 1.49316s | 999.85ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 5: 100 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 8.66s | 2;3;2;2;3 | 100 | 100 | 408440 | 2.8G | 7.3K | 94.22/s | 677K/s | 54.63ms | 146.85ms | 99.72ms | 100ms | +| 6.95s | 1;2;1;2;1 | 100 | 100 | 267820 | 2G | 7.8K | 76.99/s | 586.8K/s | 95.77ms | 295.5ms | 199.56ms | 200ms | +| 4.7s | 1;1;1;1;1 | 100 | 100 | 200412 | 408M | 2.1K | 85.12/s | 168.9K/s | 167.97ms | 430.46ms | 299.87ms | 300ms | +| 5.96s | 1;1;1;1;1 | 100 | 100 | 195934 | 355M | 1.9K | 65.74/s | 116.3K/s | 279.79ms | 723.91ms | 499.94ms | 500ms | +| 9.7s | 1;1;1;1;1 | 100 | 100 | 199823 | 353.9M | 1.8K | 41.19/s | 71.6K/s | 548.69ms | 1.46597s | 1.00015s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 6: 20 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 2.82s | 2;1;1;1;3 | 13 | 20 | 4296 | 7.4M | 1.8K | 23.40/s | 41.1K/s | 64.59ms | 136.04ms | 99.94ms | 100ms | +| 4.55s | 2;2;1;2;1 | 13 | 20 | 4296 | 7.4M | 1.8K | 14.52/s | 25.4K/s | 122.88ms | 269.19ms | 199.9ms | 200ms | +| 17.81s | 1;2;4;1;4 | 13 | 20 | 5544 | 8.8M | 1.6K | 4.79/s | 7.8K/s | 188.9ms | 410.09ms | 299.97ms | 300ms | +| 12.66s | 1;2;3;1;1 | 13 | 20 | 4296 | 7.4M | 1.8K | 5.22/s | 9.2K/s | 342.69ms | 701ms | 500.18ms | 500ms | +| 27.43s | 2;1;3;1;2 | 13 | 20 | 4608 | 7.8M | 1.7K | 2.58/s | 4.4K/s | 676.11ms | 1.34253s | 998.91ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 7: 30 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 2.9s | 2;1;1;1;3 | 20 | 30 | 10257 | 17.7M | 1.8K | 35.28/s | 61.8K/s | 62.5ms | 142.77ms | 99.83ms | 100ms | +| 5.31s | 1;2;3;1;1 | 20 | 30 | 10257 | 17.7M | 1.8K | 19.31/s | 33.9K/s | 126.07ms | 277.9ms | 200.05ms | 200ms | +| 9.53s | 2;2;2;2;3 | 20 | 30 | 12536 | 20.2M | 1.7K | 13.15/s | 21.6K/s | 176.39ms | 419.26ms | 299.67ms | 300ms | +| 23.24s | 3;2;3;2;3 | 20 | 30 | 14058 | 22M | 1.6K | 6.05/s | 9.6K/s | 306.28ms | 712.13ms | 499.88ms | 500ms | +| 24.07s | 1;2;2;2;2 | 20 | 30 | 11015 | 18.5M | 1.7K | 4.58/s | 7.8K/s | 603.76ms | 1.34469s | 1.00056s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 8: 50 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 4.17s | 4;1;1;1;1 | 33 | 50 | 28147 | 51.2M | 1.9K | 40.82/s | 75K/s | 56.12ms | 142.04ms | 100.03ms | 100ms | +| 6.41s | 2;2;2;2;2 | 33 | 50 | 32320 | 53.9M | 1.7K | 30.55/s | 51.6K/s | 115.69ms | 272.97ms | 200.11ms | 200ms | +| 6.35s | 1;1;2;2;2 | 33 | 50 | 28096 | 49.2M | 1.8K | 26.80/s | 47.5K/s | 180.47ms | 415.66ms | 300.18ms | 300ms | +| 14.32s | 1;2;2;2;3 | 33 | 50 | 32320 | 53.9M | 1.7K | 13.67/s | 23.2K/s | 298.93ms | 731.12ms | 500.17ms | 500ms | +| 16.73s | 2;1;1;1;2 | 33 | 50 | 25984 | 46.8M | 1.8K | 9.41/s | 17.1K/s | 622.68ms | 1.41558s | 1.00058s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 9: 80 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 18.11s | 5;2;4;2;6 | 53 | 80 | 140369 | 223.5M | 1.6K | 29.24/s | 47.5K/s | 59.71ms | 144.01ms | 99.96ms | 100ms | +| 4.54s | 1;1;2;2;1 | 53 | 80 | 67292 | 128.1M | 1.9K | 55.89/s | 106.1K/s | 119.26ms | 281.52ms | 200.08ms | 200ms | +| 19.31s | 5;2;1;1;1 | 53 | 80 | 83758 | 142.4M | 1.7K | 16.37/s | 28.3K/s | 182.5ms | 424.02ms | 299.79ms | 300ms | +| 18.59s | 1;1;4;1;2 | 53 | 80 | 78210 | 134M | 1.8K | 15.87/s | 27.5K/s | 266.41ms | 701.31ms | 500.3ms | 500ms | +| 18.57s | 2;1;1;2;1 | 53 | 80 | 67184 | 121.5M | 1.9K | 13.65/s | 25K/s | 561.12ms | 1.44487s | 999.79ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 10: 100 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 8.59s | 3;3;4;2;4 | 67 | 100 | 215492 | 355.4M | 1.7K | 74.85/s | 125.1K/s | 53.09ms | 141.25ms | 99.99ms | 100ms | +| 6.34s | 2;1;2;2;2 | 67 | 100 | 128020 | 271.1M | 2.2K | 60.25/s | 127K/s | 115.91ms | 293.22ms | 199.83ms | 200ms | +| 6.93s | 2;1;2;1;1 | 67 | 100 | 107506 | 199.1M | 1.9K | 46.25/s | 85.6K/s | 172.89ms | 427.05ms | 299.95ms | 300ms | +| 24.25s | 2;1;1;4;3 | 67 | 100 | 142861 | 237.1M | 1.7K | 17.58/s | 29.6K/s | 277.52ms | 721.17ms | 499.78ms | 500ms | +| 36.9s | 2;2;2;3;3 | 67 | 100 | 151668 | 244.2M | 1.6K | 12.27/s | 20K/s | 545.64ms | 1.46132s | 999.95ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 11: 50 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s incorrectly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 2.93s | 2;2;2;2;2 | 50 | 50 | 97011 | 473.6M | 5K | 132.12/s | 646.1K/s | 56.6ms | 146.01ms | 99.96ms | 50ms | +| 3.23s | 1;2;2;1;1 | 50 | 50 | 77765 | 329.7M | 4.3K | 96.03/s | 396.4K/s | 111.47ms | 280.69ms | 199.77ms | 100ms | +| 3.17s | 1;1;1;1;1 | 50 | 50 | 61241 | 118.4M | 2K | 77.04/s | 143.6K/s | 169.9ms | 451.42ms | 299.89ms | 150ms | +| 4.51s | 1;1;1;1;1 | 50 | 50 | 61517 | 101M | 1.7K | 54.44/s | 88.8K/s | 287.24ms | 723.02ms | 500.12ms | 250ms | +| 8.01s | 1;1;1;1;1 | 50 | 50 | 61073 | 100.3M | 1.7K | 30.48/s | 49.7K/s | 565.19ms | 1.43528s | 999.52ms | 500ms | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 12: 50 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s incorrectly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 4.22s | 3;4;3;2;2 | 33 | 50 | 47481 | 76.6M | 1.7K | 68.15/s | 111.1K/s | 55.71ms | 139.64ms | 100.02ms | 50ms | +| 6.05s | 4;3;2;2;2 | 33 | 50 | 43936 | 66.9M | 1.6K | 43.96/s | 67.7K/s | 117.99ms | 276.69ms | 200.08ms | 100ms | +| 8.83s | 2;3;1;2;4 | 33 | 50 | 41856 | 64.6M | 1.6K | 28.73/s | 45K/s | 162.82ms | 416.3ms | 300.1ms | 150ms | +| 12.87s | 4;1;2;1;2 | 33 | 50 | 37600 | 59.8M | 1.6K | 17.70/s | 28.6K/s | 286.24ms | 719.81ms | 500.05ms | 250ms | +| 14.24s | 2;2;1;1;2 | 33 | 50 | 33376 | 55.1M | 1.7K | 14.20/s | 23.7K/s | 550.18ms | 1.41899s | 1.00052s | 500ms | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ diff --git a/orderer/consensus/bdls/benchmarks/PI4-OVERLOAD.TXT b/orderer/consensus/bdls/benchmarks/PI4-OVERLOAD.TXT new file mode 100644 index 0000000000..68acadae2d --- /dev/null +++ b/orderer/consensus/bdls/benchmarks/PI4-OVERLOAD.TXT @@ -0,0 +1,160 @@ +DATE: 2020/03/18 +OS: Linux raspberrypi 4.19.75-v7l+ #1270 SMP Tue Sep 24 18:51:41 BST 2019 armv7l GNU/Linux +MEM: 1GB +CPU: BCM2835(4 cores) + +TESTING PURPOSE: CONVERGENCE TEST FOR BDLS CONSENSUS ALGORITHM UNDER HEAVY LOAD. + + +TERMINOLOGY: + +DECIDE.AVG = Average finalization time for each height. +DECIDE.ROUNDS = The rounds where decides has made. +PEER.NUM = Actual participantion. +PJ.NUM = Participants(Quorum) +NET.MSGS = Total network number of messages exchanged in all heights. +NET.BYTES = Total network bytes exchanged in all heights. +MSG.AVGSIZE = Average message size.(Tested with 1KB State.) +NET.MSGRATE = Network message rate(messages/second). +PEER.RATE = Peer's average bandwidth. +DELAY.MIN = Actual minimal network latency(network latency is randomized with normal distribution). +DELAY.MAX = Actual maximal network latency. +DELAY.AVG = Actual average latency. +DELAY.EXP = Expected Latency set to consensus algorithm. + +TESTING CASES: +============= + +Case 1: 20 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 3.75s | 2;1;2;2;2 | 20 | 20 | 14790 | 27.8M | 1.9K | 39.35/s | 73.3K/s | 60.96ms | 139.58ms | 99.87ms | 100ms | +| 3.48s | 1;1;1;1;1 | 20 | 20 | 10478 | 17M | 1.7K | 30.06/s | 47.9K/s | 122.84ms | 275.61ms | 199.78ms | 200ms | +| 4.11s | 1;1;1;1;1 | 20 | 20 | 9912 | 16M | 1.7K | 24.09/s | 38.5K/s | 187.74ms | 414.37ms | 299.7ms | 300ms | +| 5.73s | 1;1;1;1;1 | 20 | 20 | 9754 | 15.9M | 1.7K | 17.00/s | 27.3K/s | 317.28ms | 705.46ms | 499.4ms | 500ms | +| 9.37s | 1;1;1;1;1 | 20 | 20 | 9753 | 15.8M | 1.7K | 10.41/s | 16.9K/s | 588.93ms | 1.37534s | 1.00026s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 2: 30 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 11.55s | 2;3;3;2;2 | 30 | 30 | 51714 | 187.7M | 3.7K | 29.85/s | 109.8K/s | 60.01ms | 144.83ms | 99.98ms | 100ms | +| 8.36s | 2;2;2;2;2 | 30 | 30 | 37438 | 83.8M | 2.3K | 29.82/s | 66.7K/s | 116.52ms | 279.65ms | 199.88ms | 200ms | +| 6.65s | 1;1;2;1;1 | 30 | 30 | 25796 | 61.3M | 2.4K | 25.83/s | 60.6K/s | 175.77ms | 420.04ms | 299.61ms | 300ms | +| 7.22s | 1;1;1;1;1 | 30 | 30 | 23462 | 37.5M | 1.6K | 21.63/s | 34K/s | 310.22ms | 699.73ms | 500.22ms | 500ms | +| 11.02s | 1;1;1;1;1 | 30 | 30 | 22132 | 35.8M | 1.7K | 13.38/s | 21.5K/s | 622.43ms | 1.3779s | 999.86ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 3: 50 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 1m2.63s | 4;4;4;4;4 | 50 | 50 | 233451 | 1.5G | 6.8K | 14.91/s | 101.2K/s | 46ms | 142.93ms | 100.01ms | 100ms | +| 33.36s | 3;3;3;3;3 | 50 | 50 | 136913 | 592.1M | 4.4K | 16.41/s | 71.8K/s | 110.08ms | 290.83ms | 199.99ms | 200ms | +| 29.81s | 2;3;3;3;3 | 50 | 50 | 136037 | 553.7M | 4.2K | 18.25/s | 75.2K/s | 177.29ms | 436.52ms | 299.96ms | 300ms | +| 20.15s | 2;2;2;2;2 | 50 | 50 | 91426 | 184.4M | 2.1K | 18.15/s | 36.5K/s | 268.65ms | 698.08ms | 498.93ms | 500ms | +| 16.87s | 1;1;1;1;1 | 50 | 50 | 69318 | 113.6M | 1.7K | 16.43/s | 26.5K/s | 567.85ms | 1.4057s | 999.14ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 4: 80 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 4m51.32s | 5;6;5;5;6 | 80 | 80 | 725480 | 7G | 10.1K | 6.23/s | 62.6K/s | 52.38ms | 148.5ms | 99.99ms | 100ms | +| 3m51.4s | 4;4;4;4;5 | 80 | 80 | 574985 | 5.5G | 10.1K | 6.21/s | 62.3K/s | 102.31ms | 291.46ms | 200ms | 200ms | +| 2m10.43s | 3;4;4;4;3 | 80 | 80 | 441791 | 2.9G | 6.9K | 8.47/s | 58.1K/s | 167.07ms | 438.29ms | 299.95ms | 300ms | +| 2m35.76s | 3;3;3;3;3 | 80 | 80 | 402765 | 3.6G | 9.4K | 6.46/s | 60.6K/s | 251.75ms | 725.84ms | 499.93ms | 500ms | +| 1m8.64s | 2;3;2;2;2 | 80 | 80 | 273630 | 1.1G | 4.3K | 9.97/s | 42.6K/s | 495.14ms | 1.45231s | 999.34ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 5: 100 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 7m37.04s | 6;6;6;5;6 | 100 | 100 | 1158517 | 11.8G | 10.7K | 5.07/s | 54.1K/s | 52.5ms | 146.54ms | 99.98ms | 100ms | +| 7m27.17s | 5;5;5;5;5 | 100 | 100 | 1092993 | 14.6G | 14K | 4.89/s | 68.1K/s | 99.45ms | 296.03ms | 200.02ms | 200ms | +| 6m13.93s | 4;4;4;5;4 | 100 | 100 | 858462 | 9.3G | 11.3K | 4.59/s | 51.9K/s | 159.76ms | 441.97ms | 300.01ms | 300ms | +| 6m38.5s | 4;4;3;4;4 | 100 | 100 | 826485 | 10.7G | 13.5K | 4.15/s | 56K/s | 259.27ms | 726.32ms | 500.03ms | 500ms | +| 1m59.12s | 2;3;2;3;3 | 100 | 100 | 467686 | 2.2G | 5K | 7.85/s | 38.6K/s | 508.84ms | 1.45051s | 999.67ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 6: 20 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 7.37s | 3;3;2;2;3 | 13 | 20 | 7685 | 15.9M | 2.1K | 16.03/s | 33.6K/s | 64.44ms | 135.63ms | 100.09ms | 100ms | +| 6.59s | 2;2;3;1;1 | 13 | 20 | 5377 | 8.7M | 1.6K | 12.54/s | 20.4K/s | 111.78ms | 274.1ms | 199.6ms | 200ms | +| 11.21s | 3;1;2;3;2 | 13 | 20 | 5952 | 9.3M | 1.6K | 8.16/s | 12.9K/s | 181.75ms | 401.53ms | 300.18ms | 300ms | +| 23.4s | 2;1;4;3;1 | 13 | 20 | 5952 | 9.3M | 1.6K | 3.91/s | 6.2K/s | 290.63ms | 671.67ms | 499.92ms | 500ms | +| 58.61s | 2;3;1;5;1 | 13 | 20 | 6264 | 9.6M | 1.6K | 1.64/s | 2.6K/s | 679.76ms | 1.38604s | 1.00145s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 7: 30 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 10.34s | 3;5;3;2;2 | 20 | 30 | 22734 | 57.4M | 2.6K | 21.98/s | 56.1K/s | 60.67ms | 138.89ms | 99.92ms | 100ms | +| 6.13s | 2;1;2;1;2 | 20 | 30 | 15414 | 40.8M | 2.7K | 25.12/s | 66K/s | 115.82ms | 270.86ms | 199.64ms | 200ms | +| 22.66s | 2;1;3;5;1 | 20 | 30 | 16624 | 24.9M | 1.5K | 7.33/s | 11.2K/s | 178.08ms | 409.32ms | 299.93ms | 300ms | +| 9.94s | 1;1;2;1;2 | 20 | 30 | 11339 | 18.9M | 1.7K | 11.40/s | 19.1K/s | 319.77ms | 688.61ms | 500.2ms | 500ms | +| 17.86s | 2;2;1;1;1 | 20 | 30 | 11300 | 18.8M | 1.7K | 6.32/s | 10.6K/s | 556.13ms | 1.37776s | 1.00001s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 8: 50 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 18.26s | 4;2;4;3;4 | 33 | 50 | 75881 | 250.6M | 3.4K | 25.19/s | 83.9K/s | 56.98ms | 140.48ms | 99.98ms | 100ms | +| 16.42s | 4;3;2;3;3 | 33 | 50 | 55693 | 86.5M | 1.6K | 20.55/s | 32K/s | 111.43ms | 291.95ms | 200ms | 200ms | +| 39.53s | 4;2;3;5;4 | 33 | 50 | 62916 | 133.5M | 2.2K | 9.65/s | 20.8K/s | 181.36ms | 434.34ms | 299.83ms | 300ms | +| 28.39s | 3;3;3;3;1 | 33 | 50 | 52838 | 156.8M | 3K | 11.28/s | 33.8K/s | 282.96ms | 713.86ms | 499.58ms | 500ms | +| 37.25s | 4;1;1;1;1 | 33 | 50 | 36063 | 60.5M | 1.7K | 5.87/s | 9.9K/s | 607.98ms | 1.41903s | 1.00058s | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 9: 80 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 1m26.31s | 7;5;4;6;4 | 53 | 80 | 293454 | 2.1G | 7.3K | 12.83/s | 93.6K/s | 53.54ms | 146.09ms | 99.96ms | 100ms | +| 1m22.04s | 5;6;5;5;4 | 53 | 80 | 246294 | 1.1G | 4.8K | 11.33/s | 54.4K/s | 107.26ms | 286.39ms | 199.99ms | 200ms | +| 1m0.36s | 5;4;4;3;4 | 53 | 80 | 195944 | 592M | 3.1K | 12.25/s | 37.5K/s | 169.32ms | 439.17ms | 299.91ms | 300ms | +| 54.15s | 3;3;3;4;3 | 53 | 80 | 166929 | 571.7M | 3.5K | 11.63/s | 40.3K/s | 298.27ms | 706.16ms | 499.92ms | 500ms | +| 1m12.74s | 4;3;3;2;2 | 53 | 80 | 144462 | 379.2M | 2.7K | 7.49/s | 19.9K/s | 486.55ms | 1.4299s | 999.96ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 10: 100 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s correctly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 1m40.68s | 6;5;5;6;6 | 67 | 100 | 468680 | 1.9G | 4.2K | 13.89/s | 57.3K/s | 55.72ms | 145.67ms | 99.97ms | 100ms | +| 1m53.94s | 4;4;6;5;5 | 67 | 100 | 423539 | 2.5G | 6.3K | 11.10/s | 69.2K/s | 105.37ms | 291.69ms | 199.97ms | 200ms | +| 1m28.9s | 4;4;4;5;5 | 67 | 100 | 388470 | 1.5G | 4K | 13.04/s | 52K/s | 143.61ms | 447.5ms | 299.89ms | 300ms | +| 1m51.9s | 5;3;3;5;4 | 67 | 100 | 326453 | 1.4G | 4.6K | 8.71/s | 39.5K/s | 279.64ms | 736.14ms | 499.95ms | 500ms | +| 1m14.93s | 3;3;3;2;3 | 67 | 100 | 230441 | 683M | 3K | 9.18/s | 27.5K/s | 561.85ms | 1.45603s | 999.71ms | 1s | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 11: TestConsensusFull50LatencyHigherThanExpected, 50 Fully Connected Participants in 100ms,200ms,300ms,500ms,1s incorrectly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 1m10.47s | 5;5;5;5;5 | 50 | 50 | 280979 | 1.7G | 6.2K | 15.95/s | 98.3K/s | 57.36ms | 148.04ms | 100.01ms | 50ms | +| 1m0.11s | 4;4;4;4;4 | 50 | 50 | 226394 | 1.5G | 6.9K | 15.06/s | 103K/s | 97.24ms | 291.15ms | 200.01ms | 100ms | +| 1m2.66s | 4;4;4;3;4 | 50 | 50 | 215221 | 1.3G | 6.3K | 13.74/s | 86.3K/s | 145.65ms | 431.24ms | 299.96ms | 150ms | +| 33.19s | 3;3;4;3;3 | 50 | 50 | 150431 | 513M | 3.5K | 18.13/s | 62.3K/s | 267.14ms | 729.61ms | 499.78ms | 250ms | +| 28.37s | 3;2;2;2;2 | 50 | 50 | 113598 | 285.1M | 2.6K | 16.01/s | 40.3K/s | 567.63ms | 1.47023s | 999.95ms | 500ms | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + +Case 12: TestConsensusPartial50LatencyHigherThanExpected, 50 Partially Connected Participants in 100ms,200ms,300ms,500ms,1s incorrectly set expected delay ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| DECIDE.AVG | DECIDE.ROUNDS | PEER.NUM | PJ.NUM | NET.MSGS | NET.BYTES | MSG.AVGSIZE | NET.MSGRATE | PEER.RATE | DELAY.MIN | DELAY.MAX | DELAY.AVG | DELAY.EXP | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ +| 26.68s | 4;5;5;5;6 | 33 | 50 | 109373 | 464.5M | 4.3K | 24.85/s | 106.7K/s | 59.19ms | 145.19ms | 99.96ms | 50ms | +| 49.57s | 4;7;4;6;5 | 33 | 50 | 112521 | 556M | 5.1K | 13.76/s | 69.2K/s | 111.76ms | 291.49ms | 199.93ms | 100ms | +| 32.49s | 5;3;4;4;5 | 33 | 50 | 86129 | 358.7M | 4.3K | 16.06/s | 67.8K/s | 162.37ms | 438.48ms | 299.96ms | 150ms | +| 33.4s | 4;3;5;3;3 | 33 | 50 | 78125 | 268.7M | 3.5K | 14.17/s | 49.4K/s | 271.55ms | 715.98ms | 499.9ms | 250ms | +| 32.12s | 2;2;3;2;4 | 33 | 50 | 55073 | 122.3M | 2.3K | 10.39/s | 23.2K/s | 586.84ms | 1.42515s | 999.92ms | 500ms | ++------------+---------------+----------+--------+----------+-----------+-------------+-------------+-----------+-----------+-----------+-----------+-----------+ + + diff --git a/orderer/consensus/bdls/cmd/emucon/README.md b/orderer/consensus/bdls/cmd/emucon/README.md new file mode 100644 index 0000000000..c1951e55f0 --- /dev/null +++ b/orderer/consensus/bdls/cmd/emucon/README.md @@ -0,0 +1,100 @@ +# Consensus Emulator + +## Install + +``` +$ git clone https://github.com/Sperax/bdls +$ cd bdls/emucon +$ go build . +$ ./emucon +NAME: + BDLS consensus protocol emulator - Generate quorum then emulate participants + +USAGE: + emucon [global options] command [command options] [arguments...] + +COMMANDS: + genkeys generate quorum to participant in consensus + run start a consensus agent + help, h Shows a list of commands or help for one command + +GLOBAL OPTIONS: + --help, -h show help (default: false) +``` + + + +## GENERATE CONSENSUS GROUP KEYS + +``` +$ ./emucon genkeys --count 4 + +$ ./emucon genkeys --help +NAME: + emucon genkeys - generate quorum to participant in consensus + +USAGE: + emucon genkeys [command options] [arguments...] + +OPTIONS: + --count value number of participant in quorum (default: 4) + --config value output quorum file (default: "./quorum.json") + --help, -h show help (default: false) + +``` + + + +## NODES EMULATION + +``` +$ ./emucon run --help +NAME: + emucon run - start a consensus agent + +USAGE: + emucon run [command options] [arguments...] + +OPTIONS: + --listen value the client's listening port (default: ":4680") + --id value the node id, will use the n-th private key in quorum.json (default: 0) + --config value the shared quorum config file (default: "./quorum.json") + --peers value all peers's ip:port list to connect, as a json array (default: "./peers.json") + --help, -h show help (default: false) +``` + + + +Create a file named peers.json, like below, which contains 4 different nodes listening on different ports at localhost. + +``` +$ cat peers.json +["localhost:4680", "localhost:4681","localhost:4682", "localhost:4683"] +``` + +You can start minimum 4 nodes in 4 different terminal like below: + +``` +$./emucon run --id 0 --listen ":4680" +$./emucon run --id 1 --listen ":4681" +$./emucon run --id 2 --listen ":4682" +$./emucon run --id 3 --listen ":4683" +``` + +A succesfully running node will output something like: + +``` +$ ./emucon run --id 2 --listen ":4682" +2020/04/10 18:19:15 identity: 2 +2020/04/10 18:19:15 listening on: :4682 +2020/04/10 18:19:15 connected to peer: 127.0.0.1:4682 +2020/04/10 18:19:15 connected to peer: 127.0.0.1:4680 +2020/04/10 18:19:15 peer connected from: 127.0.0.1:49204 +2020/04/10 18:19:15 connected to peer: 127.0.0.1:4681 +2020/04/10 18:19:15 peer connected from: 127.0.0.1:49212 +2020/04/10 18:19:15 peer connected from: 127.0.0.1:49216 +2020/04/10 18:19:17 at height:1 round:1 hash:d2d583085a489287a889238229879a7bc3aef2251c39c052d165883687e06db8 +2020/04/10 18:19:18 at height:2 round:1 hash:c430bc8bb8b749717a3d90dd6ee29271694e8ad49f16e0e61b96de145f89d892 +2020/04/10 18:19:20 at height:3 round:1 hash:e21370a2f82d4b0b5a885c5a6f669890d5df9a8caffbce664e519184b1a25c64 +``` + diff --git a/orderer/consensus/bdls/cmd/emucon/emucon b/orderer/consensus/bdls/cmd/emucon/emucon new file mode 100644 index 0000000000..a232151a3d Binary files /dev/null and b/orderer/consensus/bdls/cmd/emucon/emucon differ diff --git a/orderer/consensus/bdls/cmd/emucon/main.go b/orderer/consensus/bdls/cmd/emucon/main.go new file mode 100644 index 0000000000..73a8b8a8d5 --- /dev/null +++ b/orderer/consensus/bdls/cmd/emucon/main.go @@ -0,0 +1,295 @@ +// BSD 3-Clause License +// +// Copyright (c) 2020, Sperax +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package main + +import ( + "bytes" + "crypto/ecdsa" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "math/big" + "net" + "os" + "time" + + "github.com/Sperax/bdls" + "github.com/Sperax/bdls/agent-tcp" + "github.com/Sperax/bdls/crypto/blake2b" + "github.com/urfave/cli/v2" +) + +// A quorum set for consenus +type Quorum struct { + Keys []*big.Int `json:"keys"` // pem formatted keys +} + +func main() { + app := &cli.App{ + Name: "BDLS consensus protocol emulator", + Usage: "Generate quorum then emulate participants", + EnableBashCompletion: true, + Commands: []*cli.Command{ + { + Name: "genkeys", + Usage: "generate quorum to participant in consensus", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "count", + Value: 4, + Usage: "number of participant in quorum", + }, + &cli.StringFlag{ + Name: "config", + Value: "./quorum.json", + Usage: "output quorum file", + }, + }, + Action: func(c *cli.Context) error { + count := c.Int("count") + quorum := &Quorum{} + // generate private keys + for i := 0; i < count; i++ { + privateKey, err := ecdsa.GenerateKey(bdls.S256Curve, rand.Reader) + if err != nil { + return err + } + + quorum.Keys = append(quorum.Keys, privateKey.D) + } + + file, err := os.Create(c.String("config")) + if err != nil { + return err + } + enc := json.NewEncoder(file) + enc.SetIndent("", "\t") + err = enc.Encode(quorum) + if err != nil { + return err + } + file.Close() + + log.Println("generate", c.Int("count"), "keys") + return nil + }, + }, + { + Name: "run", + Usage: "start a consensus agent", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "listen", + Value: ":4680", + Usage: "the client's listening port", + }, + &cli.IntFlag{ + Name: "id", + Value: 0, + Usage: "the node id, will use the n-th private key in quorum.json", + }, + &cli.StringFlag{ + Name: "config", + Value: "./quorum.json", + Usage: "the shared quorum config file", + }, + &cli.StringFlag{ + Name: "peers", + Value: "./peers.json", + Usage: "all peers's ip:port list to connect, as a json array", + }, + }, + Action: func(c *cli.Context) error { + // open quorum config + file, err := os.Open(c.String("config")) + if err != nil { + return err + } + defer file.Close() + + quorum := new(Quorum) + err = json.NewDecoder(file).Decode(quorum) + if err != nil { + return err + } + + id := c.Int("id") + if id >= len(quorum.Keys) { + return errors.New(fmt.Sprint("cannot locate private key for id:", id)) + } + log.Println("identity:", id) + + // create configuration + config := new(bdls.Config) + config.Epoch = time.Now() + config.CurrentHeight = 0 + config.StateCompare = func(a bdls.State, b bdls.State) int { return bytes.Compare(a, b) } + config.StateValidate = func(bdls.State) bool { return true } + + for k := range quorum.Keys { + priv := new(ecdsa.PrivateKey) + priv.PublicKey.Curve = bdls.S256Curve + priv.D = quorum.Keys[k] + priv.PublicKey.X, priv.PublicKey.Y = bdls.S256Curve.ScalarBaseMult(priv.D.Bytes()) + // myself + if id == k { + config.PrivateKey = priv + } + + // set validator sequence + config.Participants = append(config.Participants, bdls.DefaultPubKeyToIdentity(&priv.PublicKey)) + } + + if err := startConsensus(c, config); err != nil { + return err + } + return nil + }, + }, + }, + + Action: func(c *cli.Context) error { + cli.ShowAppHelp(c) + return nil + }, + } + + err := app.Run(os.Args) + if err != nil { + log.Fatal(err) + } + +} + +// consensus for one round with full procedure +func startConsensus(c *cli.Context, config *bdls.Config) error { + // create consensus + consensus, err := bdls.NewConsensus(config) + if err != nil { + return err + } + consensus.SetLatency(200 * time.Millisecond) + + // load endpoints + file, err := os.Open(c.String("peers")) + if err != nil { + return err + } + defer file.Close() + + var peers []string + err = json.NewDecoder(file).Decode(&peers) + if err != nil { + return err + } + + // start listener + tcpaddr, err := net.ResolveTCPAddr("tcp", c.String("listen")) + if err != nil { + return err + } + + l, err := net.ListenTCP("tcp", tcpaddr) + if err != nil { + return err + } + defer l.Close() + log.Println("listening on:", c.String("listen")) + + // initiate tcp agent + tagent := agent.NewTCPAgent(consensus, config.PrivateKey) + if err != nil { + return err + } + + // start updater + tagent.Update() + + // passive connection from peers + go func() { + for { + conn, err := l.Accept() + if err != nil { + return + } + log.Println("peer connected from:", conn.RemoteAddr()) + // peer endpoint created + p := agent.NewTCPPeer(conn, tagent) + tagent.AddPeer(p) + // prove my identity to this peer + p.InitiatePublicKeyAuthentication() + } + }() + + // active connections to peers + for k := range peers { + go func(raddr string) { + for { + conn, err := net.Dial("tcp", raddr) + if err == nil { + log.Println("connected to peer:", conn.RemoteAddr()) + // peer endpoint created + p := agent.NewTCPPeer(conn, tagent) + tagent.AddPeer(p) + // prove my identity to this peer + p.InitiatePublicKeyAuthentication() + return + } + <-time.After(time.Second) + } + }(peers[k]) + } + + lastHeight := uint64(0) + +NEXTHEIGHT: + for { + data := make([]byte, 1024) + io.ReadFull(rand.Reader, data) + tagent.Propose(data) + + for { + newHeight, newRound, newState := tagent.GetLatestState() + if newHeight > lastHeight { + h := blake2b.Sum256(newState) + log.Printf(" at height:%v round:%v hash:%v", newHeight, newRound, hex.EncodeToString(h[:])) + lastHeight = newHeight + continue NEXTHEIGHT + } + // wait + <-time.After(20 * time.Millisecond) + } + } +} diff --git a/orderer/consensus/bdls/cmd/emucon/peers.json b/orderer/consensus/bdls/cmd/emucon/peers.json new file mode 100644 index 0000000000..361daf4c32 --- /dev/null +++ b/orderer/consensus/bdls/cmd/emucon/peers.json @@ -0,0 +1 @@ +["localhost:4680", "localhost:4681","localhost:4682", "localhost:4683"] diff --git a/orderer/consensus/bdls/cmd/emucon/quorum.json b/orderer/consensus/bdls/cmd/emucon/quorum.json new file mode 100644 index 0000000000..c3ec8ced88 --- /dev/null +++ b/orderer/consensus/bdls/cmd/emucon/quorum.json @@ -0,0 +1,8 @@ +{ + "keys": [ + 98219006221833178226499894663828536080581259993811761082380731664557876151926, + 58630051888648454967792790648160553661274118301081136769104558537494738155681, + 100955098131707660017796861077112074015350838580117274026874229793880529586064, + 74259512097651292383407728057818570425528934348167008131707948177877498894584 + ] +} diff --git a/orderer/consensus/bdls/config.go b/orderer/consensus/bdls/config.go new file mode 100644 index 0000000000..fb473adc79 --- /dev/null +++ b/orderer/consensus/bdls/config.go @@ -0,0 +1,101 @@ +// BSD 3-Clause License +// +// Copyright (c) 2020, Sperax +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package bdls + +import ( + "crypto/ecdsa" + "time" +) + +const ( + // ConfigMinimumParticipants is the minimum number of participant allow in consensus protocol + ConfigMinimumParticipants = 4 +) + +// Config is to config the parameters of BDLS consensus protocol +type Config struct { + // the starting time point for consensus + Epoch time.Time + // CurrentHeight + CurrentHeight uint64 + // PrivateKey + PrivateKey *ecdsa.PrivateKey + // Consensus Group + Participants []Identity + // EnableCommitUnicast sets to true to enable message to be delivered via unicast + // if not(by default), message will be broadcasted + EnableCommitUnicast bool + + // StateCompare is a function from user to compare states, + // The result will be 0 if a==b, -1 if a < b, and +1 if a > b. + // Usually this will lead to block header comparsion in blockchain, or replication log in database, + // users should check fields in block header to make comparison. + StateCompare func(a State, b State) int + + // StateValidate is a function from user to validate the integrity of + // state data. + StateValidate func(State) bool + + // MessageValidator is an external validator to be called when a message inputs into ReceiveMessage + MessageValidator func(c *Consensus, m *Message, signed *SignedProto) bool + + // MessageOutCallback will be called if not nil before a message send out + MessageOutCallback func(m *Message, signed *SignedProto) + + // Identity derviation from ecdsa.PublicKey + // (optional). Default to DefaultPubKeyToIdentity + PubKeyToIdentity func(pubkey *ecdsa.PublicKey) (ret Identity) +} + +// VerifyConfig verifies the integrity of this config when creating new consensus object +func VerifyConfig(c *Config) error { + if c.Epoch.IsZero() { + return ErrConfigEpoch + } + + if c.StateCompare == nil { + return ErrConfigStateCompare + } + + if c.StateValidate == nil { + return ErrConfigStateValidate + } + + if c.PrivateKey == nil { + return ErrConfigPrivateKey + } + + if len(c.Participants) < ConfigMinimumParticipants { + return ErrConfigParticipants + } + + return nil +} diff --git a/orderer/consensus/bdls/config_test.go b/orderer/consensus/bdls/config_test.go new file mode 100644 index 0000000000..2eeb3a22ac --- /dev/null +++ b/orderer/consensus/bdls/config_test.go @@ -0,0 +1,45 @@ +package bdls + +import ( + "crypto/ecdsa" + "crypto/rand" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestVerifyConfig(t *testing.T) { + config := new(Config) + + err := VerifyConfig(config) + assert.Equal(t, ErrConfigEpoch, err) + + config.Epoch = time.Now() + err = VerifyConfig(config) + assert.Equal(t, ErrConfigStateCompare, err) + + config.StateCompare = func(State, State) int { return 0 } + err = VerifyConfig(config) + assert.Equal(t, ErrConfigStateValidate, err) + + config.StateValidate = func(State) bool { return true } + err = VerifyConfig(config) + assert.Equal(t, ErrConfigPrivateKey, err) + + randKey, err := ecdsa.GenerateKey(S256Curve, rand.Reader) + assert.Nil(t, err) + + config.PrivateKey = randKey + err = VerifyConfig(config) + assert.Equal(t, ErrConfigParticipants, err) + + for i := 0; i < ConfigMinimumParticipants; i++ { + randKey, err := ecdsa.GenerateKey(S256Curve, rand.Reader) + assert.Nil(t, err) + config.Participants = append(config.Participants, DefaultPubKeyToIdentity(&randKey.PublicKey)) + } + + err = VerifyConfig(config) + assert.Nil(t, err) +} diff --git a/orderer/consensus/bdls/consensus.go b/orderer/consensus/bdls/consensus.go new file mode 100644 index 0000000000..691871ee0f --- /dev/null +++ b/orderer/consensus/bdls/consensus.go @@ -0,0 +1,1686 @@ +// BSD 3-Clause License +// +// Copyright (c) 2020, Sperax +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package bdls + +import ( + "bytes" + "container/list" + "crypto/ecdsa" + "crypto/elliptic" + "net" + "sort" + "time" + + "github.com/Sperax/bdls/crypto/blake2b" + proto "github.com/gogo/protobuf/proto" +) + +const ( + // ProtocolVersion is the current BDLS protocol implementation version, + // version wil be sent along with messages for protocol upgrading. + ProtocolVersion = 1 + // DefaultConsensusLatency is the default propagation latency setting for + // consensus protocol, user can adjust consensus object's latency setting + // via Consensus.SetLatency() + DefaultConsensusLatency = 300 * time.Millisecond + + // MaxConsensusLatency is the ceiling of latencies + MaxConsensusLatency = 10 * time.Second +) + +type ( + // State is the data to participant in consensus + State []byte + // StateHash is a fixed size hash to identify a state + StateHash [blake2b.Size256]byte +) + +// defaultHash is the system default hash function +func defaultHash(s State) StateHash { return blake2b.Sum256(s) } + +type ( + // consensusStage defines the status of consensus automate + consensusStage byte +) + +// status definitions for consensus state machine +const ( + // stages are strictly ordered, do not change! + stageRoundChanging consensusStage = iota + stageLock + stageCommit + stageLockRelease +) + +// messageTuple contains a state hash, a decoded incoming message +// and it's encoded raw message with a signature. +type messageTuple struct { + StateHash StateHash // computed while adding + Message *Message // the decoded message + Signed *SignedProto // the encoded message with signature +} + +// a sorter for messageTuple slice +type tupleSorter struct { + tuples []messageTuple + by func(t1, t2 *messageTuple) bool +} + +// Len implements sort.Interface +func (s *tupleSorter) Len() int { return len(s.tuples) } + +// Swap implements sort.Interface +func (s *tupleSorter) Swap(i, j int) { s.tuples[i], s.tuples[j] = s.tuples[j], s.tuples[i] } + +// Less implements sort.Interface +func (s *tupleSorter) Less(i, j int) bool { return s.by(&s.tuples[i], &s.tuples[j]) } + +// consensusRound maintains exchanging messages in a round. +type consensusRound struct { + c *Consensus // the consensus object belongs to + Stage consensusStage // indicates current status in consensus automata + RoundNumber uint64 // round number + LockedState State // leader's locked state + LockedStateHash StateHash // hash of the leaders's locked state + RoundChangeSent bool // mark if the message of this round has sent + CommitSent bool // mark if this round has sent commit message once + + // NOTE: we MUST keep the original message, to re-marshal the message may + // result in different BITS LAYOUT, and different hash of course. + roundChanges []messageTuple // stores message tuples of this round + commits []messageTuple // stores message tuples of this round + + // track current max proposed state in , we don't have to compute this for + // a non-leader participant, or if there're no more than 2t+1 messages for leader. + MaxProposedState State + MaxProposedCount int +} + +// newConsensusRound creates a new round, and sets the round number +func newConsensusRound(round uint64, c *Consensus) *consensusRound { + r := new(consensusRound) + r.RoundNumber = round + r.c = c + return r +} + +// AddRoundChange adds a message to this round, and +// checks to accept only one message from one participant, +// to prevent multiple proposals attack. +func (r *consensusRound) AddRoundChange(sp *SignedProto, m *Message) bool { + for k := range r.roundChanges { + if r.roundChanges[k].Signed.X == sp.X && r.roundChanges[k].Signed.Y == sp.Y { + return false + } + } + + r.roundChanges = append(r.roundChanges, messageTuple{StateHash: r.c.stateHash(m.State), Message: m, Signed: sp}) + return true +} + +// FindRoundChange will try to find a from a given participant, +// and returns index, -1 if not found +func (r *consensusRound) FindRoundChange(X PubKeyAxis, Y PubKeyAxis) int { + for k := range r.roundChanges { + if r.roundChanges[k].Signed.X == X && r.roundChanges[k].Signed.Y == Y { + return k + } + } + return -1 +} + +// RemoveRoundChange removes the given message at idx +func (r *consensusRound) RemoveRoundChange(idx int) { + // swap to the end and shrink slice + n := len(r.roundChanges) - 1 + r.roundChanges[idx], r.roundChanges[n] = r.roundChanges[n], r.roundChanges[idx] + r.roundChanges[n] = messageTuple{} // set to nil to avoid memory leak + r.roundChanges = r.roundChanges[:n] +} + +// NumRoundChanges returns count of messages. +func (r *consensusRound) NumRoundChanges() int { return len(r.roundChanges) } + +// SignedRoundChanges converts and returns []*SignedProto(as slice) +func (r *consensusRound) SignedRoundChanges() []*SignedProto { + proof := make([]*SignedProto, 0, len(r.roundChanges)) + for k := range r.roundChanges { + proof = append(proof, r.roundChanges[k].Signed) + } + return proof +} + +// RoundChangeStates returns all non-nil state in exchanging round change message as slice +func (r *consensusRound) RoundChangeStates() []State { + states := make([]State, 0, len(r.roundChanges)) + for k := range r.roundChanges { + if r.roundChanges[k].Message.State != nil { + states = append(states, r.roundChanges[k].Message.State) + } + } + return states +} + +// AddCommit adds decoded messages along with its original signed message unchanged, +// also, messages will be de-duplicated to prevent multiple proposals attack. +func (r *consensusRound) AddCommit(sp *SignedProto, m *Message) bool { + for k := range r.commits { + if r.commits[k].Signed.X == sp.X && r.commits[k].Signed.Y == sp.Y { + return false + } + } + r.commits = append(r.commits, messageTuple{StateHash: r.c.stateHash(m.State), Message: m, Signed: sp}) + return true +} + +// NumCommitted counts messages which points to what the leader has locked. +func (r *consensusRound) NumCommitted() int { + var count int + for k := range r.commits { + if r.commits[k].StateHash == r.LockedStateHash { + count++ + } + } + return count +} + +// SignedCommits converts and returns []*SignedProto +func (r *consensusRound) SignedCommits() []*SignedProto { + proof := make([]*SignedProto, 0, len(r.commits)) + for k := range r.commits { + proof = append(proof, r.commits[k].Signed) + } + return proof +} + +// GetMaxProposed finds the most agreed-on non-nil state, if these is any. +func (r *consensusRound) GetMaxProposed() (s State, count int) { + if len(r.roundChanges) == 0 { + return nil, 0 + } + + // sort by hash, to group identical hashes together + // O(n*logn) + sorter := tupleSorter{ + tuples: r.roundChanges, + // sort by it's hash lexicographically + by: func(t1, t2 *messageTuple) bool { + return bytes.Compare(t1.StateHash[:], t2.StateHash[:]) < 0 + }, + } + sort.Sort(&sorter) + + // find the maximum occurred hash + // O(n) + maxCount := 1 + maxState := r.roundChanges[0] + curCount := 1 + + n := len(r.roundChanges) + for i := 1; i < n; i++ { + if r.roundChanges[i].StateHash == r.roundChanges[i-1].StateHash { + curCount++ + } else { + if curCount > maxCount { + maxCount = curCount + maxState = r.roundChanges[i-1] + } + curCount = 1 + } + } + + // if the last hash is the maximum occurred + if curCount > maxCount { + maxCount = curCount + maxState = r.roundChanges[n-1] + } + + return maxState.Message.State, maxCount +} + +// Consensus implements a deterministic BDLS consensus protocol. +// +// It has no internal clocking or IO, and no parallel processing. +// The runtime behavior is predictable and deterministic. +// Users should write their own timing and IO function to feed in +// messages and ticks to trigger timeouts. +type Consensus struct { + latestState State // latest confirmed state of current height + latestHeight uint64 // latest confirmed height + latestRound uint64 // latest confirmed round + latestProof *SignedProto // latest message to prove the state + + unconfirmed []State // data awaiting to be confirmed at next height + + rounds list.List // all rounds at next height(consensus round in progress) + currentRound *consensusRound // current round which has collected >=2t+1 + + // timeouts in different stage + rcTimeout time.Time // roundchange status timeout + lockTimeout time.Time // lock status timeout + commitTimeout time.Time // commit status timeout + lockReleaseTimeout time.Time // lock-release status timeout + + // locked states, along with its signatures and hashes in tuple + locks []messageTuple + + // the StateCompare function from config + stateCompare func(State, State) int + // the StateValidate function from config + stateValidate func(State) bool + // message in callback + messageValidator func(c *Consensus, m *Message, sp *SignedProto) bool + // message out callback + messageOutCallback func(m *Message, sp *SignedProto) + // public key to identity function + pubKeyToIdentity func(pubkey *ecdsa.PublicKey) Identity + + // the StateHash function to identify a state + stateHash func(State) StateHash + + // private key + privateKey *ecdsa.PrivateKey + // my publickey coodinate + identity Identity + // curve retrieved from private key + curve elliptic.Curve + + // transmission delay + latency time.Duration + + // all connected peers + peers []PeerInterface + + // participants is the consensus group, current leader is r % quorum + participants []Identity + + // count num of individual identities + numIdentities int + + // set to true to enable message unicast + enableCommitUnicast bool + + // NOTE: fixed leader for testing purpose + fixedLeader *Identity + + // broadcasting messages being sent to myself + loopback [][]byte + + // the last message which caused round change + lastRoundChangeProof []*SignedProto +} + +// NewConsensus creates a BDLS consensus object to participant in consensus procedure, +// the consensus object returned is data in memory without goroutines or other +// non-deterministic objects, and errors will be returned if there is problem, with +// the given config. +func NewConsensus(config *Config) (*Consensus, error) { + err := VerifyConfig(config) + if err != nil { + return nil, err + } + + c := new(Consensus) + c.init(config) + return c, nil +} + +// init consensus with config +func (c *Consensus) init(config *Config) { + // setting current state & height + c.latestHeight = config.CurrentHeight + c.participants = config.Participants + c.stateCompare = config.StateCompare + c.stateValidate = config.StateValidate + c.messageValidator = config.MessageValidator + c.messageOutCallback = config.MessageOutCallback + c.privateKey = config.PrivateKey + c.pubKeyToIdentity = config.PubKeyToIdentity + c.enableCommitUnicast = config.EnableCommitUnicast + + // if config has not set hash function, use the default + if c.stateHash == nil { + c.stateHash = defaultHash + } + // if config has not set public key to identity function, use the default + if c.pubKeyToIdentity == nil { + c.pubKeyToIdentity = DefaultPubKeyToIdentity + } + c.identity = c.pubKeyToIdentity(&c.privateKey.PublicKey) + c.curve = c.privateKey.Curve + + // initial default parameters settings + c.latency = DefaultConsensusLatency + + // and initiated the first proposal + c.switchRound(0) + c.currentRound.Stage = stageRoundChanging + c.broadcastRoundChange() + // set rcTimeout to lockTimeout + c.rcTimeout = config.Epoch.Add(c.roundchangeDuration(0)) + + // count number of individual identites + ids := make(map[Identity]bool) + for _, id := range c.participants { + ids[id] = true + } + c.numIdentities = len(ids) +} + +// calculates roundchangeDuration +func (c *Consensus) roundchangeDuration(round uint64) time.Duration { + d := 2 * c.latency * (1 << round) + if d > MaxConsensusLatency { + d = MaxConsensusLatency + } + return d +} + +// calculates collectDuration +func (c *Consensus) collectDuration(round uint64) time.Duration { + d := 2 * c.latency * (1 << round) + if d > MaxConsensusLatency { + d = MaxConsensusLatency + } + return d +} + +// calculates lockDuration +func (c *Consensus) lockDuration(round uint64) time.Duration { + d := 4 * c.latency * (1 << round) + if d > MaxConsensusLatency { + d = MaxConsensusLatency + } + return d +} + +// calculates commitDuration +func (c *Consensus) commitDuration(round uint64) time.Duration { + d := 2 * c.latency * (1 << round) + if d > MaxConsensusLatency { + d = MaxConsensusLatency + } + return d +} + +// calculates lockReleaseDuration +func (c *Consensus) lockReleaseDuration(round uint64) time.Duration { + d := 2 * c.latency * (1 << round) + if d > MaxConsensusLatency { + d = MaxConsensusLatency + } + return d +} + +// maximalLocked finds the maximum locked data in this round, +// with regard to StateCompare function in config. +func (c *Consensus) maximalLocked() State { + if len(c.locks) > 0 { + maxState := c.locks[0].Message.State + for i := 1; i < len(c.locks); i++ { + if c.stateCompare(maxState, c.locks[i].Message.State) < 0 { + maxState = c.locks[i].Message.State + } + } + return maxState + } + return nil +} + +// maximalUnconfirmed finds the maximal unconfirmed data with, +// regard to the StateCompare function in config. +func (c *Consensus) maximalUnconfirmed() State { + if len(c.unconfirmed) > 0 { + maxState := c.unconfirmed[0] + for i := 1; i < len(c.unconfirmed); i++ { + if c.stateCompare(maxState, c.unconfirmed[i]) < 0 { + maxState = c.unconfirmed[i] + } + } + return maxState + } + return nil +} + +// verifyMessage verifies message signature against it's & , +// and also checks if the signer is a valid participant. +// returns it's decoded 'Message' object if signature has proved authentic. +// returns nil and error if message has not been correctly signed or from an unknown participant. +func (c *Consensus) verifyMessage(signed *SignedProto) (*Message, error) { + if signed == nil { + return nil, ErrMessageIsEmpty + } + + // check signer's identity, all participants have proven + // public key + knownParticipants := false + coord := c.pubKeyToIdentity(signed.PublicKey(c.curve)) + for k := range c.participants { + if coord == c.participants[k] { + knownParticipants = true + } + } + + if !knownParticipants { + return nil, ErrMessageUnknownParticipant + } + + /* + // public key validation + p := defaultCurve.Params().P + x := new(big.Int).SetBytes(signed.X[:]) + y := new(big.Int).SetBytes(signed.Y[:]) + if x.Cmp(p) >= 0 || y.Cmp(p) >= 0 { + return nil, ErrMessageSignature + } + if !defaultCurve.IsOnCurve(x, y) { + return nil, ErrMessageSignature + } + */ + + // as public key is proven , we don't have to verify the public key + if !signed.Verify(c.curve) { + return nil, ErrMessageSignature + } + + // decode message + m := new(Message) + err := proto.Unmarshal(signed.Message, m) + if err != nil { + return nil, err + } + return m, nil +} + +// verify message +func (c *Consensus) verifyRoundChangeMessage(m *Message) error { + // check message height + if m.Height != c.latestHeight+1 { + return ErrRoundChangeHeightMismatch + } + + // check round in protocol + if m.Round < c.currentRound.RoundNumber { + return ErrRoundChangeRoundLower + } + + // state data validation for non-null + if m.State != nil { + if !c.stateValidate(m.State) { + return ErrRoundChangeStateValidation + } + } + + return nil +} + +// verifyLockMessage verifies proofs from messages, +// a lock message must contain at least 2t+1 individual +// messages on B' +func (c *Consensus) verifyLockMessage(m *Message, signed *SignedProto) error { + // check message height + if m.Height != c.latestHeight+1 { + return ErrLockHeightMismatch + } + + // check round in protocol + if m.Round < c.currentRound.RoundNumber { + return ErrLockRoundLower + } + + // a message from leader MUST include data along with the message + if m.State == nil { + return ErrLockEmptyState + } + + // state data validation + if !c.stateValidate(m.State) { + return ErrLockStateValidation + } + + // make sure this message has been signed by the leader + leaderKey := c.roundLeader(m.Round) + if c.pubKeyToIdentity(signed.PublicKey(c.curve)) != leaderKey { + return ErrLockNotSignedByLeader + } + + // validate proofs enclosed in the message one by one + rcs := make(map[Identity]State) + for _, proof := range m.Proof { + // first we need to verify the signature,and identity of this proof + mProof, err := c.verifyMessage(proof) + if err != nil { + if err == ErrMessageUnknownParticipant { + return ErrLockProofUnknownParticipant + } + return err + } + + // then we need to check the message type + if mProof.Type != MessageType_RoundChange { + return ErrLockProofTypeMismatch + } + + // and we also need to check the height & round field, + // all messages must be in the same round as the lock message + if mProof.Height != m.Height { + return ErrLockProofHeightMismatch + } + + if mProof.Round != m.Round { + return ErrLockProofRoundMismatch + } + + // state data validation in proofs + if mProof.State != nil { + if !c.stateValidate(mProof.State) { + return ErrLockProofStateValidation + } + } + + // use map to guarantee we will only accept at most 1 message from one + // individual participant + rcs[c.pubKeyToIdentity(proof.PublicKey(c.curve))] = mProof.State + } + + // count individual proofs to B', which has already guaranteed to be the maximal one. + var numValidateProofs int + mHash := c.stateHash(m.State) + for _, v := range rcs { + if c.stateHash(v) == mHash { // B' + numValidateProofs++ + } + } + + // check if valid proofs count is less that 2*t+1 + if numValidateProofs < 2*c.t()+1 { + return ErrLockProofInsufficient + } + return nil +} + +// verifyLockReleaseMessage will verify LockRelease field in a messages, +// returns the embedded message if valid +func (c *Consensus) verifyLockReleaseMessage(signed *SignedProto) (*Message, error) { + // not in lock release status, omit this message + if c.currentRound.Stage != stageLockRelease { + return nil, ErrLockReleaseStatus + } + + // verify and decode the embedded lock message + lockmsg, err := c.verifyMessage(signed) + if err != nil { + return nil, err + } + + // recursively verify proofs in lock message + err = c.verifyLockMessage(lockmsg, signed) + if err != nil { + return nil, err + } + return lockmsg, nil +} + +// verifySelectMessage verifies proofs from message MUST contain at least 2t+1 individual messages, but +// proofs from + if m.State != nil { + if !c.stateValidate(m.State) { + return ErrSelectStateValidation + } + } + + // make sure this message has been signed by the leader + leaderKey := c.roundLeader(m.Round) + if c.pubKeyToIdentity(signed.PublicKey(c.curve)) != leaderKey { + return ErrSelectNotSignedByLeader + } + + rcs := make(map[Identity]State) + for _, proof := range m.Proof { + mProof, err := c.verifyMessage(proof) + if err != nil { + if err == ErrMessageUnknownParticipant { + return ErrSelectProofUnknownParticipant + } + return err + } + + if mProof.Type != MessageType_RoundChange { + return ErrSelectProofTypeMismatch + } + + if mProof.Height != m.Height { + return ErrSelectProofHeightMismatch + } + + if mProof.Round != m.Round { + return ErrSelectProofRoundMismatch + } + + // state data validation in proofs + if mProof.State != nil { + if !c.stateValidate(mProof.State) { + return ErrSelectProofStateValidation + } + } + + // we also need to check the B'' selected by leader is the maximal one, + // if data has been proposed. + if mProof.State != nil && m.State != nil { + if c.stateCompare(m.State, mProof.State) < 0 { + return ErrSelectProofNotTheMaximal + } + } + + // we also stores B'' == NULL for counting + rcs[c.pubKeyToIdentity(proof.PublicKey(c.curve))] = mProof.State + } + + // check we have at least 2*t+1 proof + if len(rcs) < 2*c.t()+1 { + return ErrSelectProofInsufficient + } + + // count maximum proofs with B' != NULL with identical data hash, + // to prevent leader cheating on select. + dataProposals := make(map[StateHash]int) + for _, data := range rcs { + if data != nil { + dataProposals[c.stateHash(data)]++ + } + } + + // if m.State == NULL, but there are non-NULL proofs, + // the leader may be cheating + if m.State == nil && len(dataProposals) > 0 { + return ErrSelectStateMismatch + } + + // find the highest proposed B'(not NULL) + var maxProposed int + for _, count := range dataProposals { + if count > maxProposed { + maxProposed = count + } + } + + // if these are more than 2*t+1 valid proofs to B', + // this also suggests that the leader may cheat. + if maxProposed >= 2*c.t()+1 { + return ErrSelectProofExceeded + } + + return nil +} + +// verifyCommitMessage will check if this message is acceptable to consensus +func (c *Consensus) verifyCommitMessage(m *Message) error { + // the leader has to be in COMMIT status to process this message + if c.currentRound.Stage != stageCommit { + return ErrCommitStatus + } + + // a message from participants MUST includes data along with the message + if m.State == nil { + return ErrCommitEmptyState + } + + // state data validation + if !c.stateValidate(m.State) { + return ErrCommitStateValidation + } + + // check height + if m.Height != c.latestHeight+1 { + return ErrCommitHeightMismatch + } + + // only accept commits to current round + if c.currentRound.RoundNumber != m.Round { + return ErrCommitRoundMismatch + } + + // check state match + if c.stateHash(m.State) != c.currentRound.LockedStateHash { + return ErrCommitStateMismatch + } + + return nil +} + +// ValidateDecideMessage validates a message for non-participants, +// the consensus core must be correctly initialized to validate. +// the targetState is to compare the target state enclosed in decide message +func (c *Consensus) ValidateDecideMessage(bts []byte, targetState []byte) error { + signed, err := DecodeSignedMessage(bts) + if err != nil { + return err + } + + return c.validateDecideMessage(signed, targetState) +} + +// DecodeSignedMessage decodes a binary representation of signed consensus message. +func DecodeSignedMessage(bts []byte) (*SignedProto, error) { + signed := new(SignedProto) + err := proto.Unmarshal(bts, signed) + if err != nil { + return nil, err + } + return signed, nil +} + +// DecodeMessage decodes a binary representation of consensus message. +func DecodeMessage(bts []byte) (*Message, error) { + msg := new(Message) + err := proto.Unmarshal(bts, msg) + if err != nil { + return nil, err + } + return msg, nil +} + +// validateDecideMessage validates a decoded message for non-participants, +// the consensus core must be correctly initialized to validate. +func (c *Consensus) validateDecideMessage(signed *SignedProto, targetState []byte) error { + // check message version + if signed.Version != ProtocolVersion { + return ErrMessageVersion + } + + // check message signature & qualifications + m, err := c.verifyMessage(signed) + if err != nil { + return err + } + + // compare state + if !bytes.Equal(m.State, targetState) { + return ErrMismatchedTargetState + } + + // verify decide message + if m.Type == MessageType_Decide { + err := c.verifyDecideMessage(m, signed) + if err != nil { + return err + } + return nil + } + return ErrMessageUnknownMessageType +} + +// verifyDecideMessage verifies proofs from message, which MUST +// contain at least 2t+1 individual messages to B'. +func (c *Consensus) verifyDecideMessage(m *Message, signed *SignedProto) error { + // a message from leader MUST include data along with the message + if m.State == nil { + return ErrDecideEmptyState + } + + // state data validation + if !c.stateValidate(m.State) { + return ErrDecideStateValidation + } + + // check height + if m.Height <= c.latestHeight { + return ErrDecideHeightLower + } + + // make sure this message has been signed by the leader + leaderKey := c.roundLeader(m.Round) + if c.pubKeyToIdentity(signed.PublicKey(c.curve)) != leaderKey { + return ErrDecideNotSignedByLeader + } + + commits := make(map[Identity]State) + for _, proof := range m.Proof { + mProof, err := c.verifyMessage(proof) + if err != nil { + if err == ErrMessageUnknownParticipant { + return ErrDecideProofUnknownParticipant + } + return err + } + + if mProof.Type != MessageType_Commit { + return ErrDecideProofTypeMismatch + } + + if mProof.Height != m.Height { + return ErrDecideProofHeightMismatch + } + + if mProof.Round != m.Round { + return ErrDecideProofRoundMismatch + } + + if !c.stateValidate(mProof.State) { + return ErrDecideProofStateValidation + } + + // state data validation in proofs + if mProof.State != nil { + if !c.stateValidate(mProof.State) { + return ErrSelectProofStateValidation + } + } + + commits[c.pubKeyToIdentity(proof.PublicKey(c.curve))] = mProof.State + } + + // count proofs to m.State + var numValidateProofs int + mHash := c.stateHash(m.State) + for _, v := range commits { + if c.stateHash(v) == mHash { + numValidateProofs++ + } + } + + // check to see if the message has at least 2*t+1 valid proofs, + // if not, the leader may cheat. + if numValidateProofs < 2*c.t()+1 { + return ErrDecideProofInsufficient + } + return nil +} + +// broadcastRoundChange will broadcast messages on +// current round, taking the maximal B' from unconfirmed data. +func (c *Consensus) broadcastRoundChange() { + // if has sent in this round, + // then just ignore. But if we are in roundchanging state, + // we should send repeatedly, for boostrap process. + if c.currentRound.RoundChangeSent && c.currentRound.Stage != stageRoundChanging { + return + } + + // first we need to check if there is any locked data, + // locked data must be sent if there is any. + data := c.maximalLocked() + if data == nil { + // if there's none locked data, we pick the maximum unconfirmed data to propose + data = c.maximalUnconfirmed() + // if still null, return + if data == nil { + return + } + } + + var m Message + m.Type = MessageType_RoundChange + m.Height = c.latestHeight + 1 + m.Round = c.currentRound.RoundNumber + m.State = data + c.broadcast(&m) + c.currentRound.RoundChangeSent = true + //log.Println("broadcast:") +} + +// broadcastLock will broadcast messages on current round, +// the currentRound should have a chosen data in this round. +func (c *Consensus) broadcastLock() { + var m Message + m.Type = MessageType_Lock + m.Height = c.latestHeight + 1 + m.Round = c.currentRound.RoundNumber + m.State = c.currentRound.LockedState + m.Proof = c.currentRound.SignedRoundChanges() + c.broadcast(&m) + //log.Println("broadcast:") +} + +// broadcastLockRelease will broadcast messages, +func (c *Consensus) broadcastLockRelease(signed *SignedProto) { + var m Message + m.Type = MessageType_LockRelease + m.Height = c.latestHeight + 1 + m.Round = c.currentRound.RoundNumber + m.LockRelease = signed + c.broadcast(&m) + //log.Println("broadcast:") +} + +// broadcastSelect will broadcast a ", m.State) +} + +// broadcastDecide will broadcast a message by the leader, +// from current round with proofs. +func (c *Consensus) broadcastDecide() *SignedProto { + var m Message + m.Type = MessageType_Decide + m.Height = c.latestHeight + 1 + m.Round = c.currentRound.RoundNumber + m.State = c.currentRound.LockedState + m.Proof = c.currentRound.SignedCommits() + return c.broadcast(&m) + //log.Println("broadcast:") +} + +// broadcastResync will broadcast a message by the leader, +// from current round with proofs. +func (c *Consensus) broadcastResync() { + if c.lastRoundChangeProof == nil { + return + } + + var m Message + m.Type = MessageType_Resync + // we only care about messages in resync + m.Proof = c.lastRoundChangeProof + c.broadcast(&m) + //log.Println("broadcast:") +} + +// sendCommit will send a message by participants to the leader +// from received message. +func (c *Consensus) sendCommit(msgLock *Message) { + if c.currentRound.CommitSent { + return + } + + var m Message + m.Type = MessageType_Commit + m.Height = msgLock.Height // h + m.Round = msgLock.Round // r + m.State = msgLock.State // B'j + if c.enableCommitUnicast { + c.sendTo(&m, c.roundLeader(m.Round)) + } else { + c.broadcast(&m) + } + c.currentRound.CommitSent = true + //log.Println("send:") +} + +// broadcast signs the message with private key before broadcasting to all peers. +func (c *Consensus) broadcast(m *Message) *SignedProto { + // sign + sp := new(SignedProto) + sp.Version = ProtocolVersion + sp.Sign(m, c.privateKey) + + // message callback + if c.messageOutCallback != nil { + c.messageOutCallback(m, sp) + } + // protobuf marshalling + out, err := proto.Marshal(sp) + if err != nil { + panic(err) + } + + // send to peers one by one + for _, peer := range c.peers { + _ = peer.Send(out) + } + + // we also need to send this message to myself + c.loopback = append(c.loopback, out) + return sp +} + +// sendTo signs the message with private key before transmitting to the peer. +func (c *Consensus) sendTo(m *Message, leader Identity) { + // sign + sp := new(SignedProto) + sp.Version = ProtocolVersion + sp.Sign(m, c.privateKey) + + // message callback + if c.messageOutCallback != nil { + c.messageOutCallback(m, sp) + } + + // protobuf marshalling + out, err := proto.Marshal(sp) + if err != nil { + panic(err) + } + + // we need to send this message to myself (via loopback) if i'm the leader + if leader == c.identity { + c.loopback = append(c.loopback, out) + return + } + + // otherwise, find and transmit to the leader + for _, peer := range c.peers { + if pk := peer.GetPublicKey(); pk != nil { + coord := c.pubKeyToIdentity(pk) + if coord == leader { + // we do not return here to avoid missing re-connected peer. + peer.Send(out) + } + } + } +} + +// propagate broadcasts signed message UNCHANGED to peers. +func (c *Consensus) propagate(bts []byte) { + // send to peers one by one + for _, peer := range c.peers { + _ = peer.Send(bts) + } +} + +// getRound returns the consensus round with given idx, create one if not exists +// if purgeLower has set, all lower rounds will be cleared +func (c *Consensus) getRound(idx uint64, purgeLower bool) *consensusRound { + var next *list.Element + for elem := c.rounds.Front(); elem != nil; elem = next { + next = elem.Next() + r := elem.Value.(*consensusRound) + + if r.RoundNumber < idx { // lower round + // if remove flag has set, remove this round safely, + // usually used by switchRound + if purgeLower { + c.rounds.Remove(elem) + } + continue + } else if idx < r.RoundNumber { // higher round + // insert a new round entry before this round + // to make sure the list is ordered + newr := newConsensusRound(idx, c) + c.rounds.InsertBefore(newr, elem) + return newr + } else if r.RoundNumber == idx { // found entry + return r + } + } + + // looped to the end, we create and push back + newr := newConsensusRound(idx, c) + c.rounds.PushBack(newr) + return newr +} + +// lockRelease updates locks while entering lock-release status +// and will broadcast its max B' if there is any. +func (c *Consensus) lockRelease() { + // only keep the locked B' with the max round number + // while switching to lock-release status + if len(c.locks) > 0 { + max := c.locks[0] + for i := 1; i < len(c.locks); i++ { + if max.Message.Round < c.locks[i].Message.Round { + max = c.locks[i] + } + } + c.locks = []messageTuple{max} + c.broadcastLockRelease(max.Signed) + } +} + +// switchRound sets currentRound to the given idx, and creates new a consensusRound +// if it's not been initialized. +// and all lower rounds will be cleared while switching. +func (c *Consensus) switchRound(round uint64) { c.currentRound = c.getRound(round, true) } + +// roundLeader returns leader's identity for a given round +func (c *Consensus) roundLeader(round uint64) Identity { + // NOTE: fixed leader is for testing + if c.fixedLeader != nil { + return *c.fixedLeader + } + return c.participants[int(round)%len(c.participants)] +} + +// heightSync changes current height to the given height with state +// resets all fields to this new height. +func (c *Consensus) heightSync(height uint64, round uint64, s State, now time.Time) { + c.latestHeight = height // set height + c.latestRound = round // set round + c.latestState = s // set state + + c.currentRound = nil // clean current round pointer + c.lastRoundChangeProof = nil // clean round change proof + c.rounds.Init() // clean all round + c.locks = nil // clean locks + c.unconfirmed = nil // clean all unconfirmed states from previous heights + c.switchRound(0) // start new round at new height + c.currentRound.Stage = stageRoundChanging +} + +// t calculates (n-1)/3 +func (c *Consensus) t() int { return (c.numIdentities - 1) / 3 } + +// Propose adds a new state to unconfirmed queue to particpate in +// consensus at next height. +func (c *Consensus) Propose(s State) { + if s == nil { + return + } + + sHash := c.stateHash(s) + for k := range c.unconfirmed { + if c.stateHash(c.unconfirmed[k]) == sHash { + return + } + } + c.unconfirmed = append(c.unconfirmed, s) +} + +// ReceiveMessage processes incoming consensus messages, and returns error +// if message cannot be processed for some reason. +func (c *Consensus) ReceiveMessage(bts []byte, now time.Time) (err error) { + // messages broadcasted to myself may be queued recursively, and + // we only process these messages in defer to avoid side effects + // while processing. + defer func() { + for len(c.loopback) > 0 { + bts := c.loopback[0] + c.loopback = c.loopback[1:] + // NOTE: message directed to myself ignores error. + _ = c.receiveMessage(bts, now) + } + }() + + return c.receiveMessage(bts, now) +} + +func (c *Consensus) receiveMessage(bts []byte, now time.Time) error { + // unmarshal signed message + signed := new(SignedProto) + err := proto.Unmarshal(bts, signed) + if err != nil { + return err + } + + // check message version + if signed.Version != ProtocolVersion { + return ErrMessageVersion + } + + // check message signature & qualifications + m, err := c.verifyMessage(signed) + if err != nil { + return err + } + + // callback for incoming message + if c.messageValidator != nil { + if !c.messageValidator(c, m, signed) { + return ErrMessageValidator + } + } + + // message switch + switch m.Type { + case MessageType_Nop: + // nop does nothing + return nil + case MessageType_RoundChange: + err := c.verifyRoundChangeMessage(m) + if err != nil { + return err + } + + // for message, we need to find in each round + // to check if this participant has already sent + // we only keep the message from the max round. + // NOTE: we don't touch current round to prevent removing + // valid proofs. + // NOTE: the total messages are bounded to max 2*participants + // at any time, so the loop has O(n) time complexity + var next *list.Element + for elem := c.rounds.Front(); elem != nil; elem = next { + next = elem.Next() + cr := elem.Value.(*consensusRound) + if idx := cr.FindRoundChange(signed.X, signed.Y); idx != -1 { // located! + if m.Round == c.currentRound.RoundNumber { // don't remove now! + continue + } else if cr.RoundNumber > m.Round { + // existing message is higher than incoming message, + // just ignore. + return nil + } else if cr.RoundNumber < m.Round { + // existing message is lower than incoming message, + // remove the existing message from this round. + cr.RemoveRoundChange(idx) + // if no message remained in this round, release + // the round resources too, to prevent OOM attack + if cr.NumRoundChanges() == 0 { + c.rounds.Remove(elem) + } + } + } + } + + // locate to round m.Round. + // NOTE: getRound must not be called before previous checks done + // in order to prevent OOM attack by creating round objects. + round := c.getRound(m.Round, false) + // as we cleared all lower rounds message, we handle the message + // at round m.Round. if this message is not duplicated in m.Round, + // round records message along with its signed message + // to provide proofs in the future. + if round.AddRoundChange(signed, m) { + // During any time of the protocol, if a the Pacemaker of Pj (including Pi) + // receives at least 2t + 1 round-change message (including round-change + // message from himself) for round r (which is larger than its current round + // status), it enters lock status of round r + // + // NOTE: m.Round lower than currentRound.RoundNumber has been tested by + // verifyRoundChangeMessage + // NOTE: lock stage can only be entered once for a single round, malicious + // participant can keep on broadcasting increasing to everyone, + // and old messages will be removed from previous rounds in such + // case, so rounds may possibly satisify 2*t+1 more than once. + // + // Example: P sends r+1 to remove from r, and sends to r again to trigger 2t+1 once + // more to reset timeout. + if round.NumRoundChanges() == 2*c.t()+1 && round.Stage < stageLock { + // switch to this round + c.switchRound(m.Round) + // record this round change proof for resyncing + c.lastRoundChangeProof = c.currentRound.SignedRoundChanges() + + // If Pj has not broadcasted the round-change message yet, + // it broadcasts now. + c.broadcastRoundChange() + + // leader of this round MUST wait on collectDuration, + // to decide to broadcast or message + err := c.verifySelectMessage(m, signed) + if err != nil { + return err + } + + // round will be increased monotonically + if m.Round > c.currentRound.RoundNumber { + c.switchRound(m.Round) + c.lastRoundChangeProof = []*SignedProto{signed} // record this proof for resyncing + } + + // for rounds r' >= r, we must check c.stage to stageLockRelease + // only once to prevent resetting lockReleaseTimeout or shifting c.cstage + if c.currentRound.Stage < stageLockRelease { + c.currentRound.Stage = stageLockRelease + c.lockReleaseTimeout = now.Add(c.commitDuration(m.Round)) + c.lockRelease() + // add to Blockj + c.Propose(m.State) + } + + case MessageType_Lock: + // verify message + err := c.verifyLockMessage(m, signed) + if err != nil { + return err + } + + // round will be increased monotonically + if m.Round > c.currentRound.RoundNumber { + c.switchRound(m.Round) + c.lastRoundChangeProof = []*SignedProto{signed} // record this proof for resyncing + } + + // for rounds r' >= r, we must check to enter commit status + // only once to prevent resetting commitTimeout or shifting c.cstage + if c.currentRound.Stage < stageCommit { + c.currentRound.Stage = stageCommit + c.commitTimeout = now.Add(c.commitDuration(m.Round)) + + mHash := c.stateHash(m.State) + // release any potential lock on B' in this round + // in-place deletion + o := 0 + for i := 0; i < len(c.locks); i++ { + if c.locks[i].StateHash != mHash { + c.locks[o] = c.locks[i] + o++ // o is the new length of c.locks + } + } + c.locks = c.locks[:o] + // append the new element + c.locks = append(c.locks, messageTuple{StateHash: mHash, Message: m, Signed: signed}) + } + + // for any incoming message with r=r', sendCommit will send + // once. + c.sendCommit(m) + + case MessageType_LockRelease: + // verifies the LockRelease field in message. + lockmsg, err := c.verifyLockReleaseMessage(m.LockRelease) + if err != nil { + return err + } + + // length of locks is 0, append and return. + if len(c.locks) == 0 { + c.locks = append(c.locks, messageTuple{StateHash: c.stateHash(lockmsg.State), Message: lockmsg, Signed: m.LockRelease}) + return nil + } + + // remove any locks if lockmsg.r > r' and keep lockmsg.r, + o := 0 + for i := 0; i < len(c.locks); i++ { + if !(lockmsg.Round > c.locks[i].Message.Round) { + // if the round of this lock is not larger than what we + // have kept, ignore and continue. + c.locks[o] = c.locks[i] + o++ + } + } + + // some locks have been removed if o is smaller than original locks length, + // then we keep this lock. + if o < len(c.locks) { + c.locks = c.locks[:o] + c.locks = append(c.locks, messageTuple{StateHash: c.stateHash(lockmsg.State), Message: lockmsg, Signed: m.LockRelease}) + } + + case MessageType_Commit: + // leader process commits message from all participants, + // check to see if I'm the leader of this round to process this message. + leaderKey := c.roundLeader(m.Round) + if leaderKey == c.identity { + // verify commit message. + // NOTE: leader only accept commits for current height & round. + err := c.verifyCommitMessage(m) + if err != nil { + return err + } + + // verifyCommitMessage can guarantee that the message is to currentRound, + // so we're safe to process in current round. + if c.currentRound.AddCommit(signed, m) { + // NOTE: we proceed the following only when AddCommit returns true. + // NumCommitted will only return commits with locked B' + // and ignore non-B' commits. + if c.currentRound.NumCommitted() >= 2*c.t()+1 { + /* + log.Println("======= LEADER'S DECIDE=====") + log.Println("Height:", c.currentHeight+1) + log.Println("Round:", c.currentRound.RoundNumber) + log.Println("State:", State(c.currentRound.LockedState).hash()) + */ + + // broadcast decide will return what it has sent + c.latestProof = c.broadcastDecide() + c.heightSync(c.latestHeight+1, c.currentRound.RoundNumber, c.currentRound.LockedState, now) + // leader should wait for 1 more latency + c.rcTimeout = now.Add(c.roundchangeDuration(0) + c.latency) + // broadcast at new height + c.broadcastRoundChange() + } + } + } + + case MessageType_Decide: + err := c.verifyDecideMessage(m, signed) + if err != nil { + return err + } + + // record this proof for chaining + c.latestProof = signed + + // propagate this message to my neighbour. + // NOTE: verifyDecideMessage() can stop broadcast storm. + c.propagate(bts) + // passive confirmation from the leader. + c.heightSync(m.Height, m.Round, m.State, now) + // non-leader starts waiting for rcTimeout + c.rcTimeout = now.Add(c.roundchangeDuration(0)) + // we sync our height and broadcast new . + c.broadcastRoundChange() + case MessageType_Resync: + // push the proofs in loopback device + for k := range m.Proof { + // protobuf marshalling + out, err := proto.Marshal(m.Proof[k]) + if err != nil { + panic(err) + } + c.loopback = append(c.loopback, out) + } + default: + return ErrMessageUnknownMessageType + } + return nil +} + +// Update will process timing event for the state machine, callers +// from outside MUST call this function periodically(like 20ms). +func (c *Consensus) Update(now time.Time) error { + // as in ReceiveMessage, we also need to handle broadcasting messages + // directed to myself. + defer func() { + for len(c.loopback) > 0 { + bts := c.loopback[0] + c.loopback = c.loopback[1:] + _ = c.receiveMessage(bts, now) + } + }() + + // stage switch + switch c.currentRound.Stage { + case stageRoundChanging: + if c.rcTimeout.IsZero() { + panic("roundchanging stage entered, but lockTimeout not set") + } + + if now.After(c.rcTimeout) { + c.broadcastRoundChange() + c.broadcastResync() // we also need to broadcast the round change event message if there is any + c.rcTimeout = now.Add(c.roundchangeDuration(c.currentRound.RoundNumber)) + } + case stageLock: + if c.lockTimeout.IsZero() { + panic("lock stage entered, but lockTimeout not set") + } + // leader's collection, we perform periodically check for or message to participants. + // enqueue all received non-NULL data + states := c.currentRound.RoundChangeStates() + for k := range states { + c.Propose(states[k]) + } + + // broadcast this related + ErrSelectStateValidation = errors.New("the state data validation failed message has another height than expected") + ErrSelectRoundLower = errors.New("the message is not signed by leader") + ErrSelectStateMismatch = errors.New("the message has unknown participant") + ErrSelectProofTypeMismatch = errors.New("the proofs in message has mismatched height") + ErrSelectProofRoundMismatch = errors.New("the proofs in message has invalid state data") + ErrSelectProofNotTheMaximal = errors.New("the proposed state is not the maximal one in the message has insufficient overall proofs") + ErrSelectProofExceeded = errors.New("the message + MessageType_Select MessageType = 3 + // MessageCommit = message + MessageType_Commit MessageType = 4 + // MessageLockRelease = message + MessageType_LockRelease MessageType = 5 + // MessageDecide = message + MessageType_Decide MessageType = 6 + // MessageResync= message + MessageType_Resync MessageType = 7 +) + +var MessageType_name = map[int32]string{ + 0: "Nop", + 1: "RoundChange", + 2: "Lock", + 3: "Select", + 4: "Commit", + 5: "LockRelease", + 6: "Decide", + 7: "Resync", +} + +var MessageType_value = map[string]int32{ + "Nop": 0, + "RoundChange": 1, + "Lock": 2, + "Select": 3, + "Commit": 4, + "LockRelease": 5, + "Decide": 6, + "Resync": 7, +} + +func (x MessageType) String() string { + return proto.EnumName(MessageType_name, int32(x)) +} + +func (MessageType) EnumDescriptor() ([]byte, []int) { + return fileDescriptor_33c57e4bae7b9afd, []int{0} +} + +// SignedProto defines a message with signature and it's publickey +type SignedProto struct { + Version uint32 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` + // the Message encoded raw protobuf in bytes + Message []byte `protobuf:"bytes,2,opt,name=Message,proto3" json:"Message,omitempty"` + // signer's public key + X PubKeyAxis `protobuf:"bytes,3,opt,name=x,proto3,customtype=PubKeyAxis" json:"x"` + Y PubKeyAxis `protobuf:"bytes,4,opt,name=y,proto3,customtype=PubKeyAxis" json:"y"` + // signature r,s for prefix+messages+version+x+y above + R []byte `protobuf:"bytes,5,opt,name=r,proto3" json:"r,omitempty"` + S []byte `protobuf:"bytes,6,opt,name=s,proto3" json:"s,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *SignedProto) Reset() { *m = SignedProto{} } +func (m *SignedProto) String() string { return proto.CompactTextString(m) } +func (*SignedProto) ProtoMessage() {} +func (*SignedProto) Descriptor() ([]byte, []int) { + return fileDescriptor_33c57e4bae7b9afd, []int{0} +} +func (m *SignedProto) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SignedProto) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SignedProto.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SignedProto) XXX_Merge(src proto.Message) { + xxx_messageInfo_SignedProto.Merge(m, src) +} +func (m *SignedProto) XXX_Size() int { + return m.Size() +} +func (m *SignedProto) XXX_DiscardUnknown() { + xxx_messageInfo_SignedProto.DiscardUnknown(m) +} + +var xxx_messageInfo_SignedProto proto.InternalMessageInfo + +func (m *SignedProto) GetVersion() uint32 { + if m != nil { + return m.Version + } + return 0 +} + +func (m *SignedProto) GetMessage() []byte { + if m != nil { + return m.Message + } + return nil +} + +func (m *SignedProto) GetR() []byte { + if m != nil { + return m.R + } + return nil +} + +func (m *SignedProto) GetS() []byte { + if m != nil { + return m.S + } + return nil +} + +// Message defines a consensus message +type Message struct { + // Type of this message + Type MessageType `protobuf:"varint,1,opt,name=Type,proto3,enum=bdls.MessageType" json:"Type,omitempty"` + // Height in consensus + Height uint64 `protobuf:"varint,2,opt,name=Height,proto3" json:"Height,omitempty"` + // Round in consensus + Round uint64 `protobuf:"varint,3,opt,name=Round,proto3" json:"Round,omitempty"` + // Proposed state (optional) + State []byte `protobuf:"bytes,4,opt,name=State,proto3" json:"State,omitempty"` + // Proofs related + Proof []*SignedProto `protobuf:"bytes,5,rep,name=Proof,proto3" json:"Proof,omitempty"` + // for lock-release, it's an embeded message + LockRelease *SignedProto `protobuf:"bytes,6,opt,name=LockRelease,proto3" json:"LockRelease,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Message) Reset() { *m = Message{} } +func (m *Message) String() string { return proto.CompactTextString(m) } +func (*Message) ProtoMessage() {} +func (*Message) Descriptor() ([]byte, []int) { + return fileDescriptor_33c57e4bae7b9afd, []int{1} +} +func (m *Message) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Message.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Message) XXX_Merge(src proto.Message) { + xxx_messageInfo_Message.Merge(m, src) +} +func (m *Message) XXX_Size() int { + return m.Size() +} +func (m *Message) XXX_DiscardUnknown() { + xxx_messageInfo_Message.DiscardUnknown(m) +} + +var xxx_messageInfo_Message proto.InternalMessageInfo + +func (m *Message) GetType() MessageType { + if m != nil { + return m.Type + } + return MessageType_Nop +} + +func (m *Message) GetHeight() uint64 { + if m != nil { + return m.Height + } + return 0 +} + +func (m *Message) GetRound() uint64 { + if m != nil { + return m.Round + } + return 0 +} + +func (m *Message) GetState() []byte { + if m != nil { + return m.State + } + return nil +} + +func (m *Message) GetProof() []*SignedProto { + if m != nil { + return m.Proof + } + return nil +} + +func (m *Message) GetLockRelease() *SignedProto { + if m != nil { + return m.LockRelease + } + return nil +} + +func init() { + proto.RegisterEnum("bdls.MessageType", MessageType_name, MessageType_value) + proto.RegisterType((*SignedProto)(nil), "bdls.SignedProto") + proto.RegisterType((*Message)(nil), "bdls.Message") +} + +func init() { proto.RegisterFile("message.proto", fileDescriptor_33c57e4bae7b9afd) } + +var fileDescriptor_33c57e4bae7b9afd = []byte{ + // 381 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x92, 0xcd, 0xaa, 0xda, 0x40, + 0x14, 0xc7, 0x1d, 0xf3, 0x61, 0x39, 0xd1, 0x76, 0x3a, 0x94, 0x32, 0x74, 0xa1, 0x41, 0x28, 0x95, + 0x42, 0x23, 0xd4, 0x27, 0xa8, 0x76, 0x51, 0xe8, 0x07, 0x32, 0xf6, 0x05, 0xf2, 0x71, 0x8c, 0xa1, + 0x26, 0x23, 0x99, 0xa4, 0x98, 0xc7, 0xe9, 0xdb, 0xb8, 0x2c, 0x5d, 0x76, 0x21, 0x17, 0x9f, 0xe4, + 0x32, 0x13, 0xbd, 0x64, 0x71, 0xef, 0xee, 0xfc, 0xce, 0xff, 0x7f, 0xce, 0xf9, 0x27, 0x0c, 0x8c, + 0x72, 0x54, 0x2a, 0x4c, 0x31, 0x38, 0x94, 0xb2, 0x92, 0xcc, 0x8e, 0x92, 0xbd, 0x7a, 0xf3, 0x21, + 0xcd, 0xaa, 0x5d, 0x1d, 0x05, 0xb1, 0xcc, 0xe7, 0xa9, 0x4c, 0xe5, 0xdc, 0x88, 0x51, 0xbd, 0x35, + 0x64, 0xc0, 0x54, 0xed, 0xd0, 0xf4, 0x0f, 0x01, 0x6f, 0x93, 0xa5, 0x05, 0x26, 0x6b, 0xb3, 0x84, + 0xc3, 0xe0, 0x37, 0x96, 0x2a, 0x93, 0x05, 0x27, 0x3e, 0x99, 0x8d, 0xc4, 0x0d, 0xb5, 0xf2, 0xbd, + 0xbd, 0xc7, 0xfb, 0x3e, 0x99, 0x0d, 0xc5, 0x0d, 0x99, 0x0f, 0xe4, 0xc8, 0x2d, 0xdd, 0x5b, 0xb2, + 0xd3, 0x79, 0xd2, 0xfb, 0x7f, 0x9e, 0xc0, 0xba, 0x8e, 0xbe, 0x62, 0xf3, 0xe9, 0x98, 0x29, 0x41, + 0x8e, 0xda, 0xd1, 0x70, 0xfb, 0x69, 0x47, 0xc3, 0x86, 0x40, 0x4a, 0xee, 0x98, 0xbd, 0xa4, 0xd4, + 0xa4, 0xb8, 0xdb, 0x92, 0x9a, 0xfe, 0x23, 0x0f, 0xa7, 0xd9, 0x5b, 0xb0, 0x7f, 0x36, 0x07, 0x34, + 0xe1, 0x9e, 0x7f, 0x7c, 0x19, 0xe8, 0x6f, 0x0e, 0xae, 0xa2, 0x16, 0x84, 0x91, 0xd9, 0x6b, 0x70, + 0xbf, 0x60, 0x96, 0xee, 0x2a, 0x93, 0xd5, 0x16, 0x57, 0x62, 0xaf, 0xc0, 0x11, 0xb2, 0x2e, 0x12, + 0x13, 0xd7, 0x16, 0x2d, 0xe8, 0xee, 0xa6, 0x0a, 0x2b, 0x6c, 0x23, 0x8a, 0x16, 0xd8, 0x3b, 0x70, + 0xd6, 0xa5, 0x94, 0x5b, 0xee, 0xf8, 0xd6, 0xcc, 0xbb, 0xdd, 0xea, 0xfc, 0x2c, 0xd1, 0xea, 0x6c, + 0x01, 0xde, 0x37, 0x19, 0xff, 0x12, 0xb8, 0xc7, 0x50, 0xa1, 0xc9, 0xfd, 0xa8, 0xbd, 0xeb, 0x7a, + 0x5f, 0x82, 0xd7, 0x89, 0xcd, 0x06, 0x60, 0xfd, 0x90, 0x07, 0xda, 0x63, 0x2f, 0xc0, 0x33, 0xa1, + 0x56, 0xbb, 0xb0, 0x48, 0x91, 0x12, 0xf6, 0x0c, 0x6c, 0x3d, 0x47, 0xfb, 0x0c, 0xc0, 0xdd, 0xe0, + 0x1e, 0xe3, 0x8a, 0x5a, 0xba, 0x5e, 0xc9, 0x3c, 0xcf, 0x2a, 0x6a, 0xeb, 0x91, 0xce, 0x66, 0xea, + 0x68, 0xf1, 0x33, 0xc6, 0x59, 0x82, 0xd4, 0xd5, 0xb5, 0x40, 0xd5, 0x14, 0x31, 0x1d, 0x2c, 0x87, + 0xa7, 0xcb, 0x98, 0xfc, 0xbd, 0x8c, 0xc9, 0xdd, 0x65, 0x4c, 0x22, 0xd7, 0xbc, 0x80, 0xc5, 0x7d, + 0x00, 0x00, 0x00, 0xff, 0xff, 0x43, 0xc3, 0x64, 0x6c, 0x47, 0x02, 0x00, 0x00, +} + +func (m *SignedProto) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SignedProto) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SignedProto) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.S) > 0 { + i -= len(m.S) + copy(dAtA[i:], m.S) + i = encodeVarintMessage(dAtA, i, uint64(len(m.S))) + i-- + dAtA[i] = 0x32 + } + if len(m.R) > 0 { + i -= len(m.R) + copy(dAtA[i:], m.R) + i = encodeVarintMessage(dAtA, i, uint64(len(m.R))) + i-- + dAtA[i] = 0x2a + } + { + size := m.Y.Size() + i -= size + if _, err := m.Y.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + i = encodeVarintMessage(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x22 + { + size := m.X.Size() + i -= size + if _, err := m.X.MarshalTo(dAtA[i:]); err != nil { + return 0, err + } + i = encodeVarintMessage(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + if len(m.Message) > 0 { + i -= len(m.Message) + copy(dAtA[i:], m.Message) + i = encodeVarintMessage(dAtA, i, uint64(len(m.Message))) + i-- + dAtA[i] = 0x12 + } + if m.Version != 0 { + i = encodeVarintMessage(dAtA, i, uint64(m.Version)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *Message) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Message) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Message) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if m.LockRelease != nil { + { + size, err := m.LockRelease.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMessage(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } + if len(m.Proof) > 0 { + for iNdEx := len(m.Proof) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Proof[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintMessage(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x2a + } + } + if len(m.State) > 0 { + i -= len(m.State) + copy(dAtA[i:], m.State) + i = encodeVarintMessage(dAtA, i, uint64(len(m.State))) + i-- + dAtA[i] = 0x22 + } + if m.Round != 0 { + i = encodeVarintMessage(dAtA, i, uint64(m.Round)) + i-- + dAtA[i] = 0x18 + } + if m.Height != 0 { + i = encodeVarintMessage(dAtA, i, uint64(m.Height)) + i-- + dAtA[i] = 0x10 + } + if m.Type != 0 { + i = encodeVarintMessage(dAtA, i, uint64(m.Type)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func encodeVarintMessage(dAtA []byte, offset int, v uint64) int { + offset -= sovMessage(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *SignedProto) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Version != 0 { + n += 1 + sovMessage(uint64(m.Version)) + } + l = len(m.Message) + if l > 0 { + n += 1 + l + sovMessage(uint64(l)) + } + l = m.X.Size() + n += 1 + l + sovMessage(uint64(l)) + l = m.Y.Size() + n += 1 + l + sovMessage(uint64(l)) + l = len(m.R) + if l > 0 { + n += 1 + l + sovMessage(uint64(l)) + } + l = len(m.S) + if l > 0 { + n += 1 + l + sovMessage(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *Message) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Type != 0 { + n += 1 + sovMessage(uint64(m.Type)) + } + if m.Height != 0 { + n += 1 + sovMessage(uint64(m.Height)) + } + if m.Round != 0 { + n += 1 + sovMessage(uint64(m.Round)) + } + l = len(m.State) + if l > 0 { + n += 1 + l + sovMessage(uint64(l)) + } + if len(m.Proof) > 0 { + for _, e := range m.Proof { + l = e.Size() + n += 1 + l + sovMessage(uint64(l)) + } + } + if m.LockRelease != nil { + l = m.LockRelease.Size() + n += 1 + l + sovMessage(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func sovMessage(x uint64) (n int) { + return (math_bits.Len64(x|1) + 6) / 7 +} +func sozMessage(x uint64) (n int) { + return sovMessage(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *SignedProto) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SignedProto: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SignedProto: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) + } + m.Version = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Version |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Message", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Message = append(m.Message[:0], dAtA[iNdEx:postIndex]...) + if m.Message == nil { + m.Message = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field X", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.X.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Y", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if err := m.Y.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field R", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.R = append(m.R[:0], dAtA[iNdEx:postIndex]...) + if m.R == nil { + m.R = []byte{} + } + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field S", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.S = append(m.S[:0], dAtA[iNdEx:postIndex]...) + if m.S == nil { + m.S = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMessage(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Message) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Message: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Message: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + m.Type = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Type |= MessageType(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Height", wireType) + } + m.Height = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Height |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Round", wireType) + } + m.Round = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Round |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field State", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.State = append(m.State[:0], dAtA[iNdEx:postIndex]...) + if m.State == nil { + m.State = []byte{} + } + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Proof", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Proof = append(m.Proof, &SignedProto{}) + if err := m.Proof[len(m.Proof)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field LockRelease", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowMessage + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthMessage + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthMessage + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.LockRelease == nil { + m.LockRelease = &SignedProto{} + } + if err := m.LockRelease.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipMessage(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) < 0 { + return ErrInvalidLengthMessage + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipMessage(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + depth := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMessage + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMessage + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + case 1: + iNdEx += 8 + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowMessage + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if length < 0 { + return 0, ErrInvalidLengthMessage + } + iNdEx += length + case 3: + depth++ + case 4: + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupMessage + } + depth-- + case 5: + iNdEx += 4 + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + if iNdEx < 0 { + return 0, ErrInvalidLengthMessage + } + if depth == 0 { + return iNdEx, nil + } + } + return 0, io.ErrUnexpectedEOF +} + +var ( + ErrInvalidLengthMessage = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowMessage = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupMessage = fmt.Errorf("proto: unexpected end of group") +) diff --git a/orderer/consensus/bdls/message.proto b/orderer/consensus/bdls/message.proto new file mode 100644 index 0000000000..9a74ed3958 --- /dev/null +++ b/orderer/consensus/bdls/message.proto @@ -0,0 +1,83 @@ +// BSD 3-Clause License +// +// Copyright (c) 2020, Sperax +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; +package bdls; + +import "github.com/gogo/protobuf/gogoproto/gogo.proto"; + +// SignedProto defines a message with signature and it's publickey +message SignedProto { + uint32 version=1; + // the Message encoded raw protobuf in bytes + bytes Message=2; + // signer's public key + bytes x = 3 [(gogoproto.customtype) = "PubKeyAxis", (gogoproto.nullable) = false]; + bytes y = 4 [(gogoproto.customtype) = "PubKeyAxis", (gogoproto.nullable) = false]; + // signature r,s for prefix+messages+version+x+y above + bytes r = 5; + bytes s = 6; +} + +// MessageType defines supported message types +enum MessageType{ + // No operation, for default message type, and keepalive connection + Nop = 0; + // MessageRoundChange = message + RoundChange = 1; + // MessageLock = message + Lock = 2; + // MessageSelect = + m := new(Message) + m.Type = MessageType_Select + m.Height = height + m.Round = round + + var publicKeys []*ecdsa.PublicKey + for i := 0; i < numProofs; i++ { + randstate := make([]byte, 1024) + _, err := io.ReadFull(rand.Reader, randstate) + assert.Nil(t, err) + if i == 0 { + m.State = randstate + } + + // the comparison function, to make sure m.State is valid + if bytes.Compare(m.State, randstate) < 0 { + m.State = randstate + } + + // + var signedRc *SignedProto + var proofKey *ecdsa.PrivateKey + + // the first proof is signed by this message creator + if i == 0 { + _, signedRc, proofKey = createRoundChangeMessageSigner(t, proofHeight, proofRound, randstate, privateKey) + } else { + _, signedRc, proofKey = createRoundChangeMessageState(t, proofHeight, proofRound, randstate) + } + m.Proof = append(m.Proof, signedRc) + publicKeys = append(publicKeys, &proofKey.PublicKey) + } + + signed := new(SignedProto) + signed.Sign(m, privateKey) + + return m, signed, privateKey, publicKeys +} + +// createDecideMessage creates a valid message, and generate proofs based on quorum, +// the first 2t+1 roundchange proposals are the same +func createDecideMessage(t *testing.T, numProofs int, height uint64, round uint64, proofHeight uint64, proofRound uint64) (*Message, *SignedProto, *ecdsa.PrivateKey, []*ecdsa.PublicKey) { + privateKey, err := ecdsa.GenerateKey(S256Curve, rand.Reader) + assert.Nil(t, err) + valid := 2*((numProofs-1)/3) + 1 + state := make([]byte, 1024) + //_, err := io.ReadFull(rand.Reader, state) + //assert.Nil(t, err) + + // + m := new(Message) + m.Type = MessageType_Decide + m.Height = height + m.Round = round + m.State = state + + var publicKeys []*ecdsa.PublicKey + for i := 0; i < numProofs; i++ { + // + var signedRc *SignedProto + var proofKey *ecdsa.PrivateKey + if i >= valid { // only provide valid proofs + randstate := make([]byte, 1024) + _, err := io.ReadFull(rand.Reader, randstate) + assert.Nil(t, err) + _, signedRc, proofKey = createCommitMessage(t, proofHeight, proofRound, randstate) + } else { + if i == 0 { + _, signedRc, proofKey = createCommitMessageSigner(t, proofHeight, proofRound, state, privateKey) + } else { + _, signedRc, proofKey = createCommitMessage(t, proofHeight, proofRound, state) + } + } + m.Proof = append(m.Proof, signedRc) + publicKeys = append(publicKeys, &proofKey.PublicKey) + } + + signed := new(SignedProto) + signed.Sign(m, privateKey) + + return m, signed, privateKey, publicKeys +} + +/////////////////////////////////////////////////////////////////////////////// +// +// common message related tests +// +/////////////////////////////////////////////////////////////////////////////// +func TestVerifyMessage(t *testing.T) { + // signer + privateKey, err := ecdsa.GenerateKey(S256Curve, rand.Reader) + assert.Nil(t, err) + + // create consensus + consensus := createConsensus(t, 0, 0, []*ecdsa.PublicKey{&privateKey.PublicKey}) + + // verify nil message + _, err = consensus.verifyMessage(nil) + assert.Equal(t, ErrMessageIsEmpty, err) + + // check correctly signed message by a participant + message := Message{} + sp := new(SignedProto) + sp.Sign(&message, privateKey) + _, err = consensus.verifyMessage(sp) + assert.Nil(t, err) + + // change signature to random to verify incorrect signature + _, _ = io.ReadFull(rand.Reader, sp.R) + _, _ = io.ReadFull(rand.Reader, sp.S) + _, err = consensus.verifyMessage(sp) + assert.Equal(t, ErrMessageSignature, err) + + // check bad Message with correct signer + noise := make([]byte, 1024) + _, _ = io.ReadFull(rand.Reader, noise) + // hash message + sp.Message = noise + sp.X.Unmarshal(privateKey.PublicKey.X.Bytes()) + sp.Y.Unmarshal(privateKey.PublicKey.Y.Bytes()) + hash := sp.Hash() + + // sign the message + r, s, err := ecdsa.Sign(rand.Reader, privateKey, hash) + if err != nil { + panic(err) + } + sp.R = r.Bytes() + sp.S = s.Bytes() + + // unexpected EOF + _, err = consensus.verifyMessage(sp) + assert.NotNil(t, err) +} + +func TestVerifyMessageUnknownVersion(t *testing.T) { + // signer + privateKey, err := ecdsa.GenerateKey(S256Curve, rand.Reader) + assert.Nil(t, err) + + // create consensus + consensus := createConsensus(t, 0, 0, nil) + + // mock Message + message := Message{} + sp := new(SignedProto) + bts, err := proto.Marshal(&message) + if err != nil { + panic(err) + } + // hash message + sp.Version = uint32(mrand.Int31()%100 + 10) + sp.Message = bts + sp.X.Unmarshal(privateKey.PublicKey.X.Bytes()) + sp.Y.Unmarshal(privateKey.PublicKey.Y.Bytes()) + hash := sp.Hash() + + // sign the message + r, s, err := ecdsa.Sign(rand.Reader, privateKey, hash) + if err != nil { + panic(err) + } + sp.R = r.Bytes() + sp.S = s.Bytes() + + bts, err = proto.Marshal(sp) + assert.Nil(t, err) + err = consensus.ReceiveMessage(bts, time.Now()) + assert.Equal(t, ErrMessageVersion, err) +} + +func TestVerifyMessageUnknownType(t *testing.T) { + // signer + privateKey, err := ecdsa.GenerateKey(S256Curve, rand.Reader) + assert.Nil(t, err) + + // create consensus + consensus := createConsensus(t, 0, 0, []*ecdsa.PublicKey{&privateKey.PublicKey}) + message := Message{} + message.Type = MessageType(mrand.Int31()%100 + 10) + + // check correctly signed message + sp := new(SignedProto) + sp.Sign(&message, privateKey) + bts, err := proto.Marshal(sp) + assert.Nil(t, err) + err = consensus.ReceiveMessage(bts, time.Now()) + assert.Equal(t, ErrMessageUnknownMessageType, err) +} + +func TestVerifyMessageUnknownParticipant(t *testing.T) { + // signer + privateKey, err := ecdsa.GenerateKey(S256Curve, rand.Reader) + assert.Nil(t, err) + + // create consensus + consensus := createConsensus(t, 0, 0, nil) + message := Message{} + sp := new(SignedProto) + sp.Sign(&message, privateKey) + + _, err = consensus.verifyMessage(sp) + assert.Equal(t, ErrMessageUnknownParticipant, err) +} + +/////////////////////////////////////////////////////////////////////////////// +// +// message related tests +// +/////////////////////////////////////////////////////////////////////////////// +func TestVerifyRoundChangeMessageCorrect(t *testing.T) { + m, _, privateKey := createRoundChangeMessage(t, 10, 10) + consensus := createConsensus(t, 9, 10, []*ecdsa.PublicKey{&privateKey.PublicKey}) + err := consensus.verifyRoundChangeMessage(m) + assert.Nil(t, err) +} + +func TestVerifyRoundChangeMessageHeight(t *testing.T) { + m, _, privateKey := createRoundChangeMessage(t, 20, 10) + consensus := createConsensus(t, 10, 10, []*ecdsa.PublicKey{&privateKey.PublicKey}) + err := consensus.verifyRoundChangeMessage(m) + assert.Equal(t, ErrRoundChangeHeightMismatch, err) +} + +func TestVerifyRoundChangeMessageRound(t *testing.T) { + m, _, privateKey := createRoundChangeMessage(t, 20, 9) + consensus := createConsensus(t, 19, 10, []*ecdsa.PublicKey{&privateKey.PublicKey}) + err := consensus.verifyRoundChangeMessage(m) + assert.Equal(t, ErrRoundChangeRoundLower, err) +} + +/////////////////////////////////////////////////////////////////////////////// +// +// message related tests +// +/////////////////////////////////////////////////////////////////////////////// +func TestVerifyLockMessageCorrect(t *testing.T) { + m, sp, privateKey, proofKeys := createLockMessage(t, 20, 10, 10, 10, 10) + consensus := createConsensus(t, 9, 10, proofKeys) + consensus.SetLeader(&privateKey.PublicKey) + + err := consensus.verifyLockMessage(m, sp) + assert.Nil(t, err) +} + +func TestVerifyLockMessageHeight(t *testing.T) { + m, sp, privateKey, proofKeys := createLockMessage(t, 20, 10, 10, 10, 10) + consensus := createConsensus(t, 10, 10, proofKeys) + consensus.SetLeader(&privateKey.PublicKey) + + err := consensus.verifyLockMessage(m, sp) + assert.Equal(t, ErrLockHeightMismatch, err) +} + +func TestVerifyLockMessageRound(t *testing.T) { + m, sp, privateKey, proofKeys := createLockMessage(t, 20, 1, 0, 1, 0) + consensus := createConsensus(t, 0, 1, proofKeys) + consensus.SetLeader(&privateKey.PublicKey) + + err := consensus.verifyLockMessage(m, sp) + assert.Equal(t, ErrLockRoundLower, err) +} + +func TestVerifyLockMessageStateNil(t *testing.T) { + m, sp, privateKey, proofKeys := createLockMessageState(t, 20, nil, 1, 0, 1, 0) + consensus := createConsensus(t, 0, 0, proofKeys) + consensus.SetLeader(&privateKey.PublicKey) + + err := consensus.verifyLockMessage(m, sp) + assert.Equal(t, ErrLockEmptyState, err) +} + +func TestVerifyLockMessageNotSignedByLeader(t *testing.T) { + m, sp, privateKey, proofKeys := createLockMessage(t, 20, 1, 0, 1, 0) + _ = privateKey + consensus := createConsensus(t, 0, 0, proofKeys) + + // set a random leader + randKey, err := ecdsa.GenerateKey(S256Curve, rand.Reader) + assert.Nil(t, err) + consensus.SetLeader(&randKey.PublicKey) + + err = consensus.verifyLockMessage(m, sp) + assert.Equal(t, ErrLockNotSignedByLeader, err) +} + +func TestVerifyLockMessageProofSignature(t *testing.T) { + m, sp, privateKey, proofKeys := createLockMessage(t, 20, 1, 0, 1, 0) + consensus := createConsensus(t, 0, 0, proofKeys) + + consensus.SetLeader(&privateKey.PublicKey) + + // random replace with this incorrect proof + i := mrand.Int() % len(m.Proof) + _, _ = io.ReadFull(rand.Reader, m.Proof[i].R) + _, _ = io.ReadFull(rand.Reader, m.Proof[i].S) + // re-sign the sp with a incorrectly signed proof + sp.Sign(m, privateKey) + + err := consensus.verifyLockMessage(m, sp) + assert.Equal(t, ErrMessageSignature, err) +} + +func TestVerifyLockMessageProofType(t *testing.T) { + m, sp, privateKey, proofKeys := createLockMessage(t, 20, 1, 0, 1, 0) + consensus := createConsensus(t, 0, 0, proofKeys) + consensus.SetLeader(&privateKey.PublicKey) + + // create a signed random proof with incorrect type + proof, signedProof, proofKey := createRoundChangeMessageState(t, 1, 0, m.State) + proof.Type = MessageType_Lock + signedProof.Sign(proof, proofKey) + consensus.AddParticipant(&proofKey.PublicKey) + + // random replacement with this incorrect proof + i := mrand.Int() % len(m.Proof) + m.Proof[i] = signedProof + // re-sign the message + sp.Sign(m, privateKey) + + err := consensus.verifyLockMessage(m, sp) + assert.Equal(t, ErrLockProofTypeMismatch, err) +} + +func TestVerifyLockMessageProofHeight(t *testing.T) { + m, sp, privateKey, proofKeys := createLockMessage(t, 20, 1, 0, 1, 0) + consensus := createConsensus(t, 0, 0, proofKeys) + consensus.SetLeader(&privateKey.PublicKey) + + // create a signed random proof with incorrect height + _, signedProof, proofKey := createRoundChangeMessageState(t, uint64(mrand.Int31n(100000)+100), 0, m.State) + consensus.AddParticipant(&proofKey.PublicKey) + + // random replacement with this incorrect proof + i := mrand.Int() % len(m.Proof) + m.Proof[i] = signedProof + // re-sign the message + sp.Sign(m, privateKey) + + err := consensus.verifyLockMessage(m, sp) + assert.Equal(t, ErrLockProofHeightMismatch, err) +} + +func TestVerifyLockMessageProofRound(t *testing.T) { + m, sp, privateKey, proofKeys := createLockMessage(t, 20, 1, 0, 1, 0) + consensus := createConsensus(t, 0, 0, proofKeys) + + consensus.SetLeader(&privateKey.PublicKey) + + // create a signed random proof with incorrect round + _, signedProof, proofKey := createRoundChangeMessageState(t, 1, uint64(mrand.Int31n(100000)+100), m.State) + consensus.AddParticipant(&proofKey.PublicKey) + + // random replacement with this incorrect proof + i := mrand.Int() % len(m.Proof) + m.Proof[i] = signedProof + // re-sign the message + sp.Sign(m, privateKey) + + err := consensus.verifyLockMessage(m, sp) + assert.Equal(t, ErrLockProofRoundMismatch, err) +} + +func TestVerifyLockMessageUnknownParticipant(t *testing.T) { + m, sp, privateKey, proofKeys := createLockMessage(t, 20, 1, 0, 1, 0) + consensus := createConsensus(t, 0, 0, proofKeys) + + consensus.SetLeader(&privateKey.PublicKey) + + // create a signed random proof with incorrect round, but do not add to participants + _, signedProof, _ := createRoundChangeMessageState(t, 1, 0, m.State) + + // random replacement with this incorrect proof + i := mrand.Int() % len(m.Proof) + m.Proof[i] = signedProof + // re-sign the message + sp.Sign(m, privateKey) + + err := consensus.verifyLockMessage(m, sp) + assert.Equal(t, ErrLockProofUnknownParticipant, err) +} + +func TestVerifyLockMessageProofInsufficient(t *testing.T) { + quorum := 20 + m, sp, privateKey, proofKeys := createLockMessage(t, quorum, 1, 0, 1, 0) + consensus := createConsensus(t, 0, 0, proofKeys) + consensus.SetLeader(&privateKey.PublicKey) + + // only keep 20 participants,by removing the consensus's own public key + consensus.participants = consensus.participants[1:] + assert.Equal(t, quorum, len(consensus.participants)) + + // random remove a valid proof from the first 2t+1(B) + valid := 2*((quorum-1)/3) + 1 + i := mrand.Int() % valid + t.Log(i) + copy(m.Proof[i:], m.Proof[i+1:]) + m.Proof = m.Proof[:len(m.Proof)-1] + t.Log(valid, len(m.Proof)) + // re-sign the sp + sp.Sign(m, privateKey) + + err := consensus.verifyLockMessage(m, sp) + assert.Equal(t, ErrLockProofInsufficient, err) +} + +/////////////////////////////////////////////////////////////////////////////// +// +//