diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..6a543b3 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,26 @@ +name: Lint + +on: + pull_request: + branches: [ main ] + +concurrency: + group: ci-${{ github.ref }}-lint + cancel-in-progress: true + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + - name: Checkout + uses: actions/checkout@v4 + + - name: golangci-lint + uses: golangci/golangci-lint-action@v4 + with: + version: v1.58 + args: --timeout 5m diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..8c25f9a --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,30 @@ +name: "Build/Tests" +on: + push: + branches: + - master + - main + pull_request: + +concurrency: + group: ci-${{ github.ref }}-tests + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + continue-on-error: true + strategy: + fail-fast: false + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.22' + - name: Unit tests + env: + DRAND_TEST_LOGS: "${{ runner.debug == '1' && 'DEBUG' || 'INFO' }}" + CI: "true" + run: go test -v ./... diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..ab2547f --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,198 @@ +issues: + # Let us display all issues of one type at once + max-same-issues: 0 + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + - path: _test\.go + linters: + - bodyclose + - cyclop + - errcheck + - forbidigo + - goconst + - gocyclo + - mnd + - gosec + - nilnil + - noctx + - revive + - depguard + - lll # signatures are long lines + - path: _test\.go + text: "SA1019" # we still want to test deprecated functions + - path: cmd + linters: + - forbidigo # we use Println in our UX + - path: internal/lib + linters: + - forbidigo # we use Println in our UX + - path: internal/drand-cli + linters: + - forbidigo # we use Println in our UX + - goconst # we re-use some strings in our flags + - path: client/http + text: "unexported-return" +run: + skip-dirs: + - demo + - test + +linters: + enable: + - asasalint + - asciicheck + - bidichk + - bodyclose + #- containedctx #TODO could be enabled + #- contextcheck #TODO could be enabled + #- cyclop + #- deadcode # Deprecated + - decorder + # - depguard + - dogsled + - dupl + - dupword + - durationcheck + - errcheck + - errchkjson + - errname + - errorlint + # - execinquery # deprecated + - exhaustive + # - exhaustivestruct + # - exhaustruct + - exportloopref + # - forbidigo + # - forcetypeassert #TODO could be enabled + - funlen + # - gci + # - gochecknoglobals + - gochecknoinits + # - gocognit + - goconst + - gocritic + - gocyclo + # - godot + # - godox #TODO could be enabled + # - goerr113 + - gofmt + # - gofumpt + - goheader + - goimports + # - golint # Deprecated + # - gomnd # deprecated, replaced by mnd + # - gomoddirectives + - gomodguard + - goprintffuncname + - gosec + - gosimple + - govet + - grouper + # - ifshort + - importas + - ineffassign + - interfacebloat + # - interfacer # Deprecated + # - ireturn + - lll + - loggercheck + - maintidx + - makezero + # - maligned #Deprecated + #- mnd + - misspell + - nakedret + # - nestif + - nilerr + - nilnil + # - nlreturn + - noctx + - nolintlint + # - nonamedreturns + # - nosnakecase + - nosprintfhostport + # - paralleltest #TODO could be enabled + - prealloc + - predeclared + # - promlinter #TODO could be enabled + - reassign + - revive + - rowserrcheck + # - scopelint # Deprecated + - sqlclosecheck + - staticcheck + # - structcheck # Deprecated + - stylecheck + # - tagliatelle + - tenv + - testableexamples + # - testpackage + # - thelper #TODO could be enabled + - tparallel + - typecheck + - unconvert + - unparam + - unused + - usestdlibvars + # - varcheck # Deprecated + # - varnamelen + - wastedassign + - whitespace + # - wrapcheck + # - wsl + +linters-settings: + dupl: + threshold: 100 + exhaustive: + default-signifies-exhaustive: false + funlen: + lines: 100 + statements: 50 + goconst: + min-len: 3 + min-occurrences: 3 + gocritic: + enabled-tags: + - diagnostic + - experimental + - opinionated + - performance + - style + disabled-checks: + - dupImport # https://github.com/go-critic/go-critic/issues/845 + - ifElseChain + - octalLiteral + - whyNoLint + - wrapperFunc + gocyclo: + min-complexity: 15 + goimports: + local-prefixes: github.com/drand + golint: + min-confidence: 0 + mnd: + # don't include the "operation" and "assign" + checks: + - argument + - case + - condition + - return + lll: + line-length: 140 +# maligned: # Deprecated +# suggest-new: true +# govet: +# check-shadowing: true #TODO could be enabled +# enable: +# - fieldalignment #TODO could be enabled + revive: + enable: + - var-naming + misspell: + locale: US + nolintlint: + allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space) + allow-unused: false # report any unused nolint directives + require-explanation: false # don't require an explanation for nolint directives + require-specific: false # don't require nolint directives to be specific about which linter is being skipped diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7de4890 --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +drand-relay-gossip: + go build -o drand-relay-gossip ./gossip-relay/main.go diff --git a/README.md b/README.md index f3a4c45..6d39907 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,10 @@ -## New Project +## drand-cli is a set of useful binaries for the drand ecosystem + +This repo contains most notably: + - a client CLI tool to fetch and verify drand beacons from the various available sources + - a gossipsub relay to relay drand beacons on gossipsub + - Go APIs to connect to drand networks through http or gossipsub relays. + --- diff --git a/client/aggregator.go b/client/aggregator.go new file mode 100644 index 0000000..f4cbc5c --- /dev/null +++ b/client/aggregator.go @@ -0,0 +1,219 @@ +package client + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" +) + +const ( + aggregatorWatchBuffer = 5 + // defaultAutoWatchRetry is the time after which the watch channel + // created by the autoWatch is re-opened when no context error occurred. + defaultAutoWatchRetry = time.Second * 30 +) + +// newWatchAggregator maintains state of consumers calling `Watch` so that a +// single `watch` request is made to the underlying client. +// There are 3 modes taken by this aggregator. If autowatch is set, a single `watch` +// will always be invoked on the provided client. If it is not set, but a `watch client`(wc) +// is passed, a `watch` will be run on the watch client in the absence of external watchers, +// which will swap watching over to the main client. If no watch client is set and autowatch is off +// then a single watch will only run when an external watch is requested. +func newWatchAggregator(l log.Logger, c, wc client.Client, autoWatch bool, autoWatchRetry time.Duration) *watchAggregator { + if autoWatchRetry == 0 { + autoWatchRetry = defaultAutoWatchRetry + } + aggregator := &watchAggregator{ + Client: c, + passiveClient: wc, + autoWatch: autoWatch, + autoWatchRetry: autoWatchRetry, + log: l, + subscribers: make([]subscriber, 0), + } + return aggregator +} + +type subscriber struct { + ctx context.Context + c chan client.Result +} + +type watchAggregator struct { + client.Client + passiveClient client.Client + autoWatch bool + autoWatchRetry time.Duration + log log.Logger + cancelAutoWatch context.CancelFunc + + subscriberLock sync.Mutex + subscribers []subscriber + cancelPassive context.CancelFunc +} + +// Start initiates auto watching if configured to do so. +// SetLog should not be called after Start. +func (c *watchAggregator) Start() { + if c.autoWatch { + c.startAutoWatch(true) + } else if c.passiveClient != nil { + c.startAutoWatch(false) + } +} + +// SetLog configures the client log output +func (c *watchAggregator) SetLog(l log.Logger) { + c.log = l +} + +// String returns the name of this client. +func (c *watchAggregator) String() string { + return fmt.Sprintf("%s.(+aggregator)", c.Client) +} + +func (c *watchAggregator) startAutoWatch(full bool) { + ctx, cancel := context.WithCancel(context.Background()) + c.cancelAutoWatch = cancel + go func() { + for { + var results <-chan client.Result + if full { + results = c.Watch(ctx) + } else if c.passiveClient != nil { + results = c.passiveWatch(ctx) + } + LOOP: + for { + select { + case _, ok := <-results: + if !ok { + c.log.Infow("", "watch_aggregator", "auto watch ended") + break LOOP + } + case <-ctx.Done(): + return + } + } + if c.autoWatchRetry < 0 { + return + } + t := time.NewTimer(c.autoWatchRetry) + select { + case <-t.C: + case <-ctx.Done(): + if !t.Stop() { + <-t.C + } + } + c.log.Infow("", "watch_aggregator", "retrying auto watch") + } + }() +} + +// passiveWatch is a degraded form of watch, where watch only hits the 'passive client' +// unless distribution is actually needed. +func (c *watchAggregator) passiveWatch(ctx context.Context) <-chan client.Result { + c.subscriberLock.Lock() + defer c.subscriberLock.Unlock() + + if c.cancelPassive != nil { + c.log.Warnw("", "watch_aggregator", "only support one passive watch") + return nil + } + + wc := make(chan client.Result) + if len(c.subscribers) == 0 { + ctx, cancel := context.WithCancel(ctx) + c.cancelPassive = cancel + go c.sink(c.passiveClient.Watch(ctx), wc) + } else { + // trigger the startAutowatch to retry on backoff + close(wc) + } + return wc +} + +func (c *watchAggregator) Watch(ctx context.Context) <-chan client.Result { + c.subscriberLock.Lock() + defer c.subscriberLock.Unlock() + + sub := subscriber{ctx, make(chan client.Result, aggregatorWatchBuffer)} + c.subscribers = append(c.subscribers, sub) + + if len(c.subscribers) == 1 { + if c.cancelPassive != nil { + c.cancelPassive() + c.cancelPassive = nil + } + ctx, cancel := context.WithCancel(ctx) + go c.distribute(c.Client.Watch(ctx), cancel) + } + return sub.c +} + +func (c *watchAggregator) sink(in <-chan client.Result, out chan client.Result) { + defer close(out) + for range in { + continue + } +} + +func (c *watchAggregator) distribute(in <-chan client.Result, cancel context.CancelFunc) { + defer cancel() + for { + c.subscriberLock.Lock() + if len(c.subscribers) == 0 { + c.subscriberLock.Unlock() + c.log.Warnw("", "watch_aggregator", "no subscribers to distribute results to") + return + } + aCtx := c.subscribers[0].ctx + c.subscriberLock.Unlock() + + var m client.Result + var ok bool + + select { + case m, ok = <-in: + case <-aCtx.Done(): + } + + c.subscriberLock.Lock() + curr := c.subscribers + c.subscribers = c.subscribers[:0] + + for _, s := range curr { + if ok && s.ctx.Err() == nil { + c.subscribers = append(c.subscribers, s) + if m != nil { + select { + case s.c <- m: + default: + c.log.Warnw("", "watch_aggregator", "dropped watch message to subscriber. full channel") + } + } + } else { + close(s.c) + } + } + c.subscriberLock.Unlock() + + if !ok { + return + } + } +} + +func (c *watchAggregator) Close() error { + err := c.Client.Close() + if c.cancelAutoWatch != nil { + c.cancelAutoWatch() + } + return err +} diff --git a/client/aggregator_test.go b/client/aggregator_test.go new file mode 100644 index 0000000..3878793 --- /dev/null +++ b/client/aggregator_test.go @@ -0,0 +1,84 @@ +package client + +import ( + "sync" + "testing" + "time" + + clientMock "github.com/drand/drand-cli/client/mock" + "github.com/drand/drand-cli/client/test/result/mock" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" +) + +func TestAggregatorClose(t *testing.T) { + wg := sync.WaitGroup{} + wg.Add(1) + + c := &clientMock.Client{ + WatchCh: make(chan client.Result), + CloseF: func() error { + wg.Done() + return nil + }, + } + + ac := newWatchAggregator(log.New(nil, log.DebugLevel, true), c, nil, true, 0) + + err := ac.Close() // should cancel the autoWatch and close the underlying client + if err != nil { + t.Fatal(err) + } + + wg.Wait() // wait for underlying client to close +} + +func TestAggregatorPassive(t *testing.T) { + wg := sync.WaitGroup{} + wg.Add(1) + + c := &clientMock.Client{ + WatchCh: make(chan client.Result, 1), + CloseF: func() error { + wg.Done() + return nil + }, + } + + wc := &clientMock.Client{ + WatchCh: make(chan client.Result, 1), + CloseF: func() error { + return nil + }, + } + + ac := newWatchAggregator(log.New(nil, log.DebugLevel, true), c, wc, false, 0) + + wc.WatchCh <- &mock.Result{Rnd: 1234} + c.WatchCh <- &mock.Result{Rnd: 5678} + + ac.Start() + + time.Sleep(50 * time.Millisecond) + + zzz := time.NewTimer(time.Millisecond * 50) + select { + case w := <-wc.WatchCh: + t.Fatalf("passive watch should be drained, but got %v", w) + case <-zzz.C: + } + + zzz = time.NewTimer(time.Millisecond * 50) + select { + case <-c.WatchCh: + case <-zzz.C: + t.Fatalf("active watch should not have been called but was") + } + + err := ac.Close() + if err != nil { + t.Fatal(err) + } + + wg.Wait() +} diff --git a/client/cache.go b/client/cache.go new file mode 100644 index 0000000..4e04993 --- /dev/null +++ b/client/cache.go @@ -0,0 +1,123 @@ +package client + +import ( + "context" + "fmt" + + lru "github.com/hashicorp/golang-lru" + + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" +) + +// Cache provides a mechanism to check for rounds in the cache. +type Cache interface { + // TryGet provides a round beacon or nil if it is not cached. + TryGet(round uint64) client.Result + // Add adds an item to the cache + Add(uint64, client.Result) +} + +// makeCache creates a cache of a given size +func makeCache(size int) (Cache, error) { + if size == 0 { + return &nilCache{}, nil + } + c, err := lru.NewARC(size) + if err != nil { + return nil, err + } + return &typedCache{c}, nil +} + +// typedCache wraps an ARCCache containing beacon results. +type typedCache struct { + *lru.ARCCache +} + +// Add a result to the cache +func (t *typedCache) Add(round uint64, result client.Result) { + t.ARCCache.Add(round, result) +} + +// TryGet attempts to get a result from the cache +func (t *typedCache) TryGet(round uint64) client.Result { + if val, ok := t.ARCCache.Get(round); ok { + return val.(client.Result) + } + return nil +} + +// nilCache implements a cache with size 0 +type nilCache struct{} + +// Add a result to the cache +func (*nilCache) Add(_ uint64, _ client.Result) { +} + +// TryGet attempts to get ar esult from the cache +func (*nilCache) TryGet(_ uint64) client.Result { + return nil +} + +// NewCachingClient is a meta client that stores an LRU cache of +// recently fetched random values. +func NewCachingClient(l log.Logger, c client.Client, cache Cache) (client.Client, error) { + return &cachingClient{ + Client: c, + cache: cache, + log: l, + }, nil +} + +type cachingClient struct { + client.Client + + cache Cache + log log.Logger +} + +// SetLog configures the client log output +func (c *cachingClient) SetLog(l log.Logger) { + c.log = l +} + +// String returns the name of this client. +func (c *cachingClient) String() string { + if arc, ok := c.cache.(*typedCache); ok { + return fmt.Sprintf("%s.(+%d el cache)", c.Client, arc.ARCCache.Len()) + } + return fmt.Sprintf("%s.(+nil cache)", c.Client) +} + +// Get returns the randomness at `round` or an error. +func (c *cachingClient) Get(ctx context.Context, round uint64) (res client.Result, err error) { + if val := c.cache.TryGet(round); val != nil { + return val, nil + } + val, err := c.Client.Get(ctx, round) + if err == nil && val != nil { + c.cache.Add(val.GetRound(), val) + } + return val, err +} + +func (c *cachingClient) Watch(ctx context.Context) <-chan client.Result { + in := c.Client.Watch(ctx) + out := make(chan client.Result) + go func() { + for result := range in { + if ctx.Err() != nil { + break + } + c.cache.Add(result.GetRound(), result) + out <- result + } + close(out) + }() + return out +} + +func (c *cachingClient) Close() error { + return c.Client.Close() +} diff --git a/client/cache_test.go b/client/cache_test.go new file mode 100644 index 0000000..e9e8c06 --- /dev/null +++ b/client/cache_test.go @@ -0,0 +1,136 @@ +package client + +import ( + "context" + "sync" + "testing" + + clientMock "github.com/drand/drand-cli/client/mock" + "github.com/drand/drand-cli/client/test/result/mock" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" +) + +func TestCacheGet(t *testing.T) { + m := clientMock.ClientWithResults(1, 6) + cache, err := makeCache(3) + if err != nil { + t.Fatal(err) + } + c, err := NewCachingClient(log.New(nil, log.DebugLevel, true), m, cache) + if err != nil { + t.Fatal(err) + } + res, e := c.Get(context.Background(), 1) + if e != nil { + t.Fatal(e) + } + res.(*mock.Result).AssertValid(t) + + _, e = c.Get(context.Background(), 1) + if e != nil { + t.Fatal(e) + } + if len(m.Results) < 4 { + t.Fatal("multiple gets should cache.") + } + _, e = c.Get(context.Background(), 2) + if e != nil { + t.Fatal(e) + } + _, e = c.Get(context.Background(), 3) + if e != nil { + t.Fatal(e) + } + + _, e = c.Get(context.Background(), 1) + if e != nil { + t.Fatal(e) + } + if len(m.Results) != 2 { + t.Fatalf("unexpected cache size. %d", len(m.Results)) + } +} + +func TestCacheGetLatest(t *testing.T) { + m := clientMock.ClientWithResults(1, 3) + cache, err := makeCache(3) + if err != nil { + t.Fatal(err) + } + c, err := NewCachingClient(log.New(nil, log.DebugLevel, true), m, cache) + if err != nil { + t.Fatal(err) + } + + r0, e := c.Get(context.Background(), 0) + if e != nil { + t.Fatal(e) + } + r1, e := c.Get(context.Background(), 0) + if e != nil { + t.Fatal(e) + } + + if r0.GetRound() == r1.GetRound() { + t.Fatal("cached result for latest") + } +} + +func TestCacheWatch(t *testing.T) { + m := clientMock.ClientWithResults(2, 6) + rc := make(chan client.Result, 1) + m.WatchCh = rc + arcCache, err := makeCache(3) + if err != nil { + t.Fatal(err) + } + lg := log.New(nil, log.DebugLevel, true) + cache, _ := NewCachingClient(lg, m, arcCache) + c := newWatchAggregator(lg, cache, nil, false, 0) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + r1 := c.Watch(ctx) + rc <- &mock.Result{Rnd: 1, Rand: []byte{1}} + _, ok := <-r1 + if !ok { + t.Fatal("results should propagate") + } + + _, err = c.Get(context.Background(), 1) + if err != nil { + t.Fatal(err) + } + if len(m.Results) != 4 { + t.Fatalf("getting should be served by cache.") + } +} + +func TestCacheClose(t *testing.T) { + wg := sync.WaitGroup{} + wg.Add(1) + + c := &clientMock.Client{ + WatchCh: make(chan client.Result), + CloseF: func() error { + wg.Done() + return nil + }, + } + + cache, err := makeCache(1) + if err != nil { + t.Fatal(err) + } + ca, err := NewCachingClient(log.New(nil, log.DebugLevel, true), c, cache) + if err != nil { + t.Fatal(err) + } + + err = ca.Close() // should close the underlying client + if err != nil { + t.Fatal(err) + } + + wg.Wait() // wait for underlying client to close +} diff --git a/client/client.go b/client/client.go new file mode 100644 index 0000000..8e37187 --- /dev/null +++ b/client/client.go @@ -0,0 +1,329 @@ +package client + +import ( + "bytes" + "context" + "errors" + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" + "github.com/drand/drand/v2/crypto" +) + +const clientStartupTimeoutDefault = time.Second * 5 + +// New creates a client with specified configuration. +func New(ctx context.Context, l log.Logger, options ...Option) (client.Client, error) { + cfg := clientConfig{ + cacheSize: 32, + log: l, + } + + for _, opt := range options { + if err := opt(&cfg); err != nil { + return nil, err + } + } + return makeClient(ctx, l, &cfg) +} + +// Wrap provides a single entrypoint for wrapping a concrete client +// implementation with configured aggregation, caching, and retry logic +func Wrap(ctx context.Context, l log.Logger, clients []client.Client, options ...Option) (client.Client, error) { + return New(ctx, l, append(options, From(clients...))...) +} + +func trySetLog(c client.Client, l log.Logger) { + if lc, ok := c.(client.LoggingClient); ok { + lc.SetLog(l) + } +} + +// makeClient creates a client from a configuration. +func makeClient(ctx context.Context, l log.Logger, cfg *clientConfig) (client.Client, error) { + if !cfg.insecure && cfg.chainHash == nil && cfg.chainInfo == nil { + l.Errorw("no root of trust specified") + return nil, errors.New("no root of trust specified") + } + if len(cfg.clients) == 0 && cfg.watcher == nil { + l.Errorw("no points of contact specified") + return nil, errors.New("no points of contact specified") + } + + var err error + + // provision cache + cache, err := makeCache(cfg.cacheSize) + if err != nil { + return nil, err + } + + // try to populate chain info + if err := cfg.tryPopulateInfo(ctx, cfg.clients...); err != nil { + return nil, err + } + + // provision watcher client + var wc client.Client + if cfg.watcher != nil { + wc, err = makeWatcherClient(cfg, cache) + if err != nil { + return nil, err + } + cfg.clients = append(cfg.clients, wc) + } + + for _, c := range cfg.clients { + trySetLog(c, cfg.log) + } + + var c client.Client + + verifiers := make([]client.Client, 0, len(cfg.clients)) + for _, source := range cfg.clients { + sch, err := crypto.GetSchemeByID(cfg.chainInfo.Scheme) + if err != nil { + return nil, fmt.Errorf("invalid scheme name in makeClient: %w", err) + } + + nv := newVerifyingClient(source, cfg.previousResult, cfg.fullVerify, sch) + verifiers = append(verifiers, nv) + if source == wc { + wc = nv + } + } + + c, err = makeOptimizingClient(l, cfg, verifiers, wc, cache) + if err != nil { + return nil, err + } + + wa := newWatchAggregator(l, c, wc, cfg.autoWatch, cfg.autoWatchRetry) + c = wa + trySetLog(c, cfg.log) + + wa.Start() + + return c, nil +} + +//nolint:lll // This function has nicely named parameters, so it's long. +func makeOptimizingClient(l log.Logger, cfg *clientConfig, verifiers []client.Client, watcher client.Client, cache Cache) (client.Client, error) { + oc, err := newOptimizingClient(l, verifiers, 0, 0, 0, 0) + if err != nil { + return nil, err + } + if watcher != nil { + oc.MarkPassive(watcher) + } + c := client.Client(oc) + trySetLog(c, cfg.log) + + if cfg.cacheSize > 0 { + c, err = NewCachingClient(l, c, cache) + if err != nil { + return nil, err + } + trySetLog(c, cfg.log) + } + for _, v := range verifiers { + trySetLog(v, cfg.log) + v.(*verifyingClient).indirectClient = c + } + + oc.Start() + return c, nil +} + +func makeWatcherClient(cfg *clientConfig, cache Cache) (client.Client, error) { + if cfg.chainInfo == nil { + return nil, fmt.Errorf("chain info cannot be nil") + } + + w, err := cfg.watcher(cfg.chainInfo, cache) + if err != nil { + return nil, err + } + ec := EmptyClientWithInfo(cfg.chainInfo) + return &watcherClient{ec, w}, nil +} + +type clientConfig struct { + // clients is the set of options for fetching randomness + clients []client.Client + // watcher is a constructor function for generating a new partial client of randomness + watcher WatcherCtor + // from `chainInfo.Hash()` - serves as a root of trust for a given + // randomness chain. + chainHash []byte + // Full chain information - serves as a root of trust. + chainInfo *chain.Info + // A previously fetched result serving as a verification checkpoint if one exists. + previousResult client.Result + // chain signature verification back to the 1st round, or to a know result to ensure + // determinism in the event of a compromised chain. + fullVerify bool + // insecure indicates the root of trust does not need to be present. + insecure bool + // autoWatch causes the client to start watching immediately in the background so that new randomness + // is proactively fetched and added to the cache. + autoWatch bool + // cache size - how large of a cache to keep locally. + cacheSize int + // customized client log. + log log.Logger + + // autoWatchRetry specifies the time after which the watch channel + // created by the autoWatch is re-opened when no context error occurred. + autoWatchRetry time.Duration + // prometheus is an interface to a Prometheus system + prometheus prometheus.Registerer +} + +func (c *clientConfig) tryPopulateInfo(ctx context.Context, clients ...client.Client) (err error) { + if c.chainInfo == nil { + ctx, cancel := context.WithTimeout(ctx, clientStartupTimeoutDefault) + defer cancel() + + for _, cli := range clients { + c.chainInfo, err = cli.Info(ctx) + if err == nil { + return + } + + if ctx.Err() != nil { + return ctx.Err() + } + } + } + return +} + +// Option is an option configuring a client. +type Option func(cfg *clientConfig) error + +// From constructs the client from a set of clients providing randomness +func From(c ...client.Client) Option { + return func(cfg *clientConfig) error { + cfg.clients = c + return nil + } +} + +// Insecurely indicates the client should be allowed to provide randomness +// when the root of trust is not fully provided in a validate-able way. +func Insecurely() Option { + return func(cfg *clientConfig) error { + cfg.insecure = true + return nil + } +} + +// WithCacheSize specifies how large of a cache of randomness values should be +// kept locally. Default 32 +func WithCacheSize(size int) Option { + return func(cfg *clientConfig) error { + cfg.cacheSize = size + return nil + } +} + +// WithChainHash configures the client to root trust with a given randomness +// chain hash, the chain parameters will be fetched from an HTTP endpoint. +func WithChainHash(chainHash []byte) Option { + return func(cfg *clientConfig) error { + if cfg.chainInfo != nil && !bytes.Equal(cfg.chainInfo.Hash(), chainHash) { + return errors.New("refusing to override group with non-matching hash") + } + cfg.chainHash = chainHash + return nil + } +} + +// WithChainInfo configures the client to root trust in the given randomness +// chain information +func WithChainInfo(chainInfo *chain.Info) Option { + return func(cfg *clientConfig) error { + if cfg.chainHash != nil && !bytes.Equal(cfg.chainHash, chainInfo.Hash()) { + return errors.New("refusing to override hash with non-matching group") + } + cfg.chainInfo = chainInfo + return nil + } +} + +// WithVerifiedResult provides a checkpoint of randomness verified at a given round. +// Used in combination with `VerifyFullChain`, this allows for catching up only on +// previously not-yet-verified results. +func WithVerifiedResult(result client.Result) Option { + return func(cfg *clientConfig) error { + if cfg.previousResult != nil && cfg.previousResult.GetRound() > result.GetRound() { + return errors.New("refusing to override verified result with an earlier result") + } + cfg.previousResult = result + return nil + } +} + +// WithFullChainVerification validates random beacons not just as being generated correctly +// from the group signature, but ensures that the full chain is deterministic by making sure +// each round is derived correctly from the previous one. In cases of compromise where +// a single party learns sufficient shares to derive the full key, malicious randomness +// could otherwise be generated that is signed, but not properly derived from previous rounds +// according to protocol. +func WithFullChainVerification() Option { + return func(cfg *clientConfig) error { + cfg.fullVerify = true + return nil + } +} + +// Watcher supplies the `Watch` portion of the drand client interface. +type Watcher interface { + Watch(ctx context.Context) <-chan client.Result +} + +// WatcherCtor creates a Watcher once chain info is known. +type WatcherCtor func(chainInfo *chain.Info, cache Cache) (Watcher, error) + +// WithWatcher specifies a channel that can provide notifications of new +// randomness bootstrappeed from the chain info. +func WithWatcher(wc WatcherCtor) Option { + return func(cfg *clientConfig) error { + cfg.watcher = wc + return nil + } +} + +// WithAutoWatch causes the client to automatically attempt to get +// randomness for rounds, so that it will hopefully already be cached +// when `Get` is called. +func WithAutoWatch() Option { + return func(cfg *clientConfig) error { + cfg.autoWatch = true + return nil + } +} + +// WithAutoWatchRetry specifies the time after which the watch channel +// created by the autoWatch is re-opened when no context error occurred. +// Set to a negative value to disable retrying auto watch. +func WithAutoWatchRetry(interval time.Duration) Option { + return func(cfg *clientConfig) error { + cfg.autoWatchRetry = interval + return nil + } +} + +// WithPrometheus specifies a registry into which to report metrics +func WithPrometheus(r prometheus.Registerer) Option { + return func(cfg *clientConfig) error { + cfg.prometheus = r + return nil + } +} diff --git a/client/client_test.go b/client/client_test.go new file mode 100644 index 0000000..5152dda --- /dev/null +++ b/client/client_test.go @@ -0,0 +1,435 @@ +package client_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/drand/drand/v2/common/key" + "github.com/drand/drand/v2/common/log" + + clock "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + + client2 "github.com/drand/drand-cli/client" + "github.com/drand/drand-cli/client/http" + clientMock "github.com/drand/drand-cli/client/mock" + httpmock "github.com/drand/drand-cli/client/test/http/mock" + "github.com/drand/drand-cli/client/test/result/mock" + "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/crypto" +) + +func TestClientConstraints(t *testing.T) { + ctx := context.Background() + lg := log.New(nil, log.DebugLevel, true) + if _, e := client2.New(ctx, lg); e == nil { + t.Fatal("client can't be created without root of trust") + } + + if _, e := client2.New(ctx, lg, client2.WithChainHash([]byte{0})); e == nil { + t.Fatal("Client needs URLs if only a chain hash is specified") + } + + if _, e := client2.New(ctx, lg, client2.From(clientMock.ClientWithResults(0, 5))); e == nil { + t.Fatal("Client needs root of trust unless insecure specified explicitly") + } + + c := clientMock.ClientWithResults(0, 5) + // As we will run is insecurely, we will set chain info so client can fetch it + c.OptionalInfo = fakeChainInfo(t) + + if _, e := client2.New(ctx, lg, client2.From(c), client2.Insecurely()); e != nil { + t.Fatal(e) + } +} + +func TestClientMultiple(t *testing.T) { + ctx := context.Background() + lg := log.New(nil, log.DebugLevel, true) + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + clk := clock.NewFakeClockAt(time.Now()) + + addr1, chainInfo, cancel, _ := httpmock.NewMockHTTPPublicServer(t, false, sch, clk) + defer cancel() + + addr2, chaininfo2, cancel2, _ := httpmock.NewMockHTTPPublicServer(t, false, sch, clk) + defer cancel2() + + t.Log("created mockhttppublicserver", "addr", addr1, "chaininfo", chainInfo) + t.Log("created mockhttppublicserver", "addr", addr2, "chaininfo", chaininfo2) + + // TODO: review this, are we really expecting this to work when the two servers aren't serving the same chainhash? + httpClients := http.ForURLs(ctx, lg, []string{"http://" + addr1, "http://" + addr2}, chainInfo.Hash()) + if len(httpClients) == 0 { + t.Error("http clients is empty") + return + } + + var c client.Client + var e error + c, e = client2.New(ctx, + lg, + client2.From(httpClients...), + client2.WithChainHash(chainInfo.Hash())) + + if e != nil { + t.Fatal(e) + } + r, e := c.Get(ctx, 0) + if e != nil { + t.Fatal(e) + } + if r.GetRound() <= 0 { + t.Fatal("expected valid client") + } + _ = c.Close() +} + +func TestClientWithChainInfo(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + ctx := context.Background() + chainInfo := fakeChainInfo(t) + lg := log.New(nil, log.DebugLevel, true) + hc, err := http.NewWithInfo(lg, "http://nxdomain.local/", chainInfo, nil) + require.NoError(t, err) + c, err := client2.New(ctx, lg, client2.WithChainInfo(chainInfo), + client2.From(hc)) + if err != nil { + t.Fatal("existing group creation shouldn't do additional validaiton.") + } + _, err = c.Get(ctx, 0) + if err == nil { + t.Fatal("bad urls should clearly not provide randomness.") + } + _ = c.Close() +} + +func TestClientCache(t *testing.T) { + ctx := context.Background() + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + clk := clock.NewFakeClockAt(time.Now()) + addr1, chainInfo, cancel, _ := httpmock.NewMockHTTPPublicServer(t, false, sch, clk) + defer cancel() + + lg := log.New(nil, log.DebugLevel, true) + httpClients := http.ForURLs(ctx, lg, []string{"http://" + addr1}, chainInfo.Hash()) + if len(httpClients) == 0 { + t.Error("http clients is empty") + return + } + + var c client.Client + var e error + c, e = client2.New(ctx, lg, client2.From(httpClients...), + client2.WithChainHash(chainInfo.Hash()), client2.WithCacheSize(1)) + + if e != nil { + t.Fatal(e) + } + r0, e := c.Get(ctx, 0) + if e != nil { + t.Fatal(e) + } + cancel() + _, e = c.Get(ctx, r0.GetRound()) + if e != nil { + t.Fatal(e) + } + + _, e = c.Get(ctx, 4) + if e == nil { + t.Fatal("non-cached results should fail.") + } + _ = c.Close() +} + +func TestClientWithoutCache(t *testing.T) { + ctx := context.Background() + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + clk := clock.NewFakeClockAt(time.Now()) + addr1, chainInfo, cancel, _ := httpmock.NewMockHTTPPublicServer(t, false, sch, clk) + defer cancel() + + lg := log.New(nil, log.DebugLevel, true) + httpClients := http.ForURLs(ctx, lg, []string{"http://" + addr1}, chainInfo.Hash()) + if len(httpClients) == 0 { + t.Error("http clients is empty") + return + } + + var c client.Client + var e error + c, e = client2.New(ctx, + lg, + client2.From(httpClients...), + client2.WithChainHash(chainInfo.Hash()), + client2.WithCacheSize(0)) + + if e != nil { + t.Fatal(e) + } + _, e = c.Get(ctx, 0) + if e != nil { + t.Fatal(e) + } + cancel() + _, e = c.Get(ctx, 0) + if e == nil { + t.Fatal("cache should be disabled.") + } + _ = c.Close() +} + +func TestClientWithWatcher(t *testing.T) { + ctx := context.Background() + lg := log.New(nil, log.DebugLevel, true) + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + info, results := mock.VerifiableResults(2, sch) + + ch := make(chan client.Result, len(results)) + for i := range results { + ch <- &results[i] + } + close(ch) + + watcherCtor := func(chainInfo *chain.Info, _ client2.Cache) (client2.Watcher, error) { + return &clientMock.Client{WatchCh: ch}, nil + } + + var c client.Client + c, err = client2.New(ctx, + lg, + client2.WithChainInfo(info), + client2.WithWatcher(watcherCtor), + ) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + w := c.Watch(ctx) + + for i := 0; i < len(results); i++ { + r := <-w + compareResults(t, &results[i], r) + } + require.NoError(t, c.Close()) +} + +func TestClientWithWatcherCtorError(t *testing.T) { + ctx := context.Background() + lg := log.New(nil, log.DebugLevel, true) + watcherErr := errors.New("boom") + watcherCtor := func(chainInfo *chain.Info, _ client2.Cache) (client2.Watcher, error) { + return nil, watcherErr + } + + // constructor should return error returned by watcherCtor + _, err := client2.New(ctx, + lg, + client2.WithChainInfo(fakeChainInfo(t)), + client2.WithWatcher(watcherCtor), + ) + if !errors.Is(err, watcherErr) { + t.Fatal(err) + } +} + +func TestClientChainHashOverrideError(t *testing.T) { + ctx := context.Background() + lg := log.New(nil, log.DebugLevel, true) + chainInfo := fakeChainInfo(t) + _, err := client2.Wrap( + ctx, + lg, + []client.Client{client2.EmptyClientWithInfo(chainInfo)}, + client2.WithChainInfo(chainInfo), + client2.WithChainHash(fakeChainInfo(t).Hash()), + ) + if err == nil { + t.Fatal("expected error, received no error") + } + if err.Error() != "refusing to override group with non-matching hash" { + t.Fatal(err) + } +} + +func TestClientChainInfoOverrideError(t *testing.T) { + ctx := context.Background() + lg := log.New(nil, log.DebugLevel, true) + chainInfo := fakeChainInfo(t) + _, err := client2.Wrap( + ctx, + lg, + []client.Client{client2.EmptyClientWithInfo(chainInfo)}, + client2.WithChainHash(chainInfo.Hash()), + client2.WithChainInfo(fakeChainInfo(t)), + ) + if err == nil { + t.Fatal("expected error, received no error") + } + if err.Error() != "refusing to override hash with non-matching group" { + t.Fatal(err) + } +} + +func TestClientAutoWatch(t *testing.T) { + ctx := context.Background() + lg := log.New(nil, log.DebugLevel, true) + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + clk := clock.NewFakeClockAt(time.Now()) + addr1, chainInfo, cancel, _ := httpmock.NewMockHTTPPublicServer(t, false, sch, clk) + defer cancel() + + httpClient := http.ForURLs(ctx, lg, []string{"http://" + addr1}, chainInfo.Hash()) + if len(httpClient) == 0 { + t.Error("http clients is empty") + return + } + + r1, _ := httpClient[0].Get(ctx, 1) + r2, _ := httpClient[0].Get(ctx, 2) + results := []client.Result{r1, r2} + + ch := make(chan client.Result, len(results)) + for i := range results { + ch <- results[i] + } + close(ch) + + watcherCtor := func(chainInfo *chain.Info, _ client2.Cache) (client2.Watcher, error) { + return &clientMock.Client{WatchCh: ch}, nil + } + + var c client.Client + c, err = client2.New(ctx, + lg, + client2.From(clientMock.ClientWithInfo(chainInfo)), + client2.WithChainHash(chainInfo.Hash()), + client2.WithWatcher(watcherCtor), + client2.WithAutoWatch(), + ) + + if err != nil { + t.Fatal(err) + } + + time.Sleep(chainInfo.Period) + cancel() + r, err := c.Get(ctx, results[0].GetRound()) + if err != nil { + t.Fatal(err) + } + compareResults(t, r, results[0]) + _ = c.Close() +} + +func TestClientAutoWatchRetry(t *testing.T) { + ctx := context.Background() + lg := log.New(nil, log.DebugLevel, true) + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + + info, results := mock.VerifiableResults(5, sch) + resC := make(chan client.Result) + defer close(resC) + + // done is closed after all resuls have been written to resC + done := make(chan struct{}) + + // Returns a channel that yields the verifiable results above + watchF := func(ctx context.Context) <-chan client.Result { + go func() { + for i := 0; i < len(results); i++ { + select { + case resC <- &results[i]: + case <-ctx.Done(): + return + } + } + <-time.After(time.Second) + close(done) + }() + return resC + } + + var failer clientMock.Client + failer = clientMock.Client{ + WatchF: func(ctx context.Context) <-chan client.Result { + // First call returns a closed channel + ch := make(chan client.Result) + close(ch) + // Second call returns a channel that writes results + failer.WatchF = watchF + return ch + }, + } + + var c client.Client + c, err = client2.New(ctx, + lg, + client2.From(&failer, clientMock.ClientWithInfo(info)), + client2.WithChainInfo(info), + client2.WithAutoWatch(), + client2.WithAutoWatchRetry(time.Second), + client2.WithCacheSize(len(results)), + ) + + if err != nil { + t.Fatal(err) + } + defer c.Close() + + // Wait for all the results to be consumed by the autoWatch + select { + case <-done: + case <-time.After(time.Minute): + t.Fatal("timed out waiting for results to be consumed") + } + + // We should be able to retrieve all the results from the cache. + for i := range results { + r, err := c.Get(ctx, results[i].GetRound()) + if err != nil { + t.Fatal(err) + } + compareResults(t, &results[i], r) + } +} + +// compareResults asserts that two results are the same. +func compareResults(t *testing.T, expected, actual client.Result) { + t.Helper() + + require.NotNil(t, expected) + require.NotNil(t, actual) + require.Equal(t, expected.GetRound(), actual.GetRound()) + require.Equal(t, expected.GetRandomness(), actual.GetRandomness()) +} + +// fakeChainInfo creates a chain info object for use in tests. +func fakeChainInfo(t *testing.T) *chain.Info { + t.Helper() + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + pair, err := key.NewKeyPair("fakeChainInfo.test:1234", sch) + require.NoError(t, err) + + return &chain.Info{ + Period: time.Second, + GenesisTime: time.Now().Unix(), + PublicKey: pair.Public.Key, + Scheme: sch.Name, + } +} diff --git a/client/doc.go b/client/doc.go new file mode 100644 index 0000000..c4a5fca --- /dev/null +++ b/client/doc.go @@ -0,0 +1,70 @@ +/* +Package client provides transport-agnostic logic to retrieve and verify +randomness from drand, including retry, validation, caching and +optimization features. + +Example: + + package main + + import ( + "context" + "encoding/hex" + "fmt" + + "github.com/drand/drand/v2/client" + "github.com/drand/drand/v2/common/log" + ) + + var chainHash, _ = hex.DecodeString("8990e7a9aaed2ffed73dbd7092123d6f289930540d7651336225dc172e51b2ce") + + func main() { + ctx := context.Background() + lg := log.New(nil, log.DebugLevel, true) + + c, err := client.New(ctx, lg, + client.From("..."), // see concrete client implementations + client.WithChainHash(chainHash), + ) + + // e.g. use the client to get the latest randomness round: + r, err := c.Get(ctx, 0) + + fmt.Println(r.Round(), r.Randomness()) + } + +The "From" option allows you to specify clients that work over particular +transports. HTTP, gRPC and libp2p PubSub clients are provided as +subpackages https://pkg.go.dev/github.com/drand/drand-cli/internal/client/http, +https://pkg.go.dev/github.com/drand/drand-cli/internal/client/grpc and +https://pkg.go.dev/github.com/drand/drand-cli/internal/lp2p/clientlp2p/client +respectively. Note that you are not restricted to just one client. You can use +multiple clients of the same type or of different types. The base client will +periodically "speed test" it's clients, failover, cache results and aggregate +calls to "Watch" to reduce requests. + +WARNING: When using the client you should use the "WithChainHash" or +"WithChainInfo" option in order for your client to validate the randomness it +receives is from the correct chain. You may use the "Insecurely" option to +bypass this validation but it is not recommended. + +In an application that uses the drand client, the following options are likely +to be needed/customized: + + WithCacheSize() + should be set to something sensible for your application. + + WithVerifiedResult() + WithFullChainVerification() + both should be set for increased security if you have + persistent state and expect to be following the chain. + + WithAutoWatch() + will pre-load new results as they become available adding them + to the cache for speedy retreival when you need them. + + WithPrometheus() + enables metrics reporting on speed and performance to a + provided prometheus registry. +*/ +package client diff --git a/client/empty.go b/client/empty.go new file mode 100644 index 0000000..a920d14 --- /dev/null +++ b/client/empty.go @@ -0,0 +1,47 @@ +package client + +import ( + "context" + "time" + + "github.com/drand/drand/v2/common" + chain2 "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" +) + +const emptyClientStringerValue = "EmptyClient" + +// EmptyClientWithInfo makes a client that returns the given info but no randomness +func EmptyClientWithInfo(info *chain2.Info) client.Client { + return &emptyClient{info} +} + +type emptyClient struct { + i *chain2.Info +} + +func (m *emptyClient) String() string { + return emptyClientStringerValue +} + +func (m *emptyClient) Info(_ context.Context) (*chain2.Info, error) { + return m.i, nil +} + +func (m *emptyClient) RoundAt(t time.Time) uint64 { + return common.CurrentRound(t.Unix(), m.i.Period, m.i.GenesisTime) +} + +func (m *emptyClient) Get(_ context.Context, _ uint64) (client.Result, error) { + return nil, common.ErrEmptyClientUnsupportedGet +} + +func (m *emptyClient) Watch(_ context.Context) <-chan client.Result { + ch := make(chan client.Result, 1) + close(ch) + return ch +} + +func (m *emptyClient) Close() error { + return nil +} diff --git a/client/empty_test.go b/client/empty_test.go new file mode 100644 index 0000000..8b672a8 --- /dev/null +++ b/client/empty_test.go @@ -0,0 +1,70 @@ +package client + +import ( + "context" + "fmt" + "testing" + "time" + + commonutils "github.com/drand/drand/v2/common" + + chain2 "github.com/drand/drand/v2/common/client" +) + +func TestEmptyClient(t *testing.T) { + chainInfo := fakeChainInfo(t) + c := EmptyClientWithInfo(chainInfo) + + // should be able to retrieve Info + i, err := c.Info(context.Background()) + if err != nil { + t.Fatal(err) + } + if i != chainInfo { + t.Fatal("unexpected chain info", i) + } + + // should be able to retrieve RoundAt + now := time.Now() + rnd := c.RoundAt(now) + if rnd != commonutils.CurrentRound(now.Unix(), chainInfo.Period, chainInfo.GenesisTime) { + t.Fatal("unexpected RoundAt return value", rnd) + } + + // should be fmt.Stringer + sc, ok := c.(fmt.Stringer) + if !ok { + t.Fatal("expected Stringer interface") + } + if sc.String() != emptyClientStringerValue { + t.Fatal("unexpected string value") + } + + // but Get does not work + _, err = c.Get(context.Background(), 0) + if err == nil { + t.Fatal("expected an error") + } + if err.Error() != "not supported" { + t.Fatal("unexpected error from Get", err) + } + + // and Watch returns an empty closed channel + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + ch := c.Watch(ctx) + //nolint + var rs []chain2.Result + for r := range ch { + rs = append(rs, r) + } + + if len(rs) > 0 { + t.Fatal("unexpected results in watch channel", rs) + } + + if err := c.Close(); err != nil { + t.Fatal("unexpected error closing client", err) + } +} diff --git a/client/http/doc.go b/client/http/doc.go new file mode 100644 index 0000000..e246c41 --- /dev/null +++ b/client/http/doc.go @@ -0,0 +1,45 @@ +/* +Package http provides a drand client implementation that uses drand's HTTP API. + +The HTTP client uses drand's JSON HTTP API +(https://drand.love/developer/http-api/) to fetch randomness. Watching is +implemented by polling the endpoint at the expected round time. + +Example: + + package main + + import ( + "context" + "encoding/hex" + + "github.com/drand/drand/v2/client" + "github.com/drand/drand/v2/client/http" + "github.com/drand/drand/v2/common/log" + ) + + var urls = []string{ + "https://api.drand.sh", + "https://drand.cloudflare.com", + } + + var chainHash, _ = hex.DecodeString("8990e7a9aaed2ffed73dbd7092123d6f289930540d7651336225dc172e51b2ce") + + func main() { + ctx := context.Background() + lg := log.New(nil, log.DebugLevel, true) + + c, err := client.New(ctx, lg, + client.From(http.ForURLs(ctx, lg, urls, chainHash)...), + client.WithChainHash(chainHash), + ) + } + +The "ForURLs" helper creates multiple HTTP clients from a list of +URLs. Alternatively you can use the "New" or "NewWithInfo" constructor to +create clients. + +Tip: Provide multiple URLs to enable failover and speed optimized URL +selection. +*/ +package http diff --git a/client/http/example_test.go b/client/http/example_test.go new file mode 100644 index 0000000..c9daa76 --- /dev/null +++ b/client/http/example_test.go @@ -0,0 +1,52 @@ +package http_test + +import ( + "context" + "encoding/hex" + "fmt" + + client2 "github.com/drand/drand-cli/client" + "github.com/drand/drand-cli/client/http" + "github.com/drand/drand/v2/crypto" +) + +func Example_http_New() { + chainhash, err := hex.DecodeString("52db9ba70e0cc0f6eaf7803dd07447a1f5477735fd3f661792ba94600c84e971") + if err != nil { + // we recommend to handle errors as you wish rather than panicking + panic(err) + } + + client, err := http.New(context.Background(), nil, "http://api.drand.sh", chainhash, nil) + if err != nil { + panic(err) + } + + result, err := client.Get(context.Background(), 1234) + if err != nil { + panic(err) + } + + info, err := client.Info(context.Background()) + if err != nil { + panic(err) + } + + scheme, err := crypto.SchemeFromName(info.GetSchemeName()) + if err != nil { + panic(err) + } + + // make sure to verify the beacons when using the raw http client without a verifying client + err = scheme.VerifyBeacon(&client2.RandomData{ + Rnd: result.GetRound(), + Sig: result.GetSignature(), + }, info.PublicKey) + if err != nil { + panic(err) + } + + fmt.Printf("got beacon: round=%d; randomness=%x\n", result.GetRound(), result.GetRandomness()) + + //output: got beacon: round=1234; randomness=9ead58abb451d8f521338c43ba5595610642a0c07d0e9babeaae6a98787629de +} diff --git a/client/http/http.go b/client/http/http.go new file mode 100644 index 0000000..2970466 --- /dev/null +++ b/client/http/http.go @@ -0,0 +1,381 @@ +package http + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + nhttp "net/http" + "os" + "path" + "strings" + "time" + + client2 "github.com/drand/drand-cli/client" + "github.com/drand/drand/v2/crypto" + + json "github.com/nikkolasg/hexjson" + + "github.com/drand/drand/v2/common" + chain2 "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" +) + +var _ client.Client = &httpClient{} + +var errClientClosed = fmt.Errorf("client closed") + +const defaultClientExec = "unknown" +const defaultHTTTPTimeout = 60 * time.Second + +const httpWaitMaxCounter = 20 +const httpWaitInterval = 2 * time.Second +const maxTimeoutHTTPRequest = 5 * time.Second + +// New creates a new client pointing to an HTTP endpoint +func New(ctx context.Context, l log.Logger, url string, chainHash []byte, transport nhttp.RoundTripper) (*httpClient, error) { + if l == nil { + l = log.DefaultLogger() + } + if transport == nil { + transport = nhttp.DefaultTransport + } + if !strings.HasSuffix(url, "/") { + url += "/" + } + pn, err := os.Executable() + if err != nil { + pn = defaultClientExec + } + agent := fmt.Sprintf("drand-client-%s/1.0", path.Base(pn)) + c := &httpClient{ + root: url, + client: createClient(transport), + l: l, + Agent: agent, + done: make(chan struct{}), + } + + chainInfo, err := c.FetchChainInfo(ctx, chainHash) + if err != nil { + return nil, fmt.Errorf("FetchChainInfo err: %w", err) + } + c.chainInfo = chainInfo + + return c, nil +} + +// NewWithInfo constructs an http client when the group parameters are already known. +func NewWithInfo(l log.Logger, url string, info *chain2.Info, transport nhttp.RoundTripper) (*httpClient, error) { + if transport == nil { + transport = nhttp.DefaultTransport + } + if !strings.HasSuffix(url, "/") { + url += "/" + } + + pn, err := os.Executable() + if err != nil { + pn = defaultClientExec + } + agent := fmt.Sprintf("drand-client-%s/1.0", path.Base(pn)) + c := &httpClient{ + root: url, + chainInfo: info, + client: createClient(transport), + l: l, + Agent: agent, + done: make(chan struct{}), + } + return c, nil +} + +// ForURLs provides a shortcut for creating a set of HTTP clients for a set of URLs. +func ForURLs(ctx context.Context, l log.Logger, urls []string, chainHash []byte) []client.Client { + clients := make([]client.Client, 0) + var info *chain2.Info + var skipped []string + for _, u := range urls { + if info == nil { + if c, err := New(ctx, l, u, chainHash, nil); err == nil { + // Note: this wrapper assumes the current behavior that if `New` succeeds, + // Info will have been fetched. + info, _ = c.Info(ctx) + clients = append(clients, c) + } else { + skipped = append(skipped, u) + } + } else { + if c, err := NewWithInfo(l, u, info, nil); err == nil { + clients = append(clients, c) + } + } + } + if info != nil { + for _, u := range skipped { + if c, err := NewWithInfo(l, u, info, nil); err == nil { + clients = append(clients, c) + } + } + } + return clients +} + +func Ping(ctx context.Context, root string) error { + url := fmt.Sprintf("%s/health", root) + + ctx, cancel := context.WithTimeout(ctx, maxTimeoutHTTPRequest) + defer cancel() + + req, err := nhttp.NewRequestWithContext(ctx, nhttp.MethodGet, url, nhttp.NoBody) + if err != nil { + return fmt.Errorf("creating request: %w", err) + } + + response, err := nhttp.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("creating request: %w", err) + } + + defer response.Body.Close() + + return nil +} + +// createClient creates an HTTP client around a transport, allows to easily instrument it later +func createClient(transport nhttp.RoundTripper) *nhttp.Client { + hc := nhttp.Client{} + hc.Timeout = defaultHTTTPTimeout + hc.Jar = nhttp.DefaultClient.Jar + hc.CheckRedirect = nhttp.DefaultClient.CheckRedirect + hc.Transport = transport + + return &hc +} + +func IsServerReady(ctx context.Context, addr string) error { + counter := 0 + for { + // Ping is wrapping its context with a Timeout on maxTimeoutHTTPRequest anyway. + err := Ping(ctx, "http://"+addr) + if err == nil { + return nil + } + + counter++ + if counter == httpWaitMaxCounter { + return fmt.Errorf("timeout waiting http server to be ready") + } + + time.Sleep(httpWaitInterval) + } +} + +// httpClient implements Client through http requests to a Drand relay. +type httpClient struct { + root string + client *nhttp.Client + Agent string + chainInfo *chain2.Info + l log.Logger + done chan struct{} +} + +// SetLog configures the client log output +func (h *httpClient) SetLog(l log.Logger) { + h.l = l +} + +// SetUserAgent sets the user agent used by the client +func (h *httpClient) SetUserAgent(ua string) { + h.Agent = ua +} + +// String returns the name of this client. +func (h *httpClient) String() string { + return fmt.Sprintf("HTTP(%q)", h.root) +} + +// MarshalText implements encoding.TextMarshaller interface +func (h *httpClient) MarshalText() ([]byte, error) { + return json.Marshal(h.String()) +} + +type httpInfoResponse struct { + chainInfo *chain2.Info + err error +} + +// FetchChainInfo attempts to initialize an httpClient when +// it does not know the full group parameters for a drand group. The chain hash +// is the hash of the chain info. +func (h *httpClient) FetchChainInfo(ctx context.Context, chainHash []byte) (*chain2.Info, error) { + if h.chainInfo != nil { + return h.chainInfo, nil + } + + resC := make(chan httpInfoResponse, 1) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + go func() { + var url string + if len(chainHash) > 0 { + url = fmt.Sprintf("%s%x/info", h.root, chainHash) + } else { + url = fmt.Sprintf("%sinfo", h.root) + } + + req, err := nhttp.NewRequestWithContext(ctx, nhttp.MethodGet, url, nhttp.NoBody) + if err != nil { + resC <- httpInfoResponse{nil, fmt.Errorf("creating request: %w", err)} + return + } + req.Header.Set("User-Agent", h.Agent) + + infoBody, err := h.client.Do(req) + if err != nil { + resC <- httpInfoResponse{nil, fmt.Errorf("doing request: %w", err)} + return + } + defer infoBody.Body.Close() + + chainInfo, err := chain2.InfoFromJSON(infoBody.Body) + if err != nil { + resC <- httpInfoResponse{nil, fmt.Errorf("decoding response [InfoFromJSON]: %w", err)} + return + } + + if chainInfo.PublicKey == nil { + resC <- httpInfoResponse{nil, fmt.Errorf("group does not have a valid key for validation")} + return + } + + if len(chainHash) == 0 { + h.l.Warnw("", "http_client", "instantiated without trustroot", "chainHash", hex.EncodeToString(chainInfo.Hash())) + if !common.IsDefaultBeaconID(chainInfo.ID) { + err := fmt.Errorf("%s does not advertise the default drand for the default chainHash (got %x)", h.root, chainInfo.Hash()) + resC <- httpInfoResponse{nil, err} + return + } + } else if !bytes.Equal(chainInfo.Hash(), chainHash) { + err := fmt.Errorf("%s does not advertise the expected drand group (%x vs %x)", h.root, chainInfo.Hash(), chainHash) + resC <- httpInfoResponse{nil, err} + return + } + + resC <- httpInfoResponse{chainInfo, nil} + }() + + select { + case res := <-resC: + if res.err != nil { + return nil, res.err + } + return res.chainInfo, nil + case <-h.done: + return nil, errClientClosed + } +} + +type httpGetResponse struct { + result client.Result + err error +} + +// Get returns the randomness at `round` or an error. +func (h *httpClient) Get(ctx context.Context, round uint64) (client.Result, error) { + var url string + if round == 0 { + url = fmt.Sprintf("%s%x/public/latest", h.root, h.chainInfo.Hash()) + } else { + url = fmt.Sprintf("%s%x/public/%d", h.root, h.chainInfo.Hash(), round) + } + + resC := make(chan httpGetResponse, 1) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + go func() { + req, err := nhttp.NewRequestWithContext(ctx, nhttp.MethodGet, url, nhttp.NoBody) + if err != nil { + resC <- httpGetResponse{nil, fmt.Errorf("creating request: %w", err)} + return + } + req.Header.Set("User-Agent", h.Agent) + + randResponse, err := h.client.Do(req) + if err != nil || randResponse.StatusCode != nhttp.StatusOK { + resC <- httpGetResponse{nil, fmt.Errorf("doing request %v: %w", req, err)} + return + } + defer randResponse.Body.Close() + + randResp := client2.RandomData{} + if err := json.NewDecoder(randResponse.Body).Decode(&randResp); err != nil { + resC <- httpGetResponse{nil, fmt.Errorf("decoding response: %w", err)} + return + } + + if len(randResp.Sig) == 0 { + resC <- httpGetResponse{nil, fmt.Errorf("insufficient response - signature is not present")} + return + } + + randResp.Random = crypto.RandomnessFromSignature(randResp.GetSignature()) + + resC <- httpGetResponse{&randResp, nil} + }() + + select { + case res := <-resC: + if res.err != nil { + return nil, res.err + } + return res.result, nil + case <-h.done: + return nil, errClientClosed + } +} + +// Watch returns new randomness as it becomes available. +func (h *httpClient) Watch(ctx context.Context) <-chan client.Result { + out := make(chan client.Result) + go func() { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + defer close(out) + + in := client2.PollingWatcher(ctx, h, h.chainInfo, h.l) + for { + select { + case res, ok := <-in: + if !ok { + return + } + out <- res + case <-h.done: + return + } + } + }() + return out +} + +// Info returns information about the chain. +func (h *httpClient) Info(_ context.Context) (*chain2.Info, error) { + return h.chainInfo, nil +} + +// RoundAt will return the most recent round of randomness that will be available +// at time for the current client. +func (h *httpClient) RoundAt(t time.Time) uint64 { + return common.CurrentRound(t.Unix(), h.chainInfo.Period, h.chainInfo.GenesisTime) +} + +func (h *httpClient) Close() error { + close(h.done) + h.client.CloseIdleConnections() + return nil +} diff --git a/client/http/http_test.go b/client/http/http_test.go new file mode 100644 index 0000000..d168462 --- /dev/null +++ b/client/http/http_test.go @@ -0,0 +1,210 @@ +package http + +import ( + "context" + "errors" + "net/http" + "sync" + "testing" + "time" + + clock "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + + "github.com/drand/drand/v2/common/log" + + "github.com/drand/drand-cli/client" + "github.com/drand/drand-cli/client/test/http/mock" + "github.com/drand/drand/v2/crypto" +) + +func TestHTTPClient(t *testing.T) { + ctx := context.Background() + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + clk := clock.NewFakeClockAt(time.Now()) + addr, chainInfo, cancel, _ := mock.NewMockHTTPPublicServer(t, true, sch, clk) + defer cancel() + + err = IsServerReady(ctx, addr) + if err != nil { + t.Fatal(err) + } + + l := log.New(nil, log.DebugLevel, true) + httpClient, err := New(ctx, l, "http://"+addr, chainInfo.Hash(), http.DefaultTransport) + if err != nil { + t.Fatal(err) + } + + ctx1, cancel1 := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel1() + result, err := httpClient.Get(ctx1, 0) + if err != nil { + t.Fatal(err) + } + if len(result.GetRandomness()) == 0 { + t.Fatal("no randomness provided") + } + full, ok := (result).(*client.RandomData) + if !ok { + t.Fatal("Should be able to restore concrete type") + } + if len(full.Sig) == 0 { + t.Fatal("no signature provided") + } + + ctx2, cancel2 := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel2() + if _, err := httpClient.Get(ctx2, full.Rnd+1); err != nil { + t.Fatalf("http client should not perform verification of results. err: %s", err) + } + _ = httpClient.Close() +} + +func TestHTTPGetLatest(t *testing.T) { + ctx := context.Background() + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + clk := clock.NewFakeClockAt(time.Now()) + addr, chainInfo, cancel, _ := mock.NewMockHTTPPublicServer(t, false, sch, clk) + defer cancel() + + err = IsServerReady(ctx, addr) + if err != nil { + t.Fatal(err) + } + + l := log.New(nil, log.DebugLevel, true) + httpClient, err := New(ctx, l, "http://"+addr, chainInfo.Hash(), http.DefaultTransport) + if err != nil { + t.Fatal(err) + } + + ctx, cancel = context.WithTimeout(ctx, time.Second) + defer cancel() + r0, err := httpClient.Get(ctx, 0) + if err != nil { + t.Fatal(err) + } + + ctx, cancel = context.WithTimeout(ctx, time.Second) + defer cancel() + r1, err := httpClient.Get(ctx, 0) + if err != nil { + t.Fatal(err) + } + + if r1.GetRound() != r0.GetRound()+1 { + t.Fatal("expected round progression") + } + _ = httpClient.Close() +} + +func TestForURLsCreation(t *testing.T) { + ctx := context.Background() + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + clk := clock.NewFakeClockAt(time.Now()) + addr, chainInfo, cancel, _ := mock.NewMockHTTPPublicServer(t, false, sch, clk) + defer cancel() + + err = IsServerReady(ctx, addr) + if err != nil { + t.Fatal(err) + } + + l := log.New(nil, log.DebugLevel, true) + clients := ForURLs(ctx, l, []string{"http://invalid.domain/", "http://" + addr}, chainInfo.Hash()) + if len(clients) != 2 { + t.Fatal("expect both urls returned") + } + _ = clients[0].Close() + _ = clients[1].Close() +} + +func TestHTTPWatch(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + ctx := context.Background() + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + clk := clock.NewFakeClockAt(time.Now()) + addr, chainInfo, cancel, _ := mock.NewMockHTTPPublicServer(t, false, sch, clk) + defer cancel() + + err = IsServerReady(ctx, addr) + if err != nil { + t.Fatal(err) + } + + l := log.New(nil, log.DebugLevel, true) + httpClient, err := New(ctx, l, "http://"+addr, chainInfo.Hash(), http.DefaultTransport) + if err != nil { + t.Fatal(err) + } + + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + result := httpClient.Watch(ctx) + first, ok := <-result + if !ok { + t.Fatal("should get a result from watching") + } + if len(first.GetRandomness()) == 0 { + t.Fatal("should get randomness from watching") + } + + for range result { + } + _ = httpClient.Close() +} + +func TestHTTPClientClose(t *testing.T) { + ctx := context.Background() + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + clk := clock.NewFakeClockAt(time.Now()) + addr, chainInfo, cancel, _ := mock.NewMockHTTPPublicServer(t, false, sch, clk) + defer cancel() + + err = IsServerReady(ctx, addr) + if err != nil { + t.Fatal(err) + } + + l := log.New(nil, log.DebugLevel, true) + httpClient, err := New(ctx, l, "http://"+addr, chainInfo.Hash(), http.DefaultTransport) + if err != nil { + t.Fatal(err) + } + result, err := httpClient.Get(context.Background(), 1969) + if err != nil { + t.Fatal(err) + } + if result.GetRound() != 1969 { + t.Fatal("unexpected round.") + } + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + for range httpClient.Watch(context.Background()) { + } + wg.Done() + }() + + err = httpClient.Close() + if err != nil { + t.Fatal(err) + } + + _, err = httpClient.Get(context.Background(), 0) + if !errors.Is(err, errClientClosed) { + t.Fatal("unexpected error from closed client", err) + } + + wg.Wait() // wait for the watch to close +} diff --git a/client/http/metric.go b/client/http/metric.go new file mode 100644 index 0000000..63cadaf --- /dev/null +++ b/client/http/metric.go @@ -0,0 +1,70 @@ +package http + +import ( + "context" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/drand/drand-cli/internal/metrics" + "github.com/drand/drand/v2/common" + + chain2 "github.com/drand/drand/v2/common/client" +) + +// MeasureHeartbeats periodically tracks latency observed on a set of HTTP clients +func MeasureHeartbeats(ctx context.Context, c []chain2.Client) *HealthMetrics { + m := &HealthMetrics{ + next: 0, + clients: c, + } + if len(c) > 0 { + go m.startObserve(ctx) + } + return m +} + +// HealthMetrics is a measurement task around HTTP clients +type HealthMetrics struct { + next int + clients []chain2.Client +} + +// HeartbeatInterval is the duration between liveness heartbeats sent to an HTTP API. +const HeartbeatInterval = 10 * time.Second + +func (c *HealthMetrics) startObserve(ctx context.Context) { + // we check all clients within HeartbeatInterval + interval := time.Duration(int64(HeartbeatInterval) / int64(len(c.clients))) + for { + // check if ctx is Done + if ctx.Err() != nil { + return + } + time.Sleep(interval) + n := c.next % len(c.clients) + + httpClient, ok := c.clients[n].(*httpClient) + if !ok { + c.next++ + continue + } + + result, err := c.clients[n].Get(ctx, c.clients[n].RoundAt(time.Now())+1) + if err != nil { + metrics.ClientHTTPHeartbeatFailure.With(prometheus.Labels{"http_address": httpClient.root}).Inc() + continue + } + + metrics.ClientHTTPHeartbeatSuccess.With(prometheus.Labels{"http_address": httpClient.root}).Inc() + + // compute the latency metric + actual := time.Now().UnixNano() + expected := common.TimeOfRound(httpClient.chainInfo.Period, httpClient.chainInfo.GenesisTime, result.GetRound()) * 1e9 + // the labels of the gauge vec must already be set at the registerer level + metrics.ClientHTTPHeartbeatLatency. + With(prometheus.Labels{"http_address": httpClient.root}). + Set(float64(actual-expected) / float64(time.Millisecond)) + c.next++ + } +} diff --git a/client/lp2p/client.go b/client/lp2p/client.go new file mode 100644 index 0000000..b2c0178 --- /dev/null +++ b/client/lp2p/client.go @@ -0,0 +1,282 @@ +package lp2p + +import ( + "context" + "encoding/hex" + "fmt" + "sync" + + clock "github.com/jonboulle/clockwork" + "github.com/libp2p/go-libp2p" + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" + dnsaddr "github.com/multiformats/go-multiaddr-dns" + "google.golang.org/protobuf/proto" + + client2 "github.com/drand/drand-cli/client" + "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" + "github.com/drand/drand/v2/crypto" + "github.com/drand/drand/v2/protobuf/drand" +) + +// Client is a concrete pubsub client implementation +type Client struct { + cancel func() + latest uint64 + cache client2.Cache + bufferSize int + log log.Logger + + subs struct { + sync.Mutex + M map[*int]chan drand.PublicRandResponse + } +} + +// DefaultBufferSize controls how many incoming messages can be in-flight until they start +// to be dropped by the library +const DefaultBufferSize = 100 + +// SetLog configures the client log output +func (c *Client) SetLog(l log.Logger) { + c.log = l +} + +// WithPubsub provides an option for integrating pubsub notification +// into a drand client. +func WithPubsub(l log.Logger, ps *pubsub.PubSub, clk clock.Clock, bufferSize int) client2.Option { + return client2.WithWatcher(func(info *chain.Info, cache client2.Cache) (client2.Watcher, error) { + c, err := NewWithPubsub(l, ps, info, cache, clk, bufferSize) + if err != nil { + return nil, err + } + return c, nil + }) +} + +// PubSubTopic generates a drand pubsub topic from a chain hash. +func PubSubTopic(h string) string { + return fmt.Sprintf("/drand/pubsub/v0.0.0/%s", h) +} + +// NewWithPubsub creates a gossip randomness client. If the logger l is nil, it will default to +// a default Logger, +// +//nolint:funlen,lll,gocyclo // This is a long line +func NewWithPubsub(l log.Logger, ps *pubsub.PubSub, info *chain.Info, cache client2.Cache, clk clock.Clock, bufferSize int) (*Client, error) { + if info == nil { + return nil, fmt.Errorf("no chain supplied for joining") + } + + if l == nil { + l = log.DefaultLogger() + } + + scheme, err := crypto.SchemeFromName(info.Scheme) + if err != nil { + l.Errorw("invalid scheme in info", "info", info, "scheme", info.Scheme, "err", err) + + return nil, fmt.Errorf("invalid scheme in info: %w", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + c := &Client{ + cancel: cancel, + cache: cache, + bufferSize: bufferSize, + log: l, + } + + chainHash := hex.EncodeToString(info.Hash()) + topic := PubSubTopic(chainHash) + if err := ps.RegisterTopicValidator(topic, randomnessValidator(info, cache, c, clk)); err != nil { + cancel() + return nil, fmt.Errorf("creating topic: %w", err) + } + t, err := ps.Join(topic) + if err != nil { + cancel() + return nil, fmt.Errorf("joining pubsub: %w", err) + } + s, err := t.Subscribe() + if err != nil { + cancel() + return nil, fmt.Errorf("subscribe: %w", err) + } + + c.subs.M = make(map[*int]chan drand.PublicRandResponse) + + go func() { + for { + msg, err := s.Next(ctx) + if ctx.Err() != nil { + c.log.Debugw("NewPubSub closing because context was canceled", "msg", msg, "err", ctx.Err()) + + s.Cancel() + err := t.Close() + if err != nil { + c.log.Errorw("NewPubSub closing goroutine for topic", "err", err) + } + + c.subs.Lock() + for _, ch := range c.subs.M { + close(ch) + } + c.subs.M = make(map[*int]chan drand.PublicRandResponse) + c.subs.Unlock() + return + } + if err != nil { + c.log.Warnw("", "gossip client", "topic.Next error", "err", err) + continue + } + + var rand drand.PublicRandResponse + err = proto.Unmarshal(msg.Data, &rand) + if err != nil { + c.log.Warnw("", "gossip client", "unmarshal random error", "err", err) + continue + } + + err = scheme.VerifyBeacon(&rand, info.PublicKey) + if err != nil { + c.log.Errorw("invalid signature for beacon", "round", rand.GetRound(), "err", err) + continue + } + + if c.latest >= rand.Round { + c.log.Debugw("received round older than the latest previously received one", "latest", c.latest, "round", rand.Round) + continue + } + c.latest = rand.Round + + c.log.Debugw("newPubSub broadcasting round to listeners", "round", rand.Round) + c.subs.Lock() + for _, ch := range c.subs.M { + select { + case ch <- rand: + default: + c.log.Warnw("", "gossip client", "randomness notification dropped due to a full channel") + } + } + c.subs.Unlock() + c.log.Debugw("newPubSub finished broadcasting round to listeners", "round", rand.Round) + } + }() + + return c, nil +} + +// UnsubFunc is a cancel function for pubsub subscription +type UnsubFunc func() + +// Sub subscribes to notifications about new randomness. +// Client instance owns the channel after it is passed to Sub function, +// thus the channel should not be closed by library user +// +// It is recommended to use a buffered channel. If the channel is full, +// notification about randomness will be dropped. +// +// Notification channels will be closed when the client is Closed +func (c *Client) Sub(ch chan drand.PublicRandResponse) UnsubFunc { + id := new(int) + c.subs.Lock() + c.subs.M[id] = ch + c.subs.Unlock() + return func() { + c.log.Debugw("closing sub") + c.subs.Lock() + delete(c.subs.M, id) + close(ch) + c.subs.Unlock() + } +} + +// Watch implements the client.Watcher interface +func (c *Client) Watch(ctx context.Context) <-chan client.Result { + innerCh := make(chan drand.PublicRandResponse, c.bufferSize) + outerCh := make(chan client.Result, c.bufferSize) + end := c.Sub(innerCh) + + w := sync.WaitGroup{} + w.Add(1) + + go func() { + defer close(outerCh) + + w.Done() + + for { + select { + // TODO: do not copy by assignment any drand.PublicRandResponse + case resp, ok := <-innerCh: //nolint:govet + if !ok { + return + } + dat := &client2.RandomData{ + Rnd: resp.GetRound(), + Random: crypto.RandomnessFromSignature(resp.GetSignature()), + Sig: resp.GetSignature(), + PreviousSignature: resp.GetPreviousSignature(), + } + if c.cache != nil { + c.cache.Add(resp.GetRound(), dat) + } + select { + case outerCh <- dat: + c.log.Debugw("processed random beacon", "round", dat.GetRound()) + default: + c.log.Warnw("", "gossip client", "randomness notification dropped due to a full channel", "round", dat.GetRound()) + } + case <-ctx.Done(): + c.log.Debugw("client.Watch done") + end() + c.log.Debugw("client.Watch finished draining the innerCh") + return + } + } + }() + + w.Wait() + + return outerCh +} + +// Close stops Client, cancels PubSub subscription and closes the topic. +func (c *Client) Close() error { + c.cancel() + return nil +} + +// NewPubsub constructs a basic libp2p pubsub module for use with the drand client. +// The local libp2p host is returned as well to allow to properly close it once done. +func NewPubsub(ctx context.Context, listenAddr string, relayAddrs []string) (*pubsub.PubSub, host.Host, error) { + h, err := libp2p.New(libp2p.ListenAddrStrings(listenAddr)) + if err != nil { + return nil, nil, err + } + + peers := make([]peer.AddrInfo, 0, len(relayAddrs)) + for _, relayAddr := range relayAddrs { + // resolve the relay multiaddr to peers' AddrInfo + mas, err := dnsaddr.Resolve(ctx, multiaddr.StringCast(relayAddr)) + if err != nil { + return nil, nil, fmt.Errorf("dnsaddr.Resolve error: %w", err) + } + for _, ma := range mas { + relayAi, err := peer.AddrInfoFromP2pAddr(ma) + if err != nil { + h.Close() + return nil, nil, fmt.Errorf("peer.AddrInfoFromP2pAddr error: %w", err) + } + peers = append(peers, *relayAi) + } + } + + ps, err := pubsub.NewGossipSub(ctx, h, pubsub.WithDirectPeers(peers)) + return ps, h, err +} diff --git a/client/lp2p/client_test.go b/client/lp2p/client_test.go new file mode 100644 index 0000000..7186701 --- /dev/null +++ b/client/lp2p/client_test.go @@ -0,0 +1,231 @@ +package lp2p + +import ( + "context" + "fmt" + "net/http" + "path" + "strconv" + "testing" + "time" + + "github.com/hashicorp/consul/sdk/freeport" + clock "github.com/jonboulle/clockwork" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + ma "github.com/multiformats/go-multiaddr" + "github.com/stretchr/testify/require" + + dhttp "github.com/drand/drand-cli/client/http" + httpmock "github.com/drand/drand-cli/client/test/http/mock" + "github.com/drand/drand-cli/internal/lp2p" + chain2 "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" + "github.com/drand/drand/v2/crypto" +) + +// +// func TestGRPCClientTestFunc(t *testing.T) { +// lg := log.New(nil, log.DebugLevel, true) +// // start mock drand node +// sch, err := crypto.GetSchemeFromEnv() +// require.NoError(t, err) +// +// clk := clock.NewFakeClockAt(time.Now()) +// +// grpcLis, svc := mock.NewMockGRPCPublicServer(t, lg, "127.0.0.1:0", false, sch, clk) +// grpcAddr := grpcLis.Addr() +// go grpcLis.Start() +// defer grpcLis.Stop(context.Background()) +// +// dataDir := t.TempDir() +// identityDir := t.TempDir() +// +// infoProto, err := svc.ChainInfo(context.Background(), nil) +// require.NoError(t, err) +// +// info, err := chain2.InfoFromProto(infoProto) +// require.NoError(t, err) +// +// // start mock relay-node +// grpcClient, err := grpc.New(lg, grpcAddr, "", true, []byte("")) +// require.NoError(t, err) +// +// cfg := &lp2p.GossipRelayConfig{ +// ChainHash: info.HashString(), +// PeerWith: nil, +// Addr: "/ip4/127.0.0.1/tcp/" + test.FreePort(), +// DataDir: dataDir, +// IdentityPath: path.Join(identityDir, "identity.key"), +// Client: grpcClient, +// } +// g, err := lp2p.NewGossipRelayNode(lg, cfg) +// require.NoError(t, err, "gossip relay node") +// +// defer g.Shutdown() +// +// // start client +// c, err := newTestClient(t, g.Multiaddrs(), info, clk) +// require.NoError(t, err) +// defer func() { +// err := c.Close() +// require.NoError(t, err) +// }() +// +// // test client +// ctx, cancel := context.WithCancel(context.Background()) +// ch := c.Watch(ctx) +// +// baseRound := uint64(1969) +// +// mockService := svc.(mock.Service) +// // pub sub polls every 200ms +// wait := 250 * time.Millisecond +// for i := uint64(0); i < 3; i++ { +// time.Sleep(wait) +// mockService.EmitRand(false) +// t.Logf("round %d emitted\n", baseRound+i) +// +// select { +// case r, ok := <-ch: +// require.True(t, ok, "expected randomness, watch outer channel was closed instead") +// t.Logf("received round %d\n", r.Round()) +// require.Equal(t, baseRound+i, r.Round()) +// // the period of the mock servers is 1 second +// case <-time.After(5 * time.Second): +// t.Fatal("timeout.") +// } +// } +// +// time.Sleep(wait) +// mockService.EmitRand(true) +// cancel() +// +// drain(t, ch, 5*time.Second) +//} + +func drain(t *testing.T, ch <-chan client.Result, timeout time.Duration) { + for { + select { + case _, ok := <-ch: + if !ok { + return + } + case <-time.After(timeout): + t.Fatal("timed out closing channel.") + } + } +} + +func TestHTTPClientTestFunc(t *testing.T) { + ctx := context.Background() + lg := log.New(nil, log.DebugLevel, true) + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + + clk := clock.NewFakeClockAt(time.Now()) + + addr, chainInfo, stop, emit := httpmock.NewMockHTTPPublicServer(t, false, sch, clk) + defer stop() + + dataDir := t.TempDir() + identityDir := t.TempDir() + + httpClient, err := dhttp.New(ctx, lg, "http://"+addr, chainInfo.Hash(), http.DefaultTransport) + require.NoError(t, err) + + cfg := &lp2p.GossipRelayConfig{ + ChainHash: chainInfo.HashString(), + PeerWith: nil, + Addr: "/ip4/127.0.0.1/tcp/" + strconv.Itoa(freeport.GetOne(t)), + DataDir: dataDir, + IdentityPath: path.Join(identityDir, "identity.key"), + Client: httpClient, + } + g, err := lp2p.NewGossipRelayNode(lg, cfg) + if err != nil { + t.Fatalf("gossip relay node (%v)", err) + } + defer g.Shutdown() + + c, err := newTestClient(t, g.Multiaddrs(), chainInfo, clk) + if err != nil { + t.Fatal(err) + } + defer c.Close() + + ctx, cancel := context.WithCancel(ctx) + emit(false) + ch := c.Watch(ctx) + for i := 0; i < 3; i++ { + // pub sub polls every 200ms, but the other http one polls every period + time.Sleep(1250 * time.Millisecond) + emit(false) + select { + case r, ok := <-ch: + if !ok { + t.Fatal("expected randomness") + } else { + t.Log("received randomness", r.GetRound()) + } + case <-time.After(8 * time.Second): + t.Fatal("timeout.") + } + } + emit(true) + cancel() + drain(t, ch, 5*time.Second) +} + +func newTestClient(t *testing.T, relayMultiaddr []ma.Multiaddr, info *chain2.Info, clk clock.Clock) (*Client, error) { + identityDir := t.TempDir() + + lg := log.New(nil, log.DebugLevel, true) + priv, err := lp2p.LoadOrCreatePrivKey(path.Join(identityDir, "identity.key"), lg) + if err != nil { + return nil, err + } + h, ps, err := lp2p.ConstructHost(priv, "/ip4/0.0.0.0/tcp/"+strconv.Itoa(freeport.GetOne(t)), relayMultiaddr, lg) + if err != nil { + return nil, err + } + relayPeerID, err := peerIDFromMultiaddr(relayMultiaddr[0]) + if err != nil { + return nil, err + } + err = waitForConnection(h, relayPeerID, time.Minute) + if err != nil { + return nil, err + } + c, err := NewWithPubsub(lg, ps, info, nil, clk, 100) + if err != nil { + return nil, err + } + c.SetLog(lg) + return c, nil +} + +func peerIDFromMultiaddr(addr ma.Multiaddr) (peer.ID, error) { + ai, err := peer.AddrInfoFromP2pAddr(addr) + if err != nil { + return "", err + } + return ai.ID, nil +} + +func waitForConnection(h host.Host, id peer.ID, timeout time.Duration) error { + t := time.NewTimer(timeout) + for { + if len(h.Network().ConnsToPeer(id)) > 0 { + t.Stop() + return nil + } + select { + case <-t.C: + return fmt.Errorf("timed out waiting to be connected the relay @ %v", id) + default: + } + time.Sleep(time.Millisecond * 100) + } +} diff --git a/client/lp2p/doc.go b/client/lp2p/doc.go new file mode 100644 index 0000000..3889740 --- /dev/null +++ b/client/lp2p/doc.go @@ -0,0 +1,17 @@ +/* +Package lp2p provides a drand client implementation that retrieves +randomness by subscribing to a libp2p pubsub topic. + +WARNING: this client can only be used to "Watch" for new randomness rounds and +"Get" randomness rounds it has previously seen that are still in the cache. + +If you need to "Get" arbitrary rounds from the chain then you must combine this client with the http client. +You can Wrap multiple client together. + +The agnostic client builder must receive "WithChainInfo()" in order for it to +validate randomness rounds it receives, or "WithChainHash()" and be combined +with the HTTP client implementations so that chain information can be fetched from them. + +It is particularly important that rounds are verified since they can be delivered by any peer in the network. +*/ +package lp2p diff --git a/client/lp2p/example_test.go b/client/lp2p/example_test.go new file mode 100644 index 0000000..07a8f29 --- /dev/null +++ b/client/lp2p/example_test.go @@ -0,0 +1,68 @@ +package lp2p_test + +import ( + "bytes" + "context" + "fmt" + "time" + + clock "github.com/jonboulle/clockwork" + + gclient "github.com/drand/drand-cli/client/lp2p" + "github.com/drand/drand/v2/common" + "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/log" +) + +const ( + // relayP2PAddr is the p2p multiaddr of the drand gossipsub relay node to connect to. + relayP2PAddr = "/dnsaddr/api.drand.sh" + relayP2PAddr2 = "/dnsaddr/api2.drand.sh" + relayP2PAddr3 = "/dnsaddr/api3.drand.sh" + + // jsonQuicknetInfo, can be hardcoded since these don't change over time + jsonQuicknetInfo = `{ + "genesis_time": 1692803367, + "groupHash": "f477d5c89f21a17c863a7f937c6a6d15859414d2be09cd448d4279af331c5d3e", + "hash": "52db9ba70e0cc0f6eaf7803dd07447a1f5477735fd3f661792ba94600c84e971", + "metadata": { + "beaconID": "quicknet" + }, + "period": 3, + "public_key": "83cf0f2896adee7eb8b5f01fcad3912212c437e0073e911fb90022d3e760183c8c4b450b6a0a6c3ac6a5776a2d1064510d1fec758c921cc22b0e17e63aaf4bcb5ed66304de9cf809bd274ca73bab4af5a6e9c76a4bc09e76eae8991ef5ece45a", + "schemeID": "bls-unchained-g1-rfc9380" +} +` +) + +func ExampleNewPubsub() { + ctx := context.Background() + + // /0 to use a random free port + ps, h, err := gclient.NewPubsub(ctx, "/ip4/0.0.0.0/tcp/0", []string{relayP2PAddr, relayP2PAddr2, relayP2PAddr3}) + if err != nil { + panic(err) + } + defer h.Close() + + info, err := chain.InfoFromJSON(bytes.NewReader([]byte(jsonQuicknetInfo))) + if err != nil { + panic(err) + } + + // NewWithPubSub will automatically register the topic for the chainhash you're interested in + c, err := gclient.NewWithPubsub(log.DefaultLogger(), ps, info, nil, clock.NewRealClock(), gclient.DefaultBufferSize) + if err != nil { + panic(err) + } + + // This can be slow to "start" + for res := range c.Watch(context.Background()) { + expected := common.CurrentRound(time.Now().Unix(), info.Period, info.GenesisTime) + fmt.Println("correct round:", expected == res.GetRound(), "with", len(res.GetRandomness()), "random bytes") + // we just waited on the first one as an example + break + } + + //output: correct round: true with 32 random bytes +} diff --git a/client/lp2p/validator.go b/client/lp2p/validator.go new file mode 100644 index 0000000..8a6b99b --- /dev/null +++ b/client/lp2p/validator.go @@ -0,0 +1,90 @@ +package lp2p + +import ( + "bytes" + "context" + "time" + + "github.com/drand/drand-cli/client" + commonutils "github.com/drand/drand/v2/common" + + clock "github.com/jonboulle/clockwork" + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/peer" + "google.golang.org/protobuf/proto" + + chain2 "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/crypto" + "github.com/drand/drand/v2/protobuf/drand" +) + +func randomnessValidator(info *chain2.Info, cache client.Cache, c *Client, clk clock.Clock) pubsub.ValidatorEx { + var scheme *crypto.Scheme + if info != nil { + scheme, _ = crypto.GetSchemeByID(info.Scheme) + } + return func(_ context.Context, p peer.ID, m *pubsub.Message) pubsub.ValidationResult { + rand := &drand.PublicRandResponse{} + err := proto.Unmarshal(m.Data, rand) + if err != nil { + c.log.Warnw("", "gossip validator", "Not validating received randomness due to proto.Unmarshal error", "err", err) + return pubsub.ValidationReject + } + + c.log.Debugw("", "gossip validator", "Received new round", "round", rand.GetRound(), "fromPeerID", p.String()) + + if info == nil { + c.log.Warnw("", "gossip validator", "Not validating received randomness due to lack of trust root.") + return pubsub.ValidationAccept + } + + // Unwilling to relay beacons in the future. + timeNow := clk.Now() + timeOfRound := commonutils.TimeOfRound(info.Period, info.GenesisTime, rand.GetRound()) + if time.Unix(timeOfRound, 0).After(timeNow) { + c.log.Warnw("", + "gossip validator", "Not validating received randomness due to time of round", + "err", err, + "timeOfRound", timeOfRound, + "time.Now", timeNow.Unix(), + "info.Period", info.Period, + "info.Genesis", info.GenesisTime, + "round", rand.GetRound(), + ) + return pubsub.ValidationReject + } + + if cache != nil { + if current := cache.TryGet(rand.GetRound()); current != nil { + currentFull, ok := current.(*client.RandomData) + if !ok { + // Note: this shouldn't happen in practice, but if we have a + // degraded cache entry we can't validate the full byte + // sequence. + if bytes.Equal(rand.GetSignature(), current.GetSignature()) { + c.log.Warnw("", "gossip validator", "ignore") + return pubsub.ValidationIgnore + } + c.log.Warnw("", "gossip validator", "reject") + return pubsub.ValidationReject + } + if current.GetRound() == rand.GetRound() && + bytes.Equal(current.GetRandomness(), crypto.RandomnessFromSignature(rand.GetSignature())) && + bytes.Equal(current.GetSignature(), rand.GetSignature()) && + bytes.Equal(currentFull.PreviousSignature, rand.GetPreviousSignature()) { + c.log.Warnw("", "gossip validator", "ignore") + return pubsub.ValidationIgnore + } + c.log.Warnw("", "gossip validator", "reject") + return pubsub.ValidationReject + } + } + + err = scheme.VerifyBeacon(rand, info.PublicKey) + if err != nil { + c.log.Warnw("", "gossip validator", "reject", "err", err) + return pubsub.ValidationReject + } + return pubsub.ValidationAccept + } +} diff --git a/client/lp2p/validator_test.go b/client/lp2p/validator_test.go new file mode 100644 index 0000000..fad4caa --- /dev/null +++ b/client/lp2p/validator_test.go @@ -0,0 +1,282 @@ +package lp2p + +import ( + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "testing" + "time" + + "github.com/drand/drand/v2/common" + "github.com/drand/drand/v2/common/key" + "github.com/drand/drand/v2/common/log" + + clock "github.com/jonboulle/clockwork" + pubsub "github.com/libp2p/go-libp2p-pubsub" + pb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + "google.golang.org/protobuf/proto" + + "github.com/drand/drand-cli/client" + "github.com/drand/drand-cli/client/test/cache" + chain2 "github.com/drand/drand/v2/common/chain" + dcrypto "github.com/drand/drand/v2/crypto" + "github.com/drand/drand/v2/protobuf/drand" +) + +type randomDataWrapper struct { + data client.RandomData +} + +func (r *randomDataWrapper) GetRound() uint64 { + return r.data.Rnd +} + +func (r *randomDataWrapper) GetSignature() []byte { + return r.data.Sig +} + +func (r *randomDataWrapper) GetRandomness() []byte { + return r.data.Random +} + +func randomPeerID(t *testing.T) peer.ID { + priv, _, err := crypto.GenerateEd25519Key(rand.Reader) + if err != nil { + t.Fatal(err) + } + peerID, err := peer.IDFromPrivateKey(priv) + if err != nil { + t.Fatal(err) + } + return peerID +} + +func fakeRandomData(info *chain2.Info, clk clock.Clock) client.RandomData { + rnd := common.CurrentRound(clk.Now().Unix(), info.Period, info.GenesisTime) + + sig := make([]byte, 8) + binary.LittleEndian.PutUint64(sig, rnd) + psig := make([]byte, 8) + binary.LittleEndian.PutUint64(psig, rnd-1) + + return client.RandomData{ + Rnd: rnd, + Sig: sig, + PreviousSignature: psig, + Random: dcrypto.RandomnessFromSignature(sig), + } +} + +func fakeChainInfo() *chain2.Info { + sch, err := dcrypto.GetSchemeFromEnv() + if err != nil { + panic(err) + } + pair, err := key.NewKeyPair("fakeChainInfo.test:1234", sch) + if err != nil { + panic(err) + } + + return &chain2.Info{ + Period: time.Second, + GenesisTime: time.Now().Unix(), + PublicKey: pair.Public.Key, + Scheme: sch.Name, + } +} + +func TestRejectsUnmarshalBeaconFailure(t *testing.T) { + c := Client{log: log.New(nil, log.DebugLevel, true)} + clk := clock.NewFakeClock() + validate := randomnessValidator(fakeChainInfo(), nil, &c, clk) + + msg := pubsub.Message{Message: &pb.Message{}} + res := validate(context.Background(), randomPeerID(t), &msg) + + if res != pubsub.ValidationReject { + t.Fatal(errors.New("expected reject for invalid message")) + } +} + +func TestAcceptsWithoutTrustRoot(t *testing.T) { + c := Client{log: log.New(nil, log.DebugLevel, true)} + clk := clock.NewFakeClock() + validate := randomnessValidator(nil, nil, &c, clk) + + resp := drand.PublicRandResponse{} + data, err := proto.Marshal(&resp) + if err != nil { + t.Fatal(err) + } + msg := pubsub.Message{Message: &pb.Message{Data: data}} + res := validate(context.Background(), randomPeerID(t), &msg) + + if res != pubsub.ValidationAccept { + t.Fatal(errors.New("expected accept without trust root")) + } +} + +func TestRejectsFutureBeacons(t *testing.T) { + info := fakeChainInfo() + c := Client{log: log.New(nil, log.DebugLevel, true)} + clk := clock.NewFakeClock() + validate := randomnessValidator(info, nil, &c, clk) + + resp := drand.PublicRandResponse{ + Round: common.CurrentRound(time.Now().Unix(), info.Period, info.GenesisTime) + 5, + } + data, err := proto.Marshal(&resp) + if err != nil { + t.Fatal(err) + } + msg := pubsub.Message{Message: &pb.Message{Data: data}} + res := validate(context.Background(), randomPeerID(t), &msg) + + if res != pubsub.ValidationReject { + t.Fatal(errors.New("expected reject for future message")) + } +} + +func TestRejectsVerifyBeaconFailure(t *testing.T) { + info := fakeChainInfo() + c := Client{log: log.New(nil, log.DebugLevel, true)} + clk := clock.NewFakeClock() + validate := randomnessValidator(info, nil, &c, clk) + + resp := drand.PublicRandResponse{ + Round: common.CurrentRound(time.Now().Unix(), info.Period, info.GenesisTime), + // missing signature etc. + } + data, err := proto.Marshal(&resp) + if err != nil { + t.Fatal(err) + } + msg := pubsub.Message{Message: &pb.Message{Data: data}} + res := validate(context.Background(), randomPeerID(t), &msg) + + if res != pubsub.ValidationReject { + t.Fatal(errors.New("expected reject for beacon verification failure")) + } +} + +func TestIgnoresCachedEqualBeacon(t *testing.T) { + info := fakeChainInfo() + ca := cache.NewMapCache() + c := Client{log: log.New(nil, log.DebugLevel, true)} + clk := clock.NewFakeClockAt(time.Now()) + validate := randomnessValidator(info, ca, &c, clk) + rdata := fakeRandomData(info, clk) + + ca.Add(rdata.Rnd, &rdata) + + resp := drand.PublicRandResponse{ + Round: rdata.Rnd, + Signature: rdata.Sig, + PreviousSignature: rdata.PreviousSignature, + Randomness: rdata.Random, + } + data, err := proto.Marshal(&resp) + if err != nil { + t.Fatal(err) + } + msg := pubsub.Message{Message: &pb.Message{Data: data}} + res := validate(context.Background(), randomPeerID(t), &msg) + + if res != pubsub.ValidationIgnore { + t.Fatal(errors.New("expected ignore for cached beacon")) + } +} + +func TestRejectsCachedUnequalBeacon(t *testing.T) { + info := fakeChainInfo() + ca := cache.NewMapCache() + c := Client{log: log.New(nil, log.DebugLevel, true)} + clk := clock.NewFakeClock() + validate := randomnessValidator(info, ca, &c, clk) + rdata := fakeRandomData(info, clk) + + ca.Add(rdata.Rnd, &rdata) + + sig := make([]byte, 8) + binary.LittleEndian.PutUint64(sig, rdata.Rnd+1) + + resp := drand.PublicRandResponse{ + Round: rdata.Rnd, + Signature: rdata.Sig, + PreviousSignature: sig, // incoming message has incorrect previous sig + Randomness: rdata.Random, + } + data, err := proto.Marshal(&resp) + if err != nil { + t.Fatal(err) + } + msg := pubsub.Message{Message: &pb.Message{Data: data}} + res := validate(context.Background(), randomPeerID(t), &msg) + + if res != pubsub.ValidationReject { + t.Fatal(fmt.Errorf("expected reject for cached but unequal beacon, got: %v", res)) + } +} + +func TestIgnoresCachedEqualNonRandomDataBeacon(t *testing.T) { + info := fakeChainInfo() + ca := cache.NewMapCache() + c := Client{log: log.New(nil, log.DebugLevel, true)} + clk := clock.NewFakeClockAt(time.Now()) + validate := randomnessValidator(info, ca, &c, clk) + rdata := randomDataWrapper{fakeRandomData(info, clk)} + + ca.Add(rdata.GetRound(), &rdata) + + resp := drand.PublicRandResponse{ + Round: rdata.GetRound(), + Signature: rdata.GetSignature(), + PreviousSignature: rdata.data.PreviousSignature, + Randomness: rdata.GetRandomness(), + } + data, err := proto.Marshal(&resp) + if err != nil { + t.Fatal(err) + } + msg := pubsub.Message{Message: &pb.Message{Data: data}} + res := validate(context.Background(), randomPeerID(t), &msg) + + if res != pubsub.ValidationIgnore { + t.Fatal(errors.New("expected ignore for cached beacon")) + } +} + +func TestRejectsCachedEqualNonRandomDataBeacon(t *testing.T) { + info := fakeChainInfo() + ca := cache.NewMapCache() + c := Client{log: log.New(nil, log.DebugLevel, true)} + clk := clock.NewFakeClock() + validate := randomnessValidator(info, ca, &c, clk) + rdata := randomDataWrapper{fakeRandomData(info, clk)} + + ca.Add(rdata.GetRound(), &rdata) + + sig := make([]byte, 8) + binary.LittleEndian.PutUint64(sig, rdata.GetRound()+1) + + resp := drand.PublicRandResponse{ + Round: rdata.GetRound(), + Signature: sig, // incoming message has incorrect sig + PreviousSignature: rdata.data.PreviousSignature, + Randomness: rdata.GetRandomness(), + } + data, err := proto.Marshal(&resp) + if err != nil { + t.Fatal(err) + } + msg := pubsub.Message{Message: &pb.Message{Data: data}} + res := validate(context.Background(), randomPeerID(t), &msg) + + if res != pubsub.ValidationReject { + t.Fatal(errors.New("expected reject for cached beacon")) + } +} diff --git a/client/mock/mock.go b/client/mock/mock.go new file mode 100644 index 0000000..26ea8ea --- /dev/null +++ b/client/mock/mock.go @@ -0,0 +1,154 @@ +package mock + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/drand/drand-cli/client/test/result/mock" + commonutils "github.com/drand/drand/v2/common" + chain2 "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" +) + +var _ client.Client = &Client{} + +// Client provide a mocked client interface +// +//nolint:gocritic +type Client struct { + sync.Mutex + OptionalInfo *chain2.Info + WatchCh chan client.Result + WatchF func(context.Context) <-chan client.Result + Results []mock.Result + // Delay causes results to be delivered after this period of time has + // passed. Note that if the context is canceled a result is still consumed + // from Results. + Delay time.Duration + // CloseF is a function to call when the Close function is called on the + // mock client. + CloseF func() error + // if strict rounds is set, calls to get will scan through results to + // return the first result with the requested round, rather than simply + // popping the next result and treating it as a stack. + StrictRounds bool +} + +func (m *Client) String() string { + return "Mock" +} + +// Get returns the randomness at `round` or an error. +func (m *Client) Get(ctx context.Context, round uint64) (client.Result, error) { + m.Lock() + if len(m.Results) == 0 { + m.Unlock() + return nil, errors.New("no result available") + } + r := m.Results[0] + if m.StrictRounds { + for _, candidate := range m.Results { + if candidate.GetRound() == round { + r = candidate + break + } + } + } else { + m.Results = m.Results[1:] + } + m.Unlock() + + if m.Delay > 0 { + t := time.NewTimer(m.Delay) + select { + case <-t.C: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + + return &r, nil +} + +// Watch returns new randomness as it becomes available. +func (m *Client) Watch(ctx context.Context) <-chan client.Result { + if m.WatchCh != nil { + return m.WatchCh + } + if m.WatchF != nil { + return m.WatchF(ctx) + } + ch := make(chan client.Result, 1) + r, err := m.Get(ctx, 0) + if err == nil { + ch <- r + } + close(ch) + return ch +} + +func (m *Client) Info(_ context.Context) (*chain2.Info, error) { + if m.OptionalInfo != nil { + return m.OptionalInfo, nil + } + return nil, errors.New("not supported (mock client info)") +} + +// RoundAt will return the most recent round of randomness +func (m *Client) RoundAt(_ time.Time) uint64 { + return 0 +} + +// Close calls the optional CloseF function. +func (m *Client) Close() error { + if m.CloseF != nil { + return m.CloseF() + } + return nil +} + +// ClientWithResults returns a client on which `Get` works `m-n` times. +func ClientWithResults(n, m uint64) *Client { + c := new(Client) + for i := n; i < m; i++ { + c.Results = append(c.Results, mock.NewMockResult(i)) + } + return c +} + +// ClientWithInfo makes a client that returns the given info but no randomness +func ClientWithInfo(info *chain2.Info) client.Client { + return &InfoClient{info} +} + +type InfoClient struct { + i *chain2.Info +} + +func (m *InfoClient) String() string { + return "MockInfo" +} + +func (m *InfoClient) Info(_ context.Context) (*chain2.Info, error) { + return m.i, nil +} + +func (m *InfoClient) RoundAt(t time.Time) uint64 { + return commonutils.CurrentRound(t.Unix(), m.i.Period, m.i.GenesisTime) +} + +func (m *InfoClient) Get(_ context.Context, _ uint64) (client.Result, error) { + return nil, errors.New("not supported (mock info client get)") +} + +func (m *InfoClient) Watch(_ context.Context) <-chan client.Result { + ch := make(chan client.Result, 1) + close(ch) + return ch +} + +func (m *InfoClient) Close() error { + return nil +} diff --git a/client/optimizing.go b/client/optimizing.go new file mode 100644 index 0000000..82e24b3 --- /dev/null +++ b/client/optimizing.go @@ -0,0 +1,645 @@ +package client + +import ( + "context" + "errors" + "fmt" + "math" + "sort" + "strings" + "sync" + "time" + + "github.com/hashicorp/go-multierror" + + "github.com/drand/drand/v2/common" + "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" +) + +const ( + defaultRequestTimeout = time.Second * 5 + defaultSpeedTestInterval = time.Minute * 5 + // defaultRequestConcurrency controls both how many clients are raced + // when `Get` is called for on-demand results, and also how many watch + // clients are spun up (in addition to clients marked as passive) to + // provide results to `Watch` requests. + defaultRequestConcurrency = 2 + // defaultWatchRetryInterval is the time after which a closed watch channel + // is re-open when no context error occurred. + defaultWatchRetryInterval = time.Second * 30 + defaultChannelBuffer = 5 + + maxUnixTime = 1<<63 - 62135596801 + maxNanoSec = 999999999 +) + +type optimizingClient struct { + sync.RWMutex + clients []client.Client + passiveClients []client.Client + stats []*requestStat + requestTimeout time.Duration + requestConcurrency int + speedTestInterval time.Duration + watchRetryInterval time.Duration + log log.Logger + done chan struct{} +} + +// newOptimizingClient creates a drand client that measures the speed of clients +// and uses the fastest ones. +// +// Clients passed to the optimizing client are ordered by speed and calls to +// `Get` race the 2 fastest clients (by default) for the result. If a client +// errors then it is moved to the back of the list. +// +// A speed test is performed periodically in the background every 5 minutes (by +// default) to ensure we're still using the fastest clients. A negative speed +// test interval will disable testing. +// +// Calls to `Get` actually iterate over the speed-ordered client list with a +// concurrency of 2 (by default) until a result is retrieved. It means that the +// optimizing client will fallback to using the other slower clients in the +// event of failure(s). +// +// Additionally, calls to Get are given a timeout of 5 seconds (by default) to +// ensure no unbounded blocking occurs. +func newOptimizingClient( + l log.Logger, + clients []client.Client, + requestTimeout time.Duration, + requestConcurrency int, + speedTestInterval, + watchRetryInterval time.Duration, +) (*optimizingClient, error) { + if len(clients) == 0 { + return nil, errors.New("missing clients") + } + stats := make([]*requestStat, len(clients)) + now := time.Now() + for i, c := range clients { + stats[i] = &requestStat{client: c, rtt: 0, startTime: now} + } + done := make(chan struct{}) + if requestTimeout <= 0 { + requestTimeout = defaultRequestTimeout + } + if requestConcurrency <= 0 { + requestConcurrency = defaultRequestConcurrency + } + if speedTestInterval == 0 { + speedTestInterval = defaultSpeedTestInterval + } + if watchRetryInterval == 0 { + watchRetryInterval = defaultWatchRetryInterval + } + oc := &optimizingClient{ + clients: clients, + stats: stats, + requestTimeout: requestTimeout, + requestConcurrency: requestConcurrency, + speedTestInterval: speedTestInterval, + watchRetryInterval: watchRetryInterval, + log: l, + done: done, + } + return oc, nil +} + +// Start starts the background speed measurements of the optimizing client.Start +// SetLog should not be called after Start. +func (oc *optimizingClient) Start() { + if oc.speedTestInterval > 0 { + go oc.testSpeed() + } +} + +// MarkPassive tags a client as 'passive' - a generalization of the libp2p style gossip client. +// These clients will not participate in the speed test horse race, and will be protected from +// being stopped by the optimized watcher. +// Note: if a client marked as passive closes its results channel from a `watch` call, the +// optimizing client will not re-open it, as would be attempted with non-passive clients. +// MarkPassive must tag clients as passive before `Start` is run. +func (oc *optimizingClient) MarkPassive(c client.Client) { + oc.passiveClients = append(oc.passiveClients, c) + // push passive clients to the back of the list for `Get`s + for _, s := range oc.stats { + if s.client == c { + s.rtt = math.MaxInt64 + s.startTime = time.Unix(maxUnixTime, maxNanoSec) + } + } +} + +// String returns the name of this client. +func (oc *optimizingClient) String() string { + names := make([]string, len(oc.clients)) + for i, c := range oc.clients { + names[i] = fmt.Sprint(c) + } + return fmt.Sprintf("OptimizingClient(%s)", strings.Join(names, ", ")) +} + +type requestStat struct { + // client is the client used to make the request. + client client.Client + // rtt is the time it took to make the request. + rtt time.Duration + // startTime is the time at which the request was started. + startTime time.Time +} + +type requestResult struct { + // client is the client used to make the request. + client client.Client + // result is the return value from the call to Get. + result client.Result + // err is the error that occurred from a call to Get (not including context error). + err error + // stat is stats from the call to Get. + stat *requestStat +} + +// markedPassive checks if a client should be treated as passive +func (oc *optimizingClient) markedPassive(c client.Client) bool { + for _, p := range oc.passiveClients { + if p == c { + return true + } + } + return false +} + +func (oc *optimizingClient) testSpeed() { + clients := make([]client.Client, 0, len(oc.clients)) + for _, c := range oc.clients { + if !oc.markedPassive(c) { + clients = append(clients, c) + } + } + + for { + var stats []*requestStat + ctx, cancel := context.WithCancel(context.Background()) + ch := parallelGet(ctx, clients, 1, oc.requestTimeout, oc.requestConcurrency) + + LOOP: + for { + select { + case rr, ok := <-ch: + if !ok { + cancel() + break LOOP + } + if rr.err != nil { + oc.log.Infow("", "optimizing_client", "endpoint down when speed tested", "client", fmt.Sprintf("%s", rr.client), "err", rr.err) + } + stats = append(stats, rr.stat) + case <-oc.done: + cancel() + return + } + } + + oc.updateStats(stats) + + t := time.NewTimer(oc.speedTestInterval) + select { + case <-t.C: + case <-oc.done: + t.Stop() + return + } + } +} + +// SetLog configures the client log output. +func (oc *optimizingClient) SetLog(l log.Logger) { + oc.log = l +} + +// fastestClients returns a ordered slice of clients - fastest first. +func (oc *optimizingClient) fastestClients() []client.Client { + oc.RLock() + defer oc.RUnlock() + // copy the current ordered client list so we iterate over a stable slice + clients := make([]client.Client, 0, len(oc.stats)) + for _, s := range oc.stats { + clients = append(clients, s.client) + } + return clients +} + +// Get returns the randomness at `round` or an error. +func (oc *optimizingClient) Get(ctx context.Context, round uint64) (res client.Result, err error) { + clients := oc.fastestClients() + // no need to race clients when we have only one + if len(clients) == 1 { + return clients[0].Get(ctx, round) + } + var stats []*requestStat + ch := raceGet(ctx, clients, round, oc.requestTimeout, oc.requestConcurrency) + err = errors.New("no valid clients") + +LOOP: + for { + select { + case rr, ok := <-ch: + if !ok { + break LOOP + } + stats = append(stats, rr.stat) + res = rr.result + if rr.err != nil && !errors.Is(rr.err, common.ErrEmptyClientUnsupportedGet) { + err = errors.Join(err, rr.err) + } else if rr.err == nil { + err = nil + } + case <-ctx.Done(): + oc.updateStats(stats) + return nil, ctx.Err() + case <-oc.done: + oc.updateStats(stats) + return nil, errors.New("client closed") + } + } + + oc.updateStats(stats) + return res, err +} + +// get calls Get on the passed client and returns a requestResult or nil if the context was canceled. +func get(ctx context.Context, c client.Client, round uint64) *requestResult { + start := time.Now() + res, err := c.Get(ctx, round) + rtt := time.Since(start) + var stat requestStat + + // c failure, set a large RTT so it is sent to the back of the list + if err != nil && !errors.Is(err, ctx.Err()) { + stat = requestStat{c, math.MaxInt64, start} + return &requestResult{c, res, err, &stat} + } + + if ctx.Err() != nil { + return nil + } + + stat = requestStat{c, rtt, start} + return &requestResult{c, res, err, &stat} +} + +func raceGet(ctx context.Context, clients []client.Client, round uint64, timeout time.Duration, concurrency int) <-chan *requestResult { + results := make(chan *requestResult, len(clients)) + + go func() { + rctx, cancel := context.WithCancel(ctx) + defer cancel() + defer close(results) + ch := parallelGet(rctx, clients, round, timeout, concurrency) + + for { + select { + case rr, ok := <-ch: + if !ok { + return + } + results <- rr + if rr.err == nil { // race is won + return + } + case <-rctx.Done(): + return + } + } + }() + + return results +} + +func parallelGet(ctx context.Context, clients []client.Client, round uint64, timeout time.Duration, concurrency int) <-chan *requestResult { + results := make(chan *requestResult, len(clients)) + token := make(chan struct{}, concurrency) + + for i := 0; i < concurrency; i++ { + token <- struct{}{} + } + + go func() { + wg := sync.WaitGroup{} + LOOP: + for _, c := range clients { + c := c + select { + case <-token: + wg.Add(1) + go func(c client.Client) { + gctx, cancel := context.WithTimeout(ctx, timeout) + rr := get(gctx, c, round) + cancel() + if rr != nil { + results <- rr + } + token <- struct{}{} + wg.Done() + }(c) + case <-ctx.Done(): + break LOOP + } + } + wg.Wait() + close(results) + }() + + return results +} + +func (oc *optimizingClient) updateStats(stats []*requestStat) { + oc.Lock() + defer oc.Unlock() + + // update the round trip times with new samples + for _, next := range stats { + for _, curr := range oc.stats { + if curr.client == next.client { + if curr.startTime.Before(next.startTime) { + curr.rtt = next.rtt + curr.startTime = next.startTime + } + break + } + } + } + + // sort by fastest + sort.Slice(oc.stats, func(i, j int) bool { + return oc.stats[i].rtt < oc.stats[j].rtt + }) +} + +type watchResult struct { + client.Result + client.Client +} + +func (oc *optimizingClient) trackWatchResults(info *chain.Info, in chan watchResult, out chan client.Result) { + defer close(out) + + latest := uint64(0) + for r := range in { + round := r.Result.GetRound() + timeOfRound := time.Unix(common.TimeOfRound(info.Period, info.GenesisTime, round), 0) + stat := requestStat{ + client: r.Client, + rtt: time.Since(timeOfRound), + startTime: timeOfRound, + } + oc.updateStats([]*requestStat{&stat}) + if round > latest { + latest = round + out <- r.Result + } + } +} + +// Watch returns new randomness as it becomes available. +func (oc *optimizingClient) Watch(ctx context.Context) <-chan client.Result { + outChan := make(chan client.Result, defaultChannelBuffer) + inChan := make(chan watchResult, defaultChannelBuffer) + + info, err := oc.Info(ctx) + if err != nil { + oc.log.Errorw("", "optimizing_client", "failed to learn info", "err", err) + close(outChan) + return outChan + } + + state := watchState{ + ctx: ctx, + optimizer: oc, + active: make([]watchingClient, 0), + protected: make([]watchingClient, 0), + failed: make([]failedClient, 0), + retryInterval: oc.watchRetryInterval, + } + + closingClients := make(chan client.Client, 1) + for _, c := range oc.passiveClients { + c := c + go state.watchNext(ctx, c, inChan, closingClients) + state.protected = append(state.protected, watchingClient{c, nil}) + } + + go state.dispatchWatchingClients(inChan, closingClients) + go oc.trackWatchResults(info, inChan, outChan) + return outChan +} + +type watchingClient struct { + client.Client + context.CancelFunc +} + +type failedClient struct { + client.Client + backoffUntil time.Time +} + +type watchState struct { + ctx context.Context + optimizer *optimizingClient + active []watchingClient + protected []watchingClient + failed []failedClient + retryInterval time.Duration +} + +func (ws *watchState) dispatchWatchingClients(resultChan chan watchResult, closingClients chan client.Client) { + defer close(resultChan) + + // spin up initial watcher(s) + ws.tryRepopulate(resultChan, closingClients) + + ticker := time.NewTicker(ws.optimizer.watchRetryInterval) + defer ticker.Stop() + for { + select { + case c := <-closingClients: + // replace failed watchers + ws.done(c) + if ws.ctx.Err() == nil { + ws.tryRepopulate(resultChan, closingClients) + } + if len(ws.active) == 0 && len(ws.protected) == 0 { + return + } + case <-ticker.C: + // periodically cycle to fastest client. + clients := ws.optimizer.fastestClients() + if len(clients) == 0 { + continue + } + fastest := clients[0] + if ws.hasActive(fastest) == -1 && ws.hasProtected(fastest) == -1 { + ws.closeSlowest() + ws.tryRepopulate(resultChan, closingClients) + } + case <-ws.ctx.Done(): + // trigger client close. Will return once len(ws.active) == 0 + for _, c := range ws.active { + c.CancelFunc() + } + } + } +} + +func (ws *watchState) tryRepopulate(results chan watchResult, done chan client.Client) { + ws.clean() + + for { + if len(ws.active) >= ws.optimizer.requestConcurrency { + return + } + c := ws.nextUnwatched() + if c == nil { + return + } + cctx, cancel := context.WithCancel(ws.ctx) + + ws.active = append(ws.active, watchingClient{c, cancel}) + ws.optimizer.log.Infow("", "optimizing_client", "watching on client", "client", fmt.Sprintf("%s", c)) + go ws.watchNext(cctx, c, results, done) + } +} + +func (ws *watchState) watchNext(ctx context.Context, c client.Client, out chan watchResult, done chan client.Client) { + defer func() { done <- c }() + + resultStream := c.Watch(ctx) + for r := range resultStream { + out <- watchResult{r, c} + } + ws.optimizer.log.Infow("", "optimizing_client", "watch ended", "client", fmt.Sprintf("%s", c)) +} + +func (ws *watchState) clean() { + nf := make([]failedClient, 0, len(ws.failed)) + for _, f := range ws.failed { + if f.backoffUntil.After(time.Now()) { + nf = append(nf, f) + } + } + ws.failed = nf +} + +func (ws *watchState) close(clientIdx int) { + ws.active[clientIdx].CancelFunc() + ws.active[clientIdx] = ws.active[len(ws.active)-1] + ws.active[len(ws.active)-1] = watchingClient{} + ws.active = ws.active[:len(ws.active)-1] +} + +func (ws *watchState) done(c client.Client) { + idx := ws.hasActive(c) + if idx > -1 { + ws.close(idx) + ws.failed = append(ws.failed, failedClient{c, time.Now().Add(ws.retryInterval)}) + } else if i := ws.hasProtected(c); i > -1 { + ws.protected[i] = ws.protected[len(ws.protected)-1] + ws.protected = ws.protected[:len(ws.protected)-1] + return + } + // note: it's expected that the client may already not be active. + // this happens when the optimizing client has closed it via `closeSlowest` +} + +func (ws *watchState) hasActive(c client.Client) int { + for i, a := range ws.active { + if a.Client == c { + return i + } + } + return -1 +} + +func (ws *watchState) hasProtected(c client.Client) int { + for i, p := range ws.protected { + if p.Client == c { + return i + } + } + return -1 +} + +func (ws *watchState) closeSlowest() { + if len(ws.active) == 0 { + return + } + order := ws.optimizer.fastestClients() + idxs := make([]int, 0) + for _, c := range order { + if i := ws.hasActive(c); i > -1 { + idxs = append(idxs, i) + } + } + ws.close(idxs[len(idxs)-1]) +} + +func (ws *watchState) nextUnwatched() client.Client { + clients := ws.optimizer.fastestClients() +ClientLoop: + for _, c := range clients { + for _, a := range ws.active { + if c == a.Client { + continue ClientLoop + } + } + for _, f := range ws.failed { + if c == f.Client { + continue ClientLoop + } + } + for _, p := range ws.protected { + if c == p.Client { + continue ClientLoop + } + } + return c + } + return nil +} + +// Info returns the parameters of the chain this client is connected to. +// The public key, when it started, and how frequently it updates. +func (oc *optimizingClient) Info(ctx context.Context) (chainInfo *chain.Info, err error) { + clients := oc.fastestClients() + for _, c := range clients { + ctx, cancel := context.WithTimeout(ctx, oc.requestTimeout) + chainInfo, err = c.Info(ctx) + cancel() + if err == nil { + break + } + } + return +} + +// RoundAt will return the most recent round of randomness that will be available +// at time for the current client. +func (oc *optimizingClient) RoundAt(t time.Time) uint64 { + return oc.clients[0].RoundAt(t) +} + +// Close stops the background speed tests and closes the client and it's +// underlying clients for further use. +func (oc *optimizingClient) Close() error { + var errs *multierror.Error + for _, c := range oc.clients { + errs = multierror.Append(errs, c.Close()) + } + close(oc.done) + + return errs.ErrorOrNil() +} diff --git a/client/optimizing_test.go b/client/optimizing_test.go new file mode 100644 index 0000000..69566d4 --- /dev/null +++ b/client/optimizing_test.go @@ -0,0 +1,317 @@ +package client + +import ( + "context" + "sync" + "testing" + "time" + + clientMock "github.com/drand/drand-cli/client/mock" + "github.com/drand/drand-cli/client/test/result/mock" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" +) + +// waitForSpeedTest waits until all clients have had their initial speed test. +func waitForSpeedTest(t *testing.T, c client.Client, timeout time.Duration) { + t.Helper() + oc, ok := c.(*optimizingClient) + if !ok { + t.Fatal("client is not an optimizing client") + } + + timedOut := time.NewTimer(timeout) + defer timedOut.Stop() + for { + oc.RLock() + tested := true + for _, s := range oc.stats { + // all RTT's are zero until a speed test has been done + if s.rtt == 0 { + tested = false + break + } + } + oc.RUnlock() + + if tested { + return + } + + // try again in a bit... + zzz := time.NewTimer(time.Millisecond * 100) + select { + case <-zzz.C: + case <-timedOut.C: + zzz.Stop() + t.Fatal("timed out waiting for initial speed test to complete") + } + } +} + +func expectRound(t *testing.T, res client.Result, r uint64) { + t.Helper() + if res.GetRound() != r { + t.Fatalf("expected round %v, got %v", r, res.GetRound()) + } +} + +func closeClient(t *testing.T, c client.Client) { + t.Helper() + err := c.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestOptimizingGet(t *testing.T) { + c0 := clientMock.ClientWithResults(0, 5) + c1 := clientMock.ClientWithResults(5, 8) + + c0.Delay = time.Millisecond * 100 + c1.Delay = time.Millisecond + + lg := log.New(nil, log.DebugLevel, true) + oc, err := newOptimizingClient(lg, []client.Client{c0, c1}, time.Second*5, 2, time.Minute*5, 0) + if err != nil { + t.Fatal(err) + } + oc.Start() + defer closeClient(t, oc) + + waitForSpeedTest(t, oc, 10*time.Second) + + // speed test will consume round 0 and 5 from c0 and c1 + // then c1 will be used because it's faster + expectRound(t, latestResult(t, oc), 6) // round 6 from c1 and round 1 from c0 (discarded) + expectRound(t, latestResult(t, oc), 7) // round 7 from c1 and round 2 from c0 (discarded) + expectRound(t, latestResult(t, oc), 3) // c1 error (no results left), round 3 from c0 + expectRound(t, latestResult(t, oc), 4) // round 4 from c0 +} + +func TestOptimizingWatch(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + c0 := clientMock.ClientWithResults(0, 5) + c1 := clientMock.ClientWithResults(5, 8) + c2 := clientMock.ClientWithInfo(fakeChainInfo(t)) + + wc1 := make(chan client.Result, 5) + c1.WatchCh = wc1 + + c0.Delay = time.Millisecond + + lg := log.New(nil, log.DebugLevel, true) + oc, err := newOptimizingClient(lg, []client.Client{c0, c1, c2}, time.Second*5, 2, time.Minute*5, 0) + if err != nil { + t.Fatal(err) + } + oc.Start() + defer closeClient(t, oc) + + waitForSpeedTest(t, oc, time.Minute) + + ch := oc.Watch(ctx) + + expectRound(t, nextResult(t, ch), 1) // round 1 from c0 (after 100ms) + wc1 <- &mock.Result{Rnd: 2} + expectRound(t, nextResult(t, ch), 2) // round 2 from c1 and round 2 from c0 (discarded) + select { + case <-ch: + t.Fatal("should not get another watched result at this point") + case <-time.After(50 * time.Millisecond): + } + wc1 <- &mock.Result{Rnd: 6} + expectRound(t, nextResult(t, ch), 6) // round 6 from c1 +} + +func TestOptimizingWatchRetryOnClose(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var rnd uint64 + c := &clientMock.Client{ + // a single result for the speed test + Results: []mock.Result{mock.NewMockResult(0)}, + // return a watch channel that yields one result then closes + WatchF: func(context.Context) <-chan client.Result { + ch := make(chan client.Result, 1) + r := mock.NewMockResult(rnd) + rnd++ + ch <- &r + close(ch) + return ch + }, + } + + lg := log.New(nil, log.DebugLevel, true) + oc, err := newOptimizingClient(lg, []client.Client{c}, 0, 0, 0, time.Millisecond) + if err != nil { + t.Fatal(err) + } + oc.Start() + defer closeClient(t, oc) + + waitForSpeedTest(t, oc, time.Minute) + + ch := oc.Watch(ctx) + + var i uint64 + for r := range ch { + if r.GetRound() != i { + t.Fatal("unexpected round number") + } + i++ + if i > 2 { + break + } + } +} + +func TestOptimizingWatchFailover(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + chainInfo := fakeChainInfo(t) + + var rndlk sync.Mutex + var rnd uint64 = 1 + wf := func(context.Context) <-chan client.Result { + rndlk.Lock() + defer rndlk.Unlock() + ch := make(chan client.Result, 1) + r := mock.NewMockResult(rnd) + rnd++ + if rnd < 5 { + ch <- &r + } + close(ch) + return ch + } + c1 := &clientMock.Client{ + Results: []mock.Result{mock.NewMockResult(0)}, + WatchF: wf, + } + c2 := &clientMock.Client{ + Results: []mock.Result{mock.NewMockResult(0)}, + WatchF: wf, + } + + lg := log.New(nil, log.DebugLevel, true) + oc, err := newOptimizingClient(lg, []client.Client{clientMock.ClientWithInfo(chainInfo), c1, c2}, 0, 0, 0, time.Millisecond) + if err != nil { + t.Fatal(err) + } + oc.Start() + defer closeClient(t, oc) + + waitForSpeedTest(t, oc, time.Minute) + + ch := oc.Watch(ctx) + + var i uint64 = 1 + for r := range ch { + if r.GetRound() != i { + t.Fatalf("unexpected round number %d vs %d", r.GetRound(), i) + } + i++ + if i > 5 { + t.Fatal("there are a total of 4 rounds possible") + } + } + if i < 3 { + t.Fatalf("watching didn't flip / yield expected rounds. %d", i) + } +} + +func TestOptimizingRequiresClients(t *testing.T) { + lg := log.New(nil, log.DebugLevel, true) + _, err := newOptimizingClient(lg, []client.Client{}, 0, 0, 0, 0) + if err == nil { + t.Fatal("expected err is nil but it shouldn't be") + } + if err.Error() != "missing clients" { + t.Fatal("unexpected error", err) + } +} + +func TestOptimizingIsLogging(t *testing.T) { + lg := log.New(nil, log.DebugLevel, true) + oc, err := newOptimizingClient(lg, []client.Client{&clientMock.Client{}}, 0, 0, 0, 0) + if err != nil { + t.Fatal(err) + } + oc.SetLog(lg) +} + +func TestOptimizingIsCloser(t *testing.T) { + lg := log.New(nil, log.DebugLevel, true) + oc, err := newOptimizingClient(lg, []client.Client{&clientMock.Client{}}, 0, 0, 0, 0) + if err != nil { + t.Fatal(err) + } + oc.Start() + err = oc.Close() + if err != nil { + t.Fatal(err) + } +} + +func TestOptimizingInfo(t *testing.T) { + lg := log.New(nil, log.DebugLevel, true) + chainInfo := fakeChainInfo(t) + oc, err := newOptimizingClient(lg, []client.Client{clientMock.ClientWithInfo(chainInfo)}, 0, 0, 0, 0) + if err != nil { + t.Fatal(err) + } + oc.Start() + i, err := oc.Info(context.Background()) + if err != nil { + t.Fatal(err) + } + if i != chainInfo { + t.Fatal("wrong chain info", i) + } +} + +func TestOptimizingRoundAt(t *testing.T) { + lg := log.New(nil, log.DebugLevel, true) + oc, err := newOptimizingClient(lg, []client.Client{&clientMock.Client{}}, 0, 0, 0, 0) + if err != nil { + t.Fatal(err) + } + oc.Start() + r := oc.RoundAt(time.Now()) // mock client returns 0 always + if r != 0 { + t.Fatal("unexpected round", r) + } +} + +func TestOptimizingClose(t *testing.T) { + wg := sync.WaitGroup{} + + closeF := func() error { + wg.Done() + return nil + } + + clients := []client.Client{ + &clientMock.Client{WatchCh: make(chan client.Result), CloseF: closeF}, + &clientMock.Client{WatchCh: make(chan client.Result), CloseF: closeF}, + } + + wg.Add(len(clients)) + + lg := log.New(nil, log.DebugLevel, true) + oc, err := newOptimizingClient(lg, clients, 0, 0, 0, 0) + if err != nil { + t.Fatal(err) + } + + err = oc.Close() // should close the underlying clients + if err != nil { + t.Fatal(err) + } + + wg.Wait() // wait for underlying clients to close +} diff --git a/client/poll.go b/client/poll.go new file mode 100644 index 0000000..d9a90d5 --- /dev/null +++ b/client/poll.go @@ -0,0 +1,64 @@ +package client + +import ( + "context" + "time" + + commonutils "github.com/drand/drand/v2/common" + chain2 "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" +) + +// PollingWatcher generalizes the `Watch` interface for clients which learn new values +// by asking for them once each group period. +func PollingWatcher(ctx context.Context, c client.Client, chainInfo *chain2.Info, l log.Logger) <-chan client.Result { + ch := make(chan client.Result, 1) + r := c.RoundAt(time.Now()) + val, err := c.Get(ctx, r) + if err != nil { + l.Errorw("", "polling_client", "failed synchronous get", "from", c, "err", err) + close(ch) + return ch + } + ch <- val + + go func() { + defer close(ch) + + // Initially, wait to synchronize to the round boundary. + _, nextTime := commonutils.NextRound(time.Now().Unix(), chainInfo.Period, chainInfo.GenesisTime) + select { + case <-ctx.Done(): + return + case <-time.After(time.Duration(nextTime-time.Now().Unix()) * time.Second): + } + + r, err := c.Get(ctx, c.RoundAt(time.Now())) + if err == nil { + ch <- r + } else { + l.Errorw("", "polling_client", "failed first async get", "from", c, "err", err) + } + + // Then tick each period. + t := time.NewTicker(chainInfo.Period) + defer t.Stop() + for { + select { + case <-t.C: + r, err := c.Get(ctx, c.RoundAt(time.Now())) + if err == nil { + ch <- r + } else { + l.Errorw("", "polling_client", "failed subsequent watch poll", "from", c, "err", err) + } + // TODO: keep trying on errors? + case <-ctx.Done(): + return + } + } + }() + + return ch +} diff --git a/client/random.go b/client/random.go new file mode 100644 index 0000000..707e399 --- /dev/null +++ b/client/random.go @@ -0,0 +1,38 @@ +package client + +import ( + "github.com/drand/drand/v2/crypto" +) + +// RandomData holds the full random response from the server, including data needed +// for validation. +type RandomData struct { + Rnd uint64 `json:"round,omitempty"` + Random []byte `json:"randomness,omitempty"` + Sig []byte `json:"signature,omitempty"` + PreviousSignature []byte `json:"previous_signature,omitempty"` +} + +// GetRound provides access to the round associated with this random data. +func (r *RandomData) GetRound() uint64 { + return r.Rnd +} + +// GetSignature provides the signature over this round's randomness +func (r *RandomData) GetSignature() []byte { + return r.Sig +} + +// GetPreviousSignature provides the previous signature provided by the beacon, +// if nil, it's most likely using an unchained scheme. +func (r *RandomData) GetPreviousSignature() []byte { + return r.PreviousSignature +} + +// GetRandomness exports the randomness using the legacy SHA256 derivation path +func (r *RandomData) GetRandomness() []byte { + if r.Random != nil { + return r.Random + } + return crypto.RandomnessFromSignature(r.GetSignature()) +} diff --git a/client/test/cache/cache.go b/client/test/cache/cache.go new file mode 100644 index 0000000..8104126 --- /dev/null +++ b/client/test/cache/cache.go @@ -0,0 +1,36 @@ +package cache + +import ( + "sync" + + "github.com/drand/drand/v2/common/client" +) + +// MapCache is a simple cache that stores data in memory. +type MapCache struct { + sync.RWMutex + data map[uint64]client.Result +} + +// NewMapCache creates a new in memory cache backed by a map. +func NewMapCache() *MapCache { + return &MapCache{data: make(map[uint64]client.Result)} +} + +// TryGet provides a round beacon or nil if it is not cached. +func (mc *MapCache) TryGet(round uint64) client.Result { + mc.RLock() + defer mc.RUnlock() + r, ok := mc.data[round] + if !ok { + return nil + } + return r +} + +// Add adds an item to the cache +func (mc *MapCache) Add(round uint64, result client.Result) { + mc.Lock() + mc.data[round] = result + mc.Unlock() +} diff --git a/client/test/http/mock/httpserver.go b/client/test/http/mock/httpserver.go new file mode 100644 index 0000000..dd00c65 --- /dev/null +++ b/client/test/http/mock/httpserver.go @@ -0,0 +1,213 @@ +package mock + +import ( + "context" + "net" + "net/http" + "testing" + "time" + + clock "github.com/jonboulle/clockwork" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + + localClient "github.com/drand/drand-cli/client" + "github.com/drand/drand/v2/common" + "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" + dhttp "github.com/drand/drand/v2/handler/http" + "github.com/drand/drand/v2/protobuf/drand" + "github.com/drand/drand/v2/test/mock" + + "github.com/drand/drand/v2/crypto" +) + +// NewMockHTTPPublicServer creates a mock drand HTTP server for testing. +func NewMockHTTPPublicServer(t *testing.T, badSecondRound bool, sch *crypto.Scheme, clk clock.Clock) (string, *chain.Info, context.CancelFunc, func(bool)) { + t.Helper() + + server := mock.NewMockServer(t, badSecondRound, sch, clk) + client := Proxy(server) + + ctx, cancel := context.WithCancel(context.Background()) + + handler, err := dhttp.New(ctx, "") + if err != nil { + t.Fatal(err) + } + + var chainInfo *chain.Info + for i := 0; i < 3; i++ { + protoInfo, err := server.ChainInfo(ctx, &drand.ChainInfoRequest{}) + if err != nil { + t.Error("MockServer.ChainInfo error:", err) + time.Sleep(10 * time.Millisecond) + continue + } + chainInfo, err = chain.InfoFromProto(protoInfo) + if err != nil { + t.Error("MockServer.InfoFromProto error:", err) + time.Sleep(10 * time.Millisecond) + continue + } + + break + } + if chainInfo == nil { + t.Fatal("could not use server after 3 attempts.") + } + + t.Log("MockServer.ChainInfo:", chainInfo) + + handler.RegisterDefaultBeaconHandler(handler.RegisterNewBeaconHandler(client, chainInfo.HashString())) + + listener, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatal(err) + } + + httpServer := http.Server{Handler: handler.GetHTTPHandler(), ReadHeaderTimeout: 3 * time.Second} + go httpServer.Serve(listener) + + return listener.Addr().String(), chainInfo, func() { + httpServer.Shutdown(context.Background()) + cancel() + }, server.(mock.Service).EmitRand +} + +// drandProxy is used as a proxy between a Public service (e.g. the node as a server) +// and a Public Client (the client consumed by the HTTP API) +type drandProxy struct { + r drand.PublicServer +} + +// Proxy wraps a server interface into a client interface so it can be queried +func Proxy(s drand.PublicServer) client.Client { + return &drandProxy{s} +} + +// String returns the name of this proxy. +func (d *drandProxy) String() string { + return "Proxy" +} + +// Get returns randomness at a requested round +func (d *drandProxy) Get(ctx context.Context, round uint64) (client.Result, error) { + resp, err := d.r.PublicRand(ctx, &drand.PublicRandRequest{Round: round}) + if err != nil { + return nil, err + } + return &localClient.RandomData{ + Rnd: resp.GetRound(), + Random: crypto.RandomnessFromSignature(resp.GetSignature()), + Sig: resp.GetSignature(), + PreviousSignature: resp.GetPreviousSignature(), + }, nil +} + +// Watch returns new randomness as it becomes available. +func (d *drandProxy) Watch(ctx context.Context) <-chan client.Result { + proxy := newStreamProxy(ctx) + go func() { + err := d.r.PublicRandStream(&drand.PublicRandRequest{}, proxy) + if err != nil { + proxy.Close() + } + }() + return proxy.outgoing +} + +// Info returns the parameters of the chain this client is connected to. +// The public key, when it started, and how frequently it updates. +func (d *drandProxy) Info(ctx context.Context) (*chain.Info, error) { + info, err := d.r.ChainInfo(ctx, &drand.ChainInfoRequest{}) + if err != nil { + return nil, err + } + return chain.InfoFromProto(info) +} + +// RoundAt will return the most recent round of randomness that will be available +// at time for the current client. +func (d *drandProxy) RoundAt(t time.Time) uint64 { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + info, err := d.Info(ctx) + if err != nil { + return 0 + } + return common.CurrentRound(t.Unix(), info.Period, info.GenesisTime) +} + +func (d *drandProxy) Close() error { + return nil +} + +// streamProxy directly relays messages of the PublicRandResponse stream. +type streamProxy struct { + ctx context.Context + cancel context.CancelFunc + outgoing chan client.Result +} + +func newStreamProxy(ctx context.Context) *streamProxy { + ctx, cancel := context.WithCancel(ctx) + s := streamProxy{ + ctx: ctx, + cancel: cancel, + outgoing: make(chan client.Result, 1), + } + return &s +} + +func (s *streamProxy) Send(next *drand.PublicRandResponse) error { + d := common.Beacon{ + Round: next.Round, + Signature: next.Signature, + PreviousSig: next.PreviousSignature, + } + select { + case s.outgoing <- &d: + return nil + case <-s.ctx.Done(): + close(s.outgoing) + return s.ctx.Err() + default: + return nil + } +} + +func (s *streamProxy) Close() { + s.cancel() +} + +/* implement the grpc stream interface. not used since messages passed directly. */ + +func (s *streamProxy) SetHeader(metadata.MD) error { + return nil +} +func (s *streamProxy) SendHeader(metadata.MD) error { + return nil +} +func (s *streamProxy) SetTrailer(metadata.MD) {} + +func (s *streamProxy) Context() context.Context { + return peer.NewContext(s.ctx, &peer.Peer{Addr: &net.UnixAddr{}}) +} +func (s *streamProxy) SendMsg(_ interface{}) error { + return nil +} +func (s *streamProxy) RecvMsg(_ interface{}) error { + return nil +} + +func (s *streamProxy) Header() (metadata.MD, error) { + return nil, nil +} + +func (s *streamProxy) Trailer() metadata.MD { + return nil +} +func (s *streamProxy) CloseSend() error { + return nil +} diff --git a/client/test/result/mock/result.go b/client/test/result/mock/result.go new file mode 100644 index 0000000..7eec852 --- /dev/null +++ b/client/test/result/mock/result.go @@ -0,0 +1,140 @@ +package mock + +import ( + "bytes" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "testing" + "time" + + "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/crypto" + "github.com/drand/kyber/share" + "github.com/drand/kyber/sign/tbls" + "github.com/drand/kyber/util/random" +) + +// NewMockResult creates a mock result for testing. +func NewMockResult(round uint64) Result { + sig := make([]byte, 8) + binary.LittleEndian.PutUint64(sig, round) + return Result{ + Rnd: round, + Sig: sig, + Rand: crypto.RandomnessFromSignature(sig), + } +} + +// Result is a mock result that can be used for testing. +type Result struct { + Rnd uint64 + Rand []byte + Sig []byte + PSig []byte +} + +// GetRandomness is a hash of the signature. +func (r *Result) GetRandomness() []byte { + return r.Rand +} + +// GetSignature is the signature of the randomness for this round. +func (r *Result) GetSignature() []byte { + return r.Sig +} + +// GetPreviousSignature is the signature of the previous round. +func (r *Result) GetPreviousSignature() []byte { + return r.PSig +} + +// GetRound is the round number for this random data. +func (r *Result) GetRound() uint64 { + return r.Rnd +} + +// AssertValid checks that this result is valid. +func (r *Result) AssertValid(t *testing.T) { + t.Helper() + sigTarget := make([]byte, 8) + binary.LittleEndian.PutUint64(sigTarget, r.Rnd) + if !bytes.Equal(r.Sig, sigTarget) { + t.Fatalf("expected sig: %x, got %x", sigTarget, r.Sig) + } + randTarget := crypto.RandomnessFromSignature(sigTarget) + if !bytes.Equal(r.Rand, randTarget) { + t.Fatalf("expected rand: %x, got %x", randTarget, r.Rand) + } +} + +func sha256Hash(prev []byte, round int) []byte { + h := sha256.New() + if prev != nil { + _, _ = h.Write(prev) + } + if round > 0 { + _ = binary.Write(h, binary.BigEndian, uint64(round)) + } + return h.Sum(nil) +} + +func roundToBytes(r int) []byte { + var buff bytes.Buffer + binary.Write(&buff, binary.BigEndian, uint64(r)) + return buff.Bytes() +} + +// VerifiableResults creates a set of results that will pass a `chain.Verify` check. +func VerifiableResults(count int, sch *crypto.Scheme) (*chain.Info, []Result) { + secret := sch.KeyGroup.Scalar().Pick(random.New()) + public := sch.KeyGroup.Point().Mul(secret, nil) + previous := make([]byte, 32) + if _, err := rand.Reader.Read(previous); err != nil { + panic(err) + } + + out := make([]Result, count) + for i := range out { + var msg []byte + if sch.Name == crypto.DefaultSchemeID { + // we're in chained mode + msg = sha256Hash(previous, i+1) + } else { + // we are in unchained mode + msg = sha256Hash(nil, i+1) + } + + sshare := share.PriShare{I: 0, V: secret} + tsig, err := sch.ThresholdScheme.Sign(&sshare, msg) + if err != nil { + panic(err) + } + tshare := tbls.SigShare(tsig) + sig := tshare.Value() + + out[i] = Result{ + Sig: sig, + PSig: previous, + Rnd: uint64(i + 1), + Rand: crypto.RandomnessFromSignature(sig), + } + + // chained mode + if sch.Name == crypto.DefaultSchemeID { + previous = make([]byte, len(sig)) + copy(previous, sig) + } else { + previous = nil + } + } + info := chain.Info{ + PublicKey: public, + Period: time.Second, + GenesisTime: time.Now().Unix() - int64(count), + GenesisSeed: out[0].PSig, + Scheme: sch.Name, + } + + return &info, out +} diff --git a/client/utils_test.go b/client/utils_test.go new file mode 100644 index 0000000..8ee8631 --- /dev/null +++ b/client/utils_test.go @@ -0,0 +1,68 @@ +package client + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/key" + "github.com/drand/drand/v2/crypto" +) + +// fakeChainInfo creates a chain info object for use in tests. +func fakeChainInfo(t *testing.T) *chain.Info { + t.Helper() + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + pair, err := key.NewKeyPair("fakeChainInfo.test:1234", sch) + require.NoError(t, err) + + return &chain.Info{ + Period: time.Second, + GenesisTime: time.Now().Unix(), + PublicKey: pair.Public.Key, + Scheme: sch.Name, + } +} + +func latestResult(t *testing.T, c client.Client) client.Result { + t.Helper() + r, err := c.Get(context.Background(), 0) + if err != nil { + t.Fatal("getting latest result", err) + } + return r +} + +// nextResult reads the next result from the channel and fails the test if it closes before a value is read. +func nextResult(t *testing.T, ch <-chan client.Result) client.Result { + t.Helper() + + select { + case r, ok := <-ch: + if !ok { + t.Fatal("closed before result") + } + return r + case <-time.After(time.Second): + t.Fatal("timed out waiting for result.") + return nil + } +} + +// compareResults asserts that two results are the same. +func compareResults(t *testing.T, a, b client.Result) { + t.Helper() + + if a.GetRound() != b.GetRound() { + t.Fatal("unexpected result round", a.GetRound(), b.GetRound()) + } + if !bytes.Equal(a.GetRandomness(), b.GetRandomness()) { + t.Fatal("unexpected result randomness", a.GetRandomness(), b.GetRandomness()) + } +} diff --git a/client/verify.go b/client/verify.go new file mode 100644 index 0000000..d550deb --- /dev/null +++ b/client/verify.go @@ -0,0 +1,211 @@ +package client + +import ( + "context" + "fmt" + "sync" + + "github.com/drand/drand/v2/common" + chain2 "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" + "github.com/drand/drand/v2/crypto" +) + +type verifyingClient struct { + // Client is the wrapped client. calls to `get` and `watch` return results proxied from this client's fetch + client.Client + + // indirectClient is used to fetch other rounds of randomness needed for verification. + // it is separated so that it can provide a cache or shared pool that the direct client may not. + indirectClient client.Client + + pointOfTrust client.Result + potLk sync.Mutex + strict bool + + scheme *crypto.Scheme + log log.Logger +} + +// newVerifyingClient wraps a client to perform `chain.Verify` on emitted results. +func newVerifyingClient(c client.Client, previousResult client.Result, strict bool, sch *crypto.Scheme) client.Client { + return &verifyingClient{ + Client: c, + indirectClient: c, + pointOfTrust: previousResult, + strict: strict, + scheme: sch, + log: log.DefaultLogger(), + } +} + +// SetLog configures the client log output. +func (v *verifyingClient) SetLog(l log.Logger) { + v.log = l.Named("verifyingClient") +} + +// Get returns a requested round of randomness +func (v *verifyingClient) Get(ctx context.Context, round uint64) (client.Result, error) { + info, err := v.indirectClient.Info(ctx) + if err != nil { + return nil, err + } + r, err := v.Client.Get(ctx, round) + if err != nil { + return nil, err + } + rd := asRandomData(r) + if err := v.verify(ctx, info, rd); err != nil { + return nil, err + } + if round != 0 && rd.GetRound() != round { + return nil, fmt.Errorf("round mismatch (malicious relay): %d != %d", rd.GetRound(), round) + } + return rd, nil +} + +// Watch returns new randomness as it becomes available. +func (v *verifyingClient) Watch(ctx context.Context) <-chan client.Result { + outCh := make(chan client.Result, 1) + + info, err := v.indirectClient.Info(ctx) + if err != nil { + v.log.Errorw("", "verifying_client", "could not get info", "err", err) + close(outCh) + return outCh + } + + inCh := v.Client.Watch(ctx) + go func() { + defer close(outCh) + for r := range inCh { + if err := v.verify(ctx, info, asRandomData(r)); err != nil { + v.log.Errorw("failed signature verification, something nefarious could be going on!", + "round", r.GetRound(), "signature", r.GetSignature(), "err", err) + continue + } + outCh <- r + } + }() + return outCh +} + +type resultWithPreviousSignature interface { + GetPreviousSignature() []byte +} + +func asRandomData(r client.Result) *RandomData { + rd, ok := r.(*RandomData) + if ok { + rd.Random = crypto.RandomnessFromSignature(rd.GetSignature()) + return rd + } + rd = &RandomData{ + Rnd: r.GetRound(), + Random: crypto.RandomnessFromSignature(r.GetSignature()), + Sig: r.GetSignature(), + } + if rp, ok := r.(resultWithPreviousSignature); ok { + rd.PreviousSignature = rp.GetPreviousSignature() + } + + return rd +} + +func (v *verifyingClient) getTrustedPreviousSignature(ctx context.Context, round uint64) ([]byte, error) { + info, err := v.indirectClient.Info(ctx) + if err != nil { + v.log.Errorw("", "drand_client", "could not get info to verify round 1", "err", err) + return []byte{}, fmt.Errorf("could not get info: %w", err) + } + + if round == 1 { + return info.GenesisSeed, nil + } + + trustRound := uint64(1) + var trustPrevSig []byte + + v.potLk.Lock() + if v.pointOfTrust == nil || v.pointOfTrust.GetRound() > round { + // slow path + v.potLk.Unlock() + trustPrevSig, err = v.getTrustedPreviousSignature(ctx, 1) + if err != nil { + return nil, err + } + } else { + trustRound = v.pointOfTrust.GetRound() + trustPrevSig = v.pointOfTrust.GetSignature() + v.potLk.Unlock() + } + initialTrustRound := trustRound + + var next client.Result + for trustRound < round-1 { + trustRound++ + v.log.Warnw("", "verifying_client", "loading round to verify", "round", trustRound) + next, err = v.indirectClient.Get(ctx, trustRound) + if err != nil { + return []byte{}, fmt.Errorf("could not get round %d: %w", trustRound, err) + } + b := &common.Beacon{ + PreviousSig: trustPrevSig, + Round: trustRound, + Signature: next.GetSignature(), + } + + ipk := info.PublicKey.Clone() + + err = v.scheme.VerifyBeacon(b, ipk) + if err != nil { + v.log.Warnw("", "verifying_client", "failed to verify value", "b", b, "err", err) + return []byte{}, fmt.Errorf("verifying beacon: %w", err) + } + trustPrevSig = next.GetSignature() + } + if trustRound == round-1 && trustRound > initialTrustRound { + v.potLk.Lock() + v.pointOfTrust = next + v.potLk.Unlock() + } + + if trustRound != round-1 { + return []byte{}, fmt.Errorf("unexpected trust round %d", trustRound) + } + return trustPrevSig, nil +} + +func (v *verifyingClient) verify(ctx context.Context, info *chain2.Info, r *RandomData) (err error) { + fetchPrevSignature := v.strict // only useful for chained schemes + ps := r.GetPreviousSignature() + + if fetchPrevSignature { + ps, err = v.getTrustedPreviousSignature(ctx, r.GetRound()) + if err != nil { + return + } + } + + b := &common.Beacon{ + PreviousSig: ps, // for unchained schemes, this is not used in the VerifyBeacon function and can be nil + Round: r.GetRound(), + Signature: r.GetSignature(), + } + + ipk := info.PublicKey.Clone() + + err = v.scheme.VerifyBeacon(b, ipk) + if err != nil { + return fmt.Errorf("verification of %v failed: %w", b, err) + } + + r.Random = crypto.RandomnessFromSignature(r.Sig) + return nil +} + +// String returns the name of this client. +func (v *verifyingClient) String() string { + return fmt.Sprintf("%s.(+verifier)", v.Client) +} diff --git a/client/verify_test.go b/client/verify_test.go new file mode 100644 index 0000000..cb2dec0 --- /dev/null +++ b/client/verify_test.go @@ -0,0 +1,72 @@ +package client_test + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + client2 "github.com/drand/drand-cli/client" + clientMock "github.com/drand/drand-cli/client/mock" + "github.com/drand/drand-cli/client/test/result/mock" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" + "github.com/drand/drand/v2/crypto" +) + +func mockClientWithVerifiableResults(ctx context.Context, t *testing.T, l log.Logger, n int, strictRounds bool) (client.Client, []mock.Result) { + t.Helper() + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + + info, results := mock.VerifiableResults(n, sch) + mc := clientMock.Client{Results: results, StrictRounds: strictRounds, OptionalInfo: info} + + var c client.Client + + c, err = client2.Wrap( + ctx, + l, + []client.Client{clientMock.ClientWithInfo(info), &mc}, + client2.WithChainInfo(info), + client2.WithVerifiedResult(&results[0]), + client2.WithFullChainVerification(), + ) + require.NoError(t, err) + + return c, results +} + +func TestVerify(t *testing.T) { + VerifyFuncTest(t, 3, 1) +} + +func TestVerifyWithOldVerifiedResult(t *testing.T) { + VerifyFuncTest(t, 5, 4) +} + +func VerifyFuncTest(t *testing.T, clients, upTo int) { + ctx := context.Background() + l := log.New(nil, log.DebugLevel, true) + c, results := mockClientWithVerifiableResults(ctx, t, l, clients, true) + + res, err := c.Get(context.Background(), results[upTo].GetRound()) + require.NoError(t, err) + + if res.GetRound() != results[upTo].GetRound() { + t.Fatal("expected to get result.", results[upTo].GetRound(), res.GetRound(), fmt.Sprintf("%v", c)) + } +} + +func TestGetWithRoundMismatch(t *testing.T) { + ctx := context.Background() + l := log.New(nil, log.DebugLevel, true) + c, results := mockClientWithVerifiableResults(ctx, t, l, 5, false) + for i := 1; i < len(results); i++ { + results[i] = results[0] + } + + _, err := c.Get(context.Background(), 3) + require.ErrorContains(t, err, "round mismatch (malicious relay): 1 != 3") +} diff --git a/client/watcher.go b/client/watcher.go new file mode 100644 index 0000000..3c6a96d --- /dev/null +++ b/client/watcher.go @@ -0,0 +1,35 @@ +package client + +import ( + "context" + "fmt" + "io" + + "github.com/hashicorp/go-multierror" + + "github.com/drand/drand/v2/common/client" +) + +type watcherClient struct { + client.Client + watcher Watcher +} + +func (c *watcherClient) Watch(ctx context.Context) <-chan client.Result { + return c.watcher.Watch(ctx) +} + +func (c *watcherClient) Close() error { + var errs *multierror.Error + cw, ok := c.watcher.(io.Closer) + if ok { + errs = multierror.Append(errs, cw.Close()) + } + errs = multierror.Append(errs, c.Client.Close()) + return errs.ErrorOrNil() +} + +// String returns the name of this client. +func (c *watcherClient) String() string { + return fmt.Sprintf("%s.(+watcher)", c.Client) +} diff --git a/client/watcher_test.go b/client/watcher_test.go new file mode 100644 index 0000000..6b47640 --- /dev/null +++ b/client/watcher_test.go @@ -0,0 +1,86 @@ +package client + +import ( + "context" + "sync" + "testing" + "time" + + clientMock "github.com/drand/drand-cli/client/mock" + "github.com/drand/drand-cli/client/test/result/mock" + "github.com/drand/drand/v2/common/client" +) + +func TestWatcherWatch(t *testing.T) { + results := []mock.Result{ + {Rnd: 1, Rand: []byte{1}}, + {Rnd: 2, Rand: []byte{2}}, + } + + ch := make(chan client.Result, len(results)) + for i := range results { + ch <- &results[i] + } + close(ch) + + w := watcherClient{nil, &clientMock.Client{WatchCh: ch}} + + i := 0 + for r := range w.Watch(context.Background()) { + compareResults(t, r, &results[i]) + i++ + } +} + +func TestWatcherGet(t *testing.T) { + results := []mock.Result{ + {Rnd: 1, Rand: []byte{1}}, + {Rnd: 2, Rand: []byte{2}}, + } + + cr := make([]mock.Result, len(results)) + copy(cr, results) + + c := &clientMock.Client{Results: cr} + + w := watcherClient{c, c} + + for i := range results { + r, err := w.Get(context.Background(), 0) + if err != nil { + t.Fatal(err) + } + compareResults(t, r, &results[i]) + } +} + +func TestWatcherRoundAt(t *testing.T) { + c := &clientMock.Client{} + + w := watcherClient{c, c} + + if w.RoundAt(time.Now()) != 0 { + t.Fatal("unexpected RoundAt value") + } +} + +func TestWatcherClose(t *testing.T) { + wg := sync.WaitGroup{} + wg.Add(2) + + closeF := func() error { + wg.Done() + return nil + } + + w := &clientMock.Client{CloseF: closeF} + c := &clientMock.Client{CloseF: closeF} + + wc := &watcherClient{c, w} + err := wc.Close() // should close the underlying client AND watcher + if err != nil { + t.Fatal(err) + } + + wg.Wait() // wait for underlying client AND watcher to close +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..024a907 --- /dev/null +++ b/go.mod @@ -0,0 +1,152 @@ +module github.com/drand/drand-cli + +go 1.22 + +require ( + github.com/BurntSushi/toml v1.4.0 + github.com/drand/drand/v2 v2.0.2 + github.com/drand/kyber v1.3.1 + github.com/google/uuid v1.6.0 + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 + github.com/hashicorp/consul/sdk v0.16.1 + github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/golang-lru v1.0.2 + github.com/jonboulle/clockwork v0.4.0 + github.com/libp2p/go-libp2p v0.35.4 + github.com/libp2p/go-libp2p-pubsub v0.11.0 + github.com/multiformats/go-multiaddr v0.13.0 + github.com/multiformats/go-multiaddr-dns v0.3.1 + github.com/nikkolasg/hexjson v0.1.0 + github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.19.1 + github.com/stretchr/testify v1.9.0 + github.com/urfave/cli/v2 v2.27.2 + golang.org/x/crypto v0.25.0 + google.golang.org/grpc v1.65.0 + google.golang.org/protobuf v1.34.2 +) + +require ( + github.com/benbjohnson/clock v1.3.5 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/cgroups v1.1.0 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/drand/kyber-bls12381 v0.3.1 // indirect + github.com/elastic/gosigar v0.14.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/flynn/noise v1.1.0 // indirect + github.com/francoispqt/gojay v1.2.13 // indirect + github.com/go-chi/chi/v5 v5.1.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/gopacket v1.1.19 // indirect + github.com/google/pprof v0.0.0-20240722153945-304e4f0156b8 // indirect + github.com/gorilla/websocket v1.5.3 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/huin/goupnp v1.3.0 // indirect + github.com/ipfs/go-cid v0.4.1 // indirect + github.com/ipfs/go-log/v2 v2.5.1 // indirect + github.com/jackpal/go-nat-pmp v1.0.2 // indirect + github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect + github.com/jmoiron/sqlx v1.4.0 // indirect + github.com/kilic/bls12-381 v0.1.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/koron/go-ssdp v0.0.4 // indirect + github.com/lib/pq v1.10.9 // indirect + github.com/libp2p/go-buffer-pool v0.1.0 // indirect + github.com/libp2p/go-flow-metrics v0.1.0 // indirect + github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect + github.com/libp2p/go-msgio v0.3.0 // indirect + github.com/libp2p/go-nat v0.2.0 // indirect + github.com/libp2p/go-netroute v0.2.1 // indirect + github.com/libp2p/go-reuseport v0.4.0 // indirect + github.com/libp2p/go-yamux/v4 v4.0.1 // indirect + github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/miekg/dns v1.1.61 // indirect + github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect + github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/mr-tron/base58 v1.2.0 // indirect + github.com/multiformats/go-base32 v0.1.0 // indirect + github.com/multiformats/go-base36 v0.2.0 // indirect + github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multicodec v0.9.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect + github.com/multiformats/go-multistream v0.5.0 // indirect + github.com/multiformats/go-varint v0.0.7 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/onsi/ginkgo/v2 v2.19.0 // indirect + github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect + github.com/pion/datachannel v1.5.8 // indirect + github.com/pion/dtls/v2 v2.2.12 // indirect + github.com/pion/ice/v2 v2.3.31 // indirect + github.com/pion/interceptor v0.1.29 // indirect + github.com/pion/logging v0.2.2 // indirect + github.com/pion/mdns v0.0.12 // indirect + github.com/pion/randutil v0.1.0 // indirect + github.com/pion/rtcp v1.2.14 // indirect + github.com/pion/rtp v1.8.7 // indirect + github.com/pion/sctp v1.8.19 // indirect + github.com/pion/sdp/v3 v3.0.9 // indirect + github.com/pion/srtp/v2 v2.0.20 // indirect + github.com/pion/stun v0.6.1 // indirect + github.com/pion/transport/v2 v2.2.8 // indirect + github.com/pion/turn/v2 v2.1.6 // indirect + github.com/pion/webrtc/v3 v3.2.49 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + github.com/quic-go/qpack v0.4.0 // indirect + github.com/quic-go/quic-go v0.45.1 // indirect + github.com/quic-go/webtransport-go v0.8.0 // indirect + github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/wlynxg/anet v0.0.3 // indirect + github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect + go.dedis.ch/fixbuf v1.0.3 // indirect + go.etcd.io/bbolt v1.3.10 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/sdk v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect + go.uber.org/dig v1.17.1 // indirect + go.uber.org/fx v1.22.1 // indirect + go.uber.org/mock v0.4.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/mod v0.19.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.23.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240723171418-e6d459c13d2a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/blake3 v1.3.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..afd8860 --- /dev/null +++ b/go.sum @@ -0,0 +1,684 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.0/go.mod h1:TS1dMSSfndXH133OKGwekG838Om/cQT0BUHV3HcBgoo= +dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= +dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= +dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1:a1inKt/atXimZ4Mv927x+r7UpyzRUf4emIoiiSC2TN4= +dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/ardanlabs/darwin/v2 v2.0.0 h1:XCisQMgQ5EG+ZvSEcADEo+pyfIMKyWAGnn5o2TgriYE= +github.com/ardanlabs/darwin/v2 v2.0.0/go.mod h1:MubZ2e9DAYGaym0mClSOi183NYahrrfKxvSy1HMhoes= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= +github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lVfRxZq4= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= +github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/drand/drand/v2 v2.0.2 h1:F0cvopmZWZA8NLRnpXE2+qVR13aNQZeCElYlWswcigM= +github.com/drand/drand/v2 v2.0.2/go.mod h1:nWBj4w7TA3R8xCoyLzkmsESjTlg4QgNSFAiRR9qZXt8= +github.com/drand/kyber v1.3.1 h1:E0p6M3II+loMVwTlAp5zu4+GGZFNiRfq02qZxzw2T+Y= +github.com/drand/kyber v1.3.1/go.mod h1:f+mNHjiGT++CuueBrpeMhFNdKZAsy0tu03bKq9D5LPA= +github.com/drand/kyber-bls12381 v0.3.1 h1:KWb8l/zYTP5yrvKTgvhOrk2eNPscbMiUOIeWBnmUxGo= +github.com/drand/kyber-bls12381 v0.3.1/go.mod h1:H4y9bLPu7KZA/1efDg+jtJ7emKx+ro3PU7/jWUVt140= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/elastic/gosigar v0.14.3 h1:xwkKwPia+hSfg9GqrCUKYdId102m9qTJIIr7egmK/uo= +github.com/elastic/gosigar v0.14.3/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg= +github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= +github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= +github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= +github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= +github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20240722153945-304e4f0156b8 h1:ssNFCCVmib/GQSzx3uCWyfMgOamLGWuGqlMS77Y1m3Y= +github.com/google/pprof v0.0.0-20240722153945-304e4f0156b8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/hashicorp/consul/sdk v0.16.1 h1:V8TxTnImoPD5cj0U9Spl0TUxcytjcbbJeADFF07KdHg= +github.com/hashicorp/consul/sdk v0.16.1/go.mod h1:fSXvwxB2hmh1FMZCNl6PwX0Q/1wdWtHJcZ7Ea5tns0s= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= +github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= +github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= +github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= +github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= +github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= +github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= +github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= +github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= +github.com/libp2p/go-libp2p v0.35.4 h1:FDiBUYLkueFwsuNJUZaxKRdpKvBOWU64qQPL768bSeg= +github.com/libp2p/go-libp2p v0.35.4/go.mod h1:RKCDNt30IkFipGL0tl8wQW/3zVWEGFUZo8g2gAKxwjU= +github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= +github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= +github.com/libp2p/go-libp2p-pubsub v0.11.0 h1:+JvS8Kty0OiyUiN0i8H5JbaCgjnJTRnTHe4rU88dLFc= +github.com/libp2p/go-libp2p-pubsub v0.11.0/go.mod h1:QEb+hEV9WL9wCiUAnpY29FZR6W3zK8qYlaml8R4q6gQ= +github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= +github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= +github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= +github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= +github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= +github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= +github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= +github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= +github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= +github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= +github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCypkQ= +github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= +github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= +github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/miekg/dns v1.1.61 h1:nLxbwF3XxhwVSm8g9Dghm9MHPaUZuqhPiGL+675ZmEs= +github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eykwQ= +github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= +github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= +github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= +github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b/go.mod h1:lxPUiZwKoFL8DUUmalo2yJJUCxbPKtm8OKfqr2/FTNU= +github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc h1:PTfri+PuQmWDqERdnNMiD9ZejrlswWrCpBEZgWOiTrc= +github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc/go.mod h1:cGKTAVKx4SxOuR/czcZ/E2RSJ3sfHs8FpHhQ5CWMf9s= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= +github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= +github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo= +github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4= +github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ= +github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII= +github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A= +github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk= +github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E= +github.com/multiformats/go-multiaddr-fmt v0.1.0/go.mod h1:hGtDIW4PU4BqJ50gW2quDuPVjyWNZxToGUh/HwTZYJo= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= +github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= +github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-multistream v0.5.0 h1:5htLSLl7lvJk3xx3qT/8Zm9J4K8vEOf/QGkvOGQAyiE= +github.com/multiformats/go-multistream v0.5.0/go.mod h1:n6tMZiwiP2wUsR8DgfDWw1dydlEqV3l6N3/GBsX6ILA= +github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= +github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= +github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/nikkolasg/hexjson v0.1.0 h1:Cgi1MSZVQFoJKYeRpBNEcdF3LB+Zo4fYKsDz7h8uJYQ= +github.com/nikkolasg/hexjson v0.1.0/go.mod h1:fbGbWFZ0FmJMFbpCMtJpwb0tudVxSSZ+Es2TsCg57cA= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= +github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo= +github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/dtls/v2 v2.2.12 h1:KP7H5/c1EiVAAKUmXyCzPiQe5+bCJrpOeKg/L05dunk= +github.com/pion/dtls/v2 v2.2.12/go.mod h1:d9SYc9fch0CqK90mRk1dC7AkzzpwJj6u2GU3u+9pqFE= +github.com/pion/ice/v2 v2.3.31 h1:qag/YqiOn5qPi0kgeVdsytxjx8szuriWSIeXKu8dDQc= +github.com/pion/ice/v2 v2.3.31/go.mod h1:8fac0+qftclGy1tYd/nfwfHC729BLaxtVqMdMVCAVPU= +github.com/pion/interceptor v0.1.29 h1:39fsnlP1U8gw2JzOFWdfCU82vHvhW9o0rZnZF56wF+M= +github.com/pion/interceptor v0.1.29/go.mod h1:ri+LGNjRUc5xUNtDEPzfdkmSqISixVTBF/z/Zms/6T4= +github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/mdns v0.0.12 h1:CiMYlY+O0azojWDmxdNr7ADGrnZ+V6Ilfner+6mSVK8= +github.com/pion/mdns v0.0.12/go.mod h1:VExJjv8to/6Wqm1FXK+Ii/Z9tsVk/F5sD/N70cnYFbk= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtcp v1.2.12/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= +github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE= +github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4= +github.com/pion/rtp v1.8.3/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/rtp v1.8.7 h1:qslKkG8qxvQ7hqaxkmL7Pl0XcUm+/Er7nMnu6Vq+ZxM= +github.com/pion/rtp v1.8.7/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU= +github.com/pion/sctp v1.8.19 h1:2CYuw+SQ5vkQ9t0HdOPccsCz1GQMDuVy5PglLgKVBW8= +github.com/pion/sctp v1.8.19/go.mod h1:P6PbDVA++OJMrVNg2AL3XtYHV4uD6dvfyOovCgMs0PE= +github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY= +github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M= +github.com/pion/srtp/v2 v2.0.20 h1:HNNny4s+OUmG280ETrCdgFndp4ufx3/uy85EawYEhTk= +github.com/pion/srtp/v2 v2.0.20/go.mod h1:0KJQjA99A6/a0DOVTu1PhDSw0CXF2jTkqOoMg3ODqdA= +github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4= +github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= +github.com/pion/transport/v2 v2.2.3/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0= +github.com/pion/transport/v2 v2.2.8 h1:HzsqGBChgtF4Cj47gu51l5hONuK/NwgbZL17CMSuwS0= +github.com/pion/transport/v2 v2.2.8/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E= +github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= +github.com/pion/transport/v3 v3.0.2 h1:r+40RJR25S9w3jbA6/5uEPTzcdn7ncyU44RWCbHkLg4= +github.com/pion/transport/v3 v3.0.2/go.mod h1:nIToODoOlb5If2jF9y2Igfx3PFYWfuXi37m0IlWa/D0= +github.com/pion/turn/v2 v2.1.3/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= +github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc= +github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY= +github.com/pion/webrtc/v3 v3.2.49 h1:G0DfKWeA++CBH7RcwB7M4msg9t/euEnFz638PYB6Ipg= +github.com/pion/webrtc/v3 v3.2.49/go.mod h1:s+tiQ44KWdJJkEw/oOoW24fW0ZEHlgOd3QyT90bqHeM= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/quic-go/quic-go v0.45.1 h1:tPfeYCk+uZHjmDRwHHQmvHRYL2t44ROTujLeFVBmjCA= +github.com/quic-go/quic-go v0.45.1/go.mod h1:1dLehS7TIR64+vxGR70GDcatWTOtMX2PUtnKsjbTurI= +github.com/quic-go/webtransport-go v0.8.0 h1:HxSrwun11U+LlmwpgM1kEqIqH90IT4N8auv/cD7QFJg= +github.com/quic-go/webtransport-go v0.8.0/go.mod h1:N99tjprW432Ut5ONql/aUhSLT0YVSlwHohQsuac9WaM= +github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtBsk= +github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= +github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= +github.com/shurcooL/github_flavored_markdown v0.0.0-20181002035957-2122de532470/go.mod h1:2dOwnU2uBioM+SGy2aZoq1f/Sd1l9OkAeAUvjSyvgU0= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ= +github.com/shurcooL/gofontwoff v0.0.0-20180329035133-29b52fc0a18d/go.mod h1:05UtEgK5zq39gLST6uB0cf3NEHjETfB4Fgr3Gx5R9Vw= +github.com/shurcooL/gopherjslib v0.0.0-20160914041154-feb6d3990c2c/go.mod h1:8d3azKNyqcHP1GaQE/c6dDgjkgSx2BZ4IoEi4F1reUI= +github.com/shurcooL/highlight_diff v0.0.0-20170515013008-09bb4053de1b/go.mod h1:ZpfEhSmds4ytuByIcDnOLkTHGUI6KNqRNPDLHDk+mUU= +github.com/shurcooL/highlight_go v0.0.0-20181028180052-98c3abbbae20/go.mod h1:UDKB5a1T23gOMUJrI+uSuH0VRDStOiUVSjBTRDVBVag= +github.com/shurcooL/home v0.0.0-20181020052607-80b7ffcb30f9/go.mod h1:+rgNQw2P9ARFAs37qieuu7ohDNQ3gds9msbT2yn85sg= +github.com/shurcooL/htmlg v0.0.0-20170918183704-d01228ac9e50/go.mod h1:zPn1wHpTIePGnXSHpsVPWEktKXHr6+SS6x/IKRb7cpw= +github.com/shurcooL/httperror v0.0.0-20170206035902-86b7830d14cc/go.mod h1:aYMfkZ6DWSJPJ6c4Wwz3QtW22G7mf/PEgaB9k/ik5+Y= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= +github.com/shurcooL/httpgzip v0.0.0-20180522190206-b1c53ac65af9/go.mod h1:919LwcH0M7/W4fcZ0/jy0qGght1GIhqyS/EgWGH2j5Q= +github.com/shurcooL/issues v0.0.0-20181008053335-6292fdc1e191/go.mod h1:e2qWDig5bLteJ4fwvDAc2NHzqFEthkqn7aOZAOpj+PQ= +github.com/shurcooL/issuesapp v0.0.0-20180602232740-048589ce2241/go.mod h1:NPpHK2TI7iSaM0buivtFUc9offApnI0Alt/K8hcHy0I= +github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b5uSkrEVM1jQUspwbixRBhaIjIzL2xazXp6kntxYle0= +github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= +github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= +github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= +github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= +github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.27.2 h1:6e0H+AkS+zDckwPCUrZkKX38mRaau4nL2uipkJpbkcI= +github.com/urfave/cli/v2 v2.27.2/go.mod h1:g0+79LmHHATl7DAcHO99smiR/T7uGLw84w8Y42x+4eM= +github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= +github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= +github.com/wlynxg/anet v0.0.3 h1:PvR53psxFXstc12jelG6f1Lv4MWqE0tI76/hHGjh9rg= +github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 h1:gEOO8jv9F4OT7lGCjxCBTO/36wtF6j2nSip77qHd4x4= +github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1/go.mod h1:Ohn+xnUBiLI6FVj/9LpzZWtj1/D6lUovWYBkxHVV3aM= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= +go.dedis.ch/fixbuf v1.0.3/go.mod h1:yzJMt34Wa5xD37V5RTdmp38cz3QhMagdGoem9anUalw= +go.dedis.ch/protobuf v1.0.11 h1:FTYVIEzY/bfl37lu3pR4lIj+F9Vp1jE8oh91VmxKgLo= +go.dedis.ch/protobuf v1.0.11/go.mod h1:97QR256dnkimeNdfmURz0wAMNVbd1VmLXhG1CrTYrJ4= +go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= +go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc= +go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= +go.uber.org/fx v1.22.1 h1:nvvln7mwyT5s1q201YE29V/BFrGor6vMiDNpU/78Mys= +go.uber.org/fx v1.22.1/go.mod h1:HT2M7d7RHo+ebKGh9NRcrsrHHfpZ60nW3QRubMRfv48= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= +golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030000716-a0a13e073c7b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190306203927-b5d61aea6440/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade h1:WxZOF2yayUHpHSbUE6NMzumUzBxYc3YGwo0YHnbzsJY= +google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= +google.golang.org/genproto/googleapis/api v0.0.0-20240723171418-e6d459c13d2a h1:YIa/rzVqMEokBkPtydCkx1VLmv3An1Uw7w1P1m6EhOY= +google.golang.org/genproto/googleapis/api v0.0.0-20240723171418-e6d459c13d2a/go.mod h1:AHT0dDg3SoMOgZGnZk29b5xTbPHMoEC8qthmBLJCpys= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade h1:oCRSWfwGXQsqlVdErcyTt4A93Y8fo0/9D4b1gnI++qo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a h1:hqK4+jJZXCU4pW7jsAdGOVFIfLHQeV7LaizZKnZ84HI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240723171418-e6d459c13d2a/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +lukechampine.com/blake3 v1.3.0 h1:sJ3XhFINmHSrYCgl958hscfIa3bw8x4DqMP3u1YvoYE= +lukechampine.com/blake3 v1.3.0/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= +rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= +sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= +sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= diff --git a/gossip-relay/.gitignore b/gossip-relay/.gitignore new file mode 100644 index 0000000..4208a2c --- /dev/null +++ b/gossip-relay/.gitignore @@ -0,0 +1,4 @@ +./relay-gossip +datastore +identity.key +relay-gossip diff --git a/gossip-relay/README.md b/gossip-relay/README.md new file mode 100644 index 0000000..07a3b50 --- /dev/null +++ b/gossip-relay/README.md @@ -0,0 +1,282 @@ + + +## Table of Contents + +- [Drand Pubsub Relay](#drand-pubsub-relay) + - [Install](#install) + - [Usage](#usage) + - [Relay gRPC](#relay-grpc) + - [Relay HTTP](#relay-http) + - [Relay Gossipsub](#relay-gossipsub) + - [Other options](#other-options) + - [Bootstrap peers](#bootstrap-peers) + - [Failover](#failover) + - [Configuring the libp2p pubsub node](#configuring-the-libp2p-pubsub-node) + - [Usage from a golang drand client](#usage-from-a-golang-drand-client) + - [With Group TOML or Chain Info](#with-group-toml-or-chain-info) + - [With Known Chain Hash](#with-known-chain-hash) + - [Insecurely](#insecurely) + + + +# Drand Pubsub Relay + +A program that relays drand randomness rounds over libp2p pubsub (gossipsub) from a gRPC, HTTP, or gossipsub source (labeled as _drand gossipsub relay_ in this diagram): + +``` + +-------------------------------+ + | | + | drand server | + | | + +-------------------------------+ + | gRPC API |--| HTTP API | + +------^-+------------^-+-------+ + | | | | + | | | | + | | | | + +--+-v------------+-v---+ + | drand gossipsub relay | + +----------+------------+ + | + | +Publish topic=/drand/pubsub/v0.0.0/ data={randomness} + | + | + +-----------v--------------+ + | | + | libp2p gossipsub network | + | | + +--+--------------------+--+ + | | + | | + Subscribe topic=/drand/pubsub/v0.0.0/ + | | + | | + +----------------v--------+ +-------v-----------------+ + | drand client WithPubsub | | drand client WithPubsub | + +-------------------------+ +-------------------------+ +``` + +## Install + +```sh +# Clone this repo +git clone https://github.com/drand/drand-cli.git +cd drand +# Build the executable +make relay-gossip-relay +# Outputs a `drand-relay-gossip-relay` executable to the current directory. +``` + +## Usage + +In general, you _should_ specify either a `-hash-list` or `-group-conf-list` flag in order for your client to validate the randomness it receives is from the correct chain. + +_Note_: You can provide multiple values to both `-hash-list` and`-group-conf-list` flags to support multiple beacons. + +### Relay gRPC + +```sh +drand-relay-gossip-relay run -grpc-connect=127.0.0.1:3000 \ + -cert=/path/to/grpc-drand-cert +``` + +If you do not have gRPC transport credentials, you can use the `-insecure` flag: + +```sh +drand-relay-gossip-relay run -grpc-connect=127.0.0.1:3000 \ + -insecure +``` + +Or, with a hashlist: +```shell + drand-relay-gossip-relay run -grpc-connect=127.0.0.1:3000 \ + -insecure \ + -hash-list=6093f9e4320c285ac4aab50ba821cd5678ec7c5015d3d9d11ef89e2a99741e83,dbd506d6ef76e5f386f41c651dcb808c5bcbd75471cc4eafa3f4df7ad4e4c493 +``` + +### Relay HTTP + +The gossip relay can also relay directly from an HTTP API. You can specify multiple endpoints to enable failover. + +```sh +drand-relay-gossip-relay run -url=https://api.drand.sh \ + -url=https://api2.drand.sh \ + -hash-list=dbd506d6ef76e5f386f41c651dcb808c5bcbd75471cc4eafa3f4df7ad4e4c493 +``` + +### Relay Gossipsub + +The gossip relay can also relay directly from _other_ gossip relays. You can specify multiple peers to directly connect with. In this case, a group configuration file must be specified since there's no way to retrieve chain information over pubsub. + +```sh +drand-relay-gossip-relay run -relay=/ip4/127.0.0.1/tcp/44544/p2p/QmPeerID0 \ + -relay=/ip4/127.0.0.1/tcp/44545/p2p/QmPeerID1 \ + -group-conf-list=/home/user/.drand/groups/drand_group.toml +``` + +Alternatively, you can provide URL(s) of HTTP API(s) that can be contacted to retrieve chain information. In this case we must provide the chain `-hash` to verify the information we retrieve is for the chain we expect (or provide the `-insecure` flag): + +```sh +drand-relay-gossip-relay run -relay=/ip4/127.0.0.1/tcp/44544/p2p/QmPeerID0 \ + -relay=/ip4/127.0.0.1/tcp/44545/p2p/QmPeerID1 \ + -url=http://127.0.0.1:3002 \ + -hash-list=6093f9e4320c285ac4aab50ba821cd5678ec7c5015d3d9d11ef89e2a99741e83 +``` + +If you want to verify multiple networks, you can provide the `-hash-list` flag, e.g.: + +```shell +drand-relay-gossip-relay run -relay=/ip4/127.0.0.1/tcp/44544/p2p/QmPeerID0 \ + -relay=/ip4/127.0.0.1/tcp/44545/p2p/QmPeerID1 \ + -url=http://127.0.0.1:3002 \ + -hash-list=dbd506d6ef76e5f386f41c651dcb808c5bcbd75471cc4eafa3f4df7ad4e4c493,8990e7a9aaed2ffed73dbd7092123d6f289930540d7651336225dc172e51b2ce +``` + +### Other options + +#### Bootstrap peers + +If there is a set of peers the gossip relay should connect with and stay connected to then the `-peer-with` flag can be used to specify one or more peer multiaddrs for this purpose. + +#### Failover + +The `-url` flag provides the URL(s) of alternative HTTP API endpoints that may be able to provide randomness in the event of a failure of the gRPC connection/libp2p pubsub network. Each randomness round is raced with the HTTP endpoints when it becomes available such that if gRPC or pubsub take too long to deliver the round it'll be provided over HTTP e.g. + +```sh +drand-relay-gossip-relay run -grpc-connect=127.0.0.1:3000 \ + -insecure \ + -url=http://127.0.0.1:3102 +``` + +```sh +drand-relay-gossip-relay run -relay=/ip4/127.0.0.1/tcp/44544/p2p/QmPeerID0 \ + -relay=/ip4/127.0.0.1/tcp/44545/p2p/QmPeerID1 \ + -hash-list=6093f9e4320c285ac4aab50ba821cd5678ec7c5015d3d9d11ef89e2a99741e83 \ + -url=http://127.0.0.1:3102 +``` + +#### Configuring the libp2p pubsub node + +Starting a relay will spawn a libp2p pubsub node listening on `/ip4/0.0.0.0/tcp/44544` by default. Use the `-listen` flag to change. To effectively relay drand randomness, your node must be publicly accessible on the network. + +If not specified a libp2p identity will be generated and stored in an `identity.key` file in the current working directory. Use the `-identity` flag to override the location. + +### Usage from a golang drand client + +#### With Group TOML or Chain Info + +```go +package main + +import ( + "context" + "fmt" + + clock "github.com/jonboulle/clockwork" + + "github.com/drand/drand-cli/client" + p2pClient "github.com/drand/drand-cli/client/lp2p" + "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/log" +) + +const ( + // listenAddr is the multiaddr the local libp2p node should listen on. + listenAddr = "/ip4/0.0.0.0/tcp/4453" + // relayP2PAddr is the p2p multiaddr of the drand gossipsub relay node to connect to. + relayP2PAddr = "/ip4/192.168.1.124/tcp/44544/p2p/QmPeerID" + // groupTOMLPath is the path to the group configuration information (in TOML format). + groupTOMLPath = "/home/user/.drand/groups/drand_group.toml" +) + +func main() { + ctx := context.Background() + l := log.DefaultLogger() + clk := clock.NewRealClock() + + // Create libp2p pubsub + ps, err := p2pClient.NewPubsub(ctx, listenAddr, relayP2PAddr) + if err != nil { + l.Panicw("while creating new p2pClient.NewPubsub", "err", err) + } + + // Extract chain info from group TOML + info, err := chain.InfoFromGroupTOML(l, groupTOMLPath) + if err != nil { + l.Panicw("while extracting info from groupTOML", "err", err) + } + + c, err := client.New(ctx, l, p2pClient.WithPubsub(l, ps, clk, p2pClient.DefaultBufferSize), client.WithChainInfo(info)) + if err != nil { + l.Panicw("while creating a new client", "err", err) + } + + for res := range c.Watch(ctx) { + fmt.Printf("round=%v randomness=%v\n", res.Round(), res.Randomness()) + } +} +``` + +#### With Known Chain Hash + +You do not need to know the full group info to use the pubsub client if you know the chain hash and an HTTP endpoint then you can request the chain info from the HTTP endpoint, verifying it with the known chain hash: + +```go +package main + +import ( + "context" + "encoding/hex" + "fmt" + + clock "github.com/jonboulle/clockwork" + + "github.com/drand/drand-cli/client" + "github.com/drand/drand-cli/client/http" + gclient "github.com/drand/drand-cli/client/lp2p" + "github.com/drand/drand/v2/common/log" +) + +const ( + // listenAddr is the multiaddr the local libp2p node should listen on. + listenAddr = "/ip4/0.0.0.0/tcp/4453" + // relayP2PAddr is the p2p multiaddr of the drand gossipsub relay node to connect to. + relayP2PAddr = "/ip4/192.168.1.124/tcp/44544/p2p/12D3KooWAe637xuWdRCYkuaZZce13P1F9zJX5gzGUPWZJpsUGUSH" + // chainHash is a hash of the group chain information. + chainHash = "c599c267a0dd386606f7d6132da8327d57e1004760897c9dd4fb8495c29942b2" + // httpRelayURL is the URL of a drand HTTP API endpoint. + httpRelayURL = "http://127.0.0.1:3002" +) + +func main() { + ctx := context.Background() + lg := log.New(nil, log.DebugLevel, true) + clk := clock.NewRealClock() + + // Create libp2p pubsub + ps, err := gclient.NewPubsub(ctx, listenAddr, relayP2PAddr) + if err != nil { + panic(err) + } + + // Chain hash is used to verify endpoints + hash, err := hex.DecodeString(chainHash) + if err != nil { + panic(err) + } + + c, err := client.New(ctx, lg, + gclient.WithPubsub(lg, ps, clk, gclient.DefaultBufferSize), + client.WithChainHash(hash), + client.From(http.ForURLs(ctx, lg, []string{httpRelayURL}, hash)...), + ) + if err != nil { + panic(err) + } + + for res := range c.Watch(ctx) { + fmt.Printf("round=%v randomness=%v\n", res.Round(), res.Randomness()) + } +} +``` diff --git a/gossip-relay/main.go b/gossip-relay/main.go new file mode 100644 index 0000000..69c7a9f --- /dev/null +++ b/gossip-relay/main.go @@ -0,0 +1,254 @@ +package main + +import ( + "encoding/hex" + "fmt" + "os" + "path" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/urfave/cli/v2" + + "github.com/drand/drand-cli/internal/lib" + "github.com/drand/drand-cli/internal/lp2p" + "github.com/drand/drand/v2/common/log" +) + +// Automatically set through -ldflags +// Example: go install -ldflags "-X main.buildDate=`date -u +%d/%m/%Y@%H:%M:%S` -X main.gitCommit=`git rev-parse HEAD`" +var ( + gitCommit = "none" + buildDate = "unknown" +) + +func main() { + app := &cli.App{ + Name: "drand-relay-gossip-relay", + Version: "2.0.0", + Usage: "pubsub relay for drand randomness beacon", + Commands: []*cli.Command{runCmd, clientCmd, idCmd}, + } + + // See https://cli.urfave.org/v2/examples/bash-completions/#enabling for how to turn on. + app.EnableBashCompletion = true + + cli.VersionPrinter = func(_ *cli.Context) { + fmt.Printf("drand gossip-relay relay %s (date %v, commit %v)\n", app.Version, buildDate, gitCommit) + } + + err := app.Run(os.Args) + if err != nil { + fmt.Printf("error: %+v\n", err) + os.Exit(1) + } +} + +var ( + idFlag = &cli.StringFlag{ + Name: "identity", + Usage: "path to a file containing a libp2p identity (base64 encoded)", + Value: "identity.key", + EnvVars: []string{"DRAND_GOSSIP_IDENTITY"}, + } + peerWithFlag = &cli.StringSliceFlag{ + Name: "peer-with", + Usage: "peer multiaddr(s) for the relay to direct connect with", + EnvVars: []string{"DRAND_GOSSIP_PEER_WITH"}, + } + storeFlag = &cli.StringFlag{ + Name: "store", + Usage: "datastore directory", + Value: "./datastore", + EnvVars: []string{"DRAND_RELAY_STORE"}, + } + listenFlag = &cli.StringFlag{ + Name: "listen", + Usage: "listening address for libp2p", + Value: "/ip4/0.0.0.0/tcp/44544", + EnvVars: []string{"DRAND_RELAY_LISTEN"}, + } + metricsFlag = &cli.StringFlag{ + Name: "metrics", + Usage: "local host:port to bind a metrics servlet (optional)", + EnvVars: []string{"DRAND_RELAY_METRICS"}, + } +) + +var runCmd = &cli.Command{ + Name: "run", + Usage: "starts a drand gossip-relay relay process", + Flags: append(lib.ClientFlags, []cli.Flag{ + idFlag, + peerWithFlag, + storeFlag, + listenFlag, + metricsFlag, + lib.GRPCConnectFlag, + }...), + Action: func(cctx *cli.Context) error { + if cctx.IsSet(lib.HashFlag.Name) || cctx.IsSet(lib.GroupConfFlag.Name) { + fmt.Printf("--%s and --%s are deprecated. Use --%s or --%s instead\n", + lib.HashFlag.Name, + lib.GroupConfFlag, + lib.HashListFlag.Name, + lib.GroupConfListFlag.Name) + } + + switch { + case cctx.IsSet(lib.GroupConfListFlag.Name) && cctx.IsSet(lib.HashListFlag.Name): + return fmt.Errorf("only one of --%s and --%s are allowed", lib.GroupConfListFlag.Name, lib.HashListFlag.Name) + case cctx.IsSet(lib.GroupConfListFlag.Name): + groupConfs := cctx.StringSlice(lib.GroupConfListFlag.Name) + for _, groupConf := range groupConfs { + err := boostrapGossipRelayNode(cctx, groupConf, "") + if err != nil { + return err + } + } + case cctx.IsSet(lib.HashListFlag.Name): + hashes, err := computeHashes(cctx) + if err != nil { + return err + } + + for _, hash := range hashes { + err := boostrapGossipRelayNode(cctx, "", hash) + if err != nil { + return err + } + } + case cctx.IsSet(lib.HashFlag.Name): + hash := cctx.String(lib.HashFlag.Name) + if _, err := hex.DecodeString(hash); err != nil { + return fmt.Errorf("decoding hash %q: %w", hash, err) + } + + err := boostrapGossipRelayNode(cctx, "", hash) + if err != nil { + return err + } + default: + if err := boostrapGossipRelayNode(cctx, "", ""); err != nil { + return err + } + } + + // Wait until we are signaled to shutdown. + <-cctx.Context.Done() + + return cctx.Context.Err() + }, +} + +func boostrapGossipRelayNode(cctx *cli.Context, groupConf, chainHash string) error { + err := cctx.Set(lib.GroupConfFlag.Name, groupConf) + if err != nil { + return err + } + + err = cctx.Set(lib.HashFlag.Name, chainHash) + if err != nil { + return err + } + + c, err := lib.Create(cctx, cctx.IsSet(metricsFlag.Name)) + if err != nil { + return fmt.Errorf("constructing client: %w", err) + } + + chainInfo, err := c.Info(cctx.Context) + if err != nil { + return fmt.Errorf("cannot retrieve chain info: %w", err) + } + + if chainHash == "" { + chainHash = hex.EncodeToString(chainInfo.Hash()) + } + + // Set the path to be desired 'storage path / beaconID'. + // This allows running multiple networks via the same beacon. + dataDir := path.Join(cctx.String(storeFlag.Name), chainInfo.ID) + + cfg := &lp2p.GossipRelayConfig{ + ChainHash: chainHash, + PeerWith: cctx.StringSlice(peerWithFlag.Name), + Addr: cctx.String(listenFlag.Name), + DataDir: dataDir, + IdentityPath: cctx.String(idFlag.Name), + Client: c, + } + + l := log.DefaultLogger().With("beaconID", chainInfo.ID) + + _, err = lp2p.NewGossipRelayNode(l, cfg) + if err != nil { + err = fmt.Errorf("could not initialize a new gossip-relay relay node %w", err) + } + return err +} + +func computeHashes(cctx *cli.Context) ([]string, error) { + hashes := cctx.StringSlice(lib.HashListFlag.Name) + if len(hashes) == 0 { + return nil, nil + } + + for _, hash := range hashes { + if _, err := hex.DecodeString(hash); err != nil { + return nil, fmt.Errorf("decoding hash %q: %w", hash, err) + } + } + + return hashes, nil +} + +var clientCmd = &cli.Command{ + Name: "client", + Flags: lib.ClientFlags, + Action: func(cctx *cli.Context) error { + lg := log.New(nil, log.DefaultLevel, false) + cctx.Context = log.ToContext(cctx.Context, lg) + if cctx.IsSet(lib.GroupConfListFlag.Name) { + groupConfs := cctx.StringSlice(lib.GroupConfListFlag.Name) + if len(groupConfs) != 1 { + return fmt.Errorf("please specify a single valid chain using the --%s flag with the client command", lib.GroupConfListFlag.Name) + } + + if cctx.IsSet(lib.GroupConfFlag.Name) { + return fmt.Errorf("please do not use both --%s and --%s at the same time", lib.GroupConfFlag.Name, lib.GroupConfListFlag.Name) + } + if err := cctx.Set(lib.GroupConfFlag.Name, groupConfs[0]); err != nil { + return fmt.Errorf("unable to set GroupConfFlag: %w", err) + } + } + c, err := lib.Create(cctx, false) + if err != nil { + return fmt.Errorf("constructing client: %w", err) + } + + for rand := range c.Watch(cctx.Context) { + lg.Infow("", "client", "got randomness", "round", rand.GetRound(), "signature", hex.EncodeToString(rand.GetSignature())) + } + + return nil + }, +} + +var idCmd = &cli.Command{ + Name: "peerid", + Usage: "prints the libp2p peer ID or creates one if it does not exist", + Flags: []cli.Flag{idFlag}, + Action: func(cctx *cli.Context) error { + lg := log.New(nil, log.DefaultLevel, false) + priv, err := lp2p.LoadOrCreatePrivKey(cctx.String(idFlag.Name), lg) + if err != nil { + return fmt.Errorf("loading p2p key: %w", err) + } + peerID, err := peer.IDFromPrivateKey(priv) + if err != nil { + return fmt.Errorf("computing peerid: %w", err) + } + fmt.Printf("%s\n", peerID) + return nil + }, +} diff --git a/internal/cli.go b/internal/cli.go new file mode 100644 index 0000000..da66262 --- /dev/null +++ b/internal/cli.go @@ -0,0 +1,144 @@ +package drand + +import ( + "fmt" + "log" + "strconv" + "sync" + + json "github.com/nikkolasg/hexjson" + "github.com/urfave/cli/v2" + + client "github.com/drand/drand/v2/common/client" + + "github.com/drand/drand-cli/internal/lib" + "github.com/drand/drand/v2/common" +) + +// Automatically set through -ldflags +// Example: go install -ldflags "-X main.buildDate=$(date -u +%d/%m/%Y@%H:%M:%S) -X main.gitCommit=$(git rev-parse HEAD)" +var ( + gitCommit = "none" + buildDate = "unknown" +) + +var SetVersionPrinter sync.Once + +var appCommands = []*cli.Command{ + { + Name: "get", + Usage: "get allows for public information retrieval from a remote " + + "drand http-relay.\n", + Subcommands: []*cli.Command{ + { + Name: "public", + Usage: "Get the latest public randomness from the drand " + + "relay and verify it against the collective public key " + + "as specified in the chain-info.\n", + Flags: toArray(lib.URLFlag, lib.JSONFlag, lib.InsecureFlag, lib.HashListFlag, lib.VerboseFlag), + ArgsUsage: "--url url1 --url url2 ROUND... uses the first working relay to query round number ROUND", + Action: getPublicRandomness, + }, + { + Name: "chain-info", + Usage: "Get beacon information", + ArgsUsage: "--url url1 --url url2 ... uses the first working relay", + Flags: toArray(lib.URLFlag, lib.JSONFlag, lib.InsecureFlag, lib.HashListFlag, lib.VerboseFlag), + Action: getChainInfo, + }, + }, + }, +} + +// CLI runs the drand app +func CLI() *cli.App { + version := common.GetAppVersion() + + app := cli.NewApp() + app.Name = "drand-client" + + // See https://cli.urfave.org/v2/examples/bash-completions/#enabling for how to turn on. + app.EnableBashCompletion = true + + SetVersionPrinter.Do(func() { + cli.VersionPrinter = func(c *cli.Context) { + fmt.Fprintf(c.App.Writer, "drand %s (date %v, commit %v)\n", version, buildDate, gitCommit) + } + }) + + app.ExitErrHandler = func(_ *cli.Context, _ error) { + // override to prevent default behavior of calling OS.exit(1), + // when tests expect to be able to run multiple commands. + } + app.Version = version.String() + app.Usage = "distributed randomness service" + // =====Commands===== + // we need to copy the underlying commands to avoid races, cli sadly doesn't support concurrent executions well + appComm := make([]*cli.Command, len(appCommands)) + for i, p := range appCommands { + if p == nil { + continue + } + v := *p + appComm[i] = &v + } + app.Commands = appComm + + return app +} + +func toArray(flags ...cli.Flag) []cli.Flag { + return flags +} + +func instantiateClient(cctx *cli.Context) (client.Client, error) { + c, err := lib.Create(cctx, false) + if err != nil { + return nil, fmt.Errorf("constructing client: %w", err) + } + + _, err = c.Info(cctx.Context) + if err != nil { + return nil, fmt.Errorf("cannot retrieve chain info from relay: %w", err) + } + + return c, nil +} + +func getPublicRandomness(cctx *cli.Context) error { + c, err := instantiateClient(cctx) + if err != nil { + return err + } + if cctx.Args().Len() > 1 { + log.Fatal("please specify a single round as positional argument") + } + + var r uint64 + if val := cctx.Args().Get(0); val != "" { + r, err = strconv.ParseUint(val, 10, 64) + if err != nil { + return err + } + } + + round, err := c.Get(cctx.Context, r) + if err != nil { + return err + } + return json.NewEncoder(cctx.App.Writer).Encode(round) +} + +func getChainInfo(cctx *cli.Context) error { + c, err := instantiateClient(cctx) + if err != nil { + return err + } + + info, err := c.Info(cctx.Context) + if err != nil { + return err + } + + return info.ToJSON(cctx.App.Writer, nil) +} diff --git a/internal/cli_test.go b/internal/cli_test.go new file mode 100644 index 0000000..6625422 --- /dev/null +++ b/internal/cli_test.go @@ -0,0 +1,36 @@ +package drand + +import ( + "bytes" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestClientTLS(t *testing.T) { + addr := "https://api.drand.sh" + + chainInfoCmd := []string{"drand", "get", "chain-info", "--url", addr, "--insecure"} + expectedInOutput := "868f005eb8e6e4ca0a47c8a77ceaa5309a47978a7c71bc5cce96366b5d7a569937c529eeda66c7293784a9402801af31" + testCommand(t, chainInfoCmd, expectedInOutput) + + showHash := []string{"drand", "get", "public", "--url", addr, "--insecure", "123"} + round1 := "0e4f538534f426203a4089154ff31527b9c25b37f9d6704b3ecba8d74678b4e3" + testCommand(t, showHash, round1) +} + +func testCommand(t *testing.T, args []string, exp string) { + t.Helper() + + var buff bytes.Buffer + t.Log("--------------") + cli := CLI() + cli.Writer = &buff + require.NoError(t, cli.Run(args)) + if exp == "" { + return + } + t.Logf("RUNNING: %v\n", args) + require.Contains(t, strings.Trim(buff.String(), "\n"), exp) +} diff --git a/internal/grpc/client.go b/internal/grpc/client.go new file mode 100644 index 0000000..7b30b26 --- /dev/null +++ b/internal/grpc/client.go @@ -0,0 +1,143 @@ +package grpc + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "time" + + grpcProm "github.com/grpc-ecosystem/go-grpc-prometheus" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + grpcInsec "google.golang.org/grpc/credentials/insecure" + + "github.com/drand/drand/v2/crypto" + + localClient "github.com/drand/drand-cli/client" + commonutils "github.com/drand/drand/v2/common" + "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" + "github.com/drand/drand/v2/protobuf/drand" +) + +const grpcDefaultTimeout = 5 * time.Second + +type grpcClient struct { + address string + chainHash []byte + client drand.PublicClient + conn *grpc.ClientConn + l log.Logger +} + +// New creates a drand client backed by a GRPC connection. +func New(address string, insecure bool, chainHash []byte) (client.Client, error) { + var opts []grpc.DialOption + if insecure { + opts = append(opts, grpc.WithTransportCredentials(grpcInsec.NewCredentials())) + } else { + opts = append(opts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}))) + } + opts = append(opts, + grpc.WithUnaryInterceptor(grpcProm.UnaryClientInterceptor), + grpc.WithStreamInterceptor(grpcProm.StreamClientInterceptor), + ) + conn, err := grpc.NewClient(address, opts...) + if err != nil { + return nil, err + } + + return &grpcClient{address, chainHash, drand.NewPublicClient(conn), conn, log.DefaultLogger()}, nil +} + +func asRD(r *drand.PublicRandResponse) *localClient.RandomData { + return &localClient.RandomData{ + Rnd: r.GetRound(), + Random: crypto.RandomnessFromSignature(r.GetSignature()), + Sig: r.GetSignature(), + PreviousSignature: r.GetPreviousSignature(), + } +} + +// String returns the name of this client. +func (g *grpcClient) String() string { + return fmt.Sprintf("GRPC(%q)", g.address) +} + +// Get returns a the randomness at `round` or an error. +func (g *grpcClient) Get(ctx context.Context, round uint64) (client.Result, error) { + curr, err := g.client.PublicRand(ctx, &drand.PublicRandRequest{Round: round, Metadata: g.getMetadata()}) + if err != nil { + return nil, err + } + if curr == nil { + return nil, errors.New("no received randomness - unexpected gPRC response") + } + + return asRD(curr), nil +} + +// Watch returns new randomness as it becomes available. +func (g *grpcClient) Watch(ctx context.Context) <-chan client.Result { + stream, err := g.client.PublicRandStream(ctx, &drand.PublicRandRequest{Round: 0, Metadata: g.getMetadata()}) + ch := make(chan client.Result, 1) + if err != nil { + close(ch) + return ch + } + go g.translate(stream, ch) + return ch +} + +// Info returns information about the chain. +func (g *grpcClient) Info(ctx context.Context) (*chain.Info, error) { + proto, err := g.client.ChainInfo(ctx, &drand.ChainInfoRequest{Metadata: g.getMetadata()}) + if err != nil { + return nil, err + } + if proto == nil { + return nil, errors.New("no received group - unexpected gPRC response") + } + return chain.InfoFromProto(proto) +} + +func (g *grpcClient) translate(stream drand.Public_PublicRandStreamClient, out chan<- client.Result) { + defer close(out) + for { + next, err := stream.Recv() + if err != nil || stream.Context().Err() != nil { + if stream.Context().Err() == nil { + g.l.Warnw("", "grpc_client", "public rand stream", "err", err) + } + return + } + out <- asRD(next) + } +} + +func (g *grpcClient) getMetadata() *drand.Metadata { + return &drand.Metadata{ChainHash: g.chainHash} +} + +func (g *grpcClient) RoundAt(t time.Time) uint64 { + ctx, cancel := context.WithTimeout(context.Background(), grpcDefaultTimeout) + defer cancel() + + info, err := g.client.ChainInfo(ctx, &drand.ChainInfoRequest{Metadata: g.getMetadata()}) + if err != nil { + return 0 + } + return commonutils.CurrentRound(t.Unix(), time.Second*time.Duration(info.Period), info.GenesisTime) +} + +// SetLog configures the client log output +func (g *grpcClient) SetLog(l log.Logger) { + g.l = l +} + +// Close tears down the gRPC connection and all underlying connections. +func (g *grpcClient) Close() error { + return g.conn.Close() +} diff --git a/internal/grpc/client_test.go b/internal/grpc/client_test.go new file mode 100644 index 0000000..6121bae --- /dev/null +++ b/internal/grpc/client_test.go @@ -0,0 +1,112 @@ +package grpc + +import ( + "bytes" + "context" + "sync" + "testing" + "time" + + clock "github.com/jonboulle/clockwork" + + "github.com/drand/drand/v2/common/log" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/drand/drand/v2/crypto" + "github.com/drand/drand/v2/test/mock" +) + +func TestClient(t *testing.T) { + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + l, server := mock.NewMockGRPCPublicServer(t, log.DefaultLogger(), "localhost:0", false, sch, clock.NewFakeClock()) + addr := l.Addr() + + go l.Start() + defer l.Stop(context.Background()) + + c, err := New(addr, true, []byte("")) + if err != nil { + t.Fatal(err) + } + result, err := c.Get(context.Background(), 1969) + if err != nil { + t.Fatal(err) + } + if result.GetRound() != 1969 { + t.Fatal("unexpected round.") + } + r2, err := c.Get(context.Background(), 1970) + if err != nil { + t.Fatal(err) + } + if bytes.Equal(r2.GetRandomness(), result.GetRandomness()) { + t.Fatal("unexpected equality") + } + + rat := c.RoundAt(time.Now()) + if rat == 0 { + t.Fatal("round at should function") + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + res := c.Watch(ctx) + go func() { + time.Sleep(50 * time.Millisecond) + server.(mock.Service).EmitRand(false) + }() + r3, ok := <-res + if !ok { + t.Fatal("watch should work") + } + if r3.GetRound() != 1971 { + t.Fatal("unexpected round") + } + cancel() + _ = c.Close() +} + +func TestClientClose(t *testing.T) { + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + l, _ := mock.NewMockGRPCPublicServer(t, log.DefaultLogger(), "localhost:0", false, sch, clock.NewFakeClock()) + addr := l.Addr() + + go l.Start() + defer l.Stop(context.Background()) + + c, err := New(addr, true, []byte("")) + if err != nil { + t.Fatal(err) + } + result, err := c.Get(context.Background(), 1969) + if err != nil { + t.Fatal(err) + } + if result.GetRound() != 1969 { + t.Fatal("unexpected round.") + } + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + for range c.Watch(context.Background()) { + } + wg.Done() + }() + + err = c.Close() + if err != nil { + t.Fatal(err) + } + + _, err = c.Get(context.Background(), 0) + if status.Code(err) != codes.Canceled { + t.Fatal("unexpected error from closed client", err) + } + + wg.Wait() // wait for the watch to close +} diff --git a/internal/grpc/doc.go b/internal/grpc/doc.go new file mode 100644 index 0000000..f61c586 --- /dev/null +++ b/internal/grpc/doc.go @@ -0,0 +1,39 @@ +/* +Package grpc provides a drand client implementation that uses drand's gRPC API. + +The client connects to a drand gRPC endpoint to fetch randomness. The gRPC +client has some advantages over the HTTP client - it is more compact +on-the-wire and supports streaming and authentication. + +Example: + + package main + + import ( + "encoding/hex" + + "github.com/drand/drand/v2/client" + "github.com/drand/drand/v2/client/grpc" + ) + + const ( + grpcAddr = "example.drand.grpc.server:4444" + certPath = "/path/to/drand-grpc.cert" + ) + + var chainHash, _ = hex.DecodeString("8990e7a9aaed2ffed73dbd7092123d6f289930540d7651336225dc172e51b2ce") + + func main() { + gc, err := grpc.New(grpcAddr, certPath, false) + + c, err := client.New( + client.From(gc), + client.WithChainHash(chainHash), + ) + } + +A path to a file that holds TLS credentials for the drand server is required +to validate server connections. Alternatively set the final parameter to +`true` to enable _insecure_ connections (not recommended). +*/ +package grpc diff --git a/internal/lib/cli.go b/internal/lib/cli.go new file mode 100644 index 0000000..4f5a5fc --- /dev/null +++ b/internal/lib/cli.go @@ -0,0 +1,379 @@ +package lib + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "net" + nhttp "net/http" + "os" + "path" + "strings" + + "github.com/BurntSushi/toml" + "github.com/google/uuid" + clock "github.com/jonboulle/clockwork" + pubsub "github.com/libp2p/go-libp2p-pubsub" + ma "github.com/multiformats/go-multiaddr" + "github.com/urfave/cli/v2" + + "github.com/drand/drand/v2/common/key" + + pubClient "github.com/drand/drand-cli/client" + http2 "github.com/drand/drand-cli/client/http" + gclient "github.com/drand/drand-cli/client/lp2p" + "github.com/drand/drand-cli/internal/grpc" + "github.com/drand/drand-cli/internal/lp2p" + commonutils "github.com/drand/drand/v2/common" + chainCommon "github.com/drand/drand/v2/common/chain" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" +) + +var ( + // URLFlag is the CLI flag for root URL(s) for fetching randomness. + URLFlag = &cli.StringSliceFlag{ + Name: "url", + Usage: "root URL(s) for fetching randomness", + } + // GRPCConnectFlag is the CLI flag for host:port to dial a gRPC randomness + // provider. + GRPCConnectFlag = &cli.StringFlag{ + Name: "grpc-connect", + Usage: "host:port to dial a gRPC randomness provider", + } + // HashFlag is the CLI flag for the hash (in hex) of the targeted chain. + HashFlag = &cli.StringFlag{ + Name: "hash", + Usage: "The hash (in hex) of the chain to follow. Deprecated and replaced by hash-list to support multiple chains", + Aliases: []string{"chain-hash"}, + Hidden: true, + } + // HashListFlag is the CLI flag for the hashes list (in hex) for the relay to follow. + HashListFlag = &cli.StringSliceFlag{ + Name: "hash-list", + Usage: "Specify the list (in hex) of hashes the relay should follow", + } + // GroupConfFlag is the CLI flag for specifying the path to the drand group configuration (TOML encoded) or chain info (JSON encoded). + GroupConfFlag = &cli.PathFlag{ + Name: "group-conf", + Usage: "Path to a drand group configuration (TOML encoded) or chain info (JSON encoded)," + + " can be used instead of `-hash` flag to verify the chain. Deprecated and replaced by group-conf-list to support multiple chains", + Hidden: true, + } + // GroupConfListFlag is like GroupConfFlag but for a list values. + GroupConfListFlag = &cli.StringSliceFlag{ + Name: "group-conf-list", + Usage: "Paths to at least one drand group configuration (TOML encoded) or chain info (JSON encoded)," + + fmt.Sprintf(" can be used instead of `-%s` flag to verify the chain.", HashListFlag.Name), + } + // InsecureFlag is the CLI flag to allow autodetection of the chain + // information. + InsecureFlag = &cli.BoolFlag{ + Name: "insecure", + Usage: "Allow autodetection of the chain information", + } + // RelayFlag is the CLI flag for relay peer multiaddr(s) to connect with. + RelayFlag = &cli.StringSliceFlag{ + Name: "relay", + Usage: "relay peer multiaddr(s) to connect with", + } + // PortFlag is the CLI flag for local address for client to bind to, when + // connecting to relays. (specified as a numeric port, or a host:port) + PortFlag = &cli.StringFlag{ + Name: "port", + Usage: "Local (host:)port for constructed libp2p host to listen on", + } + + // JSONFlag is the value of the CLI flag `json` enabling JSON output of the loggers + JSONFlag = &cli.BoolFlag{ + Name: "json", + Usage: "Set the output as json format", + } + + VerboseFlag = &cli.BoolFlag{ + Name: "verbose", + Usage: "If set, verbosity is at the debug level", + EnvVars: []string{"DRAND_VERBOSE"}, + } +) + +// ClientFlags is a list of common flags for client creation +var ClientFlags = []cli.Flag{ + URLFlag, + HashFlag, + HashListFlag, + GroupConfListFlag, + GroupConfFlag, + InsecureFlag, + RelayFlag, + JSONFlag, + VerboseFlag, +} + +// Create builds a client, and can be invoked from a cli action supplied +// with ClientFlags +// +//nolint:gocyclo +func Create(c *cli.Context, withInstrumentation bool, opts ...pubClient.Option) (client.Client, error) { + ctx := c.Context + clients := make([]client.Client, 0) + var level int + if c.Bool(VerboseFlag.Name) { + level = log.DebugLevel + } else { + level = log.WarnLevel + } + l := log.New(nil, level, false) + + var info *chainCommon.Info + var err error + var hash []byte + if groupPath := c.Path(GroupConfFlag.Name); groupPath != "" { + l.Debugw("parsing group-conf file") + info, err = chainInfoFromGroupTOML(groupPath) + if err != nil { + l.Infow("Got a group conf file that is not a toml file. Trying it as a ChainInfo json file.", "path", groupPath) + info, err = chainInfoFromChainInfoJSON(groupPath) + if info == nil || err != nil { + return nil, fmt.Errorf("failed to decode group (%s) : %w", groupPath, err) + } + } + l.Debugw("parsing group-conf file, successful") + + opts = append(opts, pubClient.WithChainInfo(info)) + } + + if info != nil { + hash = info.Hash() + } + + grc, info, err := buildGrpcClient(c, info) + if err != nil { + return nil, err + } + if len(grc) > 0 { + clients = append(clients, grc...) + } + l.Debugw("built GRPC Client", "successful", len(grc)) + + if c.String(HashFlag.Name) != "" { + hash, err = hex.DecodeString(c.String(HashFlag.Name)) + if err != nil { + return nil, err + } + if info != nil && !bytes.Equal(hash, info.Hash()) { + return nil, fmt.Errorf( + "%w for beacon %s %v != %v", + commonutils.ErrInvalidChainHash, + info.ID, + c.String(HashFlag.Name), + hex.EncodeToString(info.Hash()), + ) + } + opts = append(opts, pubClient.WithChainHash(hash)) + } + + if c.Bool(InsecureFlag.Name) { + opts = append(opts, pubClient.Insecurely()) + } + + gc, info, err := buildHTTPClients(c, l, hash, withInstrumentation) + if err != nil { + return nil, err + } + if len(gc) > 0 { + clients = append(clients, gc...) + } + l.Debugw("built HTTP Client", "successful", len(gc)) + + if info != nil && hash != nil && !bytes.Equal(hash, info.Hash()) { + return nil, fmt.Errorf( + "%w for beacon %s : expected %v != info %v", + commonutils.ErrInvalidChainHash, + info.ID, + hex.EncodeToString(hash), + hex.EncodeToString(info.Hash()), + ) + } + + gopt, err := buildGossipClient(c, l) + if err != nil { + return nil, err + } + opts = append(opts, gopt...) + + return pubClient.Wrap(ctx, l, clients, opts...) +} + +func buildGrpcClient(c *cli.Context, info *chainCommon.Info) ([]client.Client, *chainCommon.Info, error) { + if !c.IsSet(GRPCConnectFlag.Name) { + return nil, info, nil + } + + var hash []byte + if c.IsSet(HashFlag.Name) { + var err error + + hash, err = hex.DecodeString(c.String(HashFlag.Name)) + if err != nil { + return nil, nil, err + } + } + + if info != nil && len(hash) == 0 { + hash = info.Hash() + } + + gc, err := grpc.New(c.String(GRPCConnectFlag.Name), c.Bool(InsecureFlag.Name), hash) + if err != nil { + return nil, nil, err + } + + if info == nil { + info, err = gc.Info(c.Context) + if err != nil { + return nil, nil, err + } + } + + return []client.Client{gc}, info, nil +} + +func buildHTTPClients(c *cli.Context, l log.Logger, hash []byte, withInstrumentation bool) ([]client.Client, *chainCommon.Info, error) { + ctx := c.Context + clients := make([]client.Client, 0) + var err error + var skipped []string + var hc client.Client + var info *chainCommon.Info + + urls := c.StringSlice(URLFlag.Name) + + l.Infow("Building HTTP clients", "hash", len(hash), "urls", len(urls)) + + // we return an empty list if no URLs were provided + if len(urls) == 0 { + return clients, nil, nil + } + + for _, url := range urls { + l.Debugw("trying to instantiate http client", "url", url) + hc, err = http2.New(ctx, l, url, hash, nhttp.DefaultTransport) + if err != nil { + l.Warnw("", "client", "failed to load URL", "url", url, "err", err) + skipped = append(skipped, url) + continue + } + info, err = hc.Info(ctx) + if err != nil { + l.Warnw("", "client", "failed to load Info from URL", "url", url, "err", err) + continue + } + + clients = append(clients, hc) + } + + // do we want to error out or not if all provided URL failed to instantiate a client? + if len(skipped) == len(c.StringSlice(URLFlag.Name)) { + return nil, nil, errors.New("all URLs failed to be used for creating a http client") + } + + if info != nil { + if hash != nil && !bytes.Equal(hash, info.Hash()) { + l.Warnw("mismatch between retrieved chain info hash and provided hash", "chainInfo", info.Hash(), "provided", hash) + return nil, nil, errors.New("mismatch between retrieved chain info and provided hash") + } + + // we re-try dialing the skipped remotes, just in case, but that's the last time, we won't be dialing these again + // later in case they fail. + for _, url := range skipped { + hc, err = http2.NewWithInfo(l, url, info, nhttp.DefaultTransport) + if err != nil { + l.Warnw("", "client", "failed to load URL again", "url", url, "err", err) + continue + } + clients = append(clients, hc) + } + } + + if withInstrumentation { + http2.MeasureHeartbeats(c.Context, clients) + } + + return clients, info, nil +} + +func buildGossipClient(c *cli.Context, l log.Logger) ([]pubClient.Option, error) { + if c.IsSet(RelayFlag.Name) { + addrs := c.StringSlice(RelayFlag.Name) + if len(addrs) > 0 { + relayPeers, err := lp2p.ParseMultiaddrSlice(addrs) + if err != nil { + return nil, err + } + listen := "" + if c.IsSet(PortFlag.Name) { + listen = c.String(PortFlag.Name) + } + ps, err := buildClientHost(l, listen, relayPeers) + if err != nil { + return nil, err + } + return []pubClient.Option{gclient.WithPubsub(l, ps, clock.NewRealClock(), gclient.DefaultBufferSize)}, nil + } + } + return []pubClient.Option{}, nil +} + +func buildClientHost(l log.Logger, clientListenAddr string, relayMultiaddr []ma.Multiaddr) (*pubsub.PubSub, error) { + clientID := uuid.New().String() + priv, err := lp2p.LoadOrCreatePrivKey(path.Join(os.TempDir(), "drand-"+clientID+"-id"), l) + if err != nil { + return nil, err + } + + listen := "" + if clientListenAddr != "" { + bindHost := "0.0.0.0" + if strings.Contains(clientListenAddr, ":") { + host, port, err := net.SplitHostPort(clientListenAddr) + if err != nil { + return nil, err + } + bindHost = host + clientListenAddr = port + } + listen = fmt.Sprintf("/ip4/%s/tcp/%s", bindHost, clientListenAddr) + } + + _, ps, err := lp2p.ConstructHost(priv, listen, relayMultiaddr, l) + if err != nil { + return nil, err + } + return ps, nil +} + +// chainInfoFromGroupTOML reads a drand group TOML file and returns the chain info. +func chainInfoFromGroupTOML(filePath string) (*chainCommon.Info, error) { + gt := &key.GroupTOML{} + _, err := toml.DecodeFile(filePath, gt) + if err != nil { + return nil, err + } + g := &key.Group{} + err = g.FromTOML(gt) + if err != nil { + return nil, err + } + return chainCommon.NewChainInfo(g), nil +} + +func chainInfoFromChainInfoJSON(filePath string) (*chainCommon.Info, error) { + b, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + return chainCommon.InfoFromJSON(bytes.NewBuffer(b)) +} diff --git a/internal/lib/cli_test.go b/internal/lib/cli_test.go new file mode 100644 index 0000000..9515cdd --- /dev/null +++ b/internal/lib/cli_test.go @@ -0,0 +1,159 @@ +package lib + +import ( + "bytes" + "encoding/hex" + "errors" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + clock "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v2" + + "github.com/drand/drand-cli/client" + httpmock "github.com/drand/drand-cli/client/test/http/mock" + commonutils "github.com/drand/drand/v2/common" + "github.com/drand/drand/v2/common/log" + "github.com/drand/drand/v2/crypto" +) + +var ( + opts []client.Option +) + +const ( + fakeGossipRelayAddr = "/ip4/8.8.8.8/tcp/9/p2p/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx" + fakeChainHash = "6093f9e4320c285ac4aab50ba821cd5678ec7c5015d3d9d11ef89e2a99741e83" +) + +func mockAction(c *cli.Context) error { + _, err := Create(c, false, opts...) + return err +} + +func run(l log.Logger, args []string) error { + app := cli.NewApp() + app.Name = "mock-client" + app.Flags = ClientFlags + app.Action = func(c *cli.Context) error { + c.Context = log.ToContext(c.Context, l) + return mockAction(c) + } + + return app.Run(args) +} + +func TestClientLib(t *testing.T) { + opts = []client.Option{} + lg := log.New(nil, log.DebugLevel, true) + err := run(lg, []string{"mock-client"}) + if err == nil { + t.Fatal("need to specify a connection method.", err) + } + + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + clk := clock.NewFakeClockAt(time.Now()) + addr, info, cancel, _ := httpmock.NewMockHTTPPublicServer(t, false, sch, clk) + defer cancel() + + t.Log("Started mockserver at", addr) + + args := []string{"mock-client", "--url", "http://" + addr, "--insecure"} + err = run(lg, args) + if err != nil { + t.Fatal("HTTP should work. err:", err) + } + + args = []string{"mock-client", "--url", "https://" + addr} + err = run(lg, args) + if err == nil { + t.Fatal("http-relay needs insecure or hash", err) + } + + args = []string{"mock-client", "--url", "http://" + addr, "--hash", hex.EncodeToString(info.Hash())} + err = run(lg, args) + if err != nil { + t.Fatal("http-relay should construct", err) + } + + args = []string{"mock-client", "--relay", fakeGossipRelayAddr} + err = run(lg, args) + if err == nil { + t.Fatal("relays need URL to get chain info and hash", err) + } + + args = []string{"mock-client", "--relay", fakeGossipRelayAddr, "--hash", hex.EncodeToString(info.Hash())} + err = run(lg, args) + if err == nil { + t.Fatal("relays need URL to get chain info and hash", err) + } + + args = []string{"mock-client", "--url", "http://" + addr, "--relay", fakeGossipRelayAddr, "--hash", hex.EncodeToString(info.Hash())} + err = run(lg, args) + if err != nil { + t.Fatal("unable to get relay to work", err) + } +} + +func TestClientLibGroupConfTOML(t *testing.T) { + lg := log.New(nil, log.DebugLevel, true) + err := run(lg, []string{"mock-client", "--relay", fakeGossipRelayAddr, "--group-conf", groupTOMLPath()}) + if err != nil { + t.Fatal(err) + } +} + +func TestClientLibGroupConfJSON(t *testing.T) { + lg := log.New(nil, log.DebugLevel, true) + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + clk := clock.NewFakeClockAt(time.Now()) + + addr, info, cancel, _ := httpmock.NewMockHTTPPublicServer(t, false, sch, clk) + defer cancel() + + var b bytes.Buffer + require.NoError(t, info.ToJSON(&b, nil)) + + infoPath := filepath.Join(t.TempDir(), "info.json") + + err = os.WriteFile(infoPath, b.Bytes(), 0644) + if err != nil { + t.Fatal(err) + } + + err = run(lg, []string{"mock-client", "--url", "http://" + addr, "--group-conf", infoPath}) + if err != nil { + t.Fatal(err) + } +} + +func TestClientLibChainHashOverrideError(t *testing.T) { + lg := log.New(nil, log.DebugLevel, true) + err := run(lg, []string{ + "mock-client", + "--relay", + fakeGossipRelayAddr, + "--group-conf", + groupTOMLPath(), + "--hash", + fakeChainHash, + }) + if !errors.Is(err, commonutils.ErrInvalidChainHash) { + t.Log(fakeChainHash) + t.Fatal("expected error from mismatched chain hashes. Got: ", err) + } +} + +func groupTOMLPath() string { + _, file, _, ok := runtime.Caller(0) + if !ok { + return "" + } + return filepath.Join(filepath.Dir(file), "..", "..", "internal", "test", "default.toml") +} diff --git a/internal/lp2p/addrutil.go b/internal/lp2p/addrutil.go new file mode 100644 index 0000000..2f65f74 --- /dev/null +++ b/internal/lp2p/addrutil.go @@ -0,0 +1,77 @@ +package lp2p + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/transport" + ma "github.com/multiformats/go-multiaddr" + madns "github.com/multiformats/go-multiaddr-dns" +) + +const ( + dnsResolveTimeout = 10 * time.Second +) + +// resolveAddresses resolves addresses in parallel +func resolveAddresses(ctx context.Context, addrs []ma.Multiaddr, resolver transport.Resolver) ([]peer.AddrInfo, error) { + ctx, cancel := context.WithTimeout(ctx, dnsResolveTimeout) + defer cancel() + + if resolver == nil { + resolver = madns.DefaultResolver + } + + var maddrs []ma.Multiaddr //nolint:prealloc + var wg sync.WaitGroup + resolveErrC := make(chan error, len(addrs)) + + maddrC := make(chan ma.Multiaddr) + + for _, addr := range addrs { + addr := addr + // check whether address ends in `ipfs/Qm...` + if _, last := ma.SplitLast(addr); last.Protocol().Code == ma.P_IPFS { + maddrs = append(maddrs, addr) + continue + } + wg.Add(1) + go func(maddr ma.Multiaddr) { + defer wg.Done() + raddrs, err := resolver.Resolve(ctx, maddr) + if err != nil { + resolveErrC <- fmt.Errorf("failed to resolve \"%s\": %w)", maddr, err) + return + } + // filter out addresses that still doesn't end in `ipfs/Qm...` + found := 0 + for _, raddr := range raddrs { + if _, last := ma.SplitLast(raddr); last != nil && last.Protocol().Code == ma.P_P2P { + maddrC <- raddr + found++ + } + } + if found == 0 { + resolveErrC <- fmt.Errorf("found no ipfs peers at %s", maddr) + } + }(addr) + } + go func() { + wg.Wait() + close(maddrC) + }() + + for maddr := range maddrC { + maddrs = append(maddrs, maddr) + } + + select { + case err := <-resolveErrC: + return nil, err + default: + } + return peer.AddrInfosFromP2pAddrs(maddrs...) +} diff --git a/internal/lp2p/addrutil_test.go b/internal/lp2p/addrutil_test.go new file mode 100644 index 0000000..5d2dc21 --- /dev/null +++ b/internal/lp2p/addrutil_test.go @@ -0,0 +1,109 @@ +package lp2p + +import ( + "context" + "errors" + "net" + "strings" + "testing" + + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" + madns "github.com/multiformats/go-multiaddr-dns" + "github.com/stretchr/testify/require" +) + +const ( + peer0 = "12D3KooW9rwsuYZdWMZfu4hsog3rcDH9okeB9ayAWGzESLwpca78" + peer1 = "12D3KooW9uKWKaPxUSxJySsu2YB3PxaFaXYu8KSZuvLfQxYPu8jj" + dnsaddr0 = "/dnsaddr/example0.com" + dnsaddr1 = "/dnsaddr/example1.com" + p2pIP4Addr0 = "/ip4/192.168.0.1/tcp/44544/p2p/" + peer0 + p2pIP6Addr0 = "/ip6/2001:db8::a3/tcp/44544/p2p/" + peer0 + p2pIP4Addr1 = "/ip4/10.10.10.10/tcp/44544/p2p/" + peer1 + notP2PAddr1 = "/ip4/10.10.10.10/tcp/80" +) + +func mockResolver(t *testing.T, txtRecords map[string][]string) *madns.Resolver { + mock := &madns.MockResolver{ + IP: map[string][]net.IPAddr{}, + TXT: txtRecords, + } + resolver, err := madns.NewResolver(madns.WithDefaultResolver(mock)) + require.NoError(t, err) + return resolver +} + +func findPeer(t *testing.T, ais []peer.AddrInfo, peerIDStr string) peer.AddrInfo { + t.Helper() + peerID, err := peer.Decode(peerIDStr) + if err != nil { + t.Fatal(err) + } + for _, ai := range ais { + if ai.ID == peerID { + return ai + } + } + t.Fatal("not found", peerID) + return peer.AddrInfo{} +} + +func TestResolveDNS(t *testing.T) { + addrs := []multiaddr.Multiaddr{ + multiaddr.StringCast(dnsaddr0), + multiaddr.StringCast(dnsaddr1), + } + txtRecords := map[string][]string{ + "_dnsaddr.example0.com": {"dnsaddr=" + p2pIP4Addr0, "dnsaddr=" + p2pIP6Addr0}, + "_dnsaddr.example1.com": {"dnsaddr=" + p2pIP4Addr1, "dnsaddr=" + notP2PAddr1}, + } + ais, err := resolveAddresses(context.Background(), addrs, mockResolver(t, txtRecords)) + if err != nil { + t.Fatal(err) + } + if len(ais) != 2 { + t.Fatal("expected 2 peers", len(ais)) + } + peer0Info := findPeer(t, ais, peer0) + if len(peer0Info.Addrs) != 2 { + t.Fatal("expected 2 addrs for peer", peer0, peer0Info.Addrs) + } + peer1Info := findPeer(t, ais, peer1) + if len(peer1Info.Addrs) != 1 { + t.Fatal("expected 1 addr for peer", peer1, peer1Info.Addrs) + } +} + +func TestResolveDNSNoAddrs(t *testing.T) { + addrs := []multiaddr.Multiaddr{multiaddr.StringCast(dnsaddr0)} + txtRecords := map[string][]string{"_dnsaddr.example0.com": {}} + _, err := resolveAddresses(context.Background(), addrs, mockResolver(t, txtRecords)) + if err == nil { + t.Fatal("expected error, received no error") + } + if !strings.HasPrefix(err.Error(), "found no ipfs peers at") { + t.Fatal("unexpected error", err) + } +} + +type failBackend struct{} + +func (fb *failBackend) LookupIPAddr(context.Context, string) ([]net.IPAddr, error) { + return nil, errors.New("failBackend") +} +func (fb *failBackend) LookupTXT(context.Context, string) ([]string, error) { + return nil, errors.New("failBackend") +} + +func TestResolveDNSFailure(t *testing.T) { + addrs := []multiaddr.Multiaddr{multiaddr.StringCast(dnsaddr0)} + resolver, _ := madns.NewResolver(madns.WithDefaultResolver(&failBackend{})) + _, err := resolveAddresses(context.Background(), addrs, resolver) + if err == nil { + t.Fatal("expected error, received no error") + } + if !strings.Contains(err.Error(), "failBackend") { + t.Fatal("unexpected error", err) + } +} diff --git a/internal/lp2p/ctor.go b/internal/lp2p/ctor.go new file mode 100644 index 0000000..2929919 --- /dev/null +++ b/internal/lp2p/ctor.go @@ -0,0 +1,167 @@ +package lp2p + +import ( + "context" + "crypto/rand" + "encoding/base64" + "fmt" + mrand "math/rand" + "os" + "path" + "time" + + "github.com/libp2p/go-libp2p" + pubsub "github.com/libp2p/go-libp2p-pubsub" + pubsubpb "github.com/libp2p/go-libp2p-pubsub/pb" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" + "github.com/libp2p/go-libp2p/p2p/net/connmgr" + "github.com/libp2p/go-libp2p/p2p/security/noise" + libp2ptls "github.com/libp2p/go-libp2p/p2p/security/tls" + ma "github.com/multiformats/go-multiaddr" + "github.com/pkg/errors" + "golang.org/x/crypto/blake2b" + + dlog "github.com/drand/drand/v2/common/log" +) + +const ( + // userAgent sets the libp2p user-agent which is sent along with the identify protocol. + userAgent = "drand-relay/0.0.0" + // directConnectTicks makes pubsub check it's connected to direct peers every N seconds. + directConnectTicks uint64 = 5 + lowWater = 50 + highWater = 200 + gracePeriod = time.Minute + bootstrapTimeout = 5 * time.Second + allDirPerm = 0755 + identityFilePerm = 0600 +) + +// PubSubTopic generates a drand pubsub topic from a chain hash. +func PubSubTopic(h string) string { + return fmt.Sprintf("/drand/pubsub/v0.0.0/%s", h) +} + +// ConstructHost build a libp2p host configured for relaying drand randomness over pubsub. +func ConstructHost(priv crypto.PrivKey, listenAddr string, bootstrap []ma.Multiaddr, log dlog.Logger) (host.Host, *pubsub.PubSub, error) { + ctx := context.Background() + + pstore, err := pstoremem.NewPeerstore() + if err != nil { + return nil, nil, fmt.Errorf("creating peerstore: %w", err) + } + peerID, err := peer.IDFromPrivateKey(priv) + if err != nil { + return nil, nil, fmt.Errorf("computing peerid: %w", err) + } + err = pstore.AddPrivKey(peerID, priv) + if err != nil { + return nil, nil, fmt.Errorf("adding priv to keystore: %w", err) + } + + addrInfos, err := resolveAddresses(ctx, bootstrap, nil) + if err != nil { + return nil, nil, fmt.Errorf("parsing addrInfos: %w", err) + } + + cmgr, err := connmgr.NewConnManager(lowWater, highWater, connmgr.WithGracePeriod(gracePeriod)) + if err != nil { + return nil, nil, fmt.Errorf("constructing connmanager: %w", err) + } + + opts := []libp2p.Option{ + libp2p.Identity(priv), + libp2p.ChainOptions( + libp2p.Security(libp2ptls.ID, libp2ptls.New), + libp2p.Security(noise.ID, noise.New)), + libp2p.DisableRelay(), + libp2p.Peerstore(pstore), + libp2p.UserAgent(userAgent), + libp2p.ConnectionManager(cmgr), + } + + if listenAddr != "" { + opts = append(opts, libp2p.ListenAddrStrings(listenAddr)) + } else { + opts = append(opts, libp2p.NoListenAddrs) + } + + h, err := libp2p.New(opts...) + if err != nil { + return nil, nil, fmt.Errorf("constructing host: %w", err) + } + + p, err := pubsub.NewGossipSub(ctx, h, + pubsub.WithPeerExchange(true), + pubsub.WithMessageIdFn(func(pmsg *pubsubpb.Message) string { + hash := blake2b.Sum256(pmsg.Data) + return string(hash[:]) + }), + pubsub.WithDirectPeers(addrInfos), + pubsub.WithFloodPublish(true), + pubsub.WithDirectConnectTicks(directConnectTicks), + ) + if err != nil { + return nil, nil, fmt.Errorf("constructing pubsub: %w", err) + } + + go func() { + mrand.Shuffle(len(addrInfos), func(i, j int) { + addrInfos[i], addrInfos[j] = addrInfos[j], addrInfos[i] + }) + for _, ai := range addrInfos { + ctx, cancel := context.WithTimeout(ctx, bootstrapTimeout) + err := h.Connect(ctx, ai) + cancel() + if err != nil { + log.Warnw("", "construct_host", "could not bootstrap", "addr", ai) + } + } + }() + return h, p, nil +} + +// LoadOrCreatePrivKey loads a base64 encoded libp2p private key from a file or creates one if it does not exist. +func LoadOrCreatePrivKey(identityPath string, log dlog.Logger) (crypto.PrivKey, error) { + privB64, err := os.ReadFile(identityPath) + + var priv crypto.PrivKey + switch { + case err == nil: + privBytes, err := base64.RawStdEncoding.DecodeString(string(privB64)) + if err != nil { + return nil, fmt.Errorf("decoding base64 key: %w", err) + } + priv, err = crypto.UnmarshalEd25519PrivateKey(privBytes) + if err != nil { + return nil, fmt.Errorf("unmarshaling ed25519 key: %w", err) + } + log.Infow("", "load_or_create_priv_key", "loaded private key") + + case errors.Is(err, os.ErrNotExist): + priv, _, err = crypto.GenerateEd25519Key(rand.Reader) + if err != nil { + return nil, fmt.Errorf("generating private key: %w", err) + } + b, err := priv.Raw() + if err != nil { + return nil, fmt.Errorf("marshaling private key: %w", err) + } + err = os.MkdirAll(path.Dir(identityPath), allDirPerm) + if err != nil { + return nil, fmt.Errorf("creating identity directory and parents: %w", err) + } + err = os.WriteFile(identityPath, []byte(base64.RawStdEncoding.EncodeToString(b)), identityFilePerm) + if err != nil { + return nil, fmt.Errorf("writing identity file: %w", err) + } + + default: + return nil, fmt.Errorf("getting private key: %w", err) + } + + return priv, nil +} diff --git a/internal/lp2p/ctor_test.go b/internal/lp2p/ctor_test.go new file mode 100644 index 0000000..f42ebf9 --- /dev/null +++ b/internal/lp2p/ctor_test.go @@ -0,0 +1,55 @@ +package lp2p + +import ( + "fmt" + "path" + "testing" + + "github.com/drand/drand/v2/common/log" +) + +func TestCreateThenLoadPrivKey(t *testing.T) { + dir := t.TempDir() + + // should not exist yet... + identityPath := path.Join(dir, "identify.key") + + lg := log.DefaultLogger() + priv0, err := LoadOrCreatePrivKey(identityPath, lg) + if err != nil { + t.Fatal(err) + } + + // read again, should be the same + priv1, err := LoadOrCreatePrivKey(identityPath, lg) + if err != nil { + t.Fatal(err) + } + + if !priv0.Equals(priv1) { + t.Fatal(fmt.Errorf("private key not persisted and/or not read back properly")) + } +} + +func TestCreatePrivKeyMkdirp(t *testing.T) { + dir := t.TempDir() + + // should not exist yet and has an intermediate dir that does not exist + identityPath := path.Join(dir, "not-exists-dir", "identify.key") + + lg := log.DefaultLogger() + priv0, err := LoadOrCreatePrivKey(identityPath, lg) + if err != nil { + t.Fatal(err) + } + + // read again, should be the same + priv1, err := LoadOrCreatePrivKey(identityPath, lg) + if err != nil { + t.Fatal(err) + } + + if !priv0.Equals(priv1) { + t.Fatal(fmt.Errorf("private key not persisted and/or not read back properly")) + } +} diff --git a/internal/lp2p/relaynode.go b/internal/lp2p/relaynode.go new file mode 100644 index 0000000..68563f0 --- /dev/null +++ b/internal/lp2p/relaynode.go @@ -0,0 +1,178 @@ +package lp2p + +import ( + "context" + "fmt" + "time" + + pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/host" + ma "github.com/multiformats/go-multiaddr" + "google.golang.org/protobuf/proto" + + client2 "github.com/drand/drand-cli/client" + "github.com/drand/drand/v2/common/client" + "github.com/drand/drand/v2/common/log" + "github.com/drand/drand/v2/protobuf/drand" +) + +// GossipRelayConfig configures a gossip-relay relay node. +type GossipRelayConfig struct { + // ChainHash is a hash that uniquely identifies the drand chain. + ChainHash string + PeerWith []string + Addr string + DataDir string + IdentityPath string + CertPath string + Insecure bool + Client client.Client +} + +// GossipRelayNode is a gossip-relay relay runtime. +type GossipRelayNode struct { + l log.Logger + bootstrap []ma.Multiaddr + priv crypto.PrivKey + h host.Host + ps *pubsub.PubSub + t *pubsub.Topic + addrs []ma.Multiaddr + done chan struct{} +} + +// NewGossipRelayNode starts a new gossip-relay relay node. +func NewGossipRelayNode(l log.Logger, cfg *GossipRelayConfig) (*GossipRelayNode, error) { + if cfg.Client == nil { + return nil, fmt.Errorf("no client supplying randomness supplied") + } + + bootstrap, err := ParseMultiaddrSlice(cfg.PeerWith) + if err != nil { + return nil, fmt.Errorf("parsing peer-with: %w", err) + } + + priv, err := LoadOrCreatePrivKey(cfg.IdentityPath, l) + if err != nil { + return nil, fmt.Errorf("loading p2p key: %w", err) + } + + h, ps, err := ConstructHost(priv, cfg.Addr, bootstrap, l) + if err != nil { + return nil, fmt.Errorf("constructing host: %w", err) + } + + addrs, err := h.Network().InterfaceListenAddresses() + if err != nil { + return nil, fmt.Errorf("getting InterfaceListenAddresses: %w", err) + } + + for _, a := range addrs { + l.Infow("", "relay_node", "has addr", "addr", fmt.Sprintf("%s/p2p/%s", a, h.ID())) + } + l.Infow("Joining PubSubTopic", "chainhash", cfg.ChainHash) + t, err := ps.Join(PubSubTopic(cfg.ChainHash)) + if err != nil { + return nil, fmt.Errorf("joining topic: %w", err) + } + + g := &GossipRelayNode{ + l: l, + bootstrap: bootstrap, + priv: priv, + h: h, + ps: ps, + t: t, + addrs: addrs, + done: make(chan struct{}), + } + + go g.background(cfg.Client) + + return g, nil +} + +// Multiaddrs returns the gossipsub multiaddresses of this relay node. +func (g *GossipRelayNode) Multiaddrs() []ma.Multiaddr { + base := g.h.Addrs() + b := make([]ma.Multiaddr, len(base)) + for i, a := range base { + m, err := ma.NewMultiaddr(fmt.Sprintf("%s/p2p/%s", a, g.h.ID())) + if err != nil { + panic(err) + } + b[i] = m + } + return b +} + +// Shutdown stops the relay node. +func (g *GossipRelayNode) Shutdown() { + close(g.done) +} + +// ParseMultiaddrSlice parses a list of addresses into multiaddrs +func ParseMultiaddrSlice(peers []string) ([]ma.Multiaddr, error) { + out := make([]ma.Multiaddr, len(peers)) + for i, peer := range peers { + m, err := ma.NewMultiaddr(peer) + if err != nil { + return nil, fmt.Errorf("parsing multiaddr\"%s\": %w", peer, err) + } + out[i] = m + } + return out, nil +} + +func (g *GossipRelayNode) background(w client2.Watcher) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for { + results := w.Watch(ctx) + LOOP: + for { + select { + case res, ok := <-results: + if !ok { + g.l.Warnw("", "relay_node", "watch channel closed") + break LOOP + } + + rd, ok := res.(*client2.RandomData) + if !ok { + g.l.Errorw("", "relay_node", "unexpected client result type") + continue + } + + randB, err := proto.Marshal(&drand.PublicRandResponse{ + Round: res.GetRound(), + Signature: res.GetSignature(), + PreviousSignature: rd.GetPreviousSignature(), + Randomness: res.GetRandomness(), + }) + if err != nil { + g.l.Errorw("", "relay_node", "err marshaling", "err", err) + continue + } + + g.l.Debugw("publishing message", + "relay_node", "publish", + "round", res.GetRound(), + "time.Now", time.Now().Unix(), + ) + + err = g.t.Publish(ctx, randB) + if err != nil { + g.l.Errorw("", "relay_node", "err publishing on pubsub", "err", err) + continue + } + + g.l.Infow("", "relay_node", "Published randomness on pubsub", "round", res.GetRound()) + case <-g.done: + return + } + } + time.Sleep(time.Second) + } +} diff --git a/internal/lp2p/relaynode_test.go b/internal/lp2p/relaynode_test.go new file mode 100644 index 0000000..d79db7e --- /dev/null +++ b/internal/lp2p/relaynode_test.go @@ -0,0 +1,122 @@ +package lp2p + +import ( + "context" + "encoding/hex" + "errors" + "path" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/drand/drand/v2/common/key" + "github.com/drand/drand/v2/common/log" + "github.com/drand/drand/v2/crypto" + + "github.com/drand/drand-cli/client" + "github.com/drand/drand-cli/client/test/result/mock" + "github.com/drand/drand/v2/common/chain" + client2 "github.com/drand/drand/v2/common/client" +) + +type mockClient struct { + chainInfo *chain.Info + watchF func(context.Context) <-chan client2.Result +} + +func (c *mockClient) Get(_ context.Context, _ uint64) (client2.Result, error) { + return nil, errors.New("unsupported") +} + +func (c *mockClient) Watch(ctx context.Context) <-chan client2.Result { + return c.watchF(ctx) +} + +func (c *mockClient) Info(_ context.Context) (*chain.Info, error) { + return c.chainInfo, nil +} + +func (c *mockClient) RoundAt(_ time.Time) uint64 { + return 0 +} + +func (c *mockClient) Close() error { + return nil +} + +// toRandomDataChain converts the mock results into a chain of client.RandomData +// objects. Note that you do not get back the first result. +func toRandomDataChain(results ...mock.Result) []client.RandomData { + var randomness []client.RandomData + prevSig := results[0].GetSignature() + for i := 1; i < len(results); i++ { + randomness = append(randomness, client.RandomData{ + Rnd: results[i].GetRound(), + Random: results[i].GetRandomness(), + Sig: results[i].GetSignature(), + PreviousSignature: prevSig, + }) + prevSig = results[i].GetSignature() + } + return randomness +} + +func TestWatchRetryOnClose(t *testing.T) { + sch, err := crypto.GetSchemeFromEnv() + require.NoError(t, err) + pair, err := key.NewKeyPair("fakeChainInfo.test:1234", sch) + require.NoError(t, err) + + chainInfo := &chain.Info{ + Period: time.Second, + GenesisTime: time.Now().Unix(), + PublicKey: pair.Public.Key, + } + + results := toRandomDataChain( + mock.NewMockResult(0), + mock.NewMockResult(1), + mock.NewMockResult(2), + mock.NewMockResult(3), + ) + wg := sync.WaitGroup{} + wg.Add(len(results)) + + // return a channel that writes one result then closes + watchF := func(context.Context) <-chan client2.Result { + ch := make(chan client2.Result, 1) + if len(results) > 0 { + res := results[0] + results = results[1:] + ch <- &res + wg.Done() + } + close(ch) + return ch + } + + c := &mockClient{chainInfo, watchF} + + td := t.TempDir() + lg := log.New(nil, log.DebugLevel, true) + gr, err := NewGossipRelayNode(lg, &GossipRelayConfig{ + ChainHash: hex.EncodeToString(chainInfo.Hash()), + Addr: "/ip4/0.0.0.0/tcp/0", + DataDir: td, + IdentityPath: path.Join(td, "identity.key"), + Client: c, + }) + if err != nil { + t.Fatal(err) + } + defer gr.Shutdown() + wg.Wait() + + // even though the watch channel closed, it should have been re-opened by + // the client multiple times until no results remain. + if len(results) != 0 { + t.Fatal("random data items waiting to be consumed", len(results)) + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..3670b56 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,462 @@ +package metrics + +import ( + "context" + "fmt" + "net" + "net/http" + "runtime" + "strings" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" + + common2 "github.com/drand/drand/v2/common" + "github.com/drand/drand/v2/common/log" +) + +var ( + // PrivateMetrics about the internal world (go process, private stuff) + PrivateMetrics = prometheus.NewRegistry() + // HTTPMetrics about the public surface area (http requests, cdn stuff) + HTTPMetrics = prometheus.NewRegistry() + // GroupMetrics about the group surface (grp, group-member stuff) + GroupMetrics = prometheus.NewRegistry() + // ClientMetrics about the drand client requests to servers + ClientMetrics = prometheus.NewRegistry() + + // APICallCounter (Group) how many grpc calls + APICallCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "api_call_counter", + Help: "Number of API calls that we have received", + }, []string{"api_method"}) + + // GroupDialFailures (Group) how many failures connecting outbound + GroupDialFailures = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "dial_failures", + Help: "Number of times there have been network connection issues", + }, []string{"peer_address"}) + + // OutgoingConnections (Group) how many GrpcClient connections are present + OutgoingConnections = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "outgoing_group_connections", + Help: "Number of peers with current outgoing GrpcClient connections", + }) + + GroupSize = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "group_size", + Help: "Number of peers in the current group", + }, []string{"beacon_id"}) + + GroupThreshold = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "group_threshold", + Help: "Number of shares needed for beacon reconstruction", + }, []string{"beacon_id"}) + + // BeaconDiscrepancyLatency (Group) millisecond duration between time beacon created and + // calculated time of round. + BeaconDiscrepancyLatency = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "beacon_discrepancy_latency", + Help: "Discrepancy between beacon creation time and calculated round time", + }, []string{"beacon_id"}) + + // LastBeaconRound is the most recent round (as also seen at /health) stored. + LastBeaconRound = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "last_beacon_round", + Help: "Last locally stored beacon", + }, []string{"beacon_id"}) + + // HTTPCallCounter (HTTP) how many http requests + HTTPCallCounter = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "http_call_counter", + Help: "Number of HTTP calls received", + }, []string{"code", "method"}) + // HTTPLatency (HTTP) how long http request handling takes + HTTPLatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "http_response_duration", + Help: "histogram of request latencies", + Buckets: prometheus.DefBuckets, + ConstLabels: prometheus.Labels{"handler": "http"}, + }, []string{"method"}) + // HTTPInFlight (HTTP) how many http requests exist + HTTPInFlight = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "http_in_flight", + Help: "A gauge of requests currently being served.", + }) + + // Client observation metrics + + // ClientWatchLatency measures the latency of the watch channel from the client's perspective. + ClientWatchLatency = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "client_watch_latency", + Help: "Duration between time round received and time round expected.", + }) + + // ClientHTTPHeartbeatSuccess measures the success rate of HTTP hearbeat randomness requests. + ClientHTTPHeartbeatSuccess = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "client_http_heartbeat_success", + Help: "Number of successful HTTP heartbeats.", + }, []string{"http_address"}) + + // ClientHTTPHeartbeatFailure measures the number of times HTTP heartbeats fail. + ClientHTTPHeartbeatFailure = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "client_http_heartbeat_failure", + Help: "Number of unsuccessful HTTP heartbeats.", + }, []string{"http_address"}) + + // ClientHTTPHeartbeatLatency measures the randomness latency of an HTTP source. + ClientHTTPHeartbeatLatency = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "client_http_heartbeat_latency", + Help: "Randomness latency of an HTTP source.", + }, []string{"http_address"}) + + // ClientInFlight measures how many active requests have been made + ClientInFlight = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "client_in_flight", + Help: "A gauge of in-flight drand client http requests.", + }, + []string{"url"}, + ) + + // ClientRequests measures how many total requests have been made + ClientRequests = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "client_api_requests_total", + Help: "A counter for requests from the drand client.", + }, + []string{"code", "method", "url"}, + ) + + // ClientDNSLatencyVec tracks the observed DNS resolution times + ClientDNSLatencyVec = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "client_dns_duration_seconds", + Help: "Client drand dns latency histogram.", + Buckets: []float64{.005, .01, .025, .05}, + }, + []string{"event", "url"}, + ) + + // ClientTLSLatencyVec tracks observed TLS connection times + ClientTLSLatencyVec = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "client_tls_duration_seconds", + Help: "Client drand tls latency histogram.", + Buckets: []float64{.05, .1, .25, .5}, + }, + []string{"event", "url"}, + ) + + // ClientLatencyVec tracks raw http request latencies + ClientLatencyVec = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "client_request_duration_seconds", + Help: "A histogram of client request latencies.", + Buckets: prometheus.DefBuckets, + }, + []string{"url"}, + ) + + dkgEpoch = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "dkg_epoch", + Help: "The epoch of any currently in progress or completed DKGs", + }, + []string{"beacon_id"}, + ) + + // dkgState (Group) tracks DKG status changes + dkgState = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "dkg_state", + Help: "DKG state: 0-Not Started, 1-Waiting, 2-In Progress, 3-Done, 4-Unknown, 5-Shutdown", + }, []string{"beacon_id"}) + + // DKGStateTimestamp (Group) tracks the time when the reshare status changes + dkgStateTimestamp = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "dkg_state_timestamp", + Help: "Timestamp when the DKG state last changed", + }, []string{"beacon_id"}) + + // dkgLeader (Group) tracks whether this node is the leader during DKG + dkgLeader = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "dkg_leader", + Help: "Is this node the leader during DKG? 0-false, 1-true", + }, []string{"beacon_id"}) + + // reshareState (Group) tracks reshare status changes + reshareState = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "reshare_state", + Help: "Reshare state: 0-Idle, 1-Waiting, 2-In Progress, 3-Unknown, 4-Shutdown", + }, []string{"beacon_id"}) + + // reshareStateTimestamp (Group) tracks the time when the reshare status changes + reshareStateTimestamp = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "reshare_state_timestamp", + Help: "Timestamp when the reshare state last changed", + }, []string{"beacon_id"}) + + // reshareLeader (Group) tracks whether this node is the leader during Reshare + reshareLeader = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "reshare_leader", + Help: "Is this node the leader during Reshare? 0-false, 1-true", + }, []string{"beacon_id"}) + + // drandBuildTime (Group) emits the timestamp when the binary was built in Unix time. + drandBuildTime = prometheus.NewUntypedFunc(prometheus.UntypedOpts{ + Name: "drand_build_time", + Help: "Timestamp when the binary was built in seconds since the Epoch", + ConstLabels: map[string]string{"build": common2.COMMIT, "version": common2.GetAppVersion().String()}, + }, func() float64 { return float64(getBuildTimestamp(common2.BUILDDATE)) }) + + // IsDrandNode (Group) is 1 for drand nodes, 0 for relays + IsDrandNode = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "is_drand_node", + Help: "1 for drand nodes, not emitted for relays", + }) + + // DrandStorageBackend reports the database the node is running with + DrandStorageBackend = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "drand_node_db", + Help: "The database type the node is running with. 1=bolt-trimmed, 2=postgres, 3=memdb, 4=bolt-untrimmed", + }, []string{"beaconID", "db_type"}) + + // OutgoingConnectionState (Group) tracks the state of an outgoing connection, using the states from + // https://github.com/grpc/grpc-go/blob/8075dd35d2738b352c4355b4b353dc1e9183bea7/connectivity/connectivity.go#L51-L62 + // Due to the fact that grpc-go doesn't support adding a listener for state tracking, this is + // emitted only when getting a connection to the remote host. This means that: + // * If a non-PL host is unable to connect to a PL host, the metric will not be sent to InfluxDB + // * The state might not be up to date (e.g. the remote host is disconnected but we haven't + // tried to connect to it) + OutgoingConnectionState = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "outgoing_connection_state", + Help: "State of an outgoing connection. 0=Idle, 1=Connecting, 2=Ready, 3=Transient Failure, 4=Shutdown", + }, []string{"remote_host"}) + + // DrandStartTimestamp (group) contains the timestamp in seconds since the epoch of the drand process startup + DrandStartTimestamp = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "drand_start_timestamp", + Help: "Timestamp when the drand process started up in seconds since the Epoch", + }) + + ErrorSendingPartialCounter = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "error_sending_partial", + Help: "Number of errors sending partial beacons to nodes. A good proxy for whether nodes are up or down. " + + "1 = Error occurred, 0 = No error occurred", + }, []string{"beaconID", "address"}) + + metricsBound sync.Once +) + +func bindMetrics(l log.Logger) { + // The private go-level metrics live in private. + if err := PrivateMetrics.Register(collectors.NewGoCollector()); err != nil { + l.Errorw("error in bindMetrics", "metrics", "goCollector", "err", err) + return + } + if err := PrivateMetrics.Register(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})); err != nil { + l.Errorw("error in bindMetrics", "metrics", "processCollector", "err", err) + return + } + + // Group metrics + group := []prometheus.Collector{ + APICallCounter, + GroupDialFailures, + OutgoingConnections, + GroupSize, + GroupThreshold, + BeaconDiscrepancyLatency, + LastBeaconRound, + drandBuildTime, + dkgState, + dkgStateTimestamp, + dkgLeader, + reshareState, + reshareStateTimestamp, + reshareLeader, + OutgoingConnectionState, + IsDrandNode, + DrandStartTimestamp, + DrandStorageBackend, + ErrorSendingPartialCounter, + } + for _, c := range group { + if err := GroupMetrics.Register(c); err != nil { + l.Errorw("error in bindMetrics", "metrics", "bindMetrics", "err", err) + return + } + if err := PrivateMetrics.Register(c); err != nil { + l.Errorw("error in bindMetrics", "metrics", "bindMetrics", "err", err) + return + } + } + + // HTTP metrics + httpMetrics := []prometheus.Collector{ + HTTPCallCounter, + HTTPLatency, + HTTPInFlight, + } + for _, c := range httpMetrics { + if err := HTTPMetrics.Register(c); err != nil { + l.Errorw("error in bindMetrics", "metrics", "bindMetrics", "err", err) + return + } + if err := PrivateMetrics.Register(c); err != nil { + l.Errorw("error in bindMetrics", "metrics", "bindMetrics", "err", err) + return + } + } + + // Client metrics + if err := RegisterClientMetrics(ClientMetrics); err != nil { + l.Errorw("error in bindMetrics", "metrics", "bindMetrics", "err", err) + return + } + if err := RegisterClientMetrics(PrivateMetrics); err != nil { + l.Errorw("error in bindMetrics", "metrics", "bindMetrics", "err", err) + return + } +} + +// RegisterClientMetrics registers drand client metrics with the given registry +func RegisterClientMetrics(r prometheus.Registerer) error { + // Client metrics + client := []prometheus.Collector{ + ClientDNSLatencyVec, + ClientInFlight, + ClientLatencyVec, + ClientRequests, + ClientTLSLatencyVec, + ClientWatchLatency, + ClientHTTPHeartbeatSuccess, + ClientHTTPHeartbeatFailure, + ClientHTTPHeartbeatLatency, + } + for _, c := range client { + if err := r.Register(c); err != nil { + return err + } + } + return nil +} + +// Handler abstracts a helper for relaying http requests to a group peer +type Handler func(ctx context.Context, addr string) (http.Handler, error) + +// Client is the same as net.MetricsClient but avoids cyclic dependencies in our metric and net code. +type Client interface { + GetMetrics(ctx context.Context, p string) (string, error) +} + +// Start starts a prometheus metrics server with debug endpoints. If metricsBind is 0 it will use an available port. +func Start(logger log.Logger, metricsBind string, pprof http.Handler, cli Client) net.Listener { + logger.Infow("metrics starting", "desired_port", metricsBind) + + metricsBound.Do(func() { + bindMetrics(logger) + }) + + // handle metricsBind being just a port value + if !strings.Contains(metricsBind, ":") { + metricsBind = "127.0.0.1:" + metricsBind + } + l, err := net.Listen("tcp", metricsBind) + if err != nil { + logger.Warnw("", "metrics", "listen failed", "err", err) + return nil + } + logger.Infow("metric listener started", "addr", l.Addr()) + + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.HandlerFor(PrivateMetrics, promhttp.HandlerOpts{Registry: PrivateMetrics})) + + mux.Handle("/peer/", newRemotePeerHandler(logger, cli)) + + if pprof != nil { + mux.Handle("/debug/pprof/", pprof) + } + + mux.HandleFunc("/debug/gc", func(w http.ResponseWriter, _ *http.Request) { + runtime.GC() + fmt.Fprintf(w, "GC run complete") + }) + + s := http.Server{Addr: l.Addr().String(), ReadHeaderTimeout: 3 * time.Second, Handler: mux} + go func() { + logger.Warnw("", "metrics", "listen finished", "err", s.Serve(l)) + }() + return l +} + +// remotePeerHandler is a structure that handles all peers that +// this node is connected to regardless of which group they are a part of. +type remotePeerHandler struct { + log log.Logger + client Client +} + +// newRemotePeerHandler creates a new remotePeerHandler from a MetricsClient +func newRemotePeerHandler(logger log.Logger, cli Client) *remotePeerHandler { + return &remotePeerHandler{ + log: logger, + client: cli, + } +} + +// ServeHTTP serves the metrics for the peer whose address is given in the URI. +// It assumes that the URI is in the form /peer/ +func (l *remotePeerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + addr := strings.Replace(r.URL.Path, "/peer/", "", 1) + if index := strings.Index(addr, "/"); index != -1 { + addr = addr[:index] + } + + metrics, err := l.client.GetMetrics(r.Context(), addr) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + } + + l.log.Debugw("Received metrics through GRPC", "from", addr, "err", err) + + _, err = w.Write([]byte(metrics)) + if err != nil { + l.log.Errorw("Error serving remote metrics for peer", "addr", addr) + } +} + +func getBuildTimestamp(buildDate string) int64 { + if buildDate == "" { + return 0 + } + + layout := "02/01/2006@15:04:05" + t, err := time.Parse(layout, buildDate) + if err != nil { + return 0 + } + return t.Unix() +} + +// DKGStateChange emits appropriate dkgState, dkgStateTimestamp and dkgLeader metrics +func DKGStateChange(beaconID string, epoch uint32, leader bool, state uint32) { + leading := 0.0 + if leader { + leading = 1.0 + } + dkgEpoch.WithLabelValues(beaconID).Set(float64(epoch)) + dkgState.WithLabelValues(beaconID).Set(float64(state)) + dkgStateTimestamp.WithLabelValues(beaconID).SetToCurrentTime() + dkgLeader.WithLabelValues(beaconID).Set(leading) +} + +func ErrorSendingPartial(beaconID, address string) { + ErrorSendingPartialCounter.WithLabelValues(beaconID, address).Set(1) +} + +func SuccessfulPartial(beaconID, address string) { + ErrorSendingPartialCounter.WithLabelValues(beaconID, address).Set(0) +} diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go new file mode 100644 index 0000000..44092a6 --- /dev/null +++ b/internal/metrics/metrics_test.go @@ -0,0 +1,23 @@ +package metrics + +import ( + "testing" + "time" +) + +// Note that the remote peer metrics are tested in TestMetricsForPeer in cli_test.go + +func TestBuildTimestamp(t *testing.T) { + buildTimestamp := "29/04/2021@20:23:35" + + reference, err := time.Parse(time.RFC3339, "2021-04-29T20:23:35Z") + if err != nil { + t.Fatalf("Error parsing reference time: %s", err) + } + expected := reference.Unix() + + actual := getBuildTimestamp(buildTimestamp) + if actual != expected { + t.Fatalf("Error converting build timestamp to number. Expected %v, actual %v", expected, actual) + } +} diff --git a/internal/metrics/pprof/pprof.go b/internal/metrics/pprof/pprof.go new file mode 100644 index 0000000..31b7ae6 --- /dev/null +++ b/internal/metrics/pprof/pprof.go @@ -0,0 +1,23 @@ +// Package pprof is separated out from metrics to isolate the 'init' functionality of pprof, so that it is +// included when used by binaries, but not if other drand packages get used or integrated into clients that +// don't expect the pprof side effect to have taken effect. +package pprof + +import ( + "net/http" + "net/http/pprof" // adds default pprof endpoint at /debug/pprof +) + +// WithProfile provides an http mux setup to serve pprof endpoints. it should be mounted at /debug/pprof +func WithProfile() http.Handler { + mux := http.NewServeMux() + + mux.HandleFunc("/", pprof.Index) + // sub-path need to handle the whole path for the matching to work + mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) + mux.HandleFunc("/debug/pprof/profile", pprof.Profile) + mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) + mux.HandleFunc("/debug/pprof/trace", pprof.Trace) + + return mux +} diff --git a/internal/net/client.go b/internal/net/client.go new file mode 100644 index 0000000..be7d4bd --- /dev/null +++ b/internal/net/client.go @@ -0,0 +1,16 @@ +package net + +// HTTPClient is an optional extension to the protocol client relaying of HTTP over the GRPC connection. +import ( + "context" + "net/http" +) + +type Peer interface { + Address() string +} + +// it is currently used for relaying metrics between group members. +type HTTPClient interface { + HandleHTTP(ctx context.Context, p Peer) (http.Handler, error) +} diff --git a/internal/test/default.toml b/internal/test/default.toml new file mode 100644 index 0000000..09bd4cc --- /dev/null +++ b/internal/test/default.toml @@ -0,0 +1,32 @@ +Threshold = 2 +Period = "30s" +CatchupPeriod = "2s" +GenesisTime = 1649948280 +TransitionTime = 1662041790 +GenesisSeed = "4c194718915cb446103b4959d132286176033ef9ab31824ced291ceaf9eb9b59" +SchemeID = "pedersen-bls-chained" +ID = "default" + +[[Nodes]] + Address = "pl2-rpc.incentinet.drand.sh:443" + Key = "8467b7a234c93e7e54d4e8295605c0ccf59e56606b44c4b21714b215e4f7a240f125c66cc5e5c0a73dc9ca602070f1e4" + TLS = true + Signature = "ad1a464fb51a938c2997fada84026715c7fa999869b245d6fa7c68d7b4274f67ee6025bd9e7239375cd31643353a69b90b7a81d18781d298939944adab402046c8bb52164b012156e1487a5226c3aa9448025ac65936446763f5169777537e72" + Index = 0 + +[[Nodes]] + Address = "pl1-rpc.incentinet.drand.sh:443" + Key = "8a5d883e0b363958c74b7c476e6a3e3c29b54f37caebfaba0c24b84a7ccf3d238e91f5e5a789f0251181d8996ec79d72" + TLS = true + Signature = "b4d62c493617999ba34bab9153699a57464b3a68164c84fa2ae95e1ebfe0be3c4e1c04b166d16883d4a72251a4b1bcb207a8f15fb10de1f26927569e221db7deb666feb6b893e38470043344b76660ddd5d44c0f7de21cf12185c6b03e0d0e21" + Index = 1 + +[[Nodes]] + Address = "pl3-rpc.incentinet.drand.sh:443" + Key = "a5a93fb91b52d4840a265a119b1d7a089f294e596ee9e3b7286530e9846848805fd4aafd413524e3d9ea994b3cf34bf9" + TLS = true + Signature = "9604b1efbde1171af8281ee807ac27b9c866a22cba14ba31f53ab3247476f733b57a21f7d130fcab290c8826d56a5e8a07a1fa6936443fbd69dc673a1cf54316b5eb73b490411243c6a5ae0dab3ab2634d4a026b046f715bd46b4a9c7c3b11f7" + Index = 2 + +[PublicKey] + Coefficients = ["88a8227b75dba145599d894d33eebde3b36fef900d456ae2cc4388867adb4769c40359f783750a41b4d17e40f578bfdb", "88300382e9af80b7fb7b78fed9425a108d2fc66f264c3cacace9bea7464d9d9a036410cb546ab1949ab014f893847373"] diff --git a/main.go b/main.go new file mode 100644 index 0000000..e41ebed --- /dev/null +++ b/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "fmt" + "os" + + drand "github.com/drand/drand-cli/internal" +) + +func main() { + app := drand.CLI() + if err := app.Run(os.Args); err != nil { + fmt.Printf("%+v\n", err) + os.Exit(1) + } +}