From 8069eb765ac85152e1482723ff6af2330592dde7 Mon Sep 17 00:00:00 2001 From: gammazero Date: Fri, 25 Aug 2023 23:43:21 -0700 Subject: [PATCH] Update test to use libp2phttp publisher - Update comments - Use latest go-libp2p --- cmd/provider/internal/config/httppublisher.go | 6 ++- cmd/provider/internal/config/ingest.go | 6 +-- engine/engine_test.go | 48 +++++++++++-------- engine/options.go | 17 ++++--- go.mod | 4 +- go.sum | 8 ++-- 6 files changed, 53 insertions(+), 36 deletions(-) diff --git a/cmd/provider/internal/config/httppublisher.go b/cmd/provider/internal/config/httppublisher.go index 6ed87a3d..24bf38b2 100644 --- a/cmd/provider/internal/config/httppublisher.go +++ b/cmd/provider/internal/config/httppublisher.go @@ -11,9 +11,11 @@ type HttpPublisher struct { // specified, the ListenMultiaddr is used. AnnounceMultiaddr string // ListenMultiaddr is the address of the interface to listen for HTTP - // requests for advertisements. + // requests for advertisements. Set this to "" to disable serving plain + // HTTP if only libp2phttp is wanted. ListenMultiaddr string - // NoLibp2p disables serving HTTP over libp2p if true. + // NoLibp2p disables serving HTTP over libp2p if true. Set this to true to + // publish over plain HTTP only. NoLibp2p bool } diff --git a/cmd/provider/internal/config/ingest.go b/cmd/provider/internal/config/ingest.go index 58b0f9e8..0a972438 100644 --- a/cmd/provider/internal/config/ingest.go +++ b/cmd/provider/internal/config/ingest.go @@ -35,9 +35,9 @@ type Ingest struct { HttpPublisher HttpPublisher // PublisherKind specifies which dagsync.Publisher implementation to use. - // When set to "http", publisher serves HTTP over libp2p if - // HttpPublisher.NoLibp2p is false. If HttpPublisher.ListenMultiaddr - // publisher serves plain HTTP on that address. + // When set to "http", the publisher serves plain HTTP and libp2phttp. + // Libp2phttp is disabled by setting HttpPublisher.NoLibp2p to true, and + // plain HTTP is disabled by setting HttpPublisher.ListenMultiaddr to "". PublisherKind PublisherKind // SyncPolicy configures which indexers are allowed to sync advertisements diff --git a/engine/engine_test.go b/engine/engine_test.go index ef01ba53..839754e0 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -12,7 +12,6 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" leveldb "github.com/ipfs/go-ds-leveldb" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" @@ -21,8 +20,7 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector" selectorbuilder "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/ipni/go-libipni/announce/message" - "github.com/ipni/go-libipni/dagsync/dtsync" - "github.com/ipni/go-libipni/dagsync/p2p/protocol/head" + "github.com/ipni/go-libipni/dagsync/ipnisync" "github.com/ipni/go-libipni/ingest/schema" "github.com/ipni/go-libipni/metadata" "github.com/ipni/go-libipni/test" @@ -108,7 +106,7 @@ func TestEngine_PublishLocal(t *testing.T) { require.Equal(t, gotLatestAdCid, gotPublishedAdCid) } -func TestEngine_PublishWithDataTransferPublisher(t *testing.T) { +func TestEngine_PublishWithLibp2pHttpPublisher(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), testTimeout) t.Cleanup(cancel) @@ -118,11 +116,17 @@ func TestEngine_PublishWithDataTransferPublisher(t *testing.T) { // Use test name as gossip topic name for uniqueness per test. topic := t.Name() - subHost, err := libp2p.New() + pubHost, err := libp2p.New() require.NoError(t, err) + t.Cleanup(func() { + pubHost.Close() + }) - pubHost, err := libp2p.New() + subHost, err := libp2p.New() require.NoError(t, err) + t.Cleanup(func() { + subHost.Close() + }) // DirectConnectTicks set to 5 here, because if the gossub for the subHost // does not start within n ticks then they never peer. So, if @@ -163,7 +167,7 @@ func TestEngine_PublishWithDataTransferPublisher(t *testing.T) { http.Error(w, err.Error(), 400) return } - // Since the message is coming from a dtsync publisher, the addresses + // Since the message is coming from a libp2p publisher, the addresses // should include a p2p ID. ais, err := peer.AddrInfosFromP2pAddrs(addrs...) if err != nil { @@ -186,7 +190,9 @@ func TestEngine_PublishWithDataTransferPublisher(t *testing.T) { } w.WriteHeader(http.StatusNoContent) })) - defer ts.Close() + t.Cleanup(func() { + ts.Close() + }) pubT, err := pubG.Join(topic) require.NoError(t, err) @@ -194,7 +200,8 @@ func TestEngine_PublishWithDataTransferPublisher(t *testing.T) { subject, err := engine.New( engine.WithDirectAnnounce(ts.URL), engine.WithHost(pubHost), - engine.WithPublisherKind(engine.DataTransferPublisher), + engine.WithPublisherKind(engine.HttpPublisher), + engine.WithHttpPublisherListenAddr(""), // disable plain http engine.WithTopic(pubT), engine.WithTopicName(topic), engine.WithExtraGossipData(wantExtraGossipData), @@ -202,7 +209,9 @@ func TestEngine_PublishWithDataTransferPublisher(t *testing.T) { require.NoError(t, err) err = subject.Start(ctx) require.NoError(t, err) - defer subject.Shutdown() + t.Cleanup(func() { + subject.Shutdown() + }) subG, err := pubsub.NewGossipSub(ctx, subHost, pubsub.WithDirectConnectTicks(1), @@ -282,19 +291,20 @@ func TestEngine_PublishWithDataTransferPublisher(t *testing.T) { require.NoError(t, err) requireEqualDagsyncMessage(t, wantMessage, gotMessage) - gotRootCid, err := head.QueryRootCid(ctx, subHost, topic, pubHost.ID()) - require.NoError(t, err) - require.Equal(t, gotPublishedAdCid, gotRootCid) - - ds := dssync.MutexWrap(datastore.NewMapDatastore()) ls := cidlink.DefaultLinkSystem() store := &memstore.Store{} ls.SetReadStorage(store) ls.SetWriteStorage(store) - sync, err := dtsync.NewSync(subHost, ds, ls, nil, 0, 0) + sync := ipnisync.NewSync(ls, nil, ipnisync.ClientStreamHost(subHost)) + t.Cleanup(func() { + sync.Close() + }) + subjectInfo := peer.AddrInfo{ + ID: subject.Host().ID(), + } + syncer, err := sync.NewSyncer(subjectInfo) require.NoError(t, err) - syncer := sync.NewSyncer(subject.Host().ID(), topic) gotHead, err := syncer.GetHead(ctx) require.NoError(t, err) require.Equal(t, gotLatestAdCid, gotHead) @@ -453,10 +463,10 @@ func TestEngine_ProducesSingleChainForMultipleProviders(t *testing.T) { wantContextID1 := []byte("fish") wantContextID2 := []byte("bird") subject.RegisterMultihashLister(func(ctx context.Context, p peer.ID, contextID []byte) (provider.MultihashIterator, error) { - if string(contextID) == string(wantContextID1) && p == provider1id { return provider.SliceMultihashIterator(mhs1), nil - } else if string(contextID) == string(wantContextID2) && p == provider2id { + } + if string(contextID) == string(wantContextID2) && p == provider2id { return provider.SliceMultihashIterator(mhs2), nil } return nil, errors.New("not found") diff --git a/engine/options.go b/engine/options.go index 2b891432..9b0ba6da 100644 --- a/engine/options.go +++ b/engine/options.go @@ -227,8 +227,11 @@ func WithEntriesCacheCapacity(s int) Option { } } -// WithPublisherKind sets the kind of publisher used to announce new advertisements. -// If unset, advertisements are only stored locally and no announcements are made. +// WithPublisherKind sets the kind of publisher used to serve advertisements. +// If unset, advertisements are only stored locally and no announcements are +// made. This does not affect the methods used to send announcements of new +// advertisements, which are configured independent of this. +// // See: PublisherKind. func WithPublisherKind(k PublisherKind) Option { return func(o *options) error { @@ -237,10 +240,12 @@ func WithPublisherKind(k PublisherKind) Option { } } -// WithHttpPublisherListenAddr sets the net listen address for the HTTP publisher. -// If unset, the default net listen address of '0.0.0.0:3104' is used. +// WithHttpPublisherListenAddr sets the net listen address for the HTTP +// publisher. If unset, the default net listen address of '0.0.0.0:3104' is +// used. To disable plain HTTP and only serve libp2phttp, explicitly set this +// to "". // -// Note that this option only takes effect if the PublisherKind is set to HttpPublisher. +// This option only takes effect if the PublisherKind is set to HttpPublisher. // See: WithPublisherKind. func WithHttpPublisherListenAddr(addr string) Option { return func(o *options) error { @@ -250,7 +255,7 @@ func WithHttpPublisherListenAddr(addr string) Option { } // WithHttpNoLibp2p disables serving HTTP over libp2p if true and using an HTTP -// publisher. +// publisher. This is used to disable libp2phttp and only serve plain HTTP. func WithHttpNoLibp2p(disable bool) Option { return func(o *options) error { o.pubHttpNoLibp2p = disable diff --git a/go.mod b/go.mod index 94fdbeb5..6f77b9dc 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 github.com/ipld/go-ipld-prime v0.20.0 - github.com/ipni/go-libipni v0.4.1-0.20230824030625-0669ee8d4bbe - github.com/libp2p/go-libp2p v0.29.1-0.20230823171643-912f92d49a19 + github.com/ipni/go-libipni v0.4.1-0.20230826004428-2194d3a10198 + github.com/libp2p/go-libp2p v0.29.1-0.20230825222710-ed42d2fe6cee github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-multiaddr v0.10.1 diff --git a/go.sum b/go.sum index c6c5a2d6..e9a6adb1 100644 --- a/go.sum +++ b/go.sum @@ -350,8 +350,8 @@ github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0/go.mod h1:od github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= -github.com/ipni/go-libipni v0.4.1-0.20230824030625-0669ee8d4bbe h1:wEg//rXzpRqDE9xETCkxMCobUq6c911aS4o8veqUxys= -github.com/ipni/go-libipni v0.4.1-0.20230824030625-0669ee8d4bbe/go.mod h1:fVvfLxFC9MTiCUfaxYgvc9eoPGNYpzzVAL0taKy2n70= +github.com/ipni/go-libipni v0.4.1-0.20230826004428-2194d3a10198 h1:uc4Pey8/KQa6hidvotIwvQrr59u5kU7AzN/8bKlVkL0= +github.com/ipni/go-libipni v0.4.1-0.20230826004428-2194d3a10198/go.mod h1:fnTwBp3bJqWV6ItVmT44j9JpIKBuphBpDmACmHxavXw= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -399,8 +399,8 @@ github.com/libp2p/go-cidranger v1.1.0 h1:ewPN8EZ0dd1LSnrtuwd4709PXVcITVeuwbag38y github.com/libp2p/go-cidranger v1.1.0/go.mod h1:KWZTfSr+r9qEo9OkI9/SIEeAtw+NNoU0dXIXt15Okic= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= -github.com/libp2p/go-libp2p v0.29.1-0.20230823171643-912f92d49a19 h1:oBCMCeLFvvHhAsLy9/9ybP71Sc5/DSGtZKn9aSLrwwk= -github.com/libp2p/go-libp2p v0.29.1-0.20230823171643-912f92d49a19/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= +github.com/libp2p/go-libp2p v0.29.1-0.20230825222710-ed42d2fe6cee h1:UioCkyX/qjEs5mzVoBfZUChXtyhtXG9salRwQ8iUDfI= +github.com/libp2p/go-libp2p v0.29.1-0.20230825222710-ed42d2fe6cee/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-gostream v0.6.0 h1:QfAiWeQRce6pqnYfmIVWJFXNdDyfiR/qkCnjyaZUPYU=