Update to boxo with refactored providerQueryManager.

ipfs · Nov 22, 2024 · 936da81 · 936da81
1 parent d506003
commit 936da81
Show file tree

Hide file tree

Showing 19 changed files with 100 additions and 47 deletions.
diff --git a/core/commands/bitswap.go b/core/commands/bitswap.go
@@ -5,7 +5,6 @@ import (
 	"io"
 
 	cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
-	e "github.com/ipfs/kubo/core/commands/e"
 
 	humanize "github.com/dustin/go-humanize"
 	bitswap "github.com/ipfs/boxo/bitswap"
@@ -53,10 +52,7 @@ Print out all blocks currently on the bitswap wantlist for the local peer.`,
 			return ErrNotOnline
 		}
 
-		bs, ok := nd.Exchange.(*bitswap.Bitswap)
-		if !ok {
-			return e.TypeErr(bs, nd.Exchange)
-		}
+		bs := nd.Bitswap
 
 		pstr, found := req.Options[peerOptionName].(string)
 		if found {
@@ -112,12 +108,7 @@ var bitswapStatCmd = &cmds.Command{
 			return cmds.Errorf(cmds.ErrClient, "unable to run offline: %s", ErrNotOnline)
 		}
 
-		bs, ok := nd.Exchange.(*bitswap.Bitswap)
-		if !ok {
-			return e.TypeErr(bs, nd.Exchange)
-		}
-
-		st, err := bs.Stat()
+		st, err := nd.Bitswap.Stat()
 		if err != nil {
 			return err
 		}
@@ -134,7 +125,6 @@ var bitswapStatCmd = &cmds.Command{
 			human, _ := req.Options[bitswapHumanOptionName].(bool)
 
 			fmt.Fprintln(w, "bitswap status")
-			fmt.Fprintf(w, "\tprovides buffer: %d / %d\n", s.ProvideBufLen, bitswap.HasBlockBufferSize)
 			fmt.Fprintf(w, "\tblocks received: %d\n", s.BlocksReceived)
 			fmt.Fprintf(w, "\tblocks sent: %d\n", s.BlocksSent)
 			if human {
@@ -190,17 +180,12 @@ prints the ledger associated with a given peer.
 			return ErrNotOnline
 		}
 
-		bs, ok := nd.Exchange.(*bitswap.Bitswap)
-		if !ok {
-			return e.TypeErr(bs, nd.Exchange)
-		}
-
 		partner, err := peer.Decode(req.Arguments[0])
 		if err != nil {
 			return err
 		}
 
-		return cmds.EmitOnce(res, bs.LedgerForPeer(partner))
+		return cmds.EmitOnce(res, nd.Bitswap.LedgerForPeer(partner))
 	},
 	Encoders: cmds.EncoderMap{
 		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *server.Receipt) error {

diff --git a/core/core.go b/core/core.go
@@ -19,6 +19,7 @@ import (
 	pin "github.com/ipfs/boxo/pinning/pinner"
 	"github.com/ipfs/go-datastore"
 
+	bitswap "github.com/ipfs/boxo/bitswap"
 	bserv "github.com/ipfs/boxo/blockservice"
 	bstore "github.com/ipfs/boxo/blockstore"
 	exchange "github.com/ipfs/boxo/exchange"
@@ -102,7 +103,8 @@ type IpfsNode struct {
 	UnixFSPathResolver        pathresolver.Resolver      `name:"unixFSPathResolver"`        // The UnixFS path resolver
 	OfflineIPLDPathResolver   pathresolver.Resolver      `name:"offlineIpldPathResolver"`   // The IPLD path resolver that uses only locally available blocks
 	OfflineUnixFSPathResolver pathresolver.Resolver      `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks
-	Exchange                  exchange.Interface         // the block exchange + strategy (bitswap)
+	Exchange                  exchange.Interface         // the block exchange + strategy
+	Bitswap                   *bitswap.Bitswap           `optional:"true"` // The Bitswap instance
 	Namesys                   namesys.NameSystem         // the name system, resolves paths to hashes
 	Provider                  provider.System            // the value provider system
 	IpnsRepub                 *ipnsrp.Republisher        `optional:"true"`

diff --git a/core/coreapi/pin.go b/core/coreapi/pin.go
@@ -44,7 +44,7 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp
 		return fmt.Errorf("pin: %s", err)
 	}
 
-	if err := api.provider.Provide(dagNode.Cid()); err != nil {
+	if err := api.provider.Provide(ctx, dagNode.Cid(), true); err != nil {
 		return err
 	}
 

diff --git a/core/coreapi/unixfs.go b/core/coreapi/unixfs.go
@@ -173,7 +173,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
 	}
 
 	if !settings.OnlyHash {
-		if err := api.provider.Provide(nd.Cid()); err != nil {
+		if err := api.provider.Provide(ctx, nd.Cid(), true); err != nil {
 			return path.ImmutablePath{}, err
 		}
 	}

diff --git a/core/node/bitswap.go b/core/node/bitswap.go
@@ -5,9 +5,12 @@ import (
 	"time"
 
 	"github.com/ipfs/boxo/bitswap"
+	"github.com/ipfs/boxo/bitswap/client"
 	"github.com/ipfs/boxo/bitswap/network"
 	blockstore "github.com/ipfs/boxo/blockstore"
 	exchange "github.com/ipfs/boxo/exchange"
+	"github.com/ipfs/boxo/exchange/providing"
+	provider "github.com/ipfs/boxo/provider"
 	"github.com/ipfs/kubo/config"
 	irouting "github.com/ipfs/kubo/routing"
 	"github.com/libp2p/go-libp2p/core/host"
@@ -34,15 +37,14 @@ type bitswapOptionsOut struct {
 
 // BitswapOptions creates configuration options for Bitswap from the config file
 // and whether to provide data.
-func BitswapOptions(cfg *config.Config, provide bool) interface{} {
+func BitswapOptions(cfg *config.Config) interface{} {
 	return func() bitswapOptionsOut {
 		var internalBsCfg config.InternalBitswap
 		if cfg.Internal.Bitswap != nil {
 			internalBsCfg = *cfg.Internal.Bitswap
 		}
 
 		opts := []bitswap.Option{
-			bitswap.ProvideEnabled(provide),
 			bitswap.ProviderSearchDelay(internalBsCfg.ProviderSearchDelay.WithDefault(DefaultProviderSearchDelay)), // See https://github.com/ipfs/go-ipfs/issues/8807 for rationale
 			bitswap.EngineBlockstoreWorkerCount(int(internalBsCfg.EngineBlockstoreWorkerCount.WithDefault(DefaultEngineBlockstoreWorkerCount))),
 			bitswap.TaskWorkerCount(int(internalBsCfg.TaskWorkerCount.WithDefault(DefaultTaskWorkerCount))),
@@ -55,7 +57,7 @@ func BitswapOptions(cfg *config.Config, provide bool) interface{} {
 	}
 }
 
-type onlineExchangeIn struct {
+type bitswapIn struct {
 	fx.In
 
 	Mctx        helpers.MetricsCtx
@@ -65,19 +67,62 @@ type onlineExchangeIn struct {
 	BitswapOpts []bitswap.Option `group:"bitswap-options"`
 }
 
-// OnlineExchange creates new LibP2P backed block exchange (BitSwap).
+// Bitswap creates the BitSwap server/client instance.
 // Additional options to bitswap.New can be provided via the "bitswap-options"
 // group.
-func OnlineExchange() interface{} {
-	return func(in onlineExchangeIn, lc fx.Lifecycle) exchange.Interface {
-		bitswapNetwork := network.NewFromIpfsHost(in.Host, in.Rt)
+func Bitswap(provide bool) interface{} {
+	return func(in bitswapIn, lc fx.Lifecycle) *bitswap.Bitswap {
+		bitswapNetwork := network.NewFromIpfsHost(in.Host)
+
+		var provider client.ProviderFinder
+		if provide {
+			provider = in.Rt
+		}
+		bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, provider, in.Bs, in.BitswapOpts...)
 
-		exch := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, in.Bs, in.BitswapOpts...)
 		lc.Append(fx.Hook{
 			OnStop: func(ctx context.Context) error {
-				return exch.Close()
+				return bs.Close()
 			},
 		})
+		return bs
+	}
+}
+
+// OnlineExchange creates new LibP2P backed block exchange.
+func OnlineExchange() interface{} {
+	return func(in *bitswap.Bitswap, lc fx.Lifecycle) exchange.Interface {
+		lc.Append(fx.Hook{
+			OnStop: func(ctx context.Context) error {
+				return in.Close()
+			},
+		})
+		return in
+	}
+}
+
+type providingExchangeIn struct {
+	fx.In
+
+	BaseExch exchange.Interface
+	Provider provider.System
+}
+
+// ProvidingExchange creates a providing.Exchange with the existing exchange
+// and the provider.System.
+// We cannot do this in OnlineExchange because it causes cycles so this is for
+// a decorator.
+func ProvidingExchange(provide bool) interface{} {
+	return func(in providingExchangeIn, lc fx.Lifecycle) exchange.Interface {
+		exch := in.BaseExch
+		if provide {
+			exch = providing.New(in.BaseExch, in.Provider)
+			lc.Append(fx.Hook{
+				OnStop: func(ctx context.Context) error {
+					return exch.Close()
+				},
+			})
+		}
 		return exch
 	}
 }
diff --git a/core/node/groups.go b/core/node/groups.go
@@ -293,8 +293,11 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
 	shouldBitswapProvide := !cfg.Experimental.StrategicProviding
 
 	return fx.Options(
-		fx.Provide(BitswapOptions(cfg, shouldBitswapProvide)),
+		fx.Provide(BitswapOptions(cfg)),
+		fx.Provide(Bitswap(shouldBitswapProvide)),
 		fx.Provide(OnlineExchange()),
+		// Replace our Exchange with a Providing exchange!
+		fx.Decorate(ProvidingExchange(shouldBitswapProvide)),
 		fx.Provide(DNSResolver),
 		fx.Provide(Namesys(ipnsCacheSize, cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL))),
 		fx.Provide(Peering),

diff --git a/docs/changelogs/v0.32.md b/docs/changelogs/v0.32.md
@@ -1,6 +1,6 @@
 # Kubo changelog v0.32
 
-- [v0.32.0](#v0310)
+- [v0.32.0](#v0320)
 
 ## v0.32.0
 

diff --git a/docs/changelogs/v0.33.md b/docs/changelogs/v0.33.md
@@ -16,6 +16,8 @@
 
 #### 📦️ Dependency updates
 
+- update `boxo` to [v0.24.TODO](https://github.com/ipfs/boxo/releases/tag/v0.24.TODO)
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors
diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod
@@ -7,7 +7,7 @@ go 1.23
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f
+	github.com/ipfs/boxo v0.24.4-0.20241122141840-8b304f273785
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
 	github.com/libp2p/go-libp2p v0.37.0
 	github.com/multiformats/go-multiaddr v0.13.0

diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum
@@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy
 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f h1:3qgBUQ6BYfEAPaoSYoH90PKwVT1/iFLX7fDGGkvXZ8Y=
-github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
+github.com/ipfs/boxo v0.24.4-0.20241122141840-8b304f273785 h1:F+bW9GnUAt+pCp9YDiUkBcQRAT2y0YDnzTSALCYrRuk=
+github.com/ipfs/boxo v0.24.4-0.20241122141840-8b304f273785/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

diff --git a/go.mod b/go.mod
@@ -22,7 +22,7 @@ require (
 	github.com/hashicorp/go-version v1.7.0
 	github.com/ipfs-shipyard/nopfs v0.0.12
 	github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c
-	github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f
+	github.com/ipfs/boxo v0.24.4-0.20241122141840-8b304f273785
 	github.com/ipfs/go-block-format v0.2.0
 	github.com/ipfs/go-cid v0.4.1
 	github.com/ipfs/go-cidutil v0.1.0

diff --git a/go.sum b/go.sum
@@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy
 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f h1:3qgBUQ6BYfEAPaoSYoH90PKwVT1/iFLX7fDGGkvXZ8Y=
-github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
+github.com/ipfs/boxo v0.24.4-0.20241122141840-8b304f273785 h1:F+bW9GnUAt+pCp9YDiUkBcQRAT2y0YDnzTSALCYrRuk=
+github.com/ipfs/boxo v0.24.4-0.20241122141840-8b304f273785/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

diff --git a/test/cli/delegated_routing_v1_http_proxy_test.go b/test/cli/delegated_routing_v1_http_proxy_test.go
@@ -60,8 +60,12 @@ func TestRoutingV1Proxy(t *testing.T) {
 		nodes := setupNodes(t)
 
 		cidStr := nodes[0].IPFSAddStr(testutils.RandomStr(1000))
-
-		res := nodes[1].IPFS("routing", "findprovs", cidStr)
+		// Reprovide as initialProviderDelay still ongoing
+		res := nodes[0].IPFS("bitswap", "reprovide")
+		if res.Err != nil {
+			t.Fatal(res.Err)
+		}
+		res = nodes[1].IPFS("routing", "findprovs", cidStr)
 		assert.Equal(t, nodes[0].PeerID().String(), res.Stdout.Trimmed())
 	})
 

diff --git a/test/cli/delegated_routing_v1_http_server_test.go b/test/cli/delegated_routing_v1_http_server_test.go
@@ -38,6 +38,11 @@ func TestRoutingV1Server(t *testing.T) {
 		text := "hello world " + uuid.New().String()
 		cidStr := nodes[2].IPFSAddStr(text)
 		_ = nodes[3].IPFSAddStr(text)
+		// Reprovide as initialProviderDelay still ongoing
+		res := nodes[3].IPFS("bitswap", "reprovide")
+		if res.Err != nil {
+			t.Fatal(res.Err)
+		}
 
 		cid, err := cid.Decode(cidStr)
 		assert.NoError(t, err)

diff --git a/test/cli/provider_test.go b/test/cli/provider_test.go
@@ -42,6 +42,11 @@ func TestProvider(t *testing.T) {
 		defer nodes.StopDaemons()
 
 		cid := nodes[0].IPFSAddStr(time.Now().String())
+		// Reprovide as initialProviderDelay still ongoing
+		res := nodes[0].IPFS("bitswap", "reprovide")
+		if res.Err != nil {
+			t.Fatal(res.Err)
+		}
 		expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
 	})
 

diff --git a/test/cli/routing_dht_test.go b/test/cli/routing_dht_test.go
@@ -84,7 +84,12 @@ func testRoutingDHT(t *testing.T, enablePubsub bool) {
 		t.Run("ipfs routing findprovs", func(t *testing.T) {
 			t.Parallel()
 			hash := nodes[3].IPFSAddStr("some stuff")
-			res := nodes[4].IPFS("routing", "findprovs", hash)
+			// Reprovide as initialProviderDelay still ongoing
+			res := nodes[3].IPFS("bitswap", "reprovide")
+			if res.Err != nil {
+				t.Fatal(res.Err)
+			}
+			res = nodes[4].IPFS("routing", "findprovs", hash)
 			assert.Equal(t, nodes[3].PeerID().String(), res.Stdout.Trimmed())
 		})
 

diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod
@@ -119,7 +119,7 @@ require (
 	github.com/huin/goupnp v1.3.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/ipfs/bbloom v0.0.4 // indirect
-	github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f // indirect
+	github.com/ipfs/boxo v0.24.4-0.20241122141840-8b304f273785 // indirect
 	github.com/ipfs/go-block-format v0.2.0 // indirect
 	github.com/ipfs/go-cid v0.4.1 // indirect
 	github.com/ipfs/go-datastore v0.6.0 // indirect

diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum
@@ -318,8 +318,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f h1:3qgBUQ6BYfEAPaoSYoH90PKwVT1/iFLX7fDGGkvXZ8Y=
-github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
+github.com/ipfs/boxo v0.24.4-0.20241122141840-8b304f273785 h1:F+bW9GnUAt+pCp9YDiUkBcQRAT2y0YDnzTSALCYrRuk=
+github.com/ipfs/boxo v0.24.4-0.20241122141840-8b304f273785/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
 github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
 github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
 github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=

diff --git a/test/sharness/t0220-bitswap.sh b/test/sharness/t0220-bitswap.sh
@@ -18,7 +18,6 @@ test_expect_success "'ipfs bitswap stat' succeeds" '
 test_expect_success "'ipfs bitswap stat' output looks good" '
   cat <<EOF | unexpand -t2 >expected &&
 bitswap status
-  provides buffer: 0 / 256
   blocks received: 0
   blocks sent: 0
   data received: 0
@@ -56,7 +55,6 @@ test_expect_success "'ipfs bitswap stat' succeeds" '
 test_expect_success "'ipfs bitswap stat' output looks good" '
   cat <<EOF | unexpand -t2 >expected &&
 bitswap status
-  provides buffer: 0 / 256
   blocks received: 0
   blocks sent: 0
   data received: 0
@@ -85,7 +83,6 @@ test_expect_success "'ipfs bitswap stat --human' succeeds" '
 test_expect_success "'ipfs bitswap stat --human' output looks good" '
   cat <<EOF | unexpand -t2 >expected &&
 bitswap status
-  provides buffer: 0 / 256
   blocks received: 0
   blocks sent: 0
   data received: 0 B