Merge branch 'main' into feat/configurable-providequerymanager

.github/workflows/gateway-sharness.yml

@@ -16,9 +16,9 @@ jobs:
         shell: bash
     steps:
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: 1.22.x
+          go-version: 1.23.x
       - name: Checkout boxo
         uses: actions/checkout@v3
         with:
@@ -33,11 +33,12 @@ jobs:
       - name: Replace boxo in Kubo go.mod
         run: |
           go mod edit -replace=github.com/ipfs/boxo=../boxo
-          go mod tidy
-          cat go.mod
+          make mod_tidy
         working-directory: kubo
       - name: Install sharness dependencies
-        run: make test_sharness_deps
+        run: |
+          find . -name go.mod -execdir go mod tidy \;
+          make test_sharness_deps
         working-directory: kubo
       - name: Run Kubo Sharness Tests
         run: find . -maxdepth 1 -name "*gateway*.sh" -print0 | xargs -0 -I {} bash -c "echo {}; {}"

CHANGELOG.md

@@ -18,21 +18,109 @@ The following emojis are used to highlight certain changes:
 
 ### Changed
 
-- `bitswap/server` minor memory use and performance improvements
-- `bitswap` unify logger names to use uniform format bitswap/path/pkgname
+- No longer using `github.com/jbenet/goprocess` to avoid requiring in dependents.
 
 ### Removed
 
 ### Fixed
 
 ### Security
 
+## [v0.24.3]
+
+### Changed
+
+- `go.mod` updates
+
+### Fixed
+
+- `bitswap/client` no longer logs `"Received provider X for cid Y not requested` to ERROR level, moved to DEBUG [#771](https://github.com/ipfs/boxo/pull/711)
+
+## [v0.24.2]
+
+### Changed
+
+- updated to go-libp2p to [v0.37.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.37.0)
+- `ipns/pb`: removed use of deprecated `Exporter` (SA1019, [golang/protobuf#1640](https://github.com/golang/protobuf/issues/1640), [9a7055](https://github.com/ipfs/boxo/pull/699/commits/9a7055e444527d5aad3187503a1b84bcae44f7b9))
+
+### Fixed
+
+- `bitswap/client`: fix panic if current live count is greater than broadcast limit [#702](https://github.com/ipfs/boxo/pull/702)
+
+## [v0.24.1]
+
+### Changed
+
+- `routing/http/client`: creating delegated routing client with `New` now defaults to querying delegated routing server with `DefaultProtocolFilter`  ([IPIP-484](https://github.com/ipfs/specs/pull/484)) [#689](https://github.com/ipfs/boxo/pull/689)
+- updated go-libp2p to [v0.36.5](https://github.com/libp2p/go-libp2p/releases/tag/v0.36.5)
+- updated dependencies [#693](https://github.com/ipfs/boxo/pull/693)
+- update `go-libp2p-kad-dht` to [v0.27.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.27.0)
+
+### Fixed
+
+- `routing/http/client`: optional address and protocol filter parameters from [IPIP-484](https://github.com/ipfs/specs/pull/484) use human-readable `,` instead of `%2C`. [#688](https://github.com/ipfs/boxo/pull/688)
+- `bitswap/client` Cleanup live wants when wants are canceled. This prevents live wants from continuing to get rebroadcasted even after the wants are canceled. [#690](https://github.com/ipfs/boxo/pull/690)
+- Fix problem adding invalid CID to exhausted wants list resulting in possible performance issue. [#692](https://github.com/ipfs/boxo/pull/692)
+
+## [v0.24.0]
+
+### Added
+
+* `boxo/bitswap/server`:
+  * A new [`WithWantHaveReplaceSize(n)`](https://pkg.go.dev/github.com/ipfs/boxo/bitswap/server/#WithWantHaveReplaceSize) option can be used with `bitswap.New` to fine-tune cost-vs-performance. It sets the maximum size of a block in bytes up to which the bitswap server will replace a WantHave with a WantBlock response. Setting this to 0 disables this WantHave replacement and means that block sizes are not read when processing WantHave requests. [#672](https://github.com/ipfs/boxo/pull/672)
+* `routing/http`:
+  * added support for address and protocol filtering to the delegated routing server ([IPIP-484](https://github.com/ipfs/specs/pull/484)) [#671](https://github.com/ipfs/boxo/pull/671) [#678](https://github.com/ipfs/boxo/pull/678)
+  * added support for address and protocol filtering to the delegated routing client ([IPIP-484](https://github.com/ipfs/specs/pull/484)) [#678](https://github.com/ipfs/boxo/pull/678). To add filtering to the client, use the [`WithFilterAddrs`](https://pkg.go.dev/github.com/ipfs/boxo/routing/http/client#WithFilterAddrs) and [`WithFilterProtocols`](https://pkg.go.dev/github.com/ipfs/boxo/routing/http/client#WithFilterProtocols) options when creating the client.Client-side filtering for servers that don't support filtering is enabled by default. To disable it, use the [`disableLocalFiltering`](https://pkg.go.dev/github.com/ipfs/boxo/routing/http/client#disableLocalFiltering) option when creating the client.
+
+### Changed
+
+### Removed
+
+### Fixed
+
+- `unixfs/hamt` Log error instead of panic if both link and shard are nil [#393](https://github.com/ipfs/boxo/pull/393)
+
+### Security
+
+## [v0.23.0]
+
+### Added
+
+- `files`, `ipld/unixfs`, `mfs` and `tar` now support optional UnixFS 1.5 mode and modification time metadata [#653](https://github.com/ipfs/boxo/pull/653)
+- `gateway` deserialized responses will have `Last-Modified` set to value from optional UnixFS 1.5 modification time field (if present in DAG) and a matching `If-Modified-Since` will return `304 Not Modified` (UnixFS 1.5 files only) [#659](https://github.com/ipfs/boxo/pull/659)
+
+### Changed
+
+- updated Go in `go.mod` to 1.22 [#661](https://github.com/ipfs/boxo/pull/661)
+- updated go-libp2p to [v0.36.3](https://github.com/libp2p/go-libp2p/releases/tag/v0.36.3)
+- `chunker` refactored to reduce overall memory use by reducing heap fragmentation [#649](https://github.com/ipfs/boxo/pull/649)
+- `bitswap/server` minor performance improvements in concurrent operations [#666](https://github.com/ipfs/boxo/pull/666)
+- removed dependency on go-ipfs-blocksutil [#656](https://github.com/ipfs/boxo/pull/656)
+
+## [v0.22.0]
+
+### Changed
+
+- `go-libp2p` dependency updated to [v0.36 (release notes)](https://github.com/libp2p/go-libp2p/releases/tag/v0.36.1)
+- `bitswap/server` minor memory use and performance improvements [#634](https://github.com/ipfs/boxo/pull/634)
+- `bitswap` unify logger names to use uniform format bitswap/path/pkgname [#637](https://github.com/ipfs/boxo/pull/637)
+- `gateway` now always returns meaningful cache-control headers for generated HTML listings of UnixFS directories [#643](https://github.com/ipfs/boxo/pull/643)
+- `util` generate random test data using `ipfs/go-test` instead of internal util code [#638](https://github.com/ipfs/boxo/pull/638)
+- `bitswap/server` `PeerLedger.Wants` now returns `bool` (interface change from `Wants(p peer.ID, e wl.Entry)` to `Wants(p peer.ID, e wl.Entry) bool`) [#629](https://github.com/ipfs/boxo/pull/629)
+
+### Fixed
+
+- `boxo/gateway` now correctly returns 404 Status Not Found instead of 500 when the requested content cannot be found due to offline exchange, gateway running in no-fetch (non-recursive) mode, or a similar restriction that only serves a specific set of CIDs. [#630](https://github.com/ipfs/boxo/pull/630)
+- `bitswap/client` fix memory leak in BlockPresenceManager due to unlimited map growth. [#636](https://github.com/ipfs/boxo/pull/636)
+- `bitswap/network` fixed race condition when a timeout occurred before hole punching completed while establishing a first-time stream to a peer behind a NAT [#651](https://github.com/ipfs/boxo/pull/651)
+- `bitswap`: wantlist overflow handling now cancels existing entries to make room for newer entries. This fix prevents the wantlist from filling up with CIDs that the server does not have. [#629](https://github.com/ipfs/boxo/pull/629)
+
 ## [v0.21.0]
 
 ### Changed
 
 - `boxo/gateway` is now tested against [gateway-conformance v6](https://github.com/ipfs/gateway-conformance/releases/tag/v0.6.0)
-- `bitswap/client` supports additional tracing 
+- `bitswap/client` supports additional tracing
 
 ### Removed
 

README.md

@@ -85,7 +85,7 @@ Boxo includes high-quality components useful for interacting with IPFS protocols
 - Interacting with public and private IPFS networks
 - Working with content-addressed data
 
-Boxo aims to provide a cohesive interface into these components. Note that not all of the underlying components necessarily reside in this respository.
+Boxo aims to provide a cohesive interface into these components. Note that not all of the underlying components necessarily reside in this repository.
 
 ### Does Boxo == IPFS?
 
@@ -156,22 +156,23 @@ The code coverage of this repo is not currently representative of the actual tes
 
 ### Help
 
-If you have questions, feel free to open an issue. You can also find the Boxo maintainers in [Filecoin Slack](https://filecoin.io/slack/) at #Boxo-maintainers.  (If you would like to engage via IPFS Discord or ipfs.io Matrix, please drop into the #ipfs-implementers channel/room or file an issue, and we'll get bridging from #Boxo-maintainers to these other chat platforms.)
+If you suspect a bug or have technical questions, feel free to open an issue. 
+
+For regular support, try [Community chat](https://docs.ipfs.tech/community/#chat)'s `#ipfs-implementers` room or [help/boxo at IPFS discussion forums](https://discuss.ipfs.tech/c/help/boxo/51).
 
 ### What is the response time for issues or PRs filed?
 
-New issues and PRs to this repo are usually looked at on a weekly basis as part of [Kubo triage](https://pl-strflt.notion.site/Kubo-Issue-Triage-Notes-7d4983e8cf294e07b3cc51b0c60ede9a). However, the response time may vary.
+New issues and PRs to this repo are usually looked at on a weekly basis as part of [Shipyard's GO Triage triage](https://ipshipyard.notion.site/IPFS-Go-Triage-Boxo-Kubo-Rainbow-0ddee6b7f28d412da7dabe4f9107c29a). However, the response time may vary.
 
 ### What are some projects that depend on this project?
 
 The exhaustive list is https://github.com/ipfs/boxo/network/dependents. Some notable projects include:
 
 1. [Kubo](https://github.com/ipfs/kubo), an IPFS implementation in Go
 2. [Lotus](https://github.com/filecoin-project/lotus), a Filecoin implementation in Go
-6. [rainbow](https://github.com/ipfs/rainbow), a specialized IPFS gateway
-4. [ipfs-check](https://github.com/ipfs-shipyard/ipfs-check), checks IPFS data availability
-5. [someguy](https://github.com/ipfs-shipyard/someguy), a dedicated Delegated Routing V1 server and client
-3. [Bifrost Gateway](https://github.com/ipfs/bifrost-gateway), a dedicated IPFS Gateway daemon backed by a remote datastore
+3. [rainbow](https://github.com/ipfs/rainbow), a specialized IPFS gateway
+4. [ipfs-check](https://github.com/ipfs/ipfs-check), checks IPFS data availability
+5. [someguy](https://github.com/ipfs/someguy), a dedicated Delegated Routing V1 server and client
 
 ### Governance and Access
 

RELEASE.md

@@ -52,7 +52,7 @@ At least as of 2023-06-08, changelog test is manually copied from [the changelog
 ### Related Work
 Below are links of related/adjacent work that has informed some of the decisions in this document:
 1. https://github.com/ipfs/boxo/issues/170
-2. https://pl-strflt.notion.site/Kubo-Release-Process-5a5d066264704009a28a79cff93062c4
+2. https://ipshipyard.notion.site/Kubo-Release-Process-6dba4f5755c9458ab5685eeb28173778
 3. https://github.com/ipfs/kubo/blob/master/docs/RELEASE_ISSUE_TEMPLATE.md
 
 ## Release Process
@@ -73,7 +73,7 @@ Below are links of related/adjacent work that has informed some of the decisions
 - [ ] Ensure Boxo tests are passing
 - [ ] Ensure Kubo tests are passing
   - [ ] Go to Kubo dir and run `go get github.com/ipfs/boxo@<commit_hash>` using the commit hash of the `release-vX.Y.Z` branch
-  - [ ] Run `go mod tidy` in repo root and in `docs/examples/kubo-as-a-library`
+  - [ ] Run `make mod_tidy` in repo root (to apply `go mod tidy` to code, tests, and examples)
   - [ ] Commit the changes and open a draft PR in Kubo
   - [ ] Name the PR "Upgrade to Boxo vX.Y.Z"
   - [ ] Paste a link to the Kubo PR in the Boxo PR, so reviewers can verify the Kubo test run

bitswap/benchmarks_test.go

@@ -12,8 +12,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/ipfs/boxo/bitswap/internal/testutil"
 	blocks "github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-test/random"
 	protocol "github.com/libp2p/go-libp2p/core/protocol"
 
 	"github.com/ipfs/boxo/bitswap"
@@ -169,8 +169,8 @@ func BenchmarkFetchFromOldBitswap(b *testing.B) {
 			testinstance.ConnectInstances(instances)
 
 			// Generate blocks, with a smaller root block
-			rootBlock := testutil.GenerateBlocksOfSize(1, rootBlockSize)
-			blocks := testutil.GenerateBlocksOfSize(bch.blockCount, stdBlockSize)
+			rootBlock := random.BlocksOfSize(1, rootBlockSize)
+			blocks := random.BlocksOfSize(bch.blockCount, stdBlockSize)
 			blocks[0] = rootBlock[0]
 
 			// Run the distribution
@@ -300,7 +300,7 @@ func BenchmarkDatacenterMultiLeechMultiSeed(b *testing.B) {
 			defer ig.Close()
 
 			instances := ig.Instances(numnodes)
-			blocks := testutil.GenerateBlocksOfSize(numblks, blockSize)
+			blocks := random.BlocksOfSize(numblks, int(blockSize))
 			runDistributionMulti(b, instances[:3], instances[3:], blocks, bstoreLatency, df, ff)
 		}
 	})
@@ -317,8 +317,8 @@ func subtestDistributeAndFetch(b *testing.B, numnodes, numblks int, d delay.D, b
 		ig := testinstance.NewTestInstanceGenerator(net, nil, nil)
 
 		instances := ig.Instances(numnodes)
-		rootBlock := testutil.GenerateBlocksOfSize(1, rootBlockSize)
-		blocks := testutil.GenerateBlocksOfSize(numblks, stdBlockSize)
+		rootBlock := random.BlocksOfSize(1, rootBlockSize)
+		blocks := random.BlocksOfSize(numblks, stdBlockSize)
 		blocks[0] = rootBlock[0]
 		runDistribution(b, instances, blocks, bstoreLatency, df, ff)
 		ig.Close()
@@ -333,8 +333,8 @@ func subtestDistributeAndFetchRateLimited(b *testing.B, numnodes, numblks int, d
 		defer ig.Close()
 
 		instances := ig.Instances(numnodes)
-		rootBlock := testutil.GenerateBlocksOfSize(1, rootBlockSize)
-		blocks := testutil.GenerateBlocksOfSize(numblks, blockSize)
+		rootBlock := random.BlocksOfSize(1, rootBlockSize)
+		blocks := random.BlocksOfSize(numblks, int(blockSize))
 		blocks[0] = rootBlock[0]
 		runDistribution(b, instances, blocks, bstoreLatency, df, ff)
 	}

bitswap/bitswap.go

@@ -144,10 +144,9 @@ func (bs *Bitswap) Stat() (*Stat, error) {
 
 func (bs *Bitswap) Close() error {
 	bs.net.Stop()
-	return multierr.Combine(
-		bs.Client.Close(),
-		bs.Server.Close(),
-	)
+	bs.Client.Close()
+	bs.Server.Close()
+	return nil
 }
 
 func (bs *Bitswap) WantlistForPeer(p peer.ID) []cid.Cid {