From 9adf1539291c34bdb69c62e3bfe5ae278e64342c Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 18 Aug 2023 00:32:50 +0200 Subject: [PATCH 1/2] fix(gw): useful IPIP-402 CARs on not found errors This aims to test https://github.com/ipfs/boxo/pull/440 against https://github.com/ipfs/gateway-conformance/pull/131 --- .github/workflows/gateway-conformance.yml | 4 ++-- docs/changelogs/v0.23.md | 11 +++++++++++ docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/cli/gateway_test.go | 2 +- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 9 files changed, 23 insertions(+), 12 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index 380dd4d283f..cf95b60be56 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -24,7 +24,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.3 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@261cc49391dcd2bddd7704ad28b236dc4a63c424 with: output: fixtures @@ -90,7 +90,7 @@ jobs: # 6. Run the gateway-conformance tests - name: Run gateway-conformance tests - uses: ipfs/gateway-conformance/.github/actions/test@v0.3 + uses: ipfs/gateway-conformance/.github/actions/test@261cc49391dcd2bddd7704ad28b236dc4a63c424 with: gateway-url: http://127.0.0.1:8080 json: output.json diff --git a/docs/changelogs/v0.23.md b/docs/changelogs/v0.23.md index a53639d88a4..641f20f8bb3 100644 --- a/docs/changelogs/v0.23.md +++ b/docs/changelogs/v0.23.md @@ -7,6 +7,7 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) - [Mplex deprecation](#mplex-deprecation) + - [Gateway: meaningful CAR responses on Not Found errors](#gateway-meaningful-car-responses-on-not-found-errors) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -32,6 +33,16 @@ $ ipfs config --json Swarm.Transports.Multiplexers.Mplex 200 We will completely remove Mplex in v0.24 as it makes protocols very bad to implement, if you are in this situation you need to add yamux support to your other implementation. +#### Gateway: meaningful CAR responses on Not Found errors + +When requesting a CAR from the gateway, the root of the CAR might no longer be +meaningful. By default, the CAR root will be the last resolvable segment of the +path. However, in situations where the path cannot be resolved, such as when +the path does not exist, a CAR will be sent with a root of `bafkqaaa` (empty CID). + +This CAR will contain all blocks necessary to validate that the path does not +exist without having to trust the gateway. + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index c0806d180a4..bccc3529ca7 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f + github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.29.2 github.com/multiformats/go-multiaddr v0.10.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 710497500f6..77d403050d2 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -301,8 +301,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f h1:sXqGLIATCsBdHse7S9n6e328NhORvVM64+4IRuFlpmI= -github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= +github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547 h1:AE6pzqLUq22jKFf3648vmcCcQ6lbRhbGEQvj6TL65KA= +github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index f6d60148bb0..61ec2929283 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f + github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index 083a052ca7e..fe90bb4950f 100644 --- a/go.sum +++ b/go.sum @@ -337,8 +337,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f h1:sXqGLIATCsBdHse7S9n6e328NhORvVM64+4IRuFlpmI= -github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= +github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547 h1:AE6pzqLUq22jKFf3648vmcCcQ6lbRhbGEQvj6TL65KA= +github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/cli/gateway_test.go b/test/cli/gateway_test.go index b2729da3c44..168acfdb999 100644 --- a/test/cli/gateway_test.go +++ b/test/cli/gateway_test.go @@ -487,7 +487,7 @@ func TestGateway(t *testing.T) { t.Run("not present key from node 1", func(t *testing.T) { t.Parallel() - assert.Equal(t, 404, node1.GatewayClient().Get("/ipfs/"+cidFoo).StatusCode) + assert.Equal(t, 500, node1.GatewayClient().Get("/ipfs/"+cidFoo).StatusCode) }) t.Run("not present IPNS key from node 1", func(t *testing.T) { diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index fb8b087ad5a..9715e7751ed 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -7,7 +7,7 @@ replace github.com/ipfs/kubo => ../../ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/golangci/golangci-lint v1.49.0 - github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f + github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 github.com/ipfs/go-datastore v0.6.0 diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index 3fbd086a5f9..e228e65d422 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -395,8 +395,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f h1:sXqGLIATCsBdHse7S9n6e328NhORvVM64+4IRuFlpmI= -github.com/ipfs/boxo v0.11.1-0.20230817061817-1d2f5e511e9f/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= +github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547 h1:AE6pzqLUq22jKFf3648vmcCcQ6lbRhbGEQvj6TL65KA= +github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= From c50db80266dc597cab430ac9e5d9b8ebca3de92e Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 18 Aug 2023 08:29:34 +0200 Subject: [PATCH 2/2] chore: update CAR and revert conformance to v0.3 --- .github/workflows/gateway-conformance.yml | 4 ++-- docs/examples/kubo-as-a-library/go.mod | 2 +- docs/examples/kubo-as-a-library/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/dependencies/go.mod | 2 +- test/dependencies/go.sum | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index cf95b60be56..380dd4d283f 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -24,7 +24,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@261cc49391dcd2bddd7704ad28b236dc4a63c424 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.3 with: output: fixtures @@ -90,7 +90,7 @@ jobs: # 6. Run the gateway-conformance tests - name: Run gateway-conformance tests - uses: ipfs/gateway-conformance/.github/actions/test@261cc49391dcd2bddd7704ad28b236dc4a63c424 + uses: ipfs/gateway-conformance/.github/actions/test@v0.3 with: gateway-url: http://127.0.0.1:8080 json: output.json diff --git a/docs/examples/kubo-as-a-library/go.mod b/docs/examples/kubo-as-a-library/go.mod index bccc3529ca7..9d79aa78210 100644 --- a/docs/examples/kubo-as-a-library/go.mod +++ b/docs/examples/kubo-as-a-library/go.mod @@ -7,7 +7,7 @@ go 1.18 replace github.com/ipfs/kubo => ./../../.. require ( - github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547 + github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000 github.com/libp2p/go-libp2p v0.29.2 github.com/multiformats/go-multiaddr v0.10.1 diff --git a/docs/examples/kubo-as-a-library/go.sum b/docs/examples/kubo-as-a-library/go.sum index 77d403050d2..acfc47fcba8 100644 --- a/docs/examples/kubo-as-a-library/go.sum +++ b/docs/examples/kubo-as-a-library/go.sum @@ -301,8 +301,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547 h1:AE6pzqLUq22jKFf3648vmcCcQ6lbRhbGEQvj6TL65KA= -github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= +github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3 h1:oiMqmivloEHtlCkS+8rCq1Pkz/rf6m3sC45lL4cnwBA= +github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= diff --git a/go.mod b/go.mod index 61ec2929283..638513c7b9f 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/google/uuid v1.3.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547 + github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 diff --git a/go.sum b/go.sum index fe90bb4950f..7dac8bc22c4 100644 --- a/go.sum +++ b/go.sum @@ -337,8 +337,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547 h1:AE6pzqLUq22jKFf3648vmcCcQ6lbRhbGEQvj6TL65KA= -github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= +github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3 h1:oiMqmivloEHtlCkS+8rCq1Pkz/rf6m3sC45lL4cnwBA= +github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/test/dependencies/go.mod b/test/dependencies/go.mod index 9715e7751ed..ed17f6ad856 100644 --- a/test/dependencies/go.mod +++ b/test/dependencies/go.mod @@ -7,7 +7,7 @@ replace github.com/ipfs/kubo => ../../ require ( github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd github.com/golangci/golangci-lint v1.49.0 - github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547 + github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 github.com/ipfs/go-datastore v0.6.0 diff --git a/test/dependencies/go.sum b/test/dependencies/go.sum index e228e65d422..8e943ec0bcd 100644 --- a/test/dependencies/go.sum +++ b/test/dependencies/go.sum @@ -395,8 +395,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547 h1:AE6pzqLUq22jKFf3648vmcCcQ6lbRhbGEQvj6TL65KA= -github.com/ipfs/boxo v0.11.1-0.20230817163255-eb36550a9547/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= +github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3 h1:oiMqmivloEHtlCkS+8rCq1Pkz/rf6m3sC45lL4cnwBA= +github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo=