fix: use %-encoded headers in most compatible way

client/rpc/api.go

@@ -1,18 +1,24 @@
 package rpc
 
 import (
+	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
+	"github.com/blang/semver/v4"
 	iface "github.com/ipfs/boxo/coreiface"
 	caopts "github.com/ipfs/boxo/coreiface/options"
 	"github.com/ipfs/boxo/ipld/merkledag"
 	"github.com/ipfs/go-cid"
 	legacy "github.com/ipfs/go-ipld-legacy"
+	ipfs "github.com/ipfs/kubo"
 	dagpb "github.com/ipld/go-codec-dagpb"
 	_ "github.com/ipld/go-ipld-prime/codec/dagcbor"
 	"github.com/ipld/go-ipld-prime/node/basicnode"
@@ -42,6 +48,8 @@ type HttpApi struct {
 	Headers     http.Header
 	applyGlobal func(*requestBuilder)
 	ipldDecoder *legacy.Decoder
+	versionMu   sync.Mutex
+	version     *semver.Version
 }
 
 // NewLocalApi tries to construct new HttpApi instance communicating with local
@@ -151,6 +159,7 @@ func NewURLApiWithClient(url string, c *http.Client) (*HttpApi, error) {
 	api.httpcli.CheckRedirect = func(_ *http.Request, _ []*http.Request) error {
 		return fmt.Errorf("unexpected redirect")
 	}
+
 	return api, nil
 }
 
@@ -160,14 +169,19 @@ func (api *HttpApi) WithOptions(opts ...caopts.ApiOption) (iface.CoreAPI, error)
 		return nil, err
 	}
 
-	subApi := *api
-	subApi.applyGlobal = func(req *requestBuilder) {
-		if options.Offline {
-			req.Option("offline", options.Offline)
-		}
+	subApi := &HttpApi{
+		url:     api.url,
+		httpcli: api.httpcli,
+		Headers: api.Headers,
+		applyGlobal: func(req *requestBuilder) {
+			if options.Offline {
+				req.Option("offline", options.Offline)
+			}
+		},
+		ipldDecoder: api.ipldDecoder,
 	}
 
-	return &subApi, nil
+	return subApi, nil
 }
 
 func (api *HttpApi) Request(command string, args ...string) RequestBuilder {
@@ -228,3 +242,36 @@ func (api *HttpApi) PubSub() iface.PubSubAPI {
 func (api *HttpApi) Routing() iface.RoutingAPI {
 	return (*RoutingAPI)(api)
 }
+
+func (api *HttpApi) loadRemoteVersion() (*semver.Version, error) {
+	api.versionMu.Lock()
+	defer api.versionMu.Unlock()
+
+	if api.version == nil {
+		ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30))
+		defer cancel()
+
+		resp, err := api.Request("version").Send(ctx)
+		if err != nil {
+			return nil, err
+		}
+		if resp.Error != nil {
+			return nil, resp.Error
+		}
+		defer resp.Close()
+		var out ipfs.VersionInfo
+		dec := json.NewDecoder(resp.Output)
+		if err := dec.Decode(&out); err != nil {
+			return nil, err
+		}
+
+		remoteVersion, err := semver.New(out.Version)
+		if err != nil {
+			return nil, err
+		}
+
+		api.version = remoteVersion
+	}
+
+	return api.version, nil
+}

client/rpc/requestbuilder.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/blang/semver/v4"
 	"github.com/ipfs/boxo/files"
 )
 
@@ -23,13 +24,18 @@ type RequestBuilder interface {
 	Exec(ctx context.Context, res interface{}) error
 }
 
+// encodedAbsolutePathVersion is the version from which the absolute path header in
+// multipart requests is %-encoded. Before this version, its sent raw.
+var encodedAbsolutePathVersion = semver.MustParse("0.23.0-dev")
+
 // requestBuilder is an IPFS commands request builder.
 type requestBuilder struct {
-	command string
-	args    []string
-	opts    map[string]string
-	headers map[string]string
-	body    io.Reader
+	command    string
+	args       []string
+	opts       map[string]string
+	headers    map[string]string
+	body       io.Reader
+	buildError error
 
 	shell *HttpApi
 }
@@ -60,7 +66,18 @@ func (r *requestBuilder) Body(body io.Reader) RequestBuilder {
 func (r *requestBuilder) FileBody(body io.Reader) RequestBuilder {
 	pr, _ := files.NewReaderPathFile("/dev/stdin", io.NopCloser(body), nil)
 	d := files.NewMapDirectory(map[string]files.Node{"": pr})
-	r.body = files.NewMultiFileReader(d, false)
+
+	version, err := r.shell.loadRemoteVersion()
+	if err != nil {
+		// Unfortunately, we cannot return an error here. Changing this API is also
+		// not the best since we would otherwise have an inconsistent RequestBuilder.
+		// We save the error and return it when calling [requestBuilder.Send].
+		r.buildError = err
+		return r
+	}
+
+	useEncodedAbsPaths := version.LT(encodedAbsolutePathVersion)
+	r.body = files.NewMultiFileReader(d, false, useEncodedAbsPaths)
 
 	return r
 }
@@ -97,6 +114,10 @@ func (r *requestBuilder) Header(name, value string) RequestBuilder {
 
 // Send sends the request and return the response.
 func (r *requestBuilder) Send(ctx context.Context) (*Response, error) {
+	if r.buildError != nil {
+		return nil, r.buildError
+	}
+
 	r.shell.applyGlobal(r)
 
 	req := NewRequest(ctx, r.shell.url, r.command, r.args...)

client/rpc/unixfs.go

@@ -62,7 +62,13 @@ func (api *UnixfsAPI) Add(ctx context.Context, f files.Node, opts ...caopts.Unix
 	}
 
 	d := files.NewMapDirectory(map[string]files.Node{"": f}) // unwrapped on the other side
-	req.Body(files.NewMultiFileReader(d, false))
+
+	version, err := api.core().loadRemoteVersion()
+	if err != nil {
+		return nil, err
+	}
+	useEncodedAbsPaths := version.LT(encodedAbsolutePathVersion)
+	req.Body(files.NewMultiFileReader(d, false, useEncodedAbsPaths))
 
 	var out addEvent
 	resp, err := req.Send(ctx)

cmd/ipfs/main.go

@@ -2,22 +2,27 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"os"
 	"runtime/pprof"
 	"strings"
 	"time"
 
+	"github.com/blang/semver/v4"
 	"github.com/google/uuid"
 	u "github.com/ipfs/boxo/util"
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	"github.com/ipfs/go-ipfs-cmds/cli"
 	cmdhttp "github.com/ipfs/go-ipfs-cmds/http"
 	logging "github.com/ipfs/go-log"
+	ipfs "github.com/ipfs/kubo"
 	"github.com/ipfs/kubo/cmd/ipfs/util"
 	oldcmds "github.com/ipfs/kubo/commands"
 	"github.com/ipfs/kubo/core"
@@ -224,6 +229,10 @@ func apiAddrOption(req *cmds.Request) (ma.Multiaddr, error) {
 	return ma.NewMultiaddr(apiAddrStr)
 }
 
+// encodedAbsolutePathVersion is the version from which the absolute path header in
+// multipart requests is %-encoded. Before this version, its sent raw.
+var encodedAbsolutePathVersion = semver.MustParse("0.23.0-dev")
+
 func makeExecutor(req *cmds.Request, env interface{}) (cmds.Executor, error) {
 	exe := tracingWrappedExecutor{cmds.NewExecutor(req.Root)}
 	cctx := env.(*oldcmds.Context)
@@ -315,9 +324,18 @@ func makeExecutor(req *cmds.Request, env interface{}) (cmds.Executor, error) {
 	default:
 		return nil, fmt.Errorf("unsupported API address: %s", apiAddr)
 	}
-	opts = append(opts, cmdhttp.ClientWithHTTPClient(&http.Client{
+
+	httpClient := &http.Client{
 		Transport: otelhttp.NewTransport(tpt),
-	}))
+	}
+	opts = append(opts, cmdhttp.ClientWithHTTPClient(httpClient))
+
+	// Fetch remove version, as some feature compatibility might change depending on it.
+	remoteVersion, err := getRemoteVersion(tracingWrappedExecutor{cmdhttp.NewClient(host, opts...)})
+	if err != nil {
+		return nil, err
+	}
+	opts = append(opts, cmdhttp.ClientWithRawAbsPath(remoteVersion.LT(encodedAbsolutePathVersion)))
 
 	return tracingWrappedExecutor{cmdhttp.NewClient(host, opts...)}, nil
 }
@@ -417,3 +435,40 @@ func resolveAddr(ctx context.Context, addr ma.Multiaddr) (ma.Multiaddr, error) {
 
 	return addrs[0], nil
 }
+
+type nopWriter struct {
+	io.Writer
+}
+
+func (nw nopWriter) Close() error {
+	return nil
+}
+
+func getRemoteVersion(exe cmds.Executor) (*semver.Version, error) {
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30))
+	defer cancel()
+
+	req, err := cmds.NewRequest(ctx, []string{"version"}, nil, nil, nil, Root)
+	if err != nil {
+		return nil, err
+	}
+
+	var buf bytes.Buffer
+	re, err := cmds.NewWriterResponseEmitter(nopWriter{&buf}, req)
+	if err != nil {
+		return nil, err
+	}
+
+	err = exe.Execute(req, re, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	var out ipfs.VersionInfo
+	dec := json.NewDecoder(&buf)
+	if err := dec.Decode(&out); err != nil {
+		return nil, err
+	}
+
+	return semver.New(out.Version)
+}

docs/changelogs/v0.23.md

@@ -8,6 +8,7 @@
 - [🔦 Highlights](#-highlights)
   - [Mplex deprecation](#mplex-deprecation)
   - [Gateway: meaningful CAR responses on Not Found errors](#gateway-meaningful-car-responses-on-not-found-errors)
+  - [Binary characters in file names: no longer works with old clients and new Kubo servers](#binary-characters-in-file-names-no-longer-works-with-old-clients-and-new-kubo-servers)
 - [📝 Changelog](#-changelog)
 - [👨‍👩‍👧‍👦 Contributors](#-contributors)
 
@@ -43,6 +44,21 @@ the path does not exist, a CAR will be sent with a root of `bafkqaaa` (empty CID
 This CAR will contain all blocks necessary to validate that the path does not
 exist without having to trust the gateway.
 
+#### Binary characters in file names: no longer works with old clients and new Kubo servers
+
+In this version, we updated Kubo to support Go 1.20+. In Go 1.20, a regression
+regarding multipart headers was [introduced](https://github.com/golang/go/issues/60674).
+This only affects `ipfs add` when a file name has binary characters in its name.
+As a consequence, we had to update the encoding of the file name headers. This is
+the compatibility table:
+
+|            | New Client | Old Client  |
+|------------|------------|-------------|
+| New Server | ✅         | 🟡*         |
+| Old Server | ✅         | ✅          |
+
+*Old clients can only send Unicode file paths to the server.
+
 ### 📝 Changelog
 
 ### 👨‍👩‍👧‍👦 Contributors

docs/examples/kubo-as-a-library/go.mod

@@ -7,7 +7,7 @@ go 1.20
 replace github.com/ipfs/kubo => ./../../..
 
 require (
-	github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3
+	github.com/ipfs/boxo v0.12.0
 	github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
 	github.com/libp2p/go-libp2p v0.29.2
 	github.com/multiformats/go-multiaddr v0.10.1
@@ -91,7 +91,7 @@ require (
 	github.com/ipfs/go-unixfsnode v1.7.1 // indirect
 	github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 // indirect
 	github.com/ipld/go-codec-dagpb v1.6.0 // indirect
-	github.com/ipld/go-ipld-prime v0.20.0 // indirect
+	github.com/ipld/go-ipld-prime v0.21.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect

docs/examples/kubo-as-a-library/go.sum

@@ -159,7 +159,7 @@ github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJn
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og=
-github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
@@ -301,8 +301,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3 h1:oiMqmivloEHtlCkS+8rCq1Pkz/rf6m3sC45lL4cnwBA=
-github.com/ipfs/boxo v0.11.1-0.20230818062747-654231b2bda3/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w=
+github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ=
+github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -398,8 +398,8 @@ github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6
 github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
 github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8=
 github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
-github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
-github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M=
+github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E=
+github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
 github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
 github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
@@ -737,7 +737,7 @@ github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMI
 github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE=
 github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
 github.com/warpfork/go-testmark v0.9.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0=
-github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U=
+github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s=
 github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
 github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=