diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 2a7ee155f4a..a710eef5746 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -442,6 +442,10 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request carVersion := formatParams["version"] i.serveCAR(r.Context(), w, r, resolvedPath, contentPath, carVersion, begin) return + case "application/x-tar": + logger.Debugw("serving tar file", "path", contentPath) + i.serveTAR(r.Context(), w, r, resolvedPath, contentPath, begin, logger) + return default: // catch-all for unsuported application/vnd.* err := fmt.Errorf("unsupported format %q", responseFormat) webError(w, "failed respond with requested content type", err, http.StatusBadRequest) @@ -887,6 +891,10 @@ func getEtag(r *http.Request, cid cid.Cid) string { f := responseFormat[strings.LastIndex(responseFormat, ".")+1:] // Etag: "cid.foo" (gives us nice compression together with Content-Disposition in block (raw) and car responses) suffix = `.` + f + suffix + // Since different TAR implementations may produce different byte-for-byte responses, we define a weak Etag. + if responseFormat == "application/x-tar" { + prefix = "W/" + prefix + } } // TODO: include selector suffix when https://github.com/ipfs/kubo/issues/8769 lands return prefix + cid.String() + suffix @@ -901,11 +909,13 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string] return "application/vnd.ipld.raw", nil, nil case "car": return "application/vnd.ipld.car", nil, nil + case "tar": + return "application/x-tar", nil, nil } } // Browsers and other user agents will send Accept header with generic types like: // Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8 - // We only care about explciit, vendor-specific content-types. + // We only care about explicit, vendor-specific content-types. for _, accept := range r.Header.Values("Accept") { // respond to the very first ipld content type if strings.HasPrefix(accept, "application/vnd.ipld") { diff --git a/core/corehttp/gateway_handler_tar.go b/core/corehttp/gateway_handler_tar.go new file mode 100644 index 00000000000..bbf60cefeda --- /dev/null +++ b/core/corehttp/gateway_handler_tar.go @@ -0,0 +1,77 @@ +package corehttp + +import ( + "context" + "html" + "net/http" + "time" + + files "github.com/ipfs/go-ipfs-files" + "github.com/ipfs/go-ipfs/tracing" + ipath "github.com/ipfs/interface-go-ipfs-core/path" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" +) + +var unixEpochTime = time.Unix(0, 0) + +func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) { + ctx, span := tracing.Span(ctx, "Gateway", "ServeTAR", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) + defer span.End() + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Set Cache-Control and read optional Last-Modified time + modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) + + // Finish early if Etag match + if r.Header.Get("If-None-Match") == getEtag(r, resolvedPath.Cid()) { + w.WriteHeader(http.StatusNotModified) + return + } + + // Set Content-Disposition + var name string + if urlFilename := r.URL.Query().Get("filename"); urlFilename != "" { + name = urlFilename + } else { + name = resolvedPath.Cid().String() + ".tar" + } + setContentDispositionHeader(w, name, "attachment") + + // Get Unixfs file + file, err := i.api.Unixfs().Get(ctx, resolvedPath) + if err != nil { + webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound) + return + } + defer file.Close() + + // Construct the TAR writer + tarw, err := files.NewTarWriter(w) + if err != nil { + webError(w, "could not build tar writer", err, http.StatusInternalServerError) + return + } + defer tarw.Close() + + // Sets correct Last-Modified header. This code is borrowed from the standard + // library (net/http/server.go) as we cannot use serveFile without throwing the entire + // TAR into the memory first. + if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) { + w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) + } + + w.Header().Set("Content-Type", "application/x-tar") + + if err := tarw.WriteFile(file, name); err != nil { + // We return error as a trailer, however it is not something browsers can access + // (https://github.com/mdn/browser-compat-data/issues/14703) + // Due to this, we suggest client always verify that + // the received CAR stream response is matching requested DAG selector + w.Header().Set("X-Stream-Error", err.Error()) + return + } +} diff --git a/test/sharness/t0110-gateway.sh b/test/sharness/t0110-gateway.sh index 484eefe4a0d..e61c66ff9ab 100755 --- a/test/sharness/t0110-gateway.sh +++ b/test/sharness/t0110-gateway.sh @@ -293,6 +293,16 @@ test_expect_success "Verify gateway file" ' test_cmp gateway_daemon_actual gateway_file_actual ' +test_expect_success "GET TAR file from gateway and extract" ' + curl "http://127.0.0.1:$port/ipfs/$FOO2_HASH?format=tar" | tar -x +' + +test_expect_success "GET TAR file has expected Content-Type" ' + curl -svX GET "http://127.0.0.1:$port/ipfs/$FOO2_HASH?format=tar" > curl_output_filename 2>&1 && + cat curl_output_filename && + grep "< Content-Type: application/x-tar" curl_output_filename +' + test_kill_ipfs_daemon