From fdd19656c465f07ff6d7be653f5438d74a0c0c2f Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 5 Dec 2022 20:22:26 +0100 Subject: [PATCH] feat(gateway): JSON and CBOR response formats (IPIP-328) (#9335) https://github.com/ipfs/kubo/pull/9335 https://github.com/ipfs/specs/pull/328 Co-authored-by: Marcin Rataj --- assets/dag-index-html/README.md | 3 + assets/dag-index-html/index.go | 81 ++++ assets/dir-index-html/README.md | 2 +- assets/dir-index-html/dir-index.html | 12 +- assets/dir-index-html/src/dir-index.html | 12 +- core/corehttp/gateway_handler.go | 30 +- core/corehttp/gateway_handler_codec.go | 258 ++++++++++++ core/corehttp/gateway_test.go | 8 +- docs/changelogs/v0.18.md | 73 ++++ test/sharness/t0115-gateway-dir-listing.sh | 4 +- test/sharness/t0116-gateway-cache.sh | 91 ++-- test/sharness/t0117-gateway-block.sh | 20 +- test/sharness/t0123-gateway-json-cbor.sh | 387 ++++++++++++++++++ .../dag-cbor-traversal.car | Bin 0 -> 318 bytes .../dag-json-traversal.car | Bin 0 -> 406 bytes .../t0123-gateway-json-cbor/dag-pb.car | Bin 0 -> 392 bytes .../t0123-gateway-json-cbor/dag-pb.json | 23 ++ 17 files changed, 938 insertions(+), 66 deletions(-) create mode 100644 assets/dag-index-html/README.md create mode 100644 assets/dag-index-html/index.go create mode 100644 core/corehttp/gateway_handler_codec.go create mode 100644 docs/changelogs/v0.18.md create mode 100755 test/sharness/t0123-gateway-json-cbor.sh create mode 100644 test/sharness/t0123-gateway-json-cbor/dag-cbor-traversal.car create mode 100644 test/sharness/t0123-gateway-json-cbor/dag-json-traversal.car create mode 100644 test/sharness/t0123-gateway-json-cbor/dag-pb.car create mode 100644 test/sharness/t0123-gateway-json-cbor/dag-pb.json diff --git a/assets/dag-index-html/README.md b/assets/dag-index-html/README.md new file mode 100644 index 00000000000..de38a9504a1 --- /dev/null +++ b/assets/dag-index-html/README.md @@ -0,0 +1,3 @@ +# dag-index-html + +> HTML representation for non-UnixFS DAGs such as DAG-CBOR. diff --git a/assets/dag-index-html/index.go b/assets/dag-index-html/index.go new file mode 100644 index 00000000000..214b06a3822 --- /dev/null +++ b/assets/dag-index-html/index.go @@ -0,0 +1,81 @@ +package dagindexhtml + +import "html/template" + +// TODO: DagIndexTemplate - replace static CSS with shared one with ../dir-index-html + +// DagIndexTemplate is HTML-based template for non-UnixFS DAGs when request was +// made with Accept: text/html (web browsers). +var DagIndexTemplate = template.Must(template.New("redirect").Parse(` + + + + + + + + + + + + + + + + + + {{ .Path }} + + + +
+ +
+
About IPFS
+
Install IPFS
+
About
+
Install
+
+ + + +
+
+
+
+
+

CID: {{.CID}}
+ Codec: {{.CodecName}} ({{.CodecHex}})

+
+
+ + + + + + + +
+

Preview as JSON
(application/json)

+
+

Or download as: +

+

+
+
+
+ +`)) + +type DagIndexTemplateData struct { + Path string + CID string + CodecName string + CodecHex string +} diff --git a/assets/dir-index-html/README.md b/assets/dir-index-html/README.md index ad5fa68b2db..3dd45eb5905 100644 --- a/assets/dir-index-html/README.md +++ b/assets/dir-index-html/README.md @@ -1,6 +1,6 @@ # dir-index-html -> Directory listing HTML for `go-ipfs` gateways +> Directory listing HTML for HTTP gateway ![](https://user-images.githubusercontent.com/157609/88379209-ce6f0600-cda2-11ea-9620-20b9237bb441.png) diff --git a/assets/dir-index-html/dir-index.html b/assets/dir-index-html/dir-index.html index a2d662d19c7..1d00e5fe73f 100644 --- a/assets/dir-index-html/dir-index.html +++ b/assets/dir-index-html/dir-index.html @@ -26,12 +26,12 @@
-
About IPFS
-
Install IPFS
-
About
-
Install
+
About IPFS
+
Install IPFS
+
About
+
Install
- +
@@ -84,7 +84,7 @@ {{ if .Hash }} - + {{ .ShortHash }} {{ end }} diff --git a/assets/dir-index-html/src/dir-index.html b/assets/dir-index-html/src/dir-index.html index 1bbbb0a9a8c..a763b3e76bb 100644 --- a/assets/dir-index-html/src/dir-index.html +++ b/assets/dir-index-html/src/dir-index.html @@ -25,12 +25,12 @@
-
About IPFS
-
Install IPFS
-
About
-
Install
+
About IPFS
+
Install IPFS
+
About
+
Install
@@ -83,7 +83,7 @@ {{ if .Hash }} - + {{ .ShortHash }} {{ end }} diff --git a/core/corehttp/gateway_handler.go b/core/corehttp/gateway_handler.go index 7f0f11885e6..1222b17bcd7 100644 --- a/core/corehttp/gateway_handler.go +++ b/core/corehttp/gateway_handler.go @@ -26,6 +26,7 @@ import ( coreiface "github.com/ipfs/interface-go-ipfs-core" ipath "github.com/ipfs/interface-go-ipfs-core/path" routing "github.com/libp2p/go-libp2p/core/routing" + mc "github.com/multiformats/go-multicodec" prometheus "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -417,9 +418,15 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request // Support custom response formats passed via ?format or Accept HTTP header switch responseFormat { - case "": // The implicit response format is UnixFS - logger.Debugw("serving unixfs", "path", contentPath) - i.serveUnixFS(r.Context(), w, r, resolvedPath, contentPath, begin, logger) + case "": + switch resolvedPath.Cid().Prefix().Codec { + case uint64(mc.Json), uint64(mc.DagJson), uint64(mc.Cbor), uint64(mc.DagCbor): + logger.Debugw("serving codec", "path", contentPath) + i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat) + default: + logger.Debugw("serving unixfs", "path", contentPath) + i.serveUnixFS(r.Context(), w, r, resolvedPath, contentPath, begin, logger) + } return case "application/vnd.ipld.raw": logger.Debugw("serving raw block", "path", contentPath) @@ -434,6 +441,11 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request logger.Debugw("serving tar file", "path", contentPath) i.serveTAR(r.Context(), w, r, resolvedPath, contentPath, begin, logger) return + case "application/json", "application/vnd.ipld.dag-json", + "application/cbor", "application/vnd.ipld.dag-cbor": + logger.Debugw("serving codec", "path", contentPath) + i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat) + return default: // catch-all for unsuported application/vnd.* err := fmt.Errorf("unsupported format %q", responseFormat) webError(w, "failed respond with requested content type", err, http.StatusBadRequest) @@ -866,6 +878,14 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string] return "application/vnd.ipld.car", nil, nil case "tar": return "application/x-tar", nil, nil + case "dag-json": + return "application/vnd.ipld.dag-json", nil, nil + case "json": + return "application/json", nil, nil + case "dag-cbor": + return "application/vnd.ipld.dag-cbor", nil, nil + case "cbor": + return "application/cbor", nil, nil } } // Browsers and other user agents will send Accept header with generic types like: @@ -874,7 +894,9 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string] for _, accept := range r.Header.Values("Accept") { // respond to the very first ipld content type if strings.HasPrefix(accept, "application/vnd.ipld") || - strings.HasPrefix(accept, "application/x-tar") { + strings.HasPrefix(accept, "application/x-tar") || + strings.HasPrefix(accept, "application/json") || + strings.HasPrefix(accept, "application/cbor") { mediatype, params, err := mime.ParseMediaType(accept) if err != nil { return "", nil, err diff --git a/core/corehttp/gateway_handler_codec.go b/core/corehttp/gateway_handler_codec.go new file mode 100644 index 00000000000..95a151c7943 --- /dev/null +++ b/core/corehttp/gateway_handler_codec.go @@ -0,0 +1,258 @@ +package corehttp + +import ( + "bytes" + "context" + "fmt" + "html" + "io" + "net/http" + "strings" + "time" + + cid "github.com/ipfs/go-cid" + ipldlegacy "github.com/ipfs/go-ipld-legacy" + ipath "github.com/ipfs/interface-go-ipfs-core/path" + "github.com/ipfs/kubo/assets" + dih "github.com/ipfs/kubo/assets/dag-index-html" + "github.com/ipfs/kubo/tracing" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/multicodec" + mc "github.com/multiformats/go-multicodec" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// codecToContentType maps the supported IPLD codecs to the HTTP Content +// Type they should have. +var codecToContentType = map[uint64]string{ + uint64(mc.Json): "application/json", + uint64(mc.Cbor): "application/cbor", + uint64(mc.DagJson): "application/vnd.ipld.dag-json", + uint64(mc.DagCbor): "application/vnd.ipld.dag-cbor", +} + +// contentTypeToCodecs maps the HTTP Content Type to the respective +// possible codecs. If the original data is in one of those codecs, +// we stream the raw bytes. Otherwise, we encode in the last codec +// of the list. +var contentTypeToCodecs = map[string][]uint64{ + "application/json": {uint64(mc.Json), uint64(mc.DagJson)}, + "application/vnd.ipld.dag-json": {uint64(mc.DagJson)}, + "application/cbor": {uint64(mc.Cbor), uint64(mc.DagCbor)}, + "application/vnd.ipld.dag-cbor": {uint64(mc.DagCbor)}, +} + +// contentTypeToExtension maps the HTTP Content Type to the respective file +// extension, used in Content-Disposition header when downloading the file. +var contentTypeToExtension = map[string]string{ + "application/json": ".json", + "application/vnd.ipld.dag-json": ".json", + "application/cbor": ".cbor", + "application/vnd.ipld.dag-cbor": ".cbor", +} + +func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, requestedContentType string) { + ctx, span := tracing.Span(ctx, "Gateway", "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", requestedContentType))) + defer span.End() + + cidCodec := resolvedPath.Cid().Prefix().Codec + responseContentType := requestedContentType + + // If the resolved path still has some remainder, return error for now. + // TODO: handle this when we have IPLD Patch (https://ipld.io/specs/patch/) via HTTP PUT + // TODO: (depends on https://github.com/ipfs/kubo/issues/4801 and https://github.com/ipfs/kubo/issues/4782) + if resolvedPath.Remainder() != "" { + path := strings.TrimSuffix(resolvedPath.String(), resolvedPath.Remainder()) + err := fmt.Errorf("%q of %q could not be returned: reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented: try reading %q instead", resolvedPath.Remainder(), resolvedPath.String(), path) + webError(w, "unsupported pathing", err, http.StatusNotImplemented) + return + } + + // If no explicit content type was requested, the response will have one based on the codec from the CID + if requestedContentType == "" { + cidContentType, ok := codecToContentType[cidCodec] + if !ok { + // Should not happen unless function is called with wrong parameters. + err := fmt.Errorf("content type not found for codec: %v", cidCodec) + webError(w, "internal error", err, http.StatusInternalServerError) + return + } + responseContentType = cidContentType + } + + // Set HTTP headers (for caching etc) + modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid()) + name := setCodecContentDisposition(w, r, resolvedPath, responseContentType) + w.Header().Set("Content-Type", responseContentType) + w.Header().Set("X-Content-Type-Options", "nosniff") + + // No content type is specified by the user (via Accept, or format=). However, + // we support this format. Let's handle it. + if requestedContentType == "" { + isDAG := cidCodec == uint64(mc.DagJson) || cidCodec == uint64(mc.DagCbor) + acceptsHTML := strings.Contains(r.Header.Get("Accept"), "text/html") + download := r.URL.Query().Get("download") == "true" + + if isDAG && acceptsHTML && !download { + i.serveCodecHTML(ctx, w, r, resolvedPath, contentPath) + } else { + i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime) + } + + return + } + + // Otherwise, the user has requested a specific content type. Let's first get + // the codecs that can be used with this content type. + codecs, ok := contentTypeToCodecs[requestedContentType] + if !ok { + // This is never supposed to happen unless function is called with wrong parameters. + err := fmt.Errorf("unsupported content type: %s", requestedContentType) + webError(w, err.Error(), err, http.StatusInternalServerError) + return + } + + // If we need to convert, use the last codec (strict dag- variant) + toCodec := codecs[len(codecs)-1] + + // If the requested content type has "dag-", ALWAYS go through the encoding + // process in order to validate the content. + if strings.Contains(requestedContentType, "dag-") { + i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime) + return + } + + // Otherwise, check if the data is encoded with the requested content type. + // If so, we can directly stream the raw data. serveRawBlock cannot be directly + // used here as it sets different headers. + for _, codec := range codecs { + if resolvedPath.Cid().Prefix().Codec == codec { + i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime) + return + } + } + + // Finally, if nothing of the above is true, we have to actually convert the codec. + i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime) +} + +func (i *gatewayHandler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path) { + // A HTML directory index will be presented, be sure to set the correct + // type instead of relying on autodetection (which may fail). + w.Header().Set("Content-Type", "text/html") + + // Clear Content-Disposition -- we want HTML to be rendered inline + w.Header().Del("Content-Disposition") + + // Generated index requires custom Etag (output may change between Kubo versions) + dagEtag := getDagIndexEtag(resolvedPath.Cid()) + w.Header().Set("Etag", dagEtag) + + // Remove Cache-Control for now to match UnixFS dir-index-html responses + // (we don't want browser to cache HTML forever) + // TODO: if we ever change behavior for UnixFS dir listings, same changes should be applied here + w.Header().Del("Cache-Control") + + cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec) + if err := dih.DagIndexTemplate.Execute(w, dih.DagIndexTemplateData{ + Path: contentPath.String(), + CID: resolvedPath.Cid().String(), + CodecName: cidCodec.String(), + CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), + }); err != nil { + webError(w, "failed to generate HTML listing for this DAG: try fetching raw block with ?format=raw", err, http.StatusInternalServerError) + } +} + +func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, name string, modtime time.Time) { + blockCid := resolvedPath.Cid() + blockReader, err := i.api.Block().Get(ctx, resolvedPath) + if err != nil { + webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError) + return + } + block, err := io.ReadAll(blockReader) + if err != nil { + webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError) + return + } + content := bytes.NewReader(block) + + // ServeContent will take care of + // If-None-Match+Etag, Content-Length and range requests + _, _, _ = ServeContent(w, r, name, modtime, content) +} + +func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, toCodec uint64, modtime time.Time) { + obj, err := i.api.Dag().Get(ctx, resolvedPath.Cid()) + if err != nil { + webError(w, "ipfs dag get "+html.EscapeString(resolvedPath.String()), err, http.StatusInternalServerError) + return + } + + universal, ok := obj.(ipldlegacy.UniversalNode) + if !ok { + err = fmt.Errorf("%T is not a valid IPLD node", obj) + webError(w, err.Error(), err, http.StatusInternalServerError) + return + } + finalNode := universal.(ipld.Node) + + encoder, err := multicodec.LookupEncoder(toCodec) + if err != nil { + webError(w, err.Error(), err, http.StatusInternalServerError) + return + } + + // Ensure IPLD node conforms to the codec specification. + var buf bytes.Buffer + err = encoder(finalNode, &buf) + if err != nil { + webError(w, err.Error(), err, http.StatusInternalServerError) + return + } + + // Sets correct Last-Modified header. This code is borrowed from the standard + // library (net/http/server.go) as we cannot use serveFile. + if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) { + w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) + } + + _, _ = w.Write(buf.Bytes()) +} + +func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentType string) string { + var dispType, name string + + ext, ok := contentTypeToExtension[contentType] + if !ok { + // Should never happen. + ext = ".bin" + } + + if urlFilename := r.URL.Query().Get("filename"); urlFilename != "" { + name = urlFilename + } else { + name = resolvedPath.Cid().String() + ext + } + + // JSON should be inlined, but ?download=true should still override + if r.URL.Query().Get("download") == "true" { + dispType = "attachment" + } else { + switch ext { + case ".json": // codecs that serialize to JSON can be rendered by browsers + dispType = "inline" + default: // everything else is assumed binary / opaque bytes + dispType = "attachment" + } + } + + setContentDispositionHeader(w, name, dispType) + return name +} + +func getDagIndexEtag(dagCid cid.Cid) string { + return `"DagIndex-` + assets.AssetHash + `_CID-` + dagCid.String() + `"` +} diff --git a/core/corehttp/gateway_test.go b/core/corehttp/gateway_test.go index f4dda631e43..0d2f07dbeaf 100644 --- a/core/corehttp/gateway_test.go +++ b/core/corehttp/gateway_test.go @@ -498,9 +498,9 @@ func TestIPNSHostnameBacklinks(t *testing.T) { if !strings.Contains(s, "") { t.Fatalf("expected file in directory listing") } - if !strings.Contains(s, "") { t.Fatalf("expected file in directory listing") } - if !strings.Contains(s, "" list_response +test_expect_success "dnslink gw: hash column should be a CID link to cid.ipfs.tech" ' + test_should_contain "" list_response ' ## ============================================================================ diff --git a/test/sharness/t0116-gateway-cache.sh b/test/sharness/t0116-gateway-cache.sh index 2f13f93d896..0af4ec0eb71 100755 --- a/test/sharness/t0116-gateway-cache.sh +++ b/test/sharness/t0116-gateway-cache.sh @@ -16,10 +16,14 @@ test_launch_ipfs_daemon_without_network # - have implicit index.html for a good measure # /ipns/root1/root2/root3/ (/ipns/root1/root2/root3/index.html) -# Note: we cover important edge case here: +# Note: we cover important UnixFS-focused edge case here: +# # ROOT3_CID - dir listing (dir-index-html response) # ROOT4_CID - index.html returned as a root response (dir/), instead of generated dir-index-html # FILE_CID - index.html returned directly, as a file +# +# Caching of things like raw blocks, CARs, dag-json and dag-cbor +# is tested in their respective suites. test_expect_success "Add the test directory" ' mkdir -p root2/root3/root4 && @@ -41,30 +45,36 @@ test_expect_success "Prepare IPNS unixfs content path for testing" ' ' # GET /ipfs/ + # unixfs test_expect_success "GET for /ipfs/ unixfs dir listing succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/" >/dev/null 2>curl_ipfs_dir_listing_output && - cat curl_ipfs_dir_listing_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/" >/dev/null 2>curl_ipfs_dir_listing_output ' test_expect_success "GET for /ipfs/ unixfs dir with index.html succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/" >/dev/null 2>curl_ipfs_dir_index.html_output && - cat curl_ipfs_dir_index.html_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/" >/dev/null 2>curl_ipfs_dir_index.html_output ' test_expect_success "GET for /ipfs/ unixfs file succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/index.html" >/dev/null 2>curl_ipfs_file_output && - cat curl_ipfs_file_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/index.html" >/dev/null 2>curl_ipfs_file_output + ' + # unixfs dir as dag-json + test_expect_success "GET for /ipfs/ unixfs dir as DAG-JSON succeeds" ' + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/?format=dag-json" >/dev/null 2>curl_ipfs_dir_dag-json_output && + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT1_CID/root2/root3/root4/?format=json" >/dev/null 2>curl_ipfs_dir_json_output ' # GET /ipns/ + # unixfs test_expect_success "GET for /ipns/ unixfs dir listing succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/" >/dev/null 2>curl_ipns_dir_listing_output && - cat curl_ipns_dir_listing_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/" >/dev/null 2>curl_ipns_dir_listing_output ' test_expect_success "GET for /ipns/ unixfs dir with index.html succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/" >/dev/null 2>curl_ipns_dir_index.html_output && - cat curl_ipns_dir_index.html_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/" >/dev/null 2>curl_ipns_dir_index.html_output ' test_expect_success "GET for /ipns/ unixfs file succeeds" ' - curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" >/dev/null 2>curl_ipns_file_output && - cat curl_ipns_file_output + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" >/dev/null 2>curl_ipns_file_output + ' + # unixfs dir as dag-json + test_expect_success "GET for /ipns/ unixfs dir as DAG-JSON succeeds" ' + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/?format=dag-json" >/dev/null 2>curl_ipns_dir_dag-json_output && + curl -svX GET "http://127.0.0.1:$GWAY_PORT/ipns/$TEST_IPNS_ID/root2/root3/root4/?format=json" >/dev/null 2>curl_ipns_dir_json_output ' # Cache-Control @@ -82,19 +92,34 @@ test_expect_success "Prepare IPNS unixfs content path for testing" ' test_expect_success "GET /ipfs/ unixfs dir with index.html has expected Cache-Control" ' test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_ipfs_dir_index.html_output ' - +# Cache-Control: immutable /ipfs/ unixfs dir as dag-json + test_expect_success "GET /ipfs/ dag-json has expected Cache-Control" ' + test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_ipfs_dir_dag-json_output + ' +# Cache-Control: immutable /ipfs/ unixfs dir as json + test_expect_success "GET /ipfs/ unixfs dir as json has expected Cache-Control" ' + test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_ipfs_dir_json_output + ' # Cache-Control: mutable /ipns/ file test_expect_success "GET /ipns/ unixfs file has no Cache-Control" ' test_should_not_contain "< Cache-Control" curl_ipns_file_output ' -# Cache-Control: generated /ipns/dir/ (listing) +# Cache-Control: mutable /ipns/dir/ (generated listing) test_expect_success "GET /ipns/ unixfs dir listing has no Cache-Control" ' test_should_not_contain "< Cache-Control" curl_ipns_dir_listing_output ' -# Cache-Control: immutable /ipns/dir/ (index.html) +# Cache-Control: mutable /ipns/dir/ (index.html) test_expect_success "GET /ipns/ unixfs dir with index.html has no Cache-Control" ' test_should_not_contain "< Cache-Control" curl_ipns_dir_index.html_output ' +# Cache-Control: mutable /ipns/dir/ as dag-json + test_expect_success "GET /ipns/ unixfs dir as dag-json has no Cache-Control" ' + test_should_not_contain "< Cache-Control" curl_ipns_dir_dag-json_output + ' +# Cache-Control: mutable /ipns/dir/ as json + test_expect_success "GET /ipns/ unixfs dir as json has no Cache-Control" ' + test_should_not_contain "< Cache-Control" curl_ipns_dir_json_output + ' # Cache-Control: only-if-cached test_expect_success "HEAD for /ipfs/ with only-if-cached succeeds when in local datastore" ' @@ -122,78 +147,80 @@ test_expect_success "Prepare IPNS unixfs content path for testing" ' ## dir generated listing test_expect_success "GET /ipfs/ dir listing response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3" curl_ipfs_dir_listing_output + test_should_contain "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3" curl_ipfs_dir_listing_output ' test_expect_success "GET /ipns/ dir listing response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3" curl_ipns_dir_listing_output + test_should_contain "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3" curl_ipns_dir_listing_output ' ## dir static index.html test_expect_success "GET /ipfs/ dir index.html response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/" curl_ipfs_dir_index.html_output + test_should_contain "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/" curl_ipfs_dir_index.html_output ' test_expect_success "GET /ipns/ dir index.html response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/" curl_ipns_dir_index.html_output + test_should_contain "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/" curl_ipns_dir_index.html_output ' # file test_expect_success "GET /ipfs/ file response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/index.html" curl_ipfs_file_output + test_should_contain "< X-Ipfs-Path: /ipfs/$ROOT1_CID/root2/root3/root4/index.html" curl_ipfs_file_output ' test_expect_success "GET /ipns/ file response has original content path in X-Ipfs-Path" ' - grep "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" curl_ipns_file_output + test_should_contain "< X-Ipfs-Path: /ipns/$TEST_IPNS_ID/root2/root3/root4/index.html" curl_ipns_file_output ' # X-Ipfs-Roots ## dir generated listing test_expect_success "GET /ipfs/ dir listing response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipfs_dir_listing_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipfs_dir_listing_output ' test_expect_success "GET /ipns/ dir listing response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipns_dir_listing_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID}" curl_ipns_dir_listing_output ' ## dir static index.html test_expect_success "GET /ipfs/ dir index.html response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipfs_dir_index.html_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipfs_dir_index.html_output ' test_expect_success "GET /ipns/ dir index.html response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipns_dir_index.html_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID}" curl_ipns_dir_index.html_output ' ## file test_expect_success "GET /ipfs/ file response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipfs_file_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipfs_file_output ' test_expect_success "GET /ipns/ file response has logical CID roots in X-Ipfs-Roots" ' - grep "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipns_file_output + test_should_contain "< X-Ipfs-Roots: ${ROOT1_CID},${ROOT2_CID},${ROOT3_CID},${ROOT4_CID},${FILE_CID}" curl_ipns_file_output ' # Etag ## dir generated listing test_expect_success "GET /ipfs/ dir response has special Etag for generated dir listing" ' + test_should_contain "< Etag: \"DirIndex" curl_ipfs_dir_listing_output && grep -E "< Etag: \"DirIndex-.+_CID-${ROOT3_CID}\"" curl_ipfs_dir_listing_output ' test_expect_success "GET /ipns/ dir response has special Etag for generated dir listing" ' + test_should_contain "< Etag: \"DirIndex" curl_ipfs_dir_listing_output && grep -E "< Etag: \"DirIndex-.+_CID-${ROOT3_CID}\"" curl_ipns_dir_listing_output ' ## dir static index.html should use CID of the index.html file for improved HTTP caching test_expect_success "GET /ipfs/ dir index.html response has dir CID as Etag" ' - grep "< Etag: \"${ROOT4_CID}\"" curl_ipfs_dir_index.html_output + test_should_contain "< Etag: \"${ROOT4_CID}\"" curl_ipfs_dir_index.html_output ' test_expect_success "GET /ipns/ dir index.html response has dir CID as Etag" ' - grep "< Etag: \"${ROOT4_CID}\"" curl_ipns_dir_index.html_output + test_should_contain "< Etag: \"${ROOT4_CID}\"" curl_ipns_dir_index.html_output ' ## file test_expect_success "GET /ipfs/ response has CID as Etag for a file" ' - grep "< Etag: \"${FILE_CID}\"" curl_ipfs_file_output + test_should_contain "< Etag: \"${FILE_CID}\"" curl_ipfs_file_output ' test_expect_success "GET /ipns/ response has CID as Etag for a file" ' - grep "< Etag: \"${FILE_CID}\"" curl_ipns_file_output + test_should_contain "< Etag: \"${FILE_CID}\"" curl_ipns_file_output ' # If-None-Match (return 304 Not Modified when client sends matching Etag they already have) diff --git a/test/sharness/t0117-gateway-block.sh b/test/sharness/t0117-gateway-block.sh index 87228c1c54f..5f1d080a3d4 100755 --- a/test/sharness/t0117-gateway-block.sh +++ b/test/sharness/t0117-gateway-block.sh @@ -32,43 +32,41 @@ test_expect_success "Create text fixtures" ' test_expect_success "GET response for application/vnd.ipld.raw has expected Content-Type" ' curl -svX GET -H "Accept: application/vnd.ipld.raw" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/dir/ascii.txt" >/dev/null 2>curl_output && - cat curl_output && - grep "< Content-Type: application/vnd.ipld.raw" curl_output + test_should_contain "< Content-Type: application/vnd.ipld.raw" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes Content-Length" ' BYTES=$(ipfs block get $FILE_CID | wc --bytes) - grep "< Content-Length: $BYTES" curl_output + test_should_contain "< Content-Length: $BYTES" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes Content-Disposition" ' - grep "< Content-Disposition: attachment\; filename=\"${FILE_CID}.bin\"" curl_output + test_should_contain "< Content-Disposition: attachment\; filename=\"${FILE_CID}.bin\"" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes nosniff hint" ' - grep "< X-Content-Type-Options: nosniff" curl_output + test_should_contain "< X-Content-Type-Options: nosniff" curl_output ' test_expect_success "GET for application/vnd.ipld.raw with query filename includes Content-Disposition with custom filename" ' curl -svX GET -H "Accept: application/vnd.ipld.raw" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/dir/ascii.txt?filename=foobar.bin" >/dev/null 2>curl_output_filename && - cat curl_output_filename && - grep "< Content-Disposition: attachment\; filename=\"foobar.bin\"" curl_output_filename + test_should_contain "< Content-Disposition: attachment\; filename=\"foobar.bin\"" curl_output_filename ' # Cache control HTTP headers # (basic checks, detailed behavior is tested in t0116-gateway-cache.sh) test_expect_success "GET response for application/vnd.ipld.raw includes Etag" ' - grep "< Etag: \"${FILE_CID}.raw\"" curl_output + test_should_contain "< Etag: \"${FILE_CID}.raw\"" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes X-Ipfs-Path and X-Ipfs-Roots" ' - grep "< X-Ipfs-Path" curl_output && - grep "< X-Ipfs-Roots" curl_output + test_should_contain "< X-Ipfs-Path" curl_output && + test_should_contain "< X-Ipfs-Roots" curl_output ' test_expect_success "GET response for application/vnd.ipld.raw includes Cache-Control" ' - grep "< Cache-Control: public, max-age=29030400, immutable" curl_output + test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_output ' test_kill_ipfs_daemon diff --git a/test/sharness/t0123-gateway-json-cbor.sh b/test/sharness/t0123-gateway-json-cbor.sh new file mode 100755 index 00000000000..812d90f24b9 --- /dev/null +++ b/test/sharness/t0123-gateway-json-cbor.sh @@ -0,0 +1,387 @@ +#!/usr/bin/env bash + +test_description="Test HTTP Gateway DAG-JSON (application/vnd.ipld.dag-json) and DAG-CBOR (application/vnd.ipld.dag-cbor) Support" + +. lib/test-lib.sh + +test_init_ipfs +test_launch_ipfs_daemon_without_network + +test_expect_success "Add the test directory" ' + mkdir -p rootDir/ipfs && + mkdir -p rootDir/ipns && + mkdir -p rootDir/api && + mkdir -p rootDir/ą/ę && + echo "I am a txt file on path with utf8" > rootDir/ą/ę/file-źł.txt && + echo "I am a txt file in confusing /api dir" > rootDir/api/file.txt && + echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt && + echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt && + DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) && + FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) && + FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size) + echo "$FILE_CID / $FILE_SIZE" +' + +## Reading UnixFS (data encoded with dag-pb codec) as DAG-CBOR and DAG-JSON + +test_dag_pb_headers () { + name=$1 + format=$2 + disposition=$3 + + test_expect_success "GET UnixFS as $name with format=dag-$format has expected Content-Type" ' + curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && + test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" curl_output && + test_should_not_contain "Content-Type: application/$format" curl_output + ' + + test_expect_success "GET UnixFS as $name with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" ' + curl -sD - -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" curl_output && + test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output && + test_should_not_contain "Content-Type: application/$format" curl_output + ' + + test_expect_success "GET UnixFS as $name with format=$format has expected Content-Type" ' + curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" > curl_output 2>&1 && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" curl_output && + test_should_contain "Content-Type: application/$format" curl_output && + test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output + ' + + test_expect_success "GET UnixFS as $name with 'Accept: application/$format' has expected Content-Type" ' + curl -sD - -H "Accept: application/$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" curl_output && + test_should_contain "Content-Type: application/$format" curl_output && + test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output + ' +} + +test_dag_pb_headers "DAG-JSON" "json" "inline" +test_dag_pb_headers "DAG-CBOR" "cbor" "attachment" + +test_dag_pb () { + name=$1 + format=$2 + + test_expect_success "GET UnixFS as $name has expected output for file" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 && + ipfs dag get --output-codec dag-$format $FILE_CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output + ' + + test_expect_success "GET UnixFS as $name has expected output for directory" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output 2>&1 && + ipfs dag get --output-codec dag-$format $DIR_CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output + ' + + test_expect_success "GET UnixFS as $name with format=dag-$format and format=$format produce same output" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output_1 2>&1 && + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=$format" > curl_output_2 2>&1 && + test_cmp curl_output_1 curl_output_2 + ' +} + +test_dag_pb "DAG-JSON" "json" +test_dag_pb "DAG-CBOR" "cbor" + +## Content-Type response based on Accept header and ?format= parameter + +test_cmp_dag_get () { + name=$1 + format=$2 + disposition=$3 + + test_expect_success "GET $name without Accept or format= has expected Content-Type" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && + curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" curl_output && + test_should_contain "Content-Type: application/$format" curl_output + ' + + test_expect_success "GET $name without Accept or format= produces correct output" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 && + ipfs dag get --output-codec $format $CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output + ' + + test_expect_success "GET $name with format=$format produces expected Content-Type" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && + curl -sD- "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=$format" > curl_output 2>&1 && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" curl_output && + test_should_contain "Content-Type: application/$format" curl_output + ' + + test_expect_success "GET $name with format=$format produces correct output" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=$format" > curl_output 2>&1 && + ipfs dag get --output-codec $format $CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output + ' + + test_expect_success "GET $name with format=dag-$format produces expected Content-Type" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && + curl -sD- "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" > curl_output 2>&1 && + test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" curl_output && + test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output + ' + + test_expect_success "GET $name with format=dag-$format produces correct output" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) && + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" > curl_output 2>&1 && + ipfs dag get --output-codec dag-$format $CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output + ' +} + +test_cmp_dag_get "JSON" "json" "inline" +test_cmp_dag_get "CBOR" "cbor" "attachment" + + +## Lossless conversion between JSON and CBOR + +test_expect_success "GET JSON as CBOR produces DAG-CBOR output" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec json) && + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=cbor" > curl_output 2>&1 && + ipfs dag get --output-codec dag-cbor $CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output +' + +test_expect_success "GET CBOR as JSON produces DAG-JSON output" ' + CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec cbor) && + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=json" > curl_output 2>&1 && + ipfs dag get --output-codec dag-json $CID > ipfs_dag_get_output 2>&1 && + test_cmp ipfs_dag_get_output curl_output +' + + +## Pathing, traversal + +DAG_CBOR_TRAVERSAL_CID="bafyreibs4utpgbn7uqegmd2goqz4bkyflre2ek2iwv743fhvylwi4zeeim" +DAG_JSON_TRAVERSAL_CID="baguqeeram5ujjqrwheyaty3w5gdsmoz6vittchvhk723jjqxk7hakxkd47xq" +DAG_PB_CID="bafybeiegxwlgmoh2cny7qlolykdf7aq7g6dlommarldrbm7c4hbckhfcke" + +test_expect_success "Add CARs for path traversal and DAG-PB representation tests" ' + ipfs dag import ../t0123-gateway-json-cbor/dag-cbor-traversal.car > import_output && + test_should_contain $DAG_CBOR_TRAVERSAL_CID import_output && + ipfs dag import ../t0123-gateway-json-cbor/dag-json-traversal.car > import_output && + test_should_contain $DAG_JSON_TRAVERSAL_CID import_output && + ipfs dag import ../t0123-gateway-json-cbor/dag-pb.car > import_output && + test_should_contain $DAG_PB_CID import_output +' + +test_expect_success "GET DAG-JSON traversal returns 501 if there is path remainder" ' + curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID/foo?format=dag-json" > curl_output 2>&1 && + test_should_contain "501 Not Implemented" curl_output && + test_should_contain "reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented" curl_output +' + +test_expect_success "GET DAG-JSON traverses multiple links" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_JSON_TRAVERSAL_CID/foo/link/bar?format=dag-json" > curl_output 2>&1 && + jq --sort-keys . curl_output > actual && + echo "{ \"hello\": \"this is not a link\" }" | jq --sort-keys . > expected && + test_cmp expected actual +' + +test_expect_success "GET DAG-CBOR traversal returns 501 if there is path remainder" ' + curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo?format=dag-cbor" > curl_output 2>&1 && + test_should_contain "501 Not Implemented" curl_output && + test_should_contain "reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented" curl_output +' + +test_expect_success "GET DAG-CBOR traverses multiple links" ' + curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_TRAVERSAL_CID/foo/link/bar?format=dag-json" > curl_output 2>&1 && + jq --sort-keys . curl_output > actual && + echo "{ \"hello\": \"this is not a link\" }" | jq --sort-keys . > expected && + test_cmp expected actual +' + +# test_expect_success "GET DAG-PB has expected output" ' +# curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_PB_CID?format=dag-json" > curl_output 2>&1 && +# jq --sort-keys . curl_output > actual && +# test_cmp ../t0123-gateway-json-cbor/dag-pb.json actual +# ' + + +## NATIVE TESTS: +## DAG- regression tests for core behaviors when native DAG-(CBOR|JSON) is requested + + +test_native_dag () { + name=$1 + format=$2 + disposition=$3 + CID=$4 + + # GET without explicit format and Accept: text/html returns raw block + + test_expect_success "GET $name from /ipfs without explicit format returns the same payload as the raw block" ' + ipfs block get "/ipfs/$CID" > expected && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" -o curl_output && + test_cmp expected curl_output + ' + + # GET dag-cbor block via Accept and ?format and ensure both are the same as `ipfs block get` output + + test_expect_success "GET $name from /ipfs with format=dag-$format returns the same payload as the raw block" ' + ipfs block get "/ipfs/$CID" > expected && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o curl_ipfs_dag_param_output && + test_cmp expected curl_ipfs_dag_param_output + ' + + test_expect_success "GET $name from /ipfs with format=$format returns the same payload as format=dag-$format" ' + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o expected && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o curl_ipfs_dag_param_output && + test_cmp expected curl_ipfs_dag_param_output + ' + + test_expect_success "GET $name from /ipfs with application/vnd.ipld.dag-$format returns the same payload as the raw block" ' + ipfs block get "/ipfs/$CID" > expected_block && + curl -sX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" -o curl_ipfs_dag_block_accept_output && + test_cmp expected_block curl_ipfs_dag_block_accept_output + ' + + # Make sure expected HTTP headers are returned with the dag- block + + test_expect_success "GET response for application/vnd.ipld.dag-$format has expected Content-Type" ' + curl -svX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" >/dev/null 2>curl_output && + test_should_contain "< Content-Type: application/vnd.ipld.dag-$format" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes Content-Length" ' + BYTES=$(ipfs block get $CID | wc --bytes) + test_should_contain "< Content-Length: $BYTES" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes Content-Disposition" ' + test_should_contain "< Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes nosniff hint" ' + test_should_contain "< X-Content-Type-Options: nosniff" curl_output + ' + + test_expect_success "GET for application/vnd.ipld.dag-$format with query filename includes Content-Disposition with custom filename" ' + curl -svX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?filename=foobar.$format" >/dev/null 2>curl_output_filename && + test_should_contain "< Content-Disposition: ${disposition}\; filename=\"foobar.$format\"" curl_output_filename + ' + + test_expect_success "GET for application/vnd.ipld.dag-$format with ?download=true forces Content-Disposition: attachment" ' + curl -svX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?filename=foobar.$format&download=true" >/dev/null 2>curl_output_filename && + test_should_contain "< Content-Disposition: attachment" curl_output_filename + ' + + # Cache control HTTP headers + # (basic checks, detailed behavior is tested in t0116-gateway-cache.sh) + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes Etag" ' + test_should_contain "< Etag: \"${CID}.dag-$format\"" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes X-Ipfs-Path and X-Ipfs-Roots" ' + test_should_contain "< X-Ipfs-Path" curl_output && + test_should_contain "< X-Ipfs-Roots" curl_output + ' + + test_expect_success "GET response for application/vnd.ipld.dag-$format includes Cache-Control" ' + test_should_contain "< Cache-Control: public, max-age=29030400, immutable" curl_output + ' + + # HTTP HEAD behavior + test_expect_success "HEAD $name with no explicit format returns HTTP 200" ' + curl -I "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" -o output && + test_should_contain "HTTP/1.1 200 OK" output && + test_should_contain "Content-Type: application/vnd.ipld.dag-$format" output && + test_should_contain "Content-Length: " output + ' + test_expect_success "HEAD $name with an explicit JSON format returns HTTP 200" ' + curl -I "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=json" -o output && + test_should_contain "HTTP/1.1 200 OK" output && + test_should_contain "Etag: \"$CID.json\"" output && + test_should_contain "Content-Type: application/json" output && + test_should_contain "Content-Length: " output + ' + test_expect_success "HEAD dag-pb with ?format=$format returns HTTP 200" ' + curl -I "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" -o output && + test_should_contain "HTTP/1.1 200 OK" output && + test_should_contain "Etag: \"$FILE_CID.$format\"" output && + test_should_contain "Content-Type: application/$format" output && + test_should_contain "Content-Length: " output + ' + test_expect_success "HEAD $name with only-if-cached for missing block returns HTTP 412 Precondition Failed" ' + MISSING_CID=$(echo "{\"t\": \"$(date +%s)\"}" | ipfs dag put --store-codec=dag-${format}) && + ipfs block rm -f -q $MISSING_CID && + curl -I -H "Cache-Control: only-if-cached" "http://127.0.0.1:$GWAY_PORT/ipfs/$MISSING_CID" -o output && + test_should_contain "HTTP/1.1 412 Precondition Failed" output + ' + + # IPNS behavior (should be same as immutable /ipfs, but with different caching headers) + # To keep tests small we only confirm payload is the same, and then only test delta around caching headers. + + test_expect_success "Prepare IPNS with dag-$format" ' + IPNS_ID=$(ipfs key gen --ipns-base=base36 --type=ed25519 ${format}_test_key | head -n1 | tr -d "\n") && + ipfs name publish --key ${format}_test_key --allow-offline -Q "/ipfs/$CID" > name_publish_out && + test_check_peerid "${IPNS_ID}" && + ipfs name resolve "${IPNS_ID}" > output && + printf "/ipfs/%s\n" "$CID" > expected && + test_cmp expected output + ' + + test_expect_success "GET $name from /ipns without explicit format returns the same payload as /ipfs" ' + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" -o ipfs_output && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID" -o ipns_output && + test_cmp ipfs_output ipns_output + ' + + test_expect_success "GET $name from /ipns without explicit format returns the same payload as /ipfs" ' + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o ipfs_output && + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID?format=dag-$format" -o ipns_output && + test_cmp ipfs_output ipns_output + ' + + test_expect_success "GET $name from /ipns with explicit application/vnd.ipld.dag-$format has expected headers" ' + curl -svX GET -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID" >/dev/null 2>curl_output && + test_should_not_contain "Cache-Control" curl_output && + test_should_contain "< Content-Type: application/vnd.ipld.dag-$format" curl_output && + test_should_contain "< Etag: \"${CID}.dag-$format\"" curl_output && + test_should_contain "< X-Ipfs-Path" curl_output && + test_should_contain "< X-Ipfs-Roots" curl_output + ' + + + # When Accept header includes text/html and no explicit format is requested for DAG-(CBOR|JSON) + # The gateway returns generated HTML index (see dag-index-html) for web browsers (similar to dir-index-html returned for unixfs dirs) + # As this is generated, we don't return immutable Cache-Control, even on /ipfs (same as for dir-index-html) + + test_expect_success "GET $name on /ipfs with Accept: text/html returns HTML (dag-index-html)" ' + curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 && + test_should_not_contain "Content-Disposition" curl_output && + test_should_not_contain "Cache-Control" curl_output && + test_should_contain "Etag: \"DagIndex-" curl_output && + test_should_contain "Content-Type: text/html" curl_output && + test_should_contain "" curl_output + ' + + test_expect_success "GET $name on /ipns with Accept: text/html returns HTML (dag-index-html)" ' + curl -sD - -H "Accept: text/html" "http://127.0.0.1:$GWAY_PORT/ipns/$IPNS_ID" > curl_output 2>&1 && + test_should_not_contain "Content-Disposition" curl_output && + test_should_not_contain "Cache-Control" curl_output && + test_should_contain "Etag: \"DagIndex-" curl_output && + test_should_contain "Content-Type: text/html" curl_output && + test_should_contain "" curl_output + ' + + +} + +test_native_dag "DAG-JSON" "json" "inline" "$DAG_JSON_TRAVERSAL_CID" +test_native_dag "DAG-CBOR" "cbor" "attachment" "$DAG_CBOR_TRAVERSAL_CID" + +test_kill_ipfs_daemon + +test_done + +# vim: set ts=2 sw=2 et: diff --git a/test/sharness/t0123-gateway-json-cbor/dag-cbor-traversal.car b/test/sharness/t0123-gateway-json-cbor/dag-cbor-traversal.car new file mode 100644 index 0000000000000000000000000000000000000000..92c3d4f3e21718627007431c6b6d0cc1693be206 GIT binary patch literal 318 zcmcColvlXV(2oIMVptN{kPzW{vS&r0ub_{_K>mhu-w1 zv^b}ir4|)u=I1dM5Tklwa$0`=qLiG>yll7)cbQITWL7ctu-q3j&R(lE-F}^dl&{i> zKYDy?e%dYj{535;Oie9HtSCq+tW3+S zN-Qr(GOx-pP0uSxH_s|J&aEu1%u6yc%1f`xP0cs2$TQ8(%{47ZOe!%eRI1fc%1_El zO)dc$mz0EK_&Lb+xI#2#-t#MrbvfF8z zZSAu%HyyJ6o*#0o^a0ObkQqf7E=#I1%FigR$SF!oPs}qet|&?`PD`!KG%e4s$jmO! z&n(R|%`{C;EzUElN-r=@Ez7Mm0lKW#8EhqA$>U3}JAK&Rg?BCqN|fEat~K(+;;O!m mNqL*n-%L%jtX9fM&B*}i+sBr1Tzs1^WU$DR8C literal 0 HcmV?d00001 diff --git a/test/sharness/t0123-gateway-json-cbor/dag-pb.car b/test/sharness/t0123-gateway-json-cbor/dag-pb.car new file mode 100644 index 0000000000000000000000000000000000000000..a6bb076c7e323e449dfe3f6290adbe7e3826cf21 GIT binary patch literal 392 zcmcColv6N9G>c!tg+FvpA8Lzlk~eSLZrIRuTwwE~hlf;U z76qo4r4|)u=I1eXF%qL(NS8|mq^|IQa`O8b2I)OECVzyMewgCAPNpIter34l*}|*5 zT4#fWnA7s}C0vCJfyzUL6rzjWn-?3Nly$m#PLi+f!g;?F8kf!m7^ME`R`a{7@j*j~ z9jIKdq@qNEmy3ymF`6hVpl07>^l?*Z@mjj;ut>RWcE+qL>A(DBr2-c2SUERp>UaIG uLhMP2MPRe}KxP|(+(Sr@M`}(^zK%k9eo;<}B9|dCUVv)x3~&op