fix: use ipfs-unixfs-exporter directly (#42)

`@helia/unixfs` pulls in the importer and other deps, we only need to stat and cat a DAG so we can just use the exporter for this. --------- Co-authored-by: Russell Dempsey <[email protected]>
ipfs · Apr 15, 2024 · 4532bf1 · 4532bf1
1 parent 7771af2
commit 4532bf1
Show file tree

Hide file tree

Showing 4 changed files with 167 additions and 97 deletions.
diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json
@@ -57,13 +57,12 @@
     "release": "aegir release"
   },
   "dependencies": {
-    "@helia/block-brokers": "^2.0.3",
-    "@helia/car": "^3.1.2",
-    "@helia/http": "^1.0.3",
-    "@helia/interface": "^4.1.0",
-    "@helia/ipns": "^7.2.0",
-    "@helia/routers": "^1.0.2",
-    "@helia/unixfs": "^3.0.3",
+    "@helia/block-brokers": "^2.1.0",
+    "@helia/car": "^3.1.3",
+    "@helia/http": "^1.0.4",
+    "@helia/interface": "^4.2.0",
+    "@helia/ipns": "^7.2.1",
+    "@helia/routers": "^1.0.3",
     "@ipld/dag-cbor": "^9.2.0",
     "@ipld/dag-json": "^10.2.0",
     "@ipld/dag-pb": "^4.1.0",
@@ -85,11 +84,12 @@
     "uint8arrays": "^5.0.3"
   },
   "devDependencies": {
-    "@helia/car": "^3.1.2",
-    "@helia/dag-cbor": "^3.0.2",
-    "@helia/dag-json": "^3.0.2",
-    "@helia/json": "^3.0.2",
-    "@helia/utils": "^0.1.0",
+    "@helia/car": "^3.1.3",
+    "@helia/dag-cbor": "^3.0.3",
+    "@helia/dag-json": "^3.0.3",
+    "@helia/json": "^3.0.3",
+    "@helia/unixfs": "^3.0.4",
+    "@helia/utils": "^0.2.0",
     "@ipld/car": "^5.3.0",
     "@libp2p/interface-compliance-tests": "^5.3.4",
     "@libp2p/logger": "^4.0.9",
@@ -100,10 +100,11 @@
     "blockstore-core": "^4.4.1",
     "browser-readablestream-to-it": "^2.0.5",
     "datastore-core": "^9.2.9",
-    "helia": "^4.1.0",
+    "helia": "^4.1.1",
     "ipfs-unixfs-importer": "^15.2.5",
     "ipns": "^9.1.0",
     "it-all": "^3.0.4",
+    "it-drain": "^3.0.5",
     "it-last": "^3.0.4",
     "it-to-buffer": "^4.0.5",
     "magic-bytes.js": "^1.10.0",

diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts
@@ -594,11 +594,11 @@ import { createHeliaHTTP } from '@helia/http'
 import { delegatedHTTPRouting } from '@helia/routers'
 import { dns } from '@multiformats/dns'
 import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js'
-import type { Helia } from '@helia/interface'
+import type { GetBlockProgressEvents, Helia } from '@helia/interface'
 import type { ResolveDNSLinkProgressEvents } from '@helia/ipns'
-import type { GetEvents } from '@helia/unixfs'
 import type { DNSResolvers, DNS } from '@multiformats/dns'
 import type { DNSResolver } from '@multiformats/dns/resolvers'
+import type { ExporterProgressEvents } from 'ipfs-unixfs-exporter'
 import type { CID } from 'multiformats/cid'
 import type { ProgressEvent, ProgressOptions } from 'progress-events'
 
@@ -674,8 +674,10 @@ export interface ContentTypeParser {
 }
 
 export type BubbledProgressEvents =
-  // unixfs
-  GetEvents |
+  // unixfs-exporter
+  ExporterProgressEvents |
+  // helia blockstore
+  GetBlockProgressEvents |
   // ipns
   ResolveDNSLinkProgressEvents
 

diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts
@@ -1,13 +1,13 @@
 import { car } from '@helia/car'
 import { ipns as heliaIpns, type IPNS } from '@helia/ipns'
-import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs } from '@helia/unixfs'
 import * as ipldDagCbor from '@ipld/dag-cbor'
 import * as ipldDagJson from '@ipld/dag-json'
 import { code as dagPbCode } from '@ipld/dag-pb'
 import { type AbortOptions, type Logger, type PeerId } from '@libp2p/interface'
 import { Record as DHTRecord } from '@libp2p/kad-dht'
 import { peerIdFromString } from '@libp2p/peer-id'
 import { Key } from 'interface-datastore'
+import { exporter } from 'ipfs-unixfs-exporter'
 import toBrowserReadableStream from 'it-to-browser-readablestream'
 import { code as jsonCode } from 'multiformats/codecs/json'
 import { code as rawCode } from 'multiformats/codecs/raw'
@@ -39,7 +39,6 @@ import type { CID } from 'multiformats/cid'
 interface VerifiedFetchComponents {
   helia: Helia
   ipns?: IPNS
-  unixfs?: HeliaUnixFs
 }
 
 /**
@@ -128,15 +127,13 @@ function getOverridenRawContentType ({ headers, accept }: { headers?: HeadersIni
 export class VerifiedFetch {
   private readonly helia: Helia
   private readonly ipns: IPNS
-  private readonly unixfs: HeliaUnixFs
   private readonly log: Logger
   private readonly contentTypeParser: ContentTypeParser | undefined
 
-  constructor ({ helia, ipns, unixfs }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
+  constructor ({ helia, ipns }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
     this.helia = helia
     this.log = helia.logger.forComponent('helia:verified-fetch')
     this.ipns = ipns ?? heliaIpns(helia)
-    this.unixfs = unixfs ?? heliaUnixFs(helia)
     this.contentTypeParser = init?.contentTypeParser
     this.log.trace('created VerifiedFetch instance')
   }
@@ -350,14 +347,15 @@ export class VerifiedFetch {
       const rootFilePath = 'index.html'
       try {
         this.log.trace('found directory at %c/%s, looking for index.html', cid, path)
-        const stat = await this.unixfs.stat(dirCid, {
-          path: rootFilePath,
+
+        const entry = await exporter(`/ipfs/${dirCid}/${rootFilePath}`, this.helia.blockstore, {
           signal: options?.signal,
           onProgress: options?.onProgress
         })
-        this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, stat.cid)
+
+        this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, entry.cid)
         path = rootFilePath
-        resolvedCID = stat.cid
+        resolvedCID = entry.cid
       } catch (err: any) {
         options?.signal?.throwIfAborted()
         this.log('error loading path %c/%s', dirCid, rootFilePath, err)
@@ -375,16 +373,22 @@ export class VerifiedFetch {
     }
     const offset = byteRangeContext.offset
     const length = byteRangeContext.length
-    this.log.trace('calling unixfs.cat for %c/%s with offset=%o & length=%o', resolvedCID, path, offset, length)
-    const asyncIter = this.unixfs.cat(resolvedCID, {
-      signal: options?.signal,
-      onProgress: options?.onProgress,
-      offset,
-      length
-    })
-    this.log('got async iterator for %c/%s', cid, path)
+    this.log.trace('calling exporter for %c/%s with offset=%o & length=%o', resolvedCID, path, offset, length)
 
     try {
+      const entry = await exporter(resolvedCID, this.helia.blockstore, {
+        signal: options?.signal,
+        onProgress: options?.onProgress
+      })
+
+      const asyncIter = entry.content({
+        signal: options?.signal,
+        onProgress: options?.onProgress,
+        offset,
+        length
+      })
+      this.log('got async iterator for %c/%s', cid, path)
+
       const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, {
         onProgress: options?.onProgress,
         signal: options?.signal
@@ -400,7 +404,6 @@ export class VerifiedFetch {
       if (ipfsRoots != null) {
         response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header
       }
-
       return response
     } catch (err: any) {
       options?.signal?.throwIfAborted()