From 6a2cffac39f84b17899be915bacce1589f2477bc Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Tue, 5 Nov 2024 17:36:14 +0100 Subject: [PATCH 01/32] feat: add support for p2p retrieval --- packages/verified-fetch/src/index.ts | 107 ++++++++++++++++++++++----- 1 file changed, 90 insertions(+), 17 deletions(-) diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 01adf4cf..051c5a22 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -594,10 +594,16 @@ * 4. `AbortError` - If the content request is aborted due to user aborting provided AbortSignal. Note that this is a `AbortError` from `@libp2p/interface` and not the standard `AbortError` from the Fetch API. */ -import { trustlessGateway } from '@helia/block-brokers' +import { bitswap, trustlessGateway } from '@helia/block-brokers' +import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' import { createHeliaHTTP } from '@helia/http' -import { delegatedHTTPRouting, httpGatewayRouting } from '@helia/routers' +import { delegatedHTTPRouting, httpGatewayRouting, libp2pRouting } from '@helia/routers' +import { webRTCDirect } from '@libp2p/webrtc' +import { webSockets } from '@libp2p/websockets' +import { webTransport } from '@libp2p/webtransport' import { dns } from '@multiformats/dns' +import { createHelia, libp2pDefaults } from 'helia' +import { createLibp2p } from 'libp2p' import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' import type { GetBlockProgressEvents, Helia } from '@helia/interface' import type { ResolveDNSLinkProgressEvents } from '@helia/ipns' @@ -639,6 +645,27 @@ export interface CreateVerifiedFetchInit { gateways: string[] routers?: string[] + /** + * Whether to enable WebTransport providers. + * + * @default false + */ + webSocketsProviders?: boolean + + /** + * Whether to enable WebTransport providers. + * + * @default false + */ + webTransportProviders?: boolean + + /** + * Whether to enable WebRTC providers. + * + * @default false + */ + webRTCProviders?: boolean + /** * In order to parse DNSLink records, we need to resolve DNS queries. You can * pass a list of DNS resolvers that we will provide to the @helia/ipns @@ -789,21 +816,67 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions { if (!isHelia(init)) { - init = await createHeliaHTTP({ - blockBrokers: [ - trustlessGateway({ - allowInsecure: init?.allowInsecure, - allowLocal: init?.allowLocal - }) - ], - routers: [ - ...(init?.routers ?? ['https://delegated-ipfs.dev']).map((routerUrl) => delegatedHTTPRouting(routerUrl)), - httpGatewayRouting({ - gateways: init?.gateways ?? ['https://trustless-gateway.link'] - }) - ], - dns: createDns(init?.dnsResolvers) - }) + if (init?.webTransportProviders ?? init?.webSocketsProviders ?? init?.webRTCProviders ?? false) { + const libp2pConfig = libp2pDefaults() + + libp2pConfig.transports = [] + + if (init?.webTransportProviders === true) { + libp2pConfig.transports.push(webTransport()) + } + if (init?.webSocketsProviders === true) { + libp2pConfig.transports.push(webSockets()) + } + if (init?.webRTCProviders === true) { + libp2pConfig.transports.push(webRTCDirect()) + } + + libp2pConfig.peerDiscovery = [] // disable default bootstrap peers + libp2pConfig.addresses = {} // disable default listen addresses + + const routers = init?.routers ?? ['https://delegated-ipfs.dev'] + for (let index = 0; index < routers.length; index++) { + const routerUrl = routers[index] + libp2pConfig.services[`delegatedRouting${index}`] = () => createDelegatedRoutingV1HttpApiClient(routerUrl) + } + + libp2pConfig.dns = createDns(init?.dnsResolvers) + + const libp2p = await createLibp2p(libp2pConfig) + + init = await createHelia({ + libp2p, + blockBrokers: [ + trustlessGateway({ + allowInsecure: init?.allowInsecure, + allowLocal: init?.allowLocal + }), + bitswap() + ], + routers: [ + httpGatewayRouting({ + gateways: init?.gateways ?? ['https://trustless-gateway.link'] + }), + libp2pRouting(libp2p) + ] + }) + } else { + init = await createHeliaHTTP({ + blockBrokers: [ + trustlessGateway({ + allowInsecure: init?.allowInsecure, + allowLocal: init?.allowLocal + }) + ], + routers: [ + ...(init?.routers ?? ['https://delegated-ipfs.dev']).map((routerUrl) => delegatedHTTPRouting(routerUrl)), + httpGatewayRouting({ + gateways: init?.gateways ?? ['https://trustless-gateway.link'] + }) + ], + dns: createDns(init?.dnsResolvers) + }) + } } const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, options) From 5de4bffe9359b21b20e730a5db37ede441faa1db Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Wed, 6 Nov 2024 10:25:06 +0100 Subject: [PATCH 02/32] deps: add missing deps explicitly --- packages/verified-fetch/package.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index f08bc87e..da20bb59 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -144,6 +144,7 @@ "dependencies": { "@helia/block-brokers": "^4.0.0", "@helia/car": "^4.0.0", + "@helia/delegated-routing-v1-http-api-client": "^4.1.2", "@helia/http": "^2.0.0", "@helia/interface": "^5.0.0", "@helia/ipns": "^8.0.0", @@ -155,6 +156,9 @@ "@libp2p/interface": "^2.1.3", "@libp2p/kad-dht": "^14.0.1", "@libp2p/peer-id": "^5.0.5", + "@libp2p/webrtc": "^5.0.16", + "@libp2p/websockets": "^9.0.11", + "@libp2p/webtransport": "^5.0.16", "@multiformats/dns": "^1.0.6", "cborg": "^4.2.4", "hashlru": "^2.3.0", @@ -165,6 +169,7 @@ "it-pipe": "^3.0.1", "it-tar": "^6.0.5", "it-to-browser-readablestream": "^2.0.9", + "libp2p": "^2.2.1", "lru-cache": "^10.2.2", "multiformats": "^13.3.0", "progress-events": "^1.0.1", @@ -185,7 +190,7 @@ "blockstore-core": "^5.0.2", "browser-readablestream-to-it": "^2.0.7", "datastore-core": "^10.0.2", - "helia": "^5.0.0", + "helia": "^5.1.0", "ipfs-unixfs-importer": "^15.3.1", "ipns": "^10.0.0", "it-all": "^3.0.6", From 7bf29f3564a239e3254e1de1c1433d4d7dff7c6a Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Wed, 6 Nov 2024 10:31:23 +0100 Subject: [PATCH 03/32] refactor: simplify naming --- packages/verified-fetch/src/index.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 051c5a22..daae1fe2 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -650,21 +650,21 @@ export interface CreateVerifiedFetchInit { * * @default false */ - webSocketsProviders?: boolean + webSockets?: boolean /** * Whether to enable WebTransport providers. * * @default false */ - webTransportProviders?: boolean + webTransport?: boolean /** * Whether to enable WebRTC providers. * * @default false */ - webRTCProviders?: boolean + webRTC?: boolean /** * In order to parse DNSLink records, we need to resolve DNS queries. You can @@ -816,18 +816,18 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions { if (!isHelia(init)) { - if (init?.webTransportProviders ?? init?.webSocketsProviders ?? init?.webRTCProviders ?? false) { + if (init?.webTransport ?? init?.webSockets ?? init?.webRTC ?? false) { const libp2pConfig = libp2pDefaults() libp2pConfig.transports = [] - if (init?.webTransportProviders === true) { + if (init?.webTransport === true) { libp2pConfig.transports.push(webTransport()) } - if (init?.webSocketsProviders === true) { + if (init?.webSockets === true) { libp2pConfig.transports.push(webSockets()) } - if (init?.webRTCProviders === true) { + if (init?.webRTC === true) { libp2pConfig.transports.push(webRTCDirect()) } From ae3df4d99d10762f3e750b41c4c6dfebc54a53f8 Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Thu, 7 Nov 2024 16:29:05 +0100 Subject: [PATCH 04/32] deps: make helia prod dep --- packages/verified-fetch/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index da20bb59..03727905 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -162,6 +162,7 @@ "@multiformats/dns": "^1.0.6", "cborg": "^4.2.4", "hashlru": "^2.3.0", + "helia": "^5.1.0", "interface-blockstore": "^5.3.1", "interface-datastore": "^8.3.1", "ipfs-unixfs-exporter": "^13.6.1", @@ -190,7 +191,6 @@ "blockstore-core": "^5.0.2", "browser-readablestream-to-it": "^2.0.7", "datastore-core": "^10.0.2", - "helia": "^5.1.0", "ipfs-unixfs-importer": "^15.3.1", "ipns": "^10.0.0", "it-all": "^3.0.6", From e92b3c02203c0896488a8221242ad22290875688 Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:07:57 +0100 Subject: [PATCH 05/32] feat: simplify config and default to helia p2p --- packages/verified-fetch/src/index.ts | 116 ++++++++------------------- 1 file changed, 34 insertions(+), 82 deletions(-) diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index daae1fe2..16acfec4 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -3,11 +3,13 @@ * * `@helia/verified-fetch` provides a [fetch](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API)-like API for retrieving content from the [IPFS](https://ipfs.tech/) network. * - * All content is retrieved in a [trustless manner](https://www.techopedia.com/definition/trustless), and the integrity of all bytes are verified by comparing hashes of the data. By default, CIDs are retrieved over HTTP from [trustless gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/). + * All content is retrieved in a [trustless manner](https://www.techopedia.com/definition/trustless), and the integrity of all bytes are verified by comparing hashes of the data. + * + * By default, providers for CIDs are found with delegated routers and retrieved over HTTP from [trustless gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/), and WebTransport and WebRTC providers if available. * * This is a marked improvement over `fetch` which offers no such protections and is vulnerable to all sorts of attacks like [Content Spoofing](https://owasp.org/www-community/attacks/Content_Spoofing), [DNS Hijacking](https://en.wikipedia.org/wiki/DNS_hijacking), etc. * - * A `verifiedFetch` function is exported to get up and running quickly, and a `createVerifiedFetch` function is also available that allows customizing the underlying [Helia](https://helia.io/) node for complete control over how content is retrieved. + * A `verifiedFetch` function is exported to get up and running quickly, and a `createVerifiedFetch` function is also available that allows customizing the underlying [Helia](https://ipfs.github.io/helia/) node for complete control over how content is retrieved. * * Browser-cache-friendly [Response](https://developer.mozilla.org/en-US/docs/Web/API/Response) objects are returned which should be instantly familiar to web developers. * @@ -90,7 +92,7 @@ * * The [helia](https://www.npmjs.com/package/helia) module is configured with a libp2p node that is suited for decentralized applications, alternatively [@helia/http](https://www.npmjs.com/package/@helia/http) is available which uses HTTP gateways for all network operations. * - * You can see variations of Helia and js-libp2p configuration options at . + * You can see variations of Helia and js-libp2p configuration options at . * * ```typescript * import { trustlessGateway } from '@helia/block-brokers' @@ -596,7 +598,6 @@ import { bitswap, trustlessGateway } from '@helia/block-brokers' import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' -import { createHeliaHTTP } from '@helia/http' import { delegatedHTTPRouting, httpGatewayRouting, libp2pRouting } from '@helia/routers' import { webRTCDirect } from '@libp2p/webrtc' import { webSockets } from '@libp2p/websockets' @@ -645,27 +646,6 @@ export interface CreateVerifiedFetchInit { gateways: string[] routers?: string[] - /** - * Whether to enable WebTransport providers. - * - * @default false - */ - webSockets?: boolean - - /** - * Whether to enable WebTransport providers. - * - * @default false - */ - webTransport?: boolean - - /** - * Whether to enable WebRTC providers. - * - * @default false - */ - webRTC?: boolean - /** * In order to parse DNSLink records, we need to resolve DNS queries. You can * pass a list of DNS resolvers that we will provide to the @helia/ipns @@ -816,67 +796,39 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions { if (!isHelia(init)) { - if (init?.webTransport ?? init?.webSockets ?? init?.webRTC ?? false) { - const libp2pConfig = libp2pDefaults() - - libp2pConfig.transports = [] - - if (init?.webTransport === true) { - libp2pConfig.transports.push(webTransport()) - } - if (init?.webSockets === true) { - libp2pConfig.transports.push(webSockets()) - } - if (init?.webRTC === true) { - libp2pConfig.transports.push(webRTCDirect()) - } + const libp2pConfig = libp2pDefaults() - libp2pConfig.peerDiscovery = [] // disable default bootstrap peers - libp2pConfig.addresses = {} // disable default listen addresses + libp2pConfig.transports = [webSockets(), webRTCDirect()] - const routers = init?.routers ?? ['https://delegated-ipfs.dev'] - for (let index = 0; index < routers.length; index++) { - const routerUrl = routers[index] - libp2pConfig.services[`delegatedRouting${index}`] = () => createDelegatedRoutingV1HttpApiClient(routerUrl) - } + libp2pConfig.peerDiscovery = [] // disable default bootstrap peers + libp2pConfig.addresses = {} // disable default listen addresses - libp2pConfig.dns = createDns(init?.dnsResolvers) - - const libp2p = await createLibp2p(libp2pConfig) - - init = await createHelia({ - libp2p, - blockBrokers: [ - trustlessGateway({ - allowInsecure: init?.allowInsecure, - allowLocal: init?.allowLocal - }), - bitswap() - ], - routers: [ - httpGatewayRouting({ - gateways: init?.gateways ?? ['https://trustless-gateway.link'] - }), - libp2pRouting(libp2p) - ] - }) - } else { - init = await createHeliaHTTP({ - blockBrokers: [ - trustlessGateway({ - allowInsecure: init?.allowInsecure, - allowLocal: init?.allowLocal - }) - ], - routers: [ - ...(init?.routers ?? ['https://delegated-ipfs.dev']).map((routerUrl) => delegatedHTTPRouting(routerUrl)), - httpGatewayRouting({ - gateways: init?.gateways ?? ['https://trustless-gateway.link'] - }) - ], - dns: createDns(init?.dnsResolvers) - }) + const routers = init?.routers ?? ['https://delegated-ipfs.dev'] + for (let index = 0; index < routers.length; index++) { + const routerUrl = routers[index] + libp2pConfig.services[`delegatedRouting${index}`] = () => createDelegatedRoutingV1HttpApiClient(routerUrl) } + + libp2pConfig.dns = createDns(init?.dnsResolvers) + + const libp2p = await createLibp2p(libp2pConfig) + + init = await createHelia({ + libp2p, + blockBrokers: [ + trustlessGateway({ + allowInsecure: init?.allowInsecure, + allowLocal: init?.allowLocal + }), + bitswap() + ], + routers: [ + httpGatewayRouting({ + gateways: init?.gateways ?? ['https://trustless-gateway.link'] + }), + libp2pRouting(libp2p) + ] + }) } const verifiedFetchInstance = new VerifiedFetchClass({ helia: init }, options) From 6889a6d5f0971bd14f1f46af2a005df5fd05a47c Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:08:25 +0100 Subject: [PATCH 06/32] chore: remove unneeded dep --- packages/verified-fetch/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 03727905..fbf0c14e 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -145,7 +145,6 @@ "@helia/block-brokers": "^4.0.0", "@helia/car": "^4.0.0", "@helia/delegated-routing-v1-http-api-client": "^4.1.2", - "@helia/http": "^2.0.0", "@helia/interface": "^5.0.0", "@helia/ipns": "^8.0.0", "@helia/routers": "^2.0.0", From b30c55d872c44753b2d33381c1bb32c34ee94a7a Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 18 Nov 2024 13:17:36 -0600 Subject: [PATCH 07/32] fix: build works --- packages/gateway-conformance/package.json | 10 +++++----- packages/interop/package.json | 2 +- packages/verified-fetch/package.json | 4 ++-- packages/verified-fetch/src/index.ts | 3 +-- .../test/fixtures/create-offline-helia.ts | 9 ++++----- 5 files changed, 13 insertions(+), 15 deletions(-) diff --git a/packages/gateway-conformance/package.json b/packages/gateway-conformance/package.json index f30454e3..6a164d6a 100644 --- a/packages/gateway-conformance/package.json +++ b/packages/gateway-conformance/package.json @@ -52,11 +52,11 @@ "test": "aegir test -t node" }, "dependencies": { - "@helia/block-brokers": "^3.0.1", - "@helia/http": "^1.0.8", - "@helia/interface": "^4.3.0", - "@helia/routers": "^1.1.0", - "@helia/verified-fetch": "1.5.0", + "@helia/http": "^2.0.1", + "@helia/block-brokers": "^4.0.0", + "@helia/interface": "^5.0.0", + "@helia/routers": "^2.0.0", + "@helia/verified-fetch": "../..", "@libp2p/kad-dht": "^12.0.17", "@libp2p/logger": "^4.0.13", "@libp2p/peer-id": "^4.1.2", diff --git a/packages/interop/package.json b/packages/interop/package.json index 641aa8fa..2a388e7a 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -57,7 +57,7 @@ "test:electron-main": "aegir test -t electron-main" }, "dependencies": { - "@helia/verified-fetch": "1.5.0", + "@helia/verified-fetch": "../..", "aegir": "^44.1.4", "execa": "^9.1.0", "fast-glob": "^3.3.2", diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 736e80b2..dc421976 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -171,15 +171,15 @@ "it-to-browser-readablestream": "^2.0.9", "libp2p": "^2.2.1", "lru-cache": "^11.0.2", - "multiformats": "^13.3.0", + "multiformats": "^13.3.1", "progress-events": "^1.0.1", "uint8arrays": "^5.1.0" }, "devDependencies": { "@helia/dag-cbor": "^4.0.0", "@helia/dag-json": "^4.0.0", + "@helia/http": "^2.0.1", "@helia/json": "^4.0.0", - "@helia/utils": "^1.0.0", "@ipld/car": "^5.3.2", "@libp2p/crypto": "^5.0.5", "@libp2p/interface-compliance-tests": "^6.1.6", diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 16acfec4..40570c34 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -598,10 +598,9 @@ import { bitswap, trustlessGateway } from '@helia/block-brokers' import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' -import { delegatedHTTPRouting, httpGatewayRouting, libp2pRouting } from '@helia/routers' +import { httpGatewayRouting, libp2pRouting } from '@helia/routers' import { webRTCDirect } from '@libp2p/webrtc' import { webSockets } from '@libp2p/websockets' -import { webTransport } from '@libp2p/webtransport' import { dns } from '@multiformats/dns' import { createHelia, libp2pDefaults } from 'helia' import { createLibp2p } from 'libp2p' diff --git a/packages/verified-fetch/test/fixtures/create-offline-helia.ts b/packages/verified-fetch/test/fixtures/create-offline-helia.ts index 8b7e3d1d..9698d607 100644 --- a/packages/verified-fetch/test/fixtures/create-offline-helia.ts +++ b/packages/verified-fetch/test/fixtures/create-offline-helia.ts @@ -1,15 +1,14 @@ -import { Helia as HeliaClass } from '@helia/utils' +import { createHeliaHTTP } from '@helia/http' import { MemoryBlockstore } from 'blockstore-core' import { IdentityBlockstore } from 'blockstore-core/identity' import { MemoryDatastore } from 'datastore-core' -import type { HeliaHTTPInit } from '@helia/http' -import type { Helia } from '@helia/interface' +import { type HeliaInit } from 'helia' -export async function createHelia (init: Partial = {}): Promise { +export async function createHelia (init: Partial = {}): Promise> { const datastore = new MemoryDatastore() const blockstore = new IdentityBlockstore(new MemoryBlockstore()) - const helia = new HeliaClass({ + const helia = await createHeliaHTTP({ datastore, blockstore, blockBrokers: [], From a80278ecfee9c94ea9c1ab61d5f2e0ad931369e2 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:00:36 -0600 Subject: [PATCH 08/32] chore: remove unused dep --- packages/verified-fetch/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 244a7255..1e96e71b 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -157,7 +157,6 @@ "@libp2p/peer-id": "^5.0.5", "@libp2p/webrtc": "^5.0.16", "@libp2p/websockets": "^9.0.11", - "@libp2p/webtransport": "^5.0.16", "@multiformats/dns": "^1.0.6", "cborg": "^4.2.4", "hashlru": "^2.3.0", From 7a52f021216f3f0101eeba7a1ba80eaa90a6ea9b Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:52:08 -0600 Subject: [PATCH 09/32] test: tests no longer hang --- packages/verified-fetch/src/index.ts | 37 ++++++++++++++++--- .../test/custom-dns-resolvers.spec.ts | 16 ++------ 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 40570c34..5e9be57c 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -598,15 +598,17 @@ import { bitswap, trustlessGateway } from '@helia/block-brokers' import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' +import { ipnsSelector, ipnsValidator, type ResolveDNSLinkProgressEvents } from '@helia/ipns' import { httpGatewayRouting, libp2pRouting } from '@helia/routers' +import { type Libp2p } from '@libp2p/interface' +import { kadDHT } from '@libp2p/kad-dht' import { webRTCDirect } from '@libp2p/webrtc' import { webSockets } from '@libp2p/websockets' import { dns } from '@multiformats/dns' -import { createHelia, libp2pDefaults } from 'helia' +import { createHelia, type DefaultLibp2pServices, libp2pDefaults } from 'helia' import { createLibp2p } from 'libp2p' import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' import type { GetBlockProgressEvents, Helia } from '@helia/interface' -import type { ResolveDNSLinkProgressEvents } from '@helia/ipns' import type { DNSResolvers, DNS } from '@multiformats/dns' import type { DNSResolver } from '@multiformats/dns/resolvers' import type { ExporterProgressEvents } from 'ipfs-unixfs-exporter' @@ -790,27 +792,49 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions & Record<`delegatedRouting${number}`, any> + /** * Create and return a Helia node */ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchInit, options?: CreateVerifiedFetchOptions): Promise { + let libp2p: Libp2p | undefined if (!isHelia(init)) { const libp2pConfig = libp2pDefaults() + libp2pConfig.start = false + libp2pConfig.transports = [webSockets(), webRTCDirect()] libp2pConfig.peerDiscovery = [] // disable default bootstrap peers libp2pConfig.addresses = {} // disable default listen addresses + // We only need client/listen/fetch based services + const fetchOnlyServices = { + dcutr: libp2pConfig.services.dcutr, + dht: kadDHT({ + clientMode: true, + validators: { + ipns: ipnsValidator + }, + selectors: { + ipns: ipnsSelector + } + }), + identify: libp2pConfig.services.identify, + identifyPush: libp2pConfig.services.identifyPush, + keychain: libp2pConfig.services.keychain, + ping: libp2pConfig.services.ping + } + // @ts-expect-error - borked serviceMap types + libp2pConfig.services = fetchOnlyServices const routers = init?.routers ?? ['https://delegated-ipfs.dev'] for (let index = 0; index < routers.length; index++) { const routerUrl = routers[index] libp2pConfig.services[`delegatedRouting${index}`] = () => createDelegatedRoutingV1HttpApiClient(routerUrl) } - libp2pConfig.dns = createDns(init?.dnsResolvers) - - const libp2p = await createLibp2p(libp2pConfig) + libp2p = await createLibp2p(libp2pConfig) init = await createHelia({ libp2p, @@ -821,6 +845,7 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni }), bitswap() ], + dns: createDns(init?.dnsResolvers), routers: [ httpGatewayRouting({ gateways: init?.gateways ?? ['https://trustless-gateway.link'] @@ -834,8 +859,8 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni async function verifiedFetch (resource: Resource, options?: VerifiedFetchInit): Promise { return verifiedFetchInstance.fetch(resource, options) } - verifiedFetch.stop = verifiedFetchInstance.stop.bind(verifiedFetchInstance) verifiedFetch.start = verifiedFetchInstance.start.bind(verifiedFetchInstance) + verifiedFetch.stop = verifiedFetchInstance.stop.bind(verifiedFetchInstance) return verifiedFetch } diff --git a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts index f8ff6414..c3991a9e 100644 --- a/packages/verified-fetch/test/custom-dns-resolvers.spec.ts +++ b/packages/verified-fetch/test/custom-dns-resolvers.spec.ts @@ -5,19 +5,8 @@ import Sinon from 'sinon' import { createVerifiedFetch } from '../src/index.js' import { VerifiedFetch } from '../src/verified-fetch.js' import { createHelia } from './fixtures/create-offline-helia.js' -import type { Helia } from '@helia/interface' describe('custom dns-resolvers', () => { - let helia: Helia - - beforeEach(async () => { - helia = await createHelia() - }) - - afterEach(async () => { - await stop(helia) - }) - it('is used when passed to createVerifiedFetch', async () => { const customDnsResolver = Sinon.stub().withArgs('_dnslink.some-non-cached-domain.com').resolves({ Answer: [{ @@ -40,6 +29,7 @@ describe('custom dns-resolvers', () => { RecordType.TXT ] }]) + await stop(fetch) }) it('is used when passed to VerifiedFetch', async () => { @@ -49,8 +39,7 @@ describe('custom dns-resolvers', () => { }] }) - await stop(helia) - helia = await createHelia({ + const helia = await createHelia({ dns: dns({ resolvers: { '.': customDnsResolver @@ -73,5 +62,6 @@ describe('custom dns-resolvers', () => { RecordType.TXT ] }]) + await stop(helia, verifiedFetch) }) }) From ae3d9528453e34c624264511fcd67fe35756fed0 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:35:24 -0600 Subject: [PATCH 10/32] fix: interop tests --- packages/interop/package.json | 4 ++-- packages/interop/src/fixtures/load-fixtures.ts | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 4cb2a0d4..4dfb8680 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -57,10 +57,10 @@ "test:electron-main": "aegir test -t electron-main" }, "dependencies": { - "@helia/verified-fetch": "../..", + "@helia/verified-fetch": "2.1.2", "aegir": "^45.0.1", "execa": "^9.1.0", - "fast-glob": "^3.3.2", + "glob": "^11.0.0", "ipfsd-ctl": "^14.1.0", "kubo": "^0.32.0", "kubo-rpc-client": "^4.1.1", diff --git a/packages/interop/src/fixtures/load-fixtures.ts b/packages/interop/src/fixtures/load-fixtures.ts index 9fa5c296..7642bcb8 100644 --- a/packages/interop/src/fixtures/load-fixtures.ts +++ b/packages/interop/src/fixtures/load-fixtures.ts @@ -1,5 +1,5 @@ import { $ } from 'execa' -import fg from 'fast-glob' +import { glob } from 'glob' import { path as kuboPath } from 'kubo' /** @@ -9,7 +9,9 @@ import { path as kuboPath } from 'kubo' export async function loadFixtures (IPFS_PATH = undefined): Promise { const kuboBinary = process.env.KUBO_BINARY ?? kuboPath() - for (const carFile of await fg.glob('**/fixtures/data/*.car')) { + const files = await glob('**/fixtures/data/*.car', { cwd: process.cwd() }) + + await Promise.all(files.map(async (carFile) => { await $({ env: { IPFS_PATH } })`${kuboBinary} dag import --pin-roots=false --offline ${carFile}` - } + })) } From 3de5678a3db130b79a24fc1b3d71667788bd4a92 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:41:49 -0600 Subject: [PATCH 11/32] fix: gateway conformance runs again --- packages/gateway-conformance/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gateway-conformance/package.json b/packages/gateway-conformance/package.json index 4bca5774..2ba8acb2 100644 --- a/packages/gateway-conformance/package.json +++ b/packages/gateway-conformance/package.json @@ -56,7 +56,7 @@ "@helia/block-brokers": "^4.0.0", "@helia/interface": "^5.0.0", "@helia/routers": "^2.0.0", - "@helia/verified-fetch": "../..", + "@helia/verified-fetch": "2.1.2", "@libp2p/kad-dht": "^12.0.17", "@libp2p/logger": "^4.0.13", "@libp2p/peer-id": "^4.1.2", From 2a1afc9c6f6233be831289f668765097b8e253d8 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 18 Nov 2024 15:56:07 -0600 Subject: [PATCH 12/32] chore: skip some gateway conformance tests --- packages/gateway-conformance/src/conformance.spec.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/gateway-conformance/src/conformance.spec.ts b/packages/gateway-conformance/src/conformance.spec.ts index 62039853..97716680 100644 --- a/packages/gateway-conformance/src/conformance.spec.ts +++ b/packages/gateway-conformance/src/conformance.spec.ts @@ -171,7 +171,14 @@ const tests: TestConfig[] = [ // }, { name: 'TestGatewaySubdomainAndIPNS', - run: ['TestGatewaySubdomainAndIPNS'], + run: [ + 'TestGatewaySubdomainAndIPNS' + ], + skip: [ + 'TestGatewaySubdomainAndIPNS/request_for_a_ED25519_libp2p-key_.*', + 'TestGatewaySubdomainAndIPNS/.*redirects_to_CID_with_libp2p-key_multicodec', + 'TestGatewaySubdomainAndIPNS/.*redirects_to_CIDv1.*' + ], successRate: 31.58 }, { From 5f85aafcab33f3ba342ccb22d2fd99dbd66f6574 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Mon, 18 Nov 2024 16:02:01 -0600 Subject: [PATCH 13/32] fix: single method for getting peerId --- .../src/utils/get-peer-id-from-string.ts | 12 ++++++++++++ .../verified-fetch/src/utils/parse-url-string.ts | 9 ++------- packages/verified-fetch/src/verified-fetch.ts | 16 ++++------------ 3 files changed, 18 insertions(+), 19 deletions(-) create mode 100644 packages/verified-fetch/src/utils/get-peer-id-from-string.ts diff --git a/packages/verified-fetch/src/utils/get-peer-id-from-string.ts b/packages/verified-fetch/src/utils/get-peer-id-from-string.ts new file mode 100644 index 00000000..abe7724e --- /dev/null +++ b/packages/verified-fetch/src/utils/get-peer-id-from-string.ts @@ -0,0 +1,12 @@ +import { peerIdFromCID, peerIdFromString } from '@libp2p/peer-id' +import { CID } from 'multiformats/cid' +import type { PeerId } from '@libp2p/interface' + +export function getPeerIdFromString (peerIdString: string): PeerId { + if (peerIdString.charAt(0) === '1' || peerIdString.charAt(0) === 'Q') { + return peerIdFromString(peerIdString) + } + + // try resolving as a base36 CID + return peerIdFromCID(CID.parse(peerIdString)) +} diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts index 337fe7b6..145d82a0 100644 --- a/packages/verified-fetch/src/utils/parse-url-string.ts +++ b/packages/verified-fetch/src/utils/parse-url-string.ts @@ -1,5 +1,5 @@ -import { peerIdFromCID, peerIdFromString } from '@libp2p/peer-id' import { CID } from 'multiformats/cid' +import { getPeerIdFromString } from './get-peer-id-from-string.js' import { TLRU } from './tlru.js' import type { RequestFormatShorthand } from '../types.js' import type { DNSLinkResolveResult, IPNS, IPNSResolveResult, IPNSRoutingEvents, ResolveDNSLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns' @@ -178,12 +178,7 @@ export async function parseUrlString ({ urlString, ipns, logger }: ParseUrlStrin try { // try resolving as an IPNS name - if (cidOrPeerIdOrDnsLink.charAt(0) === '1' || cidOrPeerIdOrDnsLink.charAt(0) === 'Q') { - peerId = peerIdFromString(cidOrPeerIdOrDnsLink) - } else { - // try resolving as a base36 CID - peerId = peerIdFromCID(CID.parse(cidOrPeerIdOrDnsLink)) - } + peerId = getPeerIdFromString(cidOrPeerIdOrDnsLink) if (peerId.publicKey == null) { throw new TypeError('cidOrPeerIdOrDnsLink contains no public key') } diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 7476edcf..81311580 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -5,12 +5,11 @@ import * as ipldDagJson from '@ipld/dag-json' import { code as dagPbCode } from '@ipld/dag-pb' import { type AbortOptions, type Logger, type PeerId } from '@libp2p/interface' import { Record as DHTRecord } from '@libp2p/kad-dht' -import { peerIdFromCID, peerIdFromString } from '@libp2p/peer-id' import { Key } from 'interface-datastore' import { exporter } from 'ipfs-unixfs-exporter' import toBrowserReadableStream from 'it-to-browser-readablestream' import { LRUCache } from 'lru-cache' -import { CID } from 'multiformats/cid' +import { type CID } from 'multiformats/cid' import { code as jsonCode } from 'multiformats/codecs/json' import { code as rawCode } from 'multiformats/codecs/raw' import { identity } from 'multiformats/hashes/identity' @@ -22,6 +21,7 @@ import { ByteRangeContext } from './utils/byte-range-context.js' import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js' import { getContentDispositionFilename } from './utils/get-content-disposition-filename.js' import { getETag } from './utils/get-e-tag.js' +import { getPeerIdFromString } from './utils/get-peer-id-from-string.js' import { getResolvedAcceptHeader } from './utils/get-resolved-accept-header.js' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' import { tarStream } from './utils/get-tar-stream.js' @@ -159,19 +159,11 @@ export class VerifiedFetch { if (resource.startsWith('ipns://')) { const peerIdString = resource.replace('ipns://', '') this.log.trace('trying to parse peer id from "%s"', peerIdString) - peerId = peerIdFromString(peerIdString) + peerId = getPeerIdFromString(peerIdString) } else { const peerIdString = resource.split('.ipns.')[0].split('://')[1] this.log.trace('trying to parse peer id from "%s"', peerIdString) - let cid: CID - try { - cid = CID.parse(peerIdString) - } catch (err: any) { - this.log.error('could not construct CID from peerId string "%s"', resource, err) - return badRequestResponse(resource, err) - } - - peerId = peerIdFromCID(cid) + peerId = getPeerIdFromString(peerIdString) } } catch (err: any) { this.log.error('could not parse peer id from IPNS url %s', resource, err) From 74e27ac6dcd372aa9b4c359e6e4872bcd268ae98 Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Tue, 19 Nov 2024 11:55:42 +0100 Subject: [PATCH 14/32] fix: type error and remove identify push --- packages/verified-fetch/src/index.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 5e9be57c..6bc5e151 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -606,7 +606,7 @@ import { webRTCDirect } from '@libp2p/webrtc' import { webSockets } from '@libp2p/websockets' import { dns } from '@multiformats/dns' import { createHelia, type DefaultLibp2pServices, libp2pDefaults } from 'helia' -import { createLibp2p } from 'libp2p' +import { createLibp2p, type Libp2pOptions } from 'libp2p' import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' import type { GetBlockProgressEvents, Helia } from '@helia/interface' import type { DNSResolvers, DNS } from '@multiformats/dns' @@ -800,7 +800,7 @@ type Libp2pServiceType = Pick { let libp2p: Libp2p | undefined if (!isHelia(init)) { - const libp2pConfig = libp2pDefaults() + const libp2pConfig: Libp2pOptions = libp2pDefaults() libp2pConfig.start = false @@ -822,11 +822,10 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni } }), identify: libp2pConfig.services.identify, - identifyPush: libp2pConfig.services.identifyPush, keychain: libp2pConfig.services.keychain, ping: libp2pConfig.services.ping } - // @ts-expect-error - borked serviceMap types + libp2pConfig.services = fetchOnlyServices const routers = init?.routers ?? ['https://delegated-ipfs.dev'] for (let index = 0; index < routers.length; index++) { From e6bccdf8e6056a8916c585aeaa44347acd0f6b6f Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Tue, 19 Nov 2024 15:15:27 +0100 Subject: [PATCH 15/32] Revert "fix: type error and remove identify push" This reverts commit 74e27ac6dcd372aa9b4c359e6e4872bcd268ae98. --- packages/verified-fetch/src/index.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 6bc5e151..5e9be57c 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -606,7 +606,7 @@ import { webRTCDirect } from '@libp2p/webrtc' import { webSockets } from '@libp2p/websockets' import { dns } from '@multiformats/dns' import { createHelia, type DefaultLibp2pServices, libp2pDefaults } from 'helia' -import { createLibp2p, type Libp2pOptions } from 'libp2p' +import { createLibp2p } from 'libp2p' import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' import type { GetBlockProgressEvents, Helia } from '@helia/interface' import type { DNSResolvers, DNS } from '@multiformats/dns' @@ -800,7 +800,7 @@ type Libp2pServiceType = Pick { let libp2p: Libp2p | undefined if (!isHelia(init)) { - const libp2pConfig: Libp2pOptions = libp2pDefaults() + const libp2pConfig = libp2pDefaults() libp2pConfig.start = false @@ -822,10 +822,11 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni } }), identify: libp2pConfig.services.identify, + identifyPush: libp2pConfig.services.identifyPush, keychain: libp2pConfig.services.keychain, ping: libp2pConfig.services.ping } - + // @ts-expect-error - borked serviceMap types libp2pConfig.services = fetchOnlyServices const routers = init?.routers ?? ['https://delegated-ipfs.dev'] for (let index = 0; index < routers.length; index++) { From af90f4d956d3c774891080c0932276a41e14050b Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Tue, 19 Nov 2024 16:27:42 +0100 Subject: [PATCH 16/32] fix: reuse dns in libp2p and helia and remove kad-dht --- packages/verified-fetch/src/index.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 5e9be57c..25f5d202 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -800,6 +800,7 @@ type Libp2pServiceType = Pick { let libp2p: Libp2p | undefined if (!isHelia(init)) { + const dns = createDns(init?.dnsResolvers) const libp2pConfig = libp2pDefaults() libp2pConfig.start = false @@ -812,15 +813,6 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni // We only need client/listen/fetch based services const fetchOnlyServices = { dcutr: libp2pConfig.services.dcutr, - dht: kadDHT({ - clientMode: true, - validators: { - ipns: ipnsValidator - }, - selectors: { - ipns: ipnsSelector - } - }), identify: libp2pConfig.services.identify, identifyPush: libp2pConfig.services.identifyPush, keychain: libp2pConfig.services.keychain, @@ -834,6 +826,8 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni libp2pConfig.services[`delegatedRouting${index}`] = () => createDelegatedRoutingV1HttpApiClient(routerUrl) } + libp2pConfig.dns = dns + libp2p = await createLibp2p(libp2pConfig) init = await createHelia({ @@ -845,7 +839,7 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni }), bitswap() ], - dns: createDns(init?.dnsResolvers), + dns, routers: [ httpGatewayRouting({ gateways: init?.gateways ?? ['https://trustless-gateway.link'] From 8dbe93784e860434ece312f83da3523e1d7eaf52 Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Tue, 19 Nov 2024 17:25:32 +0100 Subject: [PATCH 17/32] feat: config for libp2p with browser override --- packages/verified-fetch/package.json | 3 + packages/verified-fetch/src/index.ts | 40 +------ .../src/utils/libp2p-defaults.browser.ts | 65 ++++++++++ .../src/utils/libp2p-defaults.ts | 111 ++++++++++++++++++ 4 files changed, 184 insertions(+), 35 deletions(-) create mode 100644 packages/verified-fetch/src/utils/libp2p-defaults.browser.ts create mode 100644 packages/verified-fetch/src/utils/libp2p-defaults.ts diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 1e96e71b..f867a14b 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -200,5 +200,8 @@ "sinon": "^18.0.0", "sinon-ts": "^2.0.0" }, + "browser": { + "./dist/src/utils/libp2p-defaults.js": "./dist/src/utils/libp2p-defaults.browser.js" + }, "sideEffects": false } diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 25f5d202..1a7a8275 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -597,16 +597,13 @@ */ import { bitswap, trustlessGateway } from '@helia/block-brokers' -import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' -import { ipnsSelector, ipnsValidator, type ResolveDNSLinkProgressEvents } from '@helia/ipns' +import { type ResolveDNSLinkProgressEvents } from '@helia/ipns' import { httpGatewayRouting, libp2pRouting } from '@helia/routers' import { type Libp2p } from '@libp2p/interface' -import { kadDHT } from '@libp2p/kad-dht' -import { webRTCDirect } from '@libp2p/webrtc' -import { webSockets } from '@libp2p/websockets' import { dns } from '@multiformats/dns' -import { createHelia, type DefaultLibp2pServices, libp2pDefaults } from 'helia' +import { createHelia } from 'helia' import { createLibp2p } from 'libp2p' +import { getLibp2pConfig, type Libp2pServices } from './utils/libp2p-defaults.js' import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' import type { GetBlockProgressEvents, Helia } from '@helia/interface' import type { DNSResolvers, DNS } from '@multiformats/dns' @@ -792,42 +789,15 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions & Record<`delegatedRouting${number}`, any> - /** * Create and return a Helia node */ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchInit, options?: CreateVerifiedFetchOptions): Promise { - let libp2p: Libp2p | undefined + let libp2p: Libp2p | undefined if (!isHelia(init)) { const dns = createDns(init?.dnsResolvers) - const libp2pConfig = libp2pDefaults() - - libp2pConfig.start = false - - libp2pConfig.transports = [webSockets(), webRTCDirect()] - - libp2pConfig.peerDiscovery = [] // disable default bootstrap peers - libp2pConfig.addresses = {} // disable default listen addresses - - // We only need client/listen/fetch based services - const fetchOnlyServices = { - dcutr: libp2pConfig.services.dcutr, - identify: libp2pConfig.services.identify, - identifyPush: libp2pConfig.services.identifyPush, - keychain: libp2pConfig.services.keychain, - ping: libp2pConfig.services.ping - } - // @ts-expect-error - borked serviceMap types - libp2pConfig.services = fetchOnlyServices - const routers = init?.routers ?? ['https://delegated-ipfs.dev'] - for (let index = 0; index < routers.length; index++) { - const routerUrl = routers[index] - libp2pConfig.services[`delegatedRouting${index}`] = () => createDelegatedRoutingV1HttpApiClient(routerUrl) - } - - libp2pConfig.dns = dns + const libp2pConfig = getLibp2pConfig({ routers: init?.routers, dns }) libp2p = await createLibp2p(libp2pConfig) init = await createHelia({ diff --git a/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts b/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts new file mode 100644 index 00000000..675f4920 --- /dev/null +++ b/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts @@ -0,0 +1,65 @@ +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { createDelegatedRoutingV1HttpApiClient, type DelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' +import { dcutr } from '@libp2p/dcutr' +import { type Identify, identify } from '@libp2p/identify' +import { keychain, type Keychain } from '@libp2p/keychain' +import { mplex } from '@libp2p/mplex' +import { ping, type PingService } from '@libp2p/ping' +import { webRTCDirect } from '@libp2p/webrtc' +import { webSockets } from '@libp2p/websockets' +import { type DNS } from '@multiformats/dns' +import * as libp2pInfo from 'libp2p/version' +import { type CreateVerifiedFetchInit } from '..' +import type { Libp2pOptions } from 'libp2p' + +export interface Libp2pServices extends Record { + dcutr: unknown + identify: Identify + keychain: Keychain + ping: PingService + [key: `delegatedRouting${number}`]: DelegatedRoutingV1HttpApiClient +} + +type Libp2pOptionsInit = Pick & { dns?: DNS } + +export function getLibp2pConfig (options: Libp2pOptionsInit): Libp2pOptions { + const agentVersion = `@helia/verified-fetch ${libp2pInfo.name}/${libp2pInfo.version} UserAgent=${globalThis.navigator.userAgent}` + + const config = { + dns: options.dns, + start: false, + addresses: { + listen: [] + }, + transports: [ + webRTCDirect(), + webSockets() + ], + connectionEncrypters: [ + noise() + ], + streamMuxers: [ + yamux(), + mplex() + ], + peerDiscovery: [], // don't connect to anyone by default + // We only need client/listen/fetch based services + services: { + dcutr: dcutr(), + identify: identify({ + agentVersion + }), + keychain: keychain(), + ping: ping() + } + } + + const routers = options?.routers ?? ['https://delegated-ipfs.dev'] + for (let index = 0; index < routers.length; index++) { + const routerUrl = routers[index] + config.services[`delegatedRouting${index}`] = () => createDelegatedRoutingV1HttpApiClient(routerUrl) // TODO: add filters + } + + return config +} diff --git a/packages/verified-fetch/src/utils/libp2p-defaults.ts b/packages/verified-fetch/src/utils/libp2p-defaults.ts new file mode 100644 index 00000000..8f5663e5 --- /dev/null +++ b/packages/verified-fetch/src/utils/libp2p-defaults.ts @@ -0,0 +1,111 @@ +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { createDelegatedRoutingV1HttpApiClient, type DelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' +import { autoNAT } from '@libp2p/autonat' +import { bootstrap } from '@libp2p/bootstrap' +import { circuitRelayTransport, circuitRelayServer, type CircuitRelayService } from '@libp2p/circuit-relay-v2' +import { dcutr } from '@libp2p/dcutr' +import { type Identify, identify, identifyPush } from '@libp2p/identify' +import { type KadDHT, kadDHT } from '@libp2p/kad-dht' +import { keychain, type Keychain } from '@libp2p/keychain' +import { mplex } from '@libp2p/mplex' +import { ping, type PingService } from '@libp2p/ping' +import { tcp } from '@libp2p/tcp' +import { tls } from '@libp2p/tls' +import { uPnPNAT } from '@libp2p/upnp-nat' +import { webRTCDirect } from '@libp2p/webrtc' +import { webSockets } from '@libp2p/websockets' +import { type DNS } from '@multiformats/dns' +import { ipnsSelector } from 'ipns/selector' +import { ipnsValidator } from 'ipns/validator' +import * as libp2pInfo from 'libp2p/version' +import { type CreateVerifiedFetchInit } from '..' +import type { Libp2pOptions } from 'libp2p' + +export interface Libp2pServices extends Record { + autoNAT: unknown + dcutr: unknown + dht: KadDHT + identify: Identify + keychain: Keychain + ping: PingService + relay: CircuitRelayService + upnp: unknown + [key: `delegatedRouting${number}`]: DelegatedRoutingV1HttpApiClient +} + +export const bootstrapConfig = { + list: [ + '/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN', + '/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb', + '/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt', + // va1 is not in the TXT records for _dnsaddr.bootstrap.libp2p.io yet + // so use the host name directly + '/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8', + '/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ' + ] +} + +type Libp2pOptionsInit = Pick & { dns?: DNS } + +export function getLibp2pConfig (options: Libp2pOptionsInit): Libp2pOptions { + const agentVersion = `@helia/verified-fetch ${libp2pInfo.name}/${libp2pInfo.version} UserAgent=${process.version}` + + const config = { + dns: options.dns, + start: false, + addresses: { + listen: [ + '/ip4/0.0.0.0/tcp/0', + '/ip6/::/tcp/0', + '/p2p-circuit' + ] + }, + transports: [ + circuitRelayTransport(), + tcp(), + webRTCDirect(), + webSockets() + ], + connectionEncrypters: [ + noise(), + tls() + ], + streamMuxers: [ + yamux(), + mplex() + ], + peerDiscovery: [ + bootstrap(bootstrapConfig) + ], + services: { + autoNAT: autoNAT(), + dcutr: dcutr(), + dht: kadDHT({ + validators: { + ipns: ipnsValidator + }, + selectors: { + ipns: ipnsSelector + } + }), + identify: identify({ + agentVersion + }), + identifyPush: identifyPush({ + agentVersion + }), + keychain: keychain(), + ping: ping(), + relay: circuitRelayServer(), + upnp: uPnPNAT() + } + } + const routers = options?.routers ?? ['https://delegated-ipfs.dev'] + for (let index = 0; index < routers.length; index++) { + const routerUrl = routers[index] + config.services[`delegatedRouting${index}`] = createDelegatedRoutingV1HttpApiClient(routerUrl) // TODO: add filters + } + + return config +} From 4ab9c997354fb8e3cae3c73da65b1ae18db22ad3 Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Tue, 19 Nov 2024 17:32:52 +0100 Subject: [PATCH 18/32] fix: type errors --- packages/verified-fetch/src/utils/libp2p-defaults.browser.ts | 3 ++- packages/verified-fetch/src/utils/libp2p-defaults.ts | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts b/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts index 675f4920..f6a3ea77 100644 --- a/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts +++ b/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts @@ -58,7 +58,8 @@ export function getLibp2pConfig (options: Libp2pOptionsInit): Libp2pOptions createDelegatedRoutingV1HttpApiClient(routerUrl) // TODO: add filters + // @ts-expect-error for some reason the types are not working here + config.services[`delegatedRouting${index}`] = createDelegatedRoutingV1HttpApiClient(routerUrl) // TODO: add filters } return config diff --git a/packages/verified-fetch/src/utils/libp2p-defaults.ts b/packages/verified-fetch/src/utils/libp2p-defaults.ts index 8f5663e5..8c1136b4 100644 --- a/packages/verified-fetch/src/utils/libp2p-defaults.ts +++ b/packages/verified-fetch/src/utils/libp2p-defaults.ts @@ -104,6 +104,7 @@ export function getLibp2pConfig (options: Libp2pOptionsInit): Libp2pOptions Date: Tue, 19 Nov 2024 13:37:42 -0600 Subject: [PATCH 19/32] fix: libp2p option customization fix --- packages/verified-fetch/src/index.ts | 14 +- .../src/utils/libp2p-defaults.browser.ts | 73 +++------- .../src/utils/libp2p-defaults.ts | 130 ++++-------------- .../verified-fetch/src/utils/libp2p-types.ts | 20 +++ 4 files changed, 74 insertions(+), 163 deletions(-) create mode 100644 packages/verified-fetch/src/utils/libp2p-types.ts diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 1a7a8275..813285f6 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -597,13 +597,14 @@ */ import { bitswap, trustlessGateway } from '@helia/block-brokers' +import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' import { type ResolveDNSLinkProgressEvents } from '@helia/ipns' import { httpGatewayRouting, libp2pRouting } from '@helia/routers' import { type Libp2p } from '@libp2p/interface' import { dns } from '@multiformats/dns' import { createHelia } from 'helia' import { createLibp2p } from 'libp2p' -import { getLibp2pConfig, type Libp2pServices } from './utils/libp2p-defaults.js' +import { getLibp2pConfig } from './utils/libp2p-defaults.js' import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' import type { GetBlockProgressEvents, Helia } from '@helia/interface' import type { DNSResolvers, DNS } from '@multiformats/dns' @@ -793,11 +794,18 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions { - let libp2p: Libp2p | undefined + let libp2p: Libp2p | undefined if (!isHelia(init)) { const dns = createDns(init?.dnsResolvers) - const libp2pConfig = getLibp2pConfig({ routers: init?.routers, dns }) + const libp2pConfig = getLibp2pConfig() + libp2pConfig.dns = dns + + const routers = init?.routers ?? ['https://delegated-ipfs.dev'] + for (let index = 0; index < routers.length; index++) { + const routerUrl = routers[index] + libp2pConfig.services[`delegatedRouting${index}`] = () => createDelegatedRoutingV1HttpApiClient(routerUrl) + } libp2p = await createLibp2p(libp2pConfig) init = await createHelia({ diff --git a/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts b/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts index f6a3ea77..b05ba90e 100644 --- a/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts +++ b/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts @@ -1,66 +1,25 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' -import { createDelegatedRoutingV1HttpApiClient, type DelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' -import { dcutr } from '@libp2p/dcutr' -import { type Identify, identify } from '@libp2p/identify' -import { keychain, type Keychain } from '@libp2p/keychain' -import { mplex } from '@libp2p/mplex' -import { ping, type PingService } from '@libp2p/ping' import { webRTCDirect } from '@libp2p/webrtc' import { webSockets } from '@libp2p/websockets' -import { type DNS } from '@multiformats/dns' -import * as libp2pInfo from 'libp2p/version' -import { type CreateVerifiedFetchInit } from '..' -import type { Libp2pOptions } from 'libp2p' +import { libp2pDefaults } from 'helia' +import type { Libp2pServices, VerifiedFetchLibp2p } from './libp2p-types.js' -export interface Libp2pServices extends Record { - dcutr: unknown - identify: Identify - keychain: Keychain - ping: PingService - [key: `delegatedRouting${number}`]: DelegatedRoutingV1HttpApiClient -} - -type Libp2pOptionsInit = Pick & { dns?: DNS } +export function getLibp2pConfig (): VerifiedFetchLibp2p { + const libp2pDefaultOptions = libp2pDefaults() -export function getLibp2pConfig (options: Libp2pOptionsInit): Libp2pOptions { - const agentVersion = `@helia/verified-fetch ${libp2pInfo.name}/${libp2pInfo.version} UserAgent=${globalThis.navigator.userAgent}` + libp2pDefaultOptions.start = false + libp2pDefaultOptions.addresses = { listen: [] } + libp2pDefaultOptions.transports = [webRTCDirect(), webSockets()] - const config = { - dns: options.dns, - start: false, - addresses: { - listen: [] - }, - transports: [ - webRTCDirect(), - webSockets() - ], - connectionEncrypters: [ - noise() - ], - streamMuxers: [ - yamux(), - mplex() - ], - peerDiscovery: [], // don't connect to anyone by default - // We only need client/listen/fetch based services - services: { - dcutr: dcutr(), - identify: identify({ - agentVersion - }), - keychain: keychain(), - ping: ping() - } + const services: Libp2pServices = { + dcutr: libp2pDefaultOptions.services.dcutr, + identify: libp2pDefaultOptions.services.identify, + keychain: libp2pDefaultOptions.services.keychain, + ping: libp2pDefaultOptions.services.ping } - const routers = options?.routers ?? ['https://delegated-ipfs.dev'] - for (let index = 0; index < routers.length; index++) { - const routerUrl = routers[index] - // @ts-expect-error for some reason the types are not working here - config.services[`delegatedRouting${index}`] = createDelegatedRoutingV1HttpApiClient(routerUrl) // TODO: add filters + return { + ...libp2pDefaultOptions, + start: false, + services } - - return config } diff --git a/packages/verified-fetch/src/utils/libp2p-defaults.ts b/packages/verified-fetch/src/utils/libp2p-defaults.ts index 8c1136b4..e87f6aca 100644 --- a/packages/verified-fetch/src/utils/libp2p-defaults.ts +++ b/packages/verified-fetch/src/utils/libp2p-defaults.ts @@ -1,112 +1,36 @@ -import { noise } from '@chainsafe/libp2p-noise' -import { yamux } from '@chainsafe/libp2p-yamux' -import { createDelegatedRoutingV1HttpApiClient, type DelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' -import { autoNAT } from '@libp2p/autonat' -import { bootstrap } from '@libp2p/bootstrap' -import { circuitRelayTransport, circuitRelayServer, type CircuitRelayService } from '@libp2p/circuit-relay-v2' -import { dcutr } from '@libp2p/dcutr' -import { type Identify, identify, identifyPush } from '@libp2p/identify' -import { type KadDHT, kadDHT } from '@libp2p/kad-dht' -import { keychain, type Keychain } from '@libp2p/keychain' -import { mplex } from '@libp2p/mplex' -import { ping, type PingService } from '@libp2p/ping' -import { tcp } from '@libp2p/tcp' -import { tls } from '@libp2p/tls' -import { uPnPNAT } from '@libp2p/upnp-nat' -import { webRTCDirect } from '@libp2p/webrtc' -import { webSockets } from '@libp2p/websockets' -import { type DNS } from '@multiformats/dns' +import { kadDHT } from '@libp2p/kad-dht' +import { libp2pDefaults } from 'helia' import { ipnsSelector } from 'ipns/selector' import { ipnsValidator } from 'ipns/validator' -import * as libp2pInfo from 'libp2p/version' -import { type CreateVerifiedFetchInit } from '..' -import type { Libp2pOptions } from 'libp2p' +import type { Libp2pServices, VerifiedFetchLibp2p } from './libp2p-types.js' -export interface Libp2pServices extends Record { - autoNAT: unknown - dcutr: unknown - dht: KadDHT - identify: Identify - keychain: Keychain - ping: PingService - relay: CircuitRelayService - upnp: unknown - [key: `delegatedRouting${number}`]: DelegatedRoutingV1HttpApiClient -} - -export const bootstrapConfig = { - list: [ - '/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN', - '/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb', - '/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt', - // va1 is not in the TXT records for _dnsaddr.bootstrap.libp2p.io yet - // so use the host name directly - '/dnsaddr/va1.bootstrap.libp2p.io/p2p/12D3KooWKnDdG3iXw9eTFijk3EWSunZcFi54Zka4wmtqtt6rPxc8', - '/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ' - ] -} +export function getLibp2pConfig (): VerifiedFetchLibp2p { + const libp2pDefaultOptions = libp2pDefaults() -type Libp2pOptionsInit = Pick & { dns?: DNS } + libp2pDefaultOptions.start = false -export function getLibp2pConfig (options: Libp2pOptionsInit): Libp2pOptions { - const agentVersion = `@helia/verified-fetch ${libp2pInfo.name}/${libp2pInfo.version} UserAgent=${process.version}` - - const config = { - dns: options.dns, - start: false, - addresses: { - listen: [ - '/ip4/0.0.0.0/tcp/0', - '/ip6/::/tcp/0', - '/p2p-circuit' - ] - }, - transports: [ - circuitRelayTransport(), - tcp(), - webRTCDirect(), - webSockets() - ], - connectionEncrypters: [ - noise(), - tls() - ], - streamMuxers: [ - yamux(), - mplex() - ], - peerDiscovery: [ - bootstrap(bootstrapConfig) - ], - services: { - autoNAT: autoNAT(), - dcutr: dcutr(), - dht: kadDHT({ - validators: { - ipns: ipnsValidator - }, - selectors: { - ipns: ipnsSelector - } - }), - identify: identify({ - agentVersion - }), - identifyPush: identifyPush({ - agentVersion - }), - keychain: keychain(), - ping: ping(), - relay: circuitRelayServer(), - upnp: uPnPNAT() - } + const services: Libp2pServices = { + autoNAT: libp2pDefaultOptions.services.autoNAT, + dcutr: libp2pDefaultOptions.services.dcutr, + identify: libp2pDefaultOptions.services.identify, + keychain: libp2pDefaultOptions.services.keychain, + ping: libp2pDefaultOptions.services.ping, + upnp: libp2pDefaultOptions.services.upnp, + dht: kadDHT({ + clientMode: true, + validators: { + ipns: ipnsValidator + }, + selectors: { + ipns: ipnsSelector + } + }) } - const routers = options?.routers ?? ['https://delegated-ipfs.dev'] - for (let index = 0; index < routers.length; index++) { - const routerUrl = routers[index] - // @ts-expect-error for some reason the types are not working here - config.services[`delegatedRouting${index}`] = createDelegatedRoutingV1HttpApiClient(routerUrl) // TODO: add filters + const libp2pOptions: VerifiedFetchLibp2p = { + ...libp2pDefaultOptions, + start: false, + services } - return config + return libp2pOptions } diff --git a/packages/verified-fetch/src/utils/libp2p-types.ts b/packages/verified-fetch/src/utils/libp2p-types.ts new file mode 100644 index 00000000..d7604d35 --- /dev/null +++ b/packages/verified-fetch/src/utils/libp2p-types.ts @@ -0,0 +1,20 @@ +import type { DelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' +import type { ServiceMap } from '@libp2p/interface' +import type { DefaultLibp2pServices } from 'helia' +import type { Libp2pOptions } from 'libp2p' + +type BaseServiceMap = Record +type DelegatedRoutingServices = Record<`delegatedRouting${number}`, unknown | ((components?: unknown) => DelegatedRoutingV1HttpApiClient)> + +// eslint-disable-next-line @typescript-eslint/consistent-indexed-object-style +export interface Libp2pServices extends BaseServiceMap, DelegatedRoutingServices { + [key: string]: any +} +// type Optional = Pick, K> & Omit; + +interface Libp2pInitOverride extends Omit, 'services'> { + services: Libp2pServices + start?: boolean +} + +export type VerifiedFetchLibp2p = Libp2pInitOverride & Pick, 'services'> From fe9ab4c83d94b79ee00f7f2313ec4272bbde4578 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 19 Nov 2024 13:38:47 -0600 Subject: [PATCH 20/32] chore: fix unnecessary code --- packages/verified-fetch/src/utils/libp2p-types.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/verified-fetch/src/utils/libp2p-types.ts b/packages/verified-fetch/src/utils/libp2p-types.ts index d7604d35..049a7b37 100644 --- a/packages/verified-fetch/src/utils/libp2p-types.ts +++ b/packages/verified-fetch/src/utils/libp2p-types.ts @@ -6,11 +6,8 @@ import type { Libp2pOptions } from 'libp2p' type BaseServiceMap = Record type DelegatedRoutingServices = Record<`delegatedRouting${number}`, unknown | ((components?: unknown) => DelegatedRoutingV1HttpApiClient)> -// eslint-disable-next-line @typescript-eslint/consistent-indexed-object-style -export interface Libp2pServices extends BaseServiceMap, DelegatedRoutingServices { - [key: string]: any +export interface Libp2pServices extends BaseServiceMap, DelegatedRoutingServices, Record { } -// type Optional = Pick, K> & Omit; interface Libp2pInitOverride extends Omit, 'services'> { services: Libp2pServices From 0916ccf543aba6afe17c3493e67d1c3ffd2176a1 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 19 Nov 2024 13:39:55 -0600 Subject: [PATCH 21/32] fix: eslint rule forces style that breaks types --- packages/verified-fetch/src/utils/libp2p-types.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/verified-fetch/src/utils/libp2p-types.ts b/packages/verified-fetch/src/utils/libp2p-types.ts index 049a7b37..ebdf13e1 100644 --- a/packages/verified-fetch/src/utils/libp2p-types.ts +++ b/packages/verified-fetch/src/utils/libp2p-types.ts @@ -6,7 +6,9 @@ import type { Libp2pOptions } from 'libp2p' type BaseServiceMap = Record type DelegatedRoutingServices = Record<`delegatedRouting${number}`, unknown | ((components?: unknown) => DelegatedRoutingV1HttpApiClient)> -export interface Libp2pServices extends BaseServiceMap, DelegatedRoutingServices, Record { +// eslint-disable-next-line @typescript-eslint/consistent-indexed-object-style +export interface Libp2pServices extends BaseServiceMap, DelegatedRoutingServices { + [key: string]: any } interface Libp2pInitOverride extends Omit, 'services'> { From 310dd088b832a4fabab3e60f983fe9a8b0c36550 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 19 Nov 2024 13:52:07 -0600 Subject: [PATCH 22/32] fix: ipns is prod dep now --- packages/verified-fetch/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index f867a14b..da599984 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -164,6 +164,7 @@ "interface-blockstore": "^5.3.1", "interface-datastore": "^8.3.1", "ipfs-unixfs-exporter": "^13.6.1", + "ipns": "^10.0.0", "it-map": "^3.1.1", "it-pipe": "^3.0.1", "it-tar": "^6.0.5", @@ -190,7 +191,6 @@ "browser-readablestream-to-it": "^2.0.7", "datastore-core": "^10.0.2", "ipfs-unixfs-importer": "^15.3.1", - "ipns": "^10.0.0", "it-all": "^3.0.6", "it-drain": "^3.0.7", "it-last": "^3.0.6", From 30fb2741650486cb14acc72994a2b09750584d60 Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:55:25 +0100 Subject: [PATCH 23/32] deps: bump delegated routing client --- packages/verified-fetch/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index da599984..8b159a21 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -144,7 +144,7 @@ "dependencies": { "@helia/block-brokers": "^4.0.0", "@helia/car": "^4.0.0", - "@helia/delegated-routing-v1-http-api-client": "^4.1.2", + "@helia/delegated-routing-v1-http-api-client": "^4.2.1", "@helia/interface": "^5.0.0", "@helia/ipns": "^8.0.0", "@helia/routers": "^2.0.0", From 5f2f12762f75ac6216b6b79738454213f8824733 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 20 Nov 2024 10:35:11 -0600 Subject: [PATCH 24/32] test: add direct-retrieval test --- packages/interop/.aegir.js | 36 ++++++++++++++++++- packages/interop/package.json | 1 + packages/interop/src/direct-retrieval.spec.ts | 32 +++++++++++++++++ .../interop/src/fixtures/load-fixtures.ts | 2 +- packages/verified-fetch/src/index.ts | 35 +++++++++--------- 5 files changed, 87 insertions(+), 19 deletions(-) create mode 100644 packages/interop/src/direct-retrieval.spec.ts diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js index a02b8beb..a717cb41 100644 --- a/packages/interop/.aegir.js +++ b/packages/interop/.aegir.js @@ -1,5 +1,7 @@ import { resolve } from 'node:path' import { tmpdir } from 'node:os' +import { createDelegatedRoutingV1HttpApiServer } from '@helia/delegated-routing-v1-http-api-server' +import { stubInterface } from 'sinon-ts' const IPFS_PATH = resolve(tmpdir(), 'verified-fetch-interop-ipfs-repo') @@ -19,12 +21,44 @@ export default { await loadFixtures(IPFS_PATH) + const multiaddrs = (await kuboNode.api.id()).addresses + const id = (await kuboNode.api.id()).id + + console.log(multiaddrs.map(ma => ma.toString()).join('\n')) + + const helia = stubInterface({ + routing: stubInterface({ + findProviders: async function * findProviders () { + yield { + multiaddrs, + id, + protocols: ['transport-bitswap'] + } + } + }) + }) + const routingServer = await createDelegatedRoutingV1HttpApiServer(helia, { + listen: { + host: '127.0.0.1', + port: 0 + } + }) + await routingServer.ready() + + const address = routingServer.server.address() + const port = typeof address === 'string' ? address : address?.port + return { - kuboNode + kuboNode, + routingServer, + env: { + KUBO_DIRECT_RETRIEVAL_ROUTER: `http://127.0.0.1:${port}` + } } }, after: async (_options, beforeResult) => { await beforeResult.kuboNode.stop() + await beforeResult.routingServer.close() } } } diff --git a/packages/interop/package.json b/packages/interop/package.json index 4dfb8680..d6095b8b 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -57,6 +57,7 @@ "test:electron-main": "aegir test -t electron-main" }, "dependencies": { + "@helia/delegated-routing-v1-http-api-server": "^4.0.4", "@helia/verified-fetch": "2.1.2", "aegir": "^45.0.1", "execa": "^9.1.0", diff --git a/packages/interop/src/direct-retrieval.spec.ts b/packages/interop/src/direct-retrieval.spec.ts new file mode 100644 index 00000000..b1414d81 --- /dev/null +++ b/packages/interop/src/direct-retrieval.spec.ts @@ -0,0 +1,32 @@ +import { createVerifiedFetch } from '@helia/verified-fetch' +import { expect } from 'aegir/chai' + +describe('@helia/verified-fetch - direct retrieval', () => { + let directRetrievalRouterUrl: string + + beforeEach(async () => { + if (process.env.KUBO_DIRECT_RETRIEVAL_ROUTER == null || process.env.KUBO_DIRECT_RETRIEVAL_ROUTER === '') { + throw new Error('KUBO_DIRECT_RETRIEVAL_ROUTER environment variable is required') + } + directRetrievalRouterUrl = process.env.KUBO_DIRECT_RETRIEVAL_ROUTER + }) + + it('can fetch content directly from another node', async () => { + // console.log(process.env.KUBO_MADDRS) + // we want to disable trustless gateways and use direct retrieval + // const customRouter = stubInterface> + + const fetch = await createVerifiedFetch({ + gateways: [], + routers: [directRetrievalRouterUrl] + }) + + const foo = await fetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1 - alt.txt') + + expect(foo.status).to.equal(200) + const body = await foo.text() + expect(body).to.equal('Don\'t we all.') + + await fetch.stop() + }) +}) diff --git a/packages/interop/src/fixtures/load-fixtures.ts b/packages/interop/src/fixtures/load-fixtures.ts index 7642bcb8..f95eb218 100644 --- a/packages/interop/src/fixtures/load-fixtures.ts +++ b/packages/interop/src/fixtures/load-fixtures.ts @@ -11,7 +11,7 @@ export async function loadFixtures (IPFS_PATH = undefined): Promise { const files = await glob('**/fixtures/data/*.car', { cwd: process.cwd() }) - await Promise.all(files.map(async (carFile) => { + await Promise.allSettled(files.map(async (carFile) => { await $({ env: { IPFS_PATH } })`${kuboBinary} dag import --pin-roots=false --offline ${carFile}` })) } diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 813285f6..ea513866 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -606,7 +606,7 @@ import { createHelia } from 'helia' import { createLibp2p } from 'libp2p' import { getLibp2pConfig } from './utils/libp2p-defaults.js' import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' -import type { GetBlockProgressEvents, Helia } from '@helia/interface' +import type { GetBlockProgressEvents, Helia, Routing } from '@helia/interface' import type { DNSResolvers, DNS } from '@multiformats/dns' import type { DNSResolver } from '@multiformats/dns/resolvers' import type { ExporterProgressEvents } from 'ipfs-unixfs-exporter' @@ -801,29 +801,30 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni const libp2pConfig = getLibp2pConfig() libp2pConfig.dns = dns - const routers = init?.routers ?? ['https://delegated-ipfs.dev'] - for (let index = 0; index < routers.length; index++) { - const routerUrl = routers[index] + const delegatedRouters = init?.routers ?? ['https://delegated-ipfs.dev'] + for (let index = 0; index < delegatedRouters.length; index++) { + const routerUrl = delegatedRouters[index] libp2pConfig.services[`delegatedRouting${index}`] = () => createDelegatedRoutingV1HttpApiClient(routerUrl) } libp2p = await createLibp2p(libp2pConfig) + const blockBrokers = [ + bitswap() + ] + const routers: Array> = [ + libp2pRouting(libp2p) + ] + if (init?.gateways == null || init.gateways.length > 0) { + // if gateways is null, or set to a non-empty array, use trustless gateways. + blockBrokers.push(trustlessGateway({ allowInsecure: init?.allowInsecure, allowLocal: init?.allowLocal })) + routers.push(httpGatewayRouting({ gateways: init?.gateways ?? ['https://trustless-gateway.link'] })) + } + init = await createHelia({ libp2p, - blockBrokers: [ - trustlessGateway({ - allowInsecure: init?.allowInsecure, - allowLocal: init?.allowLocal - }), - bitswap() - ], + blockBrokers, dns, - routers: [ - httpGatewayRouting({ - gateways: init?.gateways ?? ['https://trustless-gateway.link'] - }), - libp2pRouting(libp2p) - ] + routers }) } From c8f670943821a11eb29b2e85a065b8d657d48638 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:39:45 -0600 Subject: [PATCH 25/32] test: fix browser testing --- packages/interop/.aegir.js | 2 -- packages/interop/src/direct-retrieval.spec.ts | 35 ++++++++++++------- packages/interop/src/fixtures/create-kubo.ts | 6 +++- packages/verified-fetch/src/index.ts | 18 ++++++++-- 4 files changed, 44 insertions(+), 17 deletions(-) diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js index a717cb41..7cc155d7 100644 --- a/packages/interop/.aegir.js +++ b/packages/interop/.aegir.js @@ -24,8 +24,6 @@ export default { const multiaddrs = (await kuboNode.api.id()).addresses const id = (await kuboNode.api.id()).id - console.log(multiaddrs.map(ma => ma.toString()).join('\n')) - const helia = stubInterface({ routing: stubInterface({ findProviders: async function * findProviders () { diff --git a/packages/interop/src/direct-retrieval.spec.ts b/packages/interop/src/direct-retrieval.spec.ts index b1414d81..da082b2d 100644 --- a/packages/interop/src/direct-retrieval.spec.ts +++ b/packages/interop/src/direct-retrieval.spec.ts @@ -1,30 +1,41 @@ -import { createVerifiedFetch } from '@helia/verified-fetch' +import { createVerifiedFetch, type CreateVerifiedFetchInit } from '@helia/verified-fetch' import { expect } from 'aegir/chai' +import { isNode, isBrowser } from 'wherearewe' + +/** + * Currently only testing browser and node + */ +const describe = isNode || isBrowser ? global.describe : global.describe.skip describe('@helia/verified-fetch - direct retrieval', () => { let directRetrievalRouterUrl: string + let createVerifiedFetchInit: CreateVerifiedFetchInit beforeEach(async () => { if (process.env.KUBO_DIRECT_RETRIEVAL_ROUTER == null || process.env.KUBO_DIRECT_RETRIEVAL_ROUTER === '') { throw new Error('KUBO_DIRECT_RETRIEVAL_ROUTER environment variable is required') } directRetrievalRouterUrl = process.env.KUBO_DIRECT_RETRIEVAL_ROUTER + createVerifiedFetchInit = { + gateways: [], + routers: [directRetrievalRouterUrl] + } + if (!isNode) { + createVerifiedFetchInit.libp2pConfig = { + connectionGater: { + denyDialMultiaddr: () => false + } + } + } }) it('can fetch content directly from another node', async () => { - // console.log(process.env.KUBO_MADDRS) - // we want to disable trustless gateways and use direct retrieval - // const customRouter = stubInterface> - - const fetch = await createVerifiedFetch({ - gateways: [], - routers: [directRetrievalRouterUrl] - }) + const fetch = await createVerifiedFetch(createVerifiedFetchInit) - const foo = await fetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1 - alt.txt') + const res = await fetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR/1 - Barrel - Part 1 - alt.txt') - expect(foo.status).to.equal(200) - const body = await foo.text() + expect(res.status).to.equal(200) + const body = await res.text() expect(body).to.equal('Don\'t we all.') await fetch.stop() diff --git a/packages/interop/src/fixtures/create-kubo.ts b/packages/interop/src/fixtures/create-kubo.ts index b167d765..cca0e7bb 100644 --- a/packages/interop/src/fixtures/create-kubo.ts +++ b/packages/interop/src/fixtures/create-kubo.ts @@ -14,7 +14,11 @@ export async function createKuboNode (repoPath = undefined): Promise { Addresses: { Swarm: [ '/ip4/0.0.0.0/tcp/4001', - '/ip4/0.0.0.0/tcp/4002/ws' + '/ip4/0.0.0.0/tcp/4002/ws', + '/ip4/0.0.0.0/udp/4001/webrtc-direct', + '/ip4/0.0.0.0/udp/4001/quic-v1/webtransport', + '/ip6/::/udp/4001/webrtc-direct', + '/ip6/::/udp/4001/quic-v1/webtransport' ], Gateway: '/ip4/127.0.0.1/tcp/8180' }, diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index ea513866..c571153d 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -600,10 +600,10 @@ import { bitswap, trustlessGateway } from '@helia/block-brokers' import { createDelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' import { type ResolveDNSLinkProgressEvents } from '@helia/ipns' import { httpGatewayRouting, libp2pRouting } from '@helia/routers' -import { type Libp2p } from '@libp2p/interface' +import { type Libp2p, type ServiceMap } from '@libp2p/interface' import { dns } from '@multiformats/dns' import { createHelia } from 'helia' -import { createLibp2p } from 'libp2p' +import { createLibp2p, type Libp2pOptions } from 'libp2p' import { getLibp2pConfig } from './utils/libp2p-defaults.js' import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js' import type { GetBlockProgressEvents, Helia, Routing } from '@helia/interface' @@ -681,6 +681,16 @@ export interface CreateVerifiedFetchInit { * @default false */ allowInsecure?: boolean + + /** + * We will instantiate a libp2p node for you, but if you want to override the libp2p configuration, + * you can pass it here. + * + * **WARNING**: We use Object.assign to merge the default libp2p configuration from Helia with the one you pass here, + * which results in a shallow merge. If you need a deep merge, you should do it yourself before passing the + * configuration here. + */ + libp2pConfig?: Partial> } export interface CreateVerifiedFetchOptions { @@ -806,6 +816,10 @@ export async function createVerifiedFetch (init?: Helia | CreateVerifiedFetchIni const routerUrl = delegatedRouters[index] libp2pConfig.services[`delegatedRouting${index}`] = () => createDelegatedRoutingV1HttpApiClient(routerUrl) } + // merge any passed options from init.libp2pConfig into libp2pConfig if it exists + if (init?.libp2pConfig != null) { + Object.assign(libp2pConfig, init.libp2pConfig) + } libp2p = await createLibp2p(libp2pConfig) const blockBrokers = [ From 0cc392b0d1fd4e2125accf58b0af168d5a75cef1 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 20 Nov 2024 12:48:42 -0600 Subject: [PATCH 26/32] fix: use matching index signature for Libp2pServices --- packages/verified-fetch/src/utils/libp2p-types.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/verified-fetch/src/utils/libp2p-types.ts b/packages/verified-fetch/src/utils/libp2p-types.ts index ebdf13e1..660701e5 100644 --- a/packages/verified-fetch/src/utils/libp2p-types.ts +++ b/packages/verified-fetch/src/utils/libp2p-types.ts @@ -4,12 +4,11 @@ import type { DefaultLibp2pServices } from 'helia' import type { Libp2pOptions } from 'libp2p' type BaseServiceMap = Record -type DelegatedRoutingServices = Record<`delegatedRouting${number}`, unknown | ((components?: unknown) => DelegatedRoutingV1HttpApiClient)> +type DelegatedRoutingServices = Record<`delegatedRouting${number}`, ((components?: unknown) => DelegatedRoutingV1HttpApiClient)> -// eslint-disable-next-line @typescript-eslint/consistent-indexed-object-style -export interface Libp2pServices extends BaseServiceMap, DelegatedRoutingServices { - [key: string]: any -} +export type Libp2pServices = { + [Property in keyof T]: (components: any & T) => T[Property] +} & BaseServiceMap & DelegatedRoutingServices interface Libp2pInitOverride extends Omit, 'services'> { services: Libp2pServices From 1b876d752b6ed6425cdd20f93df1157984c170b3 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 20 Nov 2024 14:22:52 -0600 Subject: [PATCH 27/32] chore: fix dep-check --- packages/interop/.aegir.js | 7 +++++++ packages/interop/package.json | 4 +++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js index 7cc155d7..52a4211b 100644 --- a/packages/interop/.aegir.js +++ b/packages/interop/.aegir.js @@ -7,6 +7,13 @@ const IPFS_PATH = resolve(tmpdir(), 'verified-fetch-interop-ipfs-repo') /** @type {import('aegir').PartialOptions} */ export default { + dependencyCheck: { + ignore: [ + '@helia/delegated-routing-v1-http-api-server', + 'sinon-ts' + ] + + }, test: { files: './dist/src/*.spec.js', before: async () => { diff --git a/packages/interop/package.json b/packages/interop/package.json index d6095b8b..7121f989 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -66,7 +66,9 @@ "kubo": "^0.32.0", "kubo-rpc-client": "^4.1.1", "magic-bytes.js": "^1.10.0", - "multiformats": "^13.1.0" + "multiformats": "^13.1.0", + "sinon-ts": "^2.0.0", + "wherearewe": "^2.0.1" }, "browser": { "./dist/src/fixtures/create-kubo.js": "./dist/src/fixtures/create-kubo.browser.js", From 06894f85a1a8cc42b99a194cb5a0d21a65971cf4 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Wed, 20 Nov 2024 15:27:12 -0600 Subject: [PATCH 28/32] chore: fix ipns tests and dep-check --- packages/gateway-conformance/package.json | 3 +- .../src/fixtures/kubo-mgmt.ts | 9 +++-- packages/verified-fetch/package.json | 1 - .../test/utils/parse-url-string.spec.ts | 33 +++++++++---------- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/packages/gateway-conformance/package.json b/packages/gateway-conformance/package.json index 2ba8acb2..ba9db886 100644 --- a/packages/gateway-conformance/package.json +++ b/packages/gateway-conformance/package.json @@ -52,8 +52,8 @@ "test": "aegir test -t node" }, "dependencies": { - "@helia/http": "^2.0.1", "@helia/block-brokers": "^4.0.0", + "@helia/http": "^2.0.1", "@helia/interface": "^5.0.0", "@helia/routers": "^2.0.0", "@helia/verified-fetch": "2.1.2", @@ -70,7 +70,6 @@ "interface-blockstore": "^5.2.10", "interface-datastore": "^8.2.11", "ipfsd-ctl": "^14.1.0", - "ipns": "^9.1.0", "kubo": "^0.32.0", "kubo-rpc-client": "^4.1.1", "uint8arrays": "^5.1.0", diff --git a/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts b/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts index d1dd81aa..2b18fb82 100644 --- a/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts +++ b/packages/gateway-conformance/src/fixtures/kubo-mgmt.ts @@ -16,8 +16,9 @@ import { peerIdFromString } from '@libp2p/peer-id' import { $ } from 'execa' import fg from 'fast-glob' import { Key } from 'interface-datastore' -import { peerIdToRoutingKey } from 'ipns' import { path } from 'kubo' +import { concat as uint8ArrayConcat } from 'uint8arrays/concat' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { GWC_IMAGE } from '../constants.js' import { getIpnsRecordDatastore } from './ipns-record-datastore.js' @@ -92,13 +93,17 @@ export async function loadFixtures (kuboRepoDir: string): Promise { } const datastore = getIpnsRecordDatastore() + const IPNS_PREFIX = uint8ArrayFromString('/ipns/') for (const fsIpnsRecord of await fg.glob([`${GWC_FIXTURES_PATH}/**/*.ipns-record`])) { const peerIdString = basename(fsIpnsRecord, '.ipns-record').split('_')[0] const relativePath = relative(GWC_FIXTURES_PATH, fsIpnsRecord) log('Loading *.ipns-record fixture %s', relativePath) const key = peerIdFromString(peerIdString) - const customRoutingKey = peerIdToRoutingKey(key) + const customRoutingKey = uint8ArrayConcat([ + IPNS_PREFIX, + key.toBytes() + ]) const dhtKey = new Key('/dht/record/' + uint8ArrayToString(customRoutingKey, 'base32'), false) const dhtRecord = new DhtRecord(customRoutingKey, await readFile(fsIpnsRecord, null), new Date(Date.now() + 9999999)) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 8b159a21..12bd2030 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -182,7 +182,6 @@ "@helia/json": "^4.0.0", "@ipld/car": "^5.3.2", "@libp2p/crypto": "^5.0.5", - "@libp2p/interface-compliance-tests": "^6.1.6", "@libp2p/logger": "^5.1.1", "@sgtpooki/file-type": "^1.0.1", "@types/sinon": "^17.0.3", diff --git a/packages/verified-fetch/test/utils/parse-url-string.spec.ts b/packages/verified-fetch/test/utils/parse-url-string.spec.ts index 5e7d588b..97f3347e 100644 --- a/packages/verified-fetch/test/utils/parse-url-string.spec.ts +++ b/packages/verified-fetch/test/utils/parse-url-string.spec.ts @@ -1,12 +1,10 @@ import { generateKeyPair } from '@libp2p/crypto/keys' -import { matchPeerId } from '@libp2p/interface-compliance-tests/matchers' import { defaultLogger } from '@libp2p/logger' import { peerIdFromPrivateKey } from '@libp2p/peer-id' import { type Answer } from '@multiformats/dns' import { expect } from 'aegir/chai' import { base36 } from 'multiformats/bases/base36' import { CID } from 'multiformats/cid' -import { match } from 'sinon' import { stubInterface } from 'sinon-ts' import { parseUrlString } from '../../src/utils/parse-url-string.js' import { ipnsRecordStub } from '../fixtures/ipns-stubs.js' @@ -283,7 +281,7 @@ describe('parseUrlString', () => { const key = await generateKeyPair('Ed25519') const testPeerId = peerIdFromPrivateKey(key) - ipns.resolve.withArgs(matchPeerId(testPeerId)).resolves({ + ipns.resolve.withArgs(key.publicKey).resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '', record: ipnsRecordStub({ peerId: testPeerId, ttl: oneHourInNanoseconds }) @@ -492,7 +490,7 @@ describe('parseUrlString', () => { }) it('can parse a URL with PeerId only', async () => { - ipns.resolve.withArgs(matchPeerId(testPeerId)).resolves({ + ipns.resolve.withArgs(testPeerId.publicKey).resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '', record: ipnsRecordStub({ peerId: testPeerId }) @@ -509,7 +507,7 @@ describe('parseUrlString', () => { }) it('can parse a base36 PeerId CID', async () => { - ipns.resolve.withArgs(matchPeerId(testPeerId)).resolves({ + ipns.resolve.withArgs(testPeerId.publicKey).resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '', record: ipnsRecordStub({ peerId: testPeerId }) @@ -526,7 +524,7 @@ describe('parseUrlString', () => { }) it('can parse a URL with PeerId+path', async () => { - ipns.resolve.withArgs(matchPeerId(testPeerId)).resolves({ + ipns.resolve.withArgs(testPeerId.publicKey).resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '', record: ipnsRecordStub({ peerId: testPeerId }) @@ -543,7 +541,7 @@ describe('parseUrlString', () => { }) it('can parse a URL with PeerId+path with a trailing slash', async () => { - ipns.resolve.withArgs(matchPeerId(testPeerId)).resolves({ + ipns.resolve.withArgs(testPeerId.publicKey).resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '', record: ipnsRecordStub({ peerId: testPeerId }) @@ -560,7 +558,7 @@ describe('parseUrlString', () => { }) it('can parse a URL with PeerId+queryString', async () => { - ipns.resolve.withArgs(matchPeerId(testPeerId)).resolves({ + ipns.resolve.withArgs(testPeerId.publicKey).resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '', record: ipnsRecordStub({ peerId: testPeerId }) @@ -579,7 +577,7 @@ describe('parseUrlString', () => { }) it('can parse a URL with PeerId+path+queryString', async () => { - ipns.resolve.withArgs(matchPeerId(testPeerId)).resolves({ + ipns.resolve.withArgs(testPeerId.publicKey).resolves({ cid: CID.parse('QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr'), path: '', record: ipnsRecordStub({ peerId: testPeerId }) @@ -604,7 +602,7 @@ describe('parseUrlString', () => { const recordPath = 'foo' const requestPath = 'bar/baz.txt' - ipns.resolve.withArgs(matchPeerId(peerId)).resolves({ + ipns.resolve.withArgs(peerId.publicKey).resolves({ cid, path: recordPath, record: ipnsRecordStub({ peerId: testPeerId }) @@ -627,7 +625,7 @@ describe('parseUrlString', () => { const recordPath = 'foo/' const requestPath = 'bar/baz.txt' - ipns.resolve.withArgs(matchPeerId(peerId)).resolves({ + ipns.resolve.withArgs(peerId.publicKey).resolves({ cid, path: recordPath, record: ipnsRecordStub({ peerId: testPeerId }) @@ -650,7 +648,7 @@ describe('parseUrlString', () => { const recordPath = '/foo/////bar//' const requestPath = '///baz///qux.txt' - ipns.resolve.withArgs(matchPeerId(peerId)).resolves({ + ipns.resolve.withArgs(peerId.publicKey).resolves({ cid, path: recordPath, record: ipnsRecordStub({ peerId: testPeerId }) @@ -675,7 +673,7 @@ describe('parseUrlString', () => { const key = await generateKeyPair('Ed25519') peerId = peerIdFromPrivateKey(key) cid = CID.parse('QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm') - ipns.resolve.withArgs(matchPeerId(peerId)).resolves({ + ipns.resolve.withArgs(peerId.publicKey).resolves({ cid, path: '', record: ipnsRecordStub({ peerId }) @@ -764,7 +762,7 @@ describe('parseUrlString', () => { const key = await generateKeyPair('Ed25519') peerId = peerIdFromPrivateKey(key) cid = CID.parse('QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm') - ipns.resolve.withArgs(matchPeerId(peerId)).resolves({ + ipns.resolve.withArgs(peerId.publicKey).resolves({ cid, path: '', record: ipnsRecordStub({ peerId }) @@ -869,20 +867,19 @@ describe('parseUrlString', () => { value = await getVal(i++) cid = CID.parse('QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm') if (type === 'peerid') { - ipns.resolve.withArgs(matchPeerId(value as PeerId)).resolves({ + ipns.resolve.withArgs((value as PeerId).publicKey).resolves({ cid, path: '', record: ipnsRecordStub({ peerId: value as PeerId }) }) } else if (type === 'dnslink-encoded') { - const matchValue = (value as string).replace(/-/g, '.') - ipns.resolveDNSLink.withArgs(match(matchValue)).resolves({ + ipns.resolveDNSLink.withArgs(value.toString().replace(/-/g, '.')).resolves({ cid, path: '', answer: stubInterface() }) } else { - ipns.resolveDNSLink.withArgs(match(value as string)).resolves({ + ipns.resolveDNSLink.withArgs(value.toString()).resolves({ cid, path: '', answer: stubInterface() From 053d245f4455fa4ac91648df17b8957613a99b20 Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Thu, 21 Nov 2024 10:21:12 +0100 Subject: [PATCH 29/32] fix: build by using relative path --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 7121f989..310b3f0c 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -58,7 +58,7 @@ }, "dependencies": { "@helia/delegated-routing-v1-http-api-server": "^4.0.4", - "@helia/verified-fetch": "2.1.2", + "@helia/verified-fetch": "../..", "aegir": "^45.0.1", "execa": "^9.1.0", "glob": "^11.0.0", From 226bf233c8efd6358a06ddc1e2bcb55bbfeefa7b Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:47:59 +0100 Subject: [PATCH 30/32] chore: simplify types --- .../src/utils/libp2p-defaults.browser.ts | 11 +++++--- .../src/utils/libp2p-defaults.ts | 26 ++++++++++--------- .../verified-fetch/src/utils/libp2p-types.ts | 14 ++-------- 3 files changed, 23 insertions(+), 28 deletions(-) diff --git a/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts b/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts index b05ba90e..9e9d252c 100644 --- a/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts +++ b/packages/verified-fetch/src/utils/libp2p-defaults.browser.ts @@ -1,16 +1,19 @@ import { webRTCDirect } from '@libp2p/webrtc' import { webSockets } from '@libp2p/websockets' -import { libp2pDefaults } from 'helia' -import type { Libp2pServices, VerifiedFetchLibp2p } from './libp2p-types.js' +import { libp2pDefaults, type DefaultLibp2pServices } from 'helia' +import type { ServiceFactoryMap } from './libp2p-types' +import type { Libp2pOptions } from 'libp2p' -export function getLibp2pConfig (): VerifiedFetchLibp2p { +type ServiceMap = Pick + +export function getLibp2pConfig (): Libp2pOptions & Required> { const libp2pDefaultOptions = libp2pDefaults() libp2pDefaultOptions.start = false libp2pDefaultOptions.addresses = { listen: [] } libp2pDefaultOptions.transports = [webRTCDirect(), webSockets()] - const services: Libp2pServices = { + const services: ServiceFactoryMap = { dcutr: libp2pDefaultOptions.services.dcutr, identify: libp2pDefaultOptions.services.identify, keychain: libp2pDefaultOptions.services.keychain, diff --git a/packages/verified-fetch/src/utils/libp2p-defaults.ts b/packages/verified-fetch/src/utils/libp2p-defaults.ts index e87f6aca..05bfb076 100644 --- a/packages/verified-fetch/src/utils/libp2p-defaults.ts +++ b/packages/verified-fetch/src/utils/libp2p-defaults.ts @@ -1,21 +1,20 @@ import { kadDHT } from '@libp2p/kad-dht' -import { libp2pDefaults } from 'helia' +import { libp2pDefaults, type DefaultLibp2pServices } from 'helia' import { ipnsSelector } from 'ipns/selector' import { ipnsValidator } from 'ipns/validator' -import type { Libp2pServices, VerifiedFetchLibp2p } from './libp2p-types.js' +import type { ServiceFactoryMap } from './libp2p-types' +import type { Libp2pOptions } from 'libp2p' -export function getLibp2pConfig (): VerifiedFetchLibp2p { +type ServiceMap = Pick + +export function getLibp2pConfig (): Libp2pOptions & Required> { const libp2pDefaultOptions = libp2pDefaults() libp2pDefaultOptions.start = false - const services: Libp2pServices = { + const services: ServiceFactoryMap = { autoNAT: libp2pDefaultOptions.services.autoNAT, dcutr: libp2pDefaultOptions.services.dcutr, - identify: libp2pDefaultOptions.services.identify, - keychain: libp2pDefaultOptions.services.keychain, - ping: libp2pDefaultOptions.services.ping, - upnp: libp2pDefaultOptions.services.upnp, dht: kadDHT({ clientMode: true, validators: { @@ -24,13 +23,16 @@ export function getLibp2pConfig (): VerifiedFetchLibp2p { selectors: { ipns: ipnsSelector } - }) + }), + identify: libp2pDefaultOptions.services.identify, + keychain: libp2pDefaultOptions.services.keychain, + ping: libp2pDefaultOptions.services.ping, + upnp: libp2pDefaultOptions.services.upnp } - const libp2pOptions: VerifiedFetchLibp2p = { + + return { ...libp2pDefaultOptions, start: false, services } - - return libp2pOptions } diff --git a/packages/verified-fetch/src/utils/libp2p-types.ts b/packages/verified-fetch/src/utils/libp2p-types.ts index 660701e5..39f29a23 100644 --- a/packages/verified-fetch/src/utils/libp2p-types.ts +++ b/packages/verified-fetch/src/utils/libp2p-types.ts @@ -1,18 +1,8 @@ import type { DelegatedRoutingV1HttpApiClient } from '@helia/delegated-routing-v1-http-api-client' import type { ServiceMap } from '@libp2p/interface' -import type { DefaultLibp2pServices } from 'helia' -import type { Libp2pOptions } from 'libp2p' -type BaseServiceMap = Record type DelegatedRoutingServices = Record<`delegatedRouting${number}`, ((components?: unknown) => DelegatedRoutingV1HttpApiClient)> -export type Libp2pServices = { +export type ServiceFactoryMap = { [Property in keyof T]: (components: any & T) => T[Property] -} & BaseServiceMap & DelegatedRoutingServices - -interface Libp2pInitOverride extends Omit, 'services'> { - services: Libp2pServices - start?: boolean -} - -export type VerifiedFetchLibp2p = Libp2pInitOverride & Pick, 'services'> +} & DelegatedRoutingServices From 654565d4aaca26ad3499a3c461fa40f6e4797837 Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Thu, 21 Nov 2024 11:51:36 +0100 Subject: [PATCH 31/32] chore: remove unused dep --- packages/verified-fetch/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index cfcde4ce..4ff8695c 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -145,7 +145,6 @@ "@helia/block-brokers": "^4.0.2", "@helia/car": "^4.0.1", "@helia/delegated-routing-v1-http-api-client": "^4.2.1", - "@helia/http": "^2.0.2", "@helia/interface": "^5.1.0", "@helia/ipns": "^8.0.1", "@helia/routers": "^2.2.0", From dc07e56a0771c74c34e4c1bd19cb3aa941b1db48 Mon Sep 17 00:00:00 2001 From: Daniel N <2color@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:33:34 +0100 Subject: [PATCH 32/32] fix: use caret so that the sibling dep is used --- packages/interop/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/interop/package.json b/packages/interop/package.json index 310b3f0c..f72d0405 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -58,7 +58,7 @@ }, "dependencies": { "@helia/delegated-routing-v1-http-api-server": "^4.0.4", - "@helia/verified-fetch": "../..", + "@helia/verified-fetch": "^2.1.3", "aegir": "^45.0.1", "execa": "^9.1.0", "glob": "^11.0.0",