From 1561e4a106074b94e421a77b0b8776b065e48bc5 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Wed, 3 Apr 2024 19:24:51 +0100 Subject: [PATCH] fix: respect the IPNS TTL field (#482) Uses the `@libp2p/record` `timeReceived` property and the `.ttl` field of the IPNS Record to calculate the TTL of the record separately from the record EOL. This was going to be a push to #473 but it was merged prematurely. --------- Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> --- packages/ipns/package.json | 2 +- packages/ipns/src/index.ts | 45 ++++++++-- packages/ipns/src/routing/local-store.ts | 39 ++++++++- packages/ipns/src/routing/pubsub.ts | 6 +- packages/ipns/test/publish.spec.ts | 4 +- packages/ipns/test/resolve.spec.ts | 102 ++++++++++++++++++++++- 6 files changed, 180 insertions(+), 18 deletions(-) diff --git a/packages/ipns/package.json b/packages/ipns/package.json index bd34d221f..77e9a7d5d 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -171,7 +171,7 @@ "@libp2p/peer-id": "^4.0.7", "@multiformats/dns": "^1.0.1", "interface-datastore": "^8.2.11", - "ipns": "^9.0.0", + "ipns": "^9.1.0", "multiformats": "^13.1.0", "progress-events": "^1.0.0", "uint8arrays": "^5.0.2" diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index 6106434fc..1d9533b8f 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -254,6 +254,8 @@ const HOUR = 60 * MINUTE const DEFAULT_LIFETIME_MS = 24 * HOUR const DEFAULT_REPUBLISH_INTERVAL_MS = 23 * HOUR +const DEFAULT_TTL_NS = BigInt(HOUR) * 1_000_000n + export type PublishProgressEvents = ProgressEvent<'ipns:publish:start'> | ProgressEvent<'ipns:publish:success', IPNSRecord> | @@ -425,8 +427,8 @@ class DefaultIPNS implements IPNS { if (await this.localStore.has(routingKey, options)) { // if we have published under this key before, increment the sequence number - const buf = await this.localStore.get(routingKey, options) - const existingRecord = unmarshal(buf) + const { record } = await this.localStore.get(routingKey, options) + const existingRecord = unmarshal(record) sequenceNumber = existingRecord.sequence + 1n } @@ -539,17 +541,46 @@ class DefaultIPNS implements IPNS { log('record is present in the cache') if (options.nocache !== true) { - // check the local cache first - const record = await this.localStore.get(routingKey, options) - try { + // check the local cache first + const { record, created } = await this.localStore.get(routingKey, options) + + this.log('record retrieved from cache') + + // validate the record await ipnsValidator(routingKey, record) - this.log('record successfully retrieved from cache') + this.log('record was valid') + + // check the TTL + const ipnsRecord = unmarshal(record) - return unmarshal(record) + // IPNS TTL is in nanoseconds, convert to milliseconds, default to one + // hour + const ttlMs = Number((ipnsRecord.ttl ?? DEFAULT_TTL_NS) / 1_000_000n) + const ttlExpires = created.getTime() + ttlMs + + if (ttlExpires > Date.now()) { + // the TTL has not yet expired, return the cached record + this.log('record TTL was valid') + return ipnsRecord + } + + if (options.offline === true) { + // the TTL has expired but we are skipping the routing search + this.log('record TTL has been reached but we are resolving offline-only, returning record') + return ipnsRecord + } + + this.log('record TTL has been reached, searching routing for updates') + + // add the local record to our list of resolved record, and also + // search the routing for updates - the most up to date record will be + // returned + records.push(record) } catch (err) { this.log('cached record was invalid', err) + await this.localStore.delete(routingKey, options) } } else { log('ignoring local cache due to nocache=true option') diff --git a/packages/ipns/src/routing/local-store.ts b/packages/ipns/src/routing/local-store.ts index bb142e185..90dc2bbe4 100644 --- a/packages/ipns/src/routing/local-store.ts +++ b/packages/ipns/src/routing/local-store.ts @@ -1,8 +1,9 @@ import { Record } from '@libp2p/kad-dht' import { type Datastore, Key } from 'interface-datastore' import { CustomProgressEvent, type ProgressEvent } from 'progress-events' +import { equals as uint8ArrayEquals } from 'uint8arrays/equals' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import type { GetOptions, IPNSRouting, PutOptions } from '../routing' +import type { GetOptions, PutOptions } from '../routing' import type { AbortOptions } from '@libp2p/interface' function dhtRoutingKey (key: Uint8Array): Key { @@ -14,8 +15,16 @@ export type DatastoreProgressEvents = ProgressEvent<'ipns:routing:datastore:get'> | ProgressEvent<'ipns:routing:datastore:error', Error> -export interface LocalStore extends IPNSRouting { +export interface GetResult { + record: Uint8Array + created: Date +} + +export interface LocalStore { + put(routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: PutOptions): Promise + get(routingKey: Uint8Array, options?: GetOptions): Promise has(routingKey: Uint8Array, options?: AbortOptions): Promise + delete(routingKey: Uint8Array, options?: AbortOptions): Promise } /** @@ -29,6 +38,21 @@ export function localStore (datastore: Datastore): LocalStore { try { const key = dhtRoutingKey(routingKey) + // don't overwrite existing, identical records as this will affect the + // TTL + try { + const existingBuf = await datastore.get(key) + const existingRecord = Record.deserialize(existingBuf) + + if (uint8ArrayEquals(existingRecord.value, marshalledRecord)) { + return + } + } catch (err: any) { + if (err.code !== 'ERR_NOT_FOUND') { + throw err + } + } + // Marshal to libp2p record as the DHT does const record = new Record(routingKey, marshalledRecord, new Date()) @@ -39,7 +63,7 @@ export function localStore (datastore: Datastore): LocalStore { throw err } }, - async get (routingKey: Uint8Array, options: GetOptions = {}): Promise { + async get (routingKey: Uint8Array, options: GetOptions = {}): Promise { try { const key = dhtRoutingKey(routingKey) @@ -49,7 +73,10 @@ export function localStore (datastore: Datastore): LocalStore { // Unmarshal libp2p record as the DHT does const record = Record.deserialize(buf) - return record.value + return { + record: record.value, + created: record.timeReceived + } } catch (err: any) { options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:error', err)) throw err @@ -58,6 +85,10 @@ export function localStore (datastore: Datastore): LocalStore { async has (routingKey: Uint8Array, options: AbortOptions = {}): Promise { const key = dhtRoutingKey(routingKey) return datastore.has(key, options) + }, + async delete (routingKey, options): Promise { + const key = dhtRoutingKey(routingKey) + return datastore.delete(key, options) } } } diff --git a/packages/ipns/src/routing/pubsub.ts b/packages/ipns/src/routing/pubsub.ts index 43770696c..7679cb5bf 100644 --- a/packages/ipns/src/routing/pubsub.ts +++ b/packages/ipns/src/routing/pubsub.ts @@ -72,7 +72,7 @@ class PubSubRouting implements IPNSRouting { await ipnsValidator(routingKey, message.data) if (await this.localStore.has(routingKey)) { - const currentRecord = await this.localStore.get(routingKey) + const { record: currentRecord } = await this.localStore.get(routingKey) if (uint8ArrayEquals(currentRecord, message.data)) { log('not storing record as we already have it') @@ -128,7 +128,9 @@ class PubSubRouting implements IPNSRouting { } // chain through to local store - return await this.localStore.get(routingKey, options) + const { record } = await this.localStore.get(routingKey, options) + + return record } catch (err: any) { options.onProgress?.(new CustomProgressEvent('ipns:pubsub:error', err)) throw err diff --git a/packages/ipns/test/publish.spec.ts b/packages/ipns/test/publish.spec.ts index b50ea7e48..4043ec25e 100644 --- a/packages/ipns/test/publish.spec.ts +++ b/packages/ipns/test/publish.spec.ts @@ -44,7 +44,7 @@ describe('publish', () => { const ipnsEntry = await name.publish(key, cid) expect(ipnsEntry).to.have.property('sequence', 1n) - expect(ipnsEntry).to.have.property('ttl', 8640000000000n) // 24 hours + expect(ipnsEntry).to.have.property('ttl', 3_600_000_000_000n) // 1 hour }) it('should publish an IPNS record with a custom ttl params', async function () { @@ -55,7 +55,7 @@ describe('publish', () => { }) expect(ipnsEntry).to.have.property('sequence', 1n) - expect(ipnsEntry).to.have.property('ttl', BigInt(lifetime) * 100000n) + expect(ipnsEntry).to.have.property('ttl', 3_600_000_000_000n) expect(heliaRouting.put.called).to.be.true() expect(customRouting.put.called).to.be.true() diff --git a/packages/ipns/test/resolve.spec.ts b/packages/ipns/test/resolve.spec.ts index 4e478f8fc..7b37f8978 100644 --- a/packages/ipns/test/resolve.spec.ts +++ b/packages/ipns/test/resolve.spec.ts @@ -6,7 +6,7 @@ import { createEd25519PeerId } from '@libp2p/peer-id-factory' import { expect } from 'aegir/chai' import { MemoryDatastore } from 'datastore-core' import { type Datastore, Key } from 'interface-datastore' -import { create, marshal, peerIdToRoutingKey, unmarshal } from 'ipns' +import { create, createWithExpiration, marshal, peerIdToRoutingKey, unmarshal } from 'ipns' import drain from 'it-drain' import { CID } from 'multiformats/cid' import Sinon from 'sinon' @@ -78,6 +78,7 @@ describe('resolve', () => { }) it('should skip the local cache when resolving a record', async () => { + const cachePutSpy = Sinon.spy(datastore, 'put') const cacheGetSpy = Sinon.spy(datastore, 'get') const key = await createEd25519PeerId() @@ -92,7 +93,9 @@ describe('resolve', () => { expect(heliaRouting.get.called).to.be.true() expect(customRouting.get.called).to.be.true() - expect(cacheGetSpy.called).to.be.false() + + // we call `.get` during `.put` + cachePutSpy.calledBefore(cacheGetSpy) }) it('should retrieve from local cache when resolving a record', async () => { @@ -198,4 +201,99 @@ describe('resolve', () => { expect(result).to.have.deep.property('record', record) }) + + it('should not search the routing for updated IPNS records when a locally cached copy is within the TTL', async () => { + const key = await createEd25519PeerId() + const customRoutingKey = peerIdToRoutingKey(key) + const dhtKey = new Key('/dht/record/' + uint8ArrayToString(customRoutingKey, 'base32'), false) + + // create a record with a valid lifetime and a non-expired TTL + const ipnsRecord = await create(key, cid, 1, Math.pow(2, 10), { + ttlNs: 10_000_000 + }) + const dhtRecord = new Record(customRoutingKey, marshal(ipnsRecord), new Date(Date.now())) + + await datastore.put(dhtKey, dhtRecord.serialize()) + + const result = await name.resolve(key) + expect(result).to.have.deep.property('record', unmarshal(marshal(ipnsRecord))) + + // should not have searched the routing + expect(customRouting.get.called).to.be.false() + }) + + it('should search the routing for updated IPNS records when a locally cached copy has passed the TTL', async () => { + const key = await createEd25519PeerId() + + const customRoutingKey = peerIdToRoutingKey(key) + const dhtKey = new Key('/dht/record/' + uint8ArrayToString(customRoutingKey, 'base32'), false) + + // create a record with a valid lifetime but an expired ttl + const ipnsRecord = await create(key, cid, 1, Math.pow(2, 10), { + ttlNs: 10 + }) + const dhtRecord = new Record(customRoutingKey, marshal(ipnsRecord), new Date(Date.now() - 1000)) + + await datastore.put(dhtKey, dhtRecord.serialize()) + + const result = await name.resolve(key) + expect(result).to.have.deep.property('record', unmarshal(marshal(ipnsRecord))) + + // should have searched the routing + expect(customRouting.get.called).to.be.true() + }) + + it('should search the routing for updated IPNS records when a locally cached copy has passed the TTL and choose the record with a higher sequence number', async () => { + const key = await createEd25519PeerId() + + const customRoutingKey = peerIdToRoutingKey(key) + const dhtKey = new Key('/dht/record/' + uint8ArrayToString(customRoutingKey, 'base32'), false) + + // create a record with a valid lifetime but an expired ttl + const ipnsRecord = await create(key, cid, 10, Math.pow(2, 10), { + ttlNs: 10 + }) + const dhtRecord = new Record(customRoutingKey, marshal(ipnsRecord), new Date(Date.now() - 1000)) + + await datastore.put(dhtKey, dhtRecord.serialize()) + + // the routing returns a valid record with an higher sequence number + const ipnsRecordFromRouting = await create(key, cid, 11, Math.pow(2, 10), { + ttlNs: 10_000_000 + }) + customRouting.get.withArgs(customRoutingKey).resolves(marshal(ipnsRecordFromRouting)) + + const result = await name.resolve(key) + expect(result).to.have.deep.property('record', unmarshal(marshal(ipnsRecordFromRouting))) + + // should have searched the routing + expect(customRouting.get.called).to.be.true() + }) + + it('should search the routing when a locally cached copy has an expired lifetime', async () => { + const key = await createEd25519PeerId() + + const customRoutingKey = peerIdToRoutingKey(key) + const dhtKey = new Key('/dht/record/' + uint8ArrayToString(customRoutingKey, 'base32'), false) + + // create a record with an expired lifetime but valid TTL + const ipnsRecord = await createWithExpiration(key, cid, 10, new Date(Date.now() - Math.pow(2, 10)).toString(), { + ttlNs: 10_000_000 + }) + const dhtRecord = new Record(customRoutingKey, marshal(ipnsRecord), new Date(Date.now())) + + await datastore.put(dhtKey, dhtRecord.serialize()) + + // the routing returns a valid record with an higher sequence number + const ipnsRecordFromRouting = await create(key, cid, 11, Math.pow(2, 10), { + ttlNs: 10_000_000 + }) + customRouting.get.withArgs(customRoutingKey).resolves(marshal(ipnsRecordFromRouting)) + + const result = await name.resolve(key) + expect(result).to.have.deep.property('record', unmarshal(marshal(ipnsRecordFromRouting))) + + // should have searched the routing + expect(customRouting.get.called).to.be.true() + }) })