diff --git a/.changeset/kind-deers-burn.md b/.changeset/kind-deers-burn.md new file mode 100644 index 000000000..f70eb6549 --- /dev/null +++ b/.changeset/kind-deers-burn.md @@ -0,0 +1,9 @@ +--- +"@web5/user-agent": patch +"@web5/agent": patch +"@web5/dids": patch +"@web5/identity-agent": patch +"@web5/proxy-agent": patch +--- + +Implement DidResolverCache thats specific to Agent usage diff --git a/.github/workflows/alpha-npm.yml b/.github/workflows/alpha-npm.yml index 1235f307c..a722ea5fd 100644 --- a/.github/workflows/alpha-npm.yml +++ b/.github/workflows/alpha-npm.yml @@ -21,7 +21,8 @@ jobs: env: # Packages not listed here will be excluded from publishing - PACKAGES: "agent api common credentials crypto crypto-aws-kms dids identity-agent proxy-agent user-agent" + # These are currently in a specific order due to dependency requirements + PACKAGES: "crypto crypto-aws-kms common dids credentials agent identity-agent proxy-agent user-agent api" steps: - name: Checkout source @@ -29,7 +30,7 @@ jobs: # https://cashapp.github.io/hermit/usage/ci/ - name: Init Hermit - uses: cashapp/activate-hermit@31ce88b17a84941bb1b782f1b7b317856addf286 #v1.1.0 + uses: cashapp/activate-hermit@v1 with: cache: "true" @@ -63,11 +64,17 @@ jobs: node ./scripts/bump-workspace.mjs --prerelease=$ALPHA_PRERELEASE shell: bash - - name: Build all workspace packages + - name: Build all workspace packages sequentially env: NODE_AUTH_TOKEN: ${{secrets.npm_token}} - run: pnpm --recursive --stream build - + run: | + for package in $PACKAGES; do + cd packages/$package + pnpm build + cd ../.. + done + shell: bash + - name: Publish selected @web5/* packages env: NODE_AUTH_TOKEN: ${{secrets.npm_token}} diff --git a/audit-ci.json b/audit-ci.json index 0311fe9e4..12d9d2333 100644 --- a/audit-ci.json +++ b/audit-ci.json @@ -6,6 +6,7 @@ "mysql2", "braces", "GHSA-rv95-896h-c2vc", - "GHSA-952p-6rrq-rcjv" + "GHSA-952p-6rrq-rcjv", + "GHSA-4vvj-4cpr-p986" ] } \ No newline at end of file diff --git a/package.json b/package.json index 12c283353..db1067aaa 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,8 @@ "elliptic@>=4.0.0 <=6.5.6": ">=6.5.7", "elliptic@>=2.0.0 <=6.5.6": ">=6.5.7", "elliptic@>=5.2.1 <=6.5.6": ">=6.5.7", - "micromatch@<4.0.8": ">=4.0.8" + "micromatch@<4.0.8": ">=4.0.8", + "webpack@<5.94.0": ">=5.94.0" } } } diff --git a/packages/agent/package.json b/packages/agent/package.json index eb67702d6..3c04ac5c2 100644 --- a/packages/agent/package.json +++ b/packages/agent/package.json @@ -74,7 +74,7 @@ "@tbd54566975/dwn-sdk-js": "0.4.6", "@web5/common": "1.0.0", "@web5/crypto": "workspace:*", - "@web5/dids": "1.1.0", + "@web5/dids": "workspace:*", "abstract-level": "1.0.4", "ed25519-keygen": "0.4.11", "isomorphic-ws": "^5.0.0", diff --git a/packages/agent/src/agent-did-resolver-cache.ts b/packages/agent/src/agent-did-resolver-cache.ts new file mode 100644 index 000000000..ecf35b76b --- /dev/null +++ b/packages/agent/src/agent-did-resolver-cache.ts @@ -0,0 +1,72 @@ +import { DidResolutionResult, DidResolverCache, DidResolverCacheLevel, DidResolverCacheLevelParams } from '@web5/dids'; +import { Web5PlatformAgent } from './types/agent.js'; + + +/** + * AgentDidResolverCache keeps a stale copy of the Agent's managed Identity DIDs and only evicts and refreshes upon a successful resolution. + * This allows for quick and offline access to the internal DIDs used by the agent. + */ +export class AgentDidResolverCache extends DidResolverCacheLevel implements DidResolverCache { + + /** + * Holds the instance of a `Web5PlatformAgent` that represents the current execution context for + * the `AgentDidApi`. This agent is used to interact with other Web5 agent components. It's vital + * to ensure this instance is set to correctly contextualize operations within the broader Web5 + * Agent framework. + */ + private _agent?: Web5PlatformAgent; + + /** A map of DIDs that are currently in-flight. This helps avoid going into an infinite loop */ + private _resolving: Map = new Map(); + + constructor({ agent, db, location, ttl }: DidResolverCacheLevelParams & { agent?: Web5PlatformAgent }) { + super ({ db, location, ttl }); + this._agent = agent; + } + + get agent() { + if (!this._agent) { + throw new Error('Agent not initialized'); + } + return this._agent; + } + + set agent(agent: Web5PlatformAgent) { + this._agent = agent; + } + + /** + * Get the DID resolution result from the cache for the given DID. + * + * If the DID is managed by the agent, or is the agent's own DID, it will not evict it from the cache until a new resolution is successful. + * This is done to achieve quick and offline access to the agent's own managed DIDs. + */ + async get(did: string): Promise { + try { + const str = await this.cache.get(did); + const cachedResult = JSON.parse(str); + if (!this._resolving.has(did) && Date.now() >= cachedResult.ttlMillis) { + this._resolving.set(did, true); + if (this.agent.agentDid.uri === did || 'undefined' !== typeof await this.agent.identity.get({ didUri: did })) { + try { + const result = await this.agent.did.resolve(did); + if (!result.didResolutionMetadata.error) { + this.set(did, result); + } + } finally { + this._resolving.delete(did); + } + } else { + this._resolving.delete(did); + this.cache.nextTick(() => this.cache.del(did)); + } + } + return cachedResult.value; + } catch(error: any) { + if (error.notFound) { + return; + } + throw error; + } + } +} \ No newline at end of file diff --git a/packages/agent/src/did-api.ts b/packages/agent/src/did-api.ts index 8b533b635..4a17d57af 100644 --- a/packages/agent/src/did-api.ts +++ b/packages/agent/src/did-api.ts @@ -3,12 +3,12 @@ import type { DidMetadata, PortableDid, DidMethodApi, - DidResolverCache, DidDhtCreateOptions, DidJwkCreateOptions, DidResolutionResult, DidResolutionOptions, DidVerificationMethod, + DidResolverCache, } from '@web5/dids'; import { BearerDid, Did, UniversalResolver } from '@web5/dids'; @@ -18,7 +18,7 @@ import type { AgentKeyManager } from './types/key-manager.js'; import type { ResponseStatus, Web5PlatformAgent } from './types/agent.js'; import { InMemoryDidStore } from './store-did.js'; -import { DidResolverCacheMemory } from './prototyping/dids/resolver-cache-memory.js'; +import { AgentDidResolverCache } from './agent-did-resolver-cache.js'; export enum DidInterface { Create = 'Create', @@ -87,8 +87,10 @@ export interface DidApiParams { * An optional `DidResolverCache` instance used for caching resolved DID documents. * * Providing a cache implementation can significantly enhance resolution performance by avoiding - * redundant resolutions for previously resolved DIDs. If omitted, a no-operation cache is used, - * which effectively disables caching. + * redundant resolutions for previously resolved DIDs. If omitted, the default is an instance of `AgentDidResolverCache`. + * + * `AgentDidResolverCache` keeps a stale copy of the Agent's managed Identity DIDs and only refreshes upon a successful resolution. + * This allows for quick and offline access to the internal DIDs used by the agent. */ resolverCache?: DidResolverCache; @@ -120,10 +122,10 @@ export class AgentDidApi } // Initialize the DID resolver with the given DID methods and resolver cache, or use a default - // in-memory cache if none is provided. + // AgentDidResolverCache if none is provided. super({ didResolvers : didMethods, - cache : resolverCache ?? new DidResolverCacheMemory() + cache : resolverCache ?? new AgentDidResolverCache({ agent, location: 'DATA/AGENT/DID_CACHE' }) }); this._agent = agent; @@ -152,6 +154,11 @@ export class AgentDidApi set agent(agent: Web5PlatformAgent) { this._agent = agent; + + // AgentDidResolverCache should set the agent if it is the type of cache being used + if ('agent' in this.cache) { + this.cache.agent = agent; + } } public async create({ diff --git a/packages/agent/src/identity-api.ts b/packages/agent/src/identity-api.ts index c64ef46ef..b857a67e6 100644 --- a/packages/agent/src/identity-api.ts +++ b/packages/agent/src/identity-api.ts @@ -183,14 +183,13 @@ export class AgentIdentityApi { + return this.get({ didUri: metadata.uri, tenant: metadata.tenant }); + }) + ); - for (const metadata of storedIdentities) { - const identity = await this.get({ didUri: metadata.uri, tenant: metadata.tenant }); - identities.push(identity!); - } - - return identities; + return identities.filter(identity => typeof identity !== 'undefined') as BearerIdentity[]; } public async manage({ portableIdentity }: { diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 5ed8caa62..7f7457575 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -7,6 +7,7 @@ export type * from './types/permissions.js'; export type * from './types/sync.js'; export type * from './types/vc.js'; +export * from './agent-did-resolver-cache.js'; export * from './bearer-identity.js'; export * from './cached-permissions.js'; export * from './crypto-api.js'; diff --git a/packages/agent/src/store-data.ts b/packages/agent/src/store-data.ts index 6873eaa0e..7d5699022 100644 --- a/packages/agent/src/store-data.ts +++ b/packages/agent/src/store-data.ts @@ -55,16 +55,20 @@ export class DwnDataStore = Jwk> implem /** * Index for mappings from Store Identifier to DWN record ID. + * Since these values don't change, we can use a long TTL. * - * Up to 1,000 entries are retained for 2 hours. + * Up to 1,000 entries are retained for 21 days. + * NOTE: The maximum number for the ttl is 2^31 - 1 milliseconds (24.8 days), setting to 21 days to be safe. */ - protected _index = new TtlCache({ ttl: ms('2 hours'), max: 1000 }); + protected _index = new TtlCache({ ttl: ms('21 days'), max: 1000 }); /** * Cache of tenant DIDs that have been initialized with the protocol. * This is used to avoid redundant protocol initialization requests. + * + * Since these are default protocols and unlikely to change, we can use a long TTL. */ - protected _protocolInitializedCache: TtlCache = new TtlCache({ ttl: ms('1 hour'), max: 1000 }); + protected _protocolInitializedCache: TtlCache = new TtlCache({ ttl: ms('21 days'), max: 1000 }); /** * The protocol assigned to this storage instance. diff --git a/packages/agent/src/test-harness.ts b/packages/agent/src/test-harness.ts index e17dc8df0..ec2e89d4d 100644 --- a/packages/agent/src/test-harness.ts +++ b/packages/agent/src/test-harness.ts @@ -4,11 +4,12 @@ import type { AbstractLevel } from 'abstract-level'; import { Level } from 'level'; import { LevelStore, MemoryStore } from '@web5/common'; import { DataStoreLevel, Dwn, EventEmitterStream, EventLogLevel, MessageStoreLevel, ResumableTaskStoreLevel } from '@tbd54566975/dwn-sdk-js'; -import { DidDht, DidJwk, DidResolutionResult, DidResolverCache, DidResolverCacheLevel } from '@web5/dids'; +import { DidDht, DidJwk, DidResolutionResult, DidResolverCache } from '@web5/dids'; import type { Web5PlatformAgent } from './types/agent.js'; import { AgentDidApi } from './did-api.js'; +import { AgentDidResolverCache } from './agent-did-resolver-cache.js'; import { AgentDwnApi } from './dwn-api.js'; import { AgentSyncApi } from './sync-api.js'; import { Web5RpcClient } from './rpc-client.js'; @@ -287,7 +288,7 @@ export class PlatformAgentTestHarness { const { didStore, identityStore, keyStore } = stores; // Setup DID Resolver Cache - const didResolverCache = new DidResolverCacheLevel({ + const didResolverCache = new AgentDidResolverCache({ location: testDataPath('DID_RESOLVERCACHE') }); diff --git a/packages/agent/tests/agent-did-resolver-cach.spec.ts b/packages/agent/tests/agent-did-resolver-cach.spec.ts new file mode 100644 index 000000000..37b7536b0 --- /dev/null +++ b/packages/agent/tests/agent-did-resolver-cach.spec.ts @@ -0,0 +1,142 @@ +import { AgentDidResolverCache } from '../src/agent-did-resolver-cache.js'; +import { PlatformAgentTestHarness } from '../src/test-harness.js'; +import { TestAgent } from './utils/test-agent.js'; + +import sinon from 'sinon'; +import { expect } from 'chai'; +import { DidJwk } from '@web5/dids'; +import { BearerIdentity } from '../src/bearer-identity.js'; + +describe('AgentDidResolverCache', () => { + let resolverCache: AgentDidResolverCache; + let testHarness: PlatformAgentTestHarness; + + before(async () => { + testHarness = await PlatformAgentTestHarness.setup({ + agentClass : TestAgent, + agentStores : 'dwn' + }); + + resolverCache = new AgentDidResolverCache({ agent: testHarness.agent, location: '__TESTDATA__/did_cache' }); + }); + + after(async () => { + sinon.restore(); + await testHarness.clearStorage(); + await testHarness.closeStorage(); + }); + + beforeEach(async () => { + sinon.restore(); + await testHarness.clearStorage(); + await testHarness.createAgentDid(); + }); + + it('does not attempt to resolve a DID that is already resolving', async () => { + const did = testHarness.agent.agentDid.uri; + const getStub = sinon.stub(resolverCache['cache'], 'get').resolves(JSON.stringify({ ttlMillis: Date.now() - 1000, value: { didDocument: { id: did } } })); + const resolveSpy = sinon.spy(testHarness.agent.did, 'resolve'); + + await Promise.all([ + resolverCache.get(did), + resolverCache.get(did) + ]); + + // get should be called twice, but resolve should only be called once + // because the second call should be blocked by the _resolving Map + expect(getStub.callCount).to.equal(2); + expect(resolveSpy.callCount).to.equal(1); + }); + + it('should not resolve a DID if the ttl has not elapsed', async () => { + const did = testHarness.agent.agentDid.uri; + const getStub = sinon.stub(resolverCache['cache'], 'get').resolves(JSON.stringify({ ttlMillis: Date.now() + 1000, value: { didDocument: { id: did } } })); + const resolveSpy = sinon.spy(testHarness.agent.did, 'resolve'); + + await resolverCache.get(did); + + // get should be called once, but resolve should not be called + expect(getStub.callCount).to.equal(1); + expect(resolveSpy.callCount).to.equal(0); + }); + + it('should not call resolve if the DID is not the agent DID or exists as an identity in the agent', async () => { + const did = await DidJwk.create({}); + const getStub = sinon.stub(resolverCache['cache'], 'get').resolves(JSON.stringify({ ttlMillis: Date.now() - 1000, value: { didDocument: { id: did.uri } } })); + const resolveSpy = sinon.spy(testHarness.agent.did, 'resolve'); + const nextTickSpy = sinon.stub(resolverCache['cache'], 'nextTick').resolves(); + sinon.stub(testHarness.agent.identity, 'get').resolves(undefined); + + await resolverCache.get(did.uri), + + // get should be called once, but we do not resolve even though the TTL is expired + expect(getStub.callCount).to.equal(1); + expect(resolveSpy.callCount).to.equal(0); + + // we expect the nextTick of the cache to be called to trigger a delete of the cache item after returning as it's expired + expect(nextTickSpy.callCount).to.equal(1); + }); + + it('should resolve if the DID is managed by the agent', async () => { + const did = await DidJwk.create({}); + const getStub = sinon.stub(resolverCache['cache'], 'get').resolves(JSON.stringify({ ttlMillis: Date.now() - 1000, value: { didDocument: { id: did.uri } } })); + const resolveSpy = sinon.spy(testHarness.agent.did, 'resolve'); + const nextTickSpy = sinon.stub(resolverCache['cache'], 'nextTick').resolves(); + sinon.stub(testHarness.agent.identity, 'get').resolves(new BearerIdentity({ + metadata: { name: 'Some Name', uri: did.uri, tenant: did.uri }, + did, + })); + + await resolverCache.get(did.uri), + + // get should be called once, and we also resolve the DId as it's returned by the identity.get method + expect(getStub.callCount).to.equal(1); + expect(resolveSpy.callCount).to.equal(1); + }); + + it('does not cache notFound records', async () => { + const did = testHarness.agent.agentDid.uri; + const getStub = sinon.stub(resolverCache['cache'], 'get').rejects({ notFound: true }); + + const result = await resolverCache.get(did); + + // get should be called once, and resolve should be called once + expect(getStub.callCount).to.equal(1); + expect(result).to.equal(undefined); + }); + + it('throws if the error is anything other than a notFound error', async () => { + const did = testHarness.agent.agentDid.uri; + const getStub = sinon.stub(resolverCache['cache'], 'get').rejects(new Error('Some Error')); + + try { + await resolverCache.get(did); + expect.fail('Should have thrown'); + } catch (error: any) { + expect(error.message).to.equal('Some Error'); + } + }); + + it('throws if the agent is not initialized', async () => { + // close existing DB + await resolverCache['cache'].close(); + + // set resolver cache without an agent + resolverCache = new AgentDidResolverCache({ location: '__TESTDATA__/did_cache' }); + + try { + // attempt to access the agent property + resolverCache.agent; + + expect.fail('Should have thrown'); + } catch (error: any) { + expect(error.message).to.equal('Agent not initialized'); + } + + // set the agent property + resolverCache.agent = testHarness.agent; + + // should not throw + resolverCache.agent; + }); +}); \ No newline at end of file diff --git a/packages/dids/src/resolver/resolver-cache-level.ts b/packages/dids/src/resolver/resolver-cache-level.ts index 69481e143..c4d2f2f80 100644 --- a/packages/dids/src/resolver/resolver-cache-level.ts +++ b/packages/dids/src/resolver/resolver-cache-level.ts @@ -73,10 +73,10 @@ type CachedDidResolutionResult = { */ export class DidResolverCacheLevel implements DidResolverCache { /** The underlying LevelDB store used for caching. */ - private cache; + protected cache; /** The time-to-live for cache entries in milliseconds. */ - private ttl: number; + protected ttl: number; constructor({ db, diff --git a/packages/dids/src/resolver/universal-resolver.ts b/packages/dids/src/resolver/universal-resolver.ts index 93ff11fb8..e938613ff 100644 --- a/packages/dids/src/resolver/universal-resolver.ts +++ b/packages/dids/src/resolver/universal-resolver.ts @@ -66,7 +66,7 @@ export class UniversalResolver implements DidResolver, DidUrlDereferencer { /** * A cache for storing resolved DID documents. */ - private cache: DidResolverCache; + protected cache: DidResolverCache; /** * A map to store method resolvers against method names. diff --git a/packages/user-agent/src/user-agent.ts b/packages/user-agent/src/user-agent.ts index 426a0668d..9568959fb 100644 --- a/packages/user-agent/src/user-agent.ts +++ b/packages/user-agent/src/user-agent.ts @@ -12,6 +12,7 @@ import { ProcessDwnRequest, Web5PlatformAgent, AgentPermissionsApi, + AgentDidResolverCache, } from '@web5/agent'; import { LevelStore } from '@web5/common'; @@ -152,7 +153,7 @@ export class Web5UserAgent=2.0.0 <=6.5.6: '>=6.5.7' elliptic@>=5.2.1 <=6.5.6: '>=6.5.7' micromatch@<4.0.8: '>=4.0.8' + webpack@<5.94.0: '>=5.94.0' importers: @@ -65,8 +66,8 @@ importers: specifier: workspace:* version: link:../crypto '@web5/dids': - specifier: 1.1.0 - version: 1.1.0 + specifier: workspace:* + version: link:../dids abstract-level: specifier: 1.0.4 version: 1.0.4 @@ -5323,13 +5324,13 @@ packages: resolution: {integrity: sha512-oYwAqCuL0OZhBoSgmdrLa7mv9MjommVMiQIWgcztf+eS4+8BfcUee6nenFnDhKOhzAVnk5gpZdfnz1iiBv+5sg==} engines: {node: '>= 14.15.0'} peerDependencies: - webpack: ^5.72.1 + webpack: '>=5.94.0' source-map-loader@5.0.0: resolution: {integrity: sha512-k2Dur7CbSLcAH73sBcIkV5xjPV4SzqO1NJ7+XaQl8if3VODDUj3FNchNGpqgJSKbvUfJuhVdv8K2Eu8/TNl2eA==} engines: {node: '>= 18.12.0'} peerDependencies: - webpack: ^5.72.1 + webpack: '>=5.94.0' source-map-support@0.5.21: resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} @@ -5507,7 +5508,7 @@ packages: '@swc/core': '*' esbuild: '*' uglify-js: '*' - webpack: ^5.1.0 + webpack: '>=5.94.0' peerDependenciesMeta: '@swc/core': optional: true