From 91319bdaf91b703c8b1947df991b14dfcb3dff0e Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Mon, 22 Jan 2024 11:10:03 -0600 Subject: [PATCH 01/35] Adding changes to src --- packages/agent/src/dwn-manager.ts | 47 ++++++-- packages/agent/src/types/agent.ts | 4 +- packages/api/src/dwn-api.ts | 1 + packages/api/src/record.ts | 189 ++++++++++++++++++++++++++++-- 4 files changed, 217 insertions(+), 24 deletions(-) diff --git a/packages/agent/src/dwn-manager.ts b/packages/agent/src/dwn-manager.ts index 44a6be195..201caf54c 100644 --- a/packages/agent/src/dwn-manager.ts +++ b/packages/agent/src/dwn-manager.ts @@ -63,17 +63,26 @@ type DwnMessage = { data?: Blob; } -const dwnMessageCreators = { +const recordsWriteType = DwnInterfaceName.Records + DwnMethodName.Write; +const dwnMessageConstructors = { [DwnInterfaceName.Events + DwnMethodName.Get] : EventsGet, [DwnInterfaceName.Messages + DwnMethodName.Get] : MessagesGet, [DwnInterfaceName.Records + DwnMethodName.Read] : RecordsRead, [DwnInterfaceName.Records + DwnMethodName.Query] : RecordsQuery, - [DwnInterfaceName.Records + DwnMethodName.Write] : RecordsWrite, + [recordsWriteType] : RecordsWrite, [DwnInterfaceName.Records + DwnMethodName.Delete] : RecordsDelete, [DwnInterfaceName.Protocols + DwnMethodName.Query] : ProtocolsQuery, [DwnInterfaceName.Protocols + DwnMethodName.Configure] : ProtocolsConfigure, }; +(RecordsWrite.prototype as any).forceSet = function(prop: string, value: unknown){ + this[prop] = value; +}; + +(RecordsWrite.prototype as any).forceGet = function(prop: string){ + return this[prop]; +}; + export type DwnManagerOptions = { agent?: Web5ManagedAgent; dwn: Dwn; @@ -187,7 +196,9 @@ export class DwnManager { messageData = data; } else { + console.log('start constructDwnMessage'); const { message } = await this.constructDwnMessage({ request }); + console.log('passed constructDwnMessage'); dwnRpcRequest.message = message; messageData = request.dataStream; } @@ -242,9 +253,12 @@ export class DwnManager { } private async constructDwnMessage(options: { - request: ProcessDwnRequest + request : ProcessDwnRequest }) { const { request } = options; + const rawMessage = (request as any).rawMessage; + + console.log('259', rawMessage); let readableStream: Readable | undefined; @@ -252,7 +266,7 @@ export class DwnManager { if (request.messageType === 'RecordsWrite') { const messageOptions = request.messageOptions as RecordsWriteOptions; - if (request.dataStream && !messageOptions.data) { + if (request.dataStream && !messageOptions?.data) { const { dataStream } = request; let isomorphicNodeReadable: Readable; @@ -266,22 +280,29 @@ export class DwnManager { readableStream = webReadableToIsomorphicNodeReadable(forProcessMessage); } - // @ts-ignore - messageOptions.dataCid = await Cid.computeDagPbCidFromStream(isomorphicNodeReadable); - // @ts-ignore - messageOptions.dataSize ??= isomorphicNodeReadable['bytesRead']; + if (!rawMessage) { + // @ts-ignore + messageOptions.dataCid = await Cid.computeDagPbCidFromStream(isomorphicNodeReadable); + // @ts-ignore + messageOptions.dataSize ??= isomorphicNodeReadable['bytesRead']; + } } } const dwnSigner = await this.constructDwnSigner(request.author); - - const messageCreator = dwnMessageCreators[request.messageType]; - const dwnMessage = await messageCreator.create({ + const dwnMessageConstructor = dwnMessageConstructors[request.messageType]; + const dwnMessage = rawMessage ? await dwnMessageConstructor.parse(rawMessage) : await dwnMessageConstructor.create({ ...request.messageOptions, signer: dwnSigner }); - return { message: dwnMessage.message, dataStream: readableStream }; + if (dwnMessageConstructor === RecordsWrite){ + if (request.import) { + await (dwnMessage as RecordsWrite).signAsOwner(dwnSigner); + } + } + + return { message: (dwnMessage as any).message, dataStream: readableStream }; } private async getAuthorSigningKeyId(options: { @@ -411,7 +432,7 @@ export class DwnManager { const dwnSigner = await this.constructDwnSigner(author); - const messageCreator = dwnMessageCreators[messageType]; + const messageCreator = dwnMessageConstructors[messageType]; const dwnMessage = await messageCreator.create({ ...messageOptions, diff --git a/packages/agent/src/types/agent.ts b/packages/agent/src/types/agent.ts index 97691f724..f10a07b93 100644 --- a/packages/agent/src/types/agent.ts +++ b/packages/agent/src/types/agent.ts @@ -75,8 +75,10 @@ export type DwnRequest = { */ export type ProcessDwnRequest = DwnRequest & { dataStream?: Blob | ReadableStream | Readable; - messageOptions: unknown; + rawMessage?: unknown; + messageOptions?: unknown; store?: boolean; + import?: boolean; }; export type SendDwnRequest = DwnRequest & (ProcessDwnRequest | { messageCid: string }) diff --git a/packages/api/src/dwn-api.ts b/packages/api/src/dwn-api.ts index bc387a4dc..b74e63c06 100644 --- a/packages/api/src/dwn-api.ts +++ b/packages/api/src/dwn-api.ts @@ -366,6 +366,7 @@ export class DwnApi { const { entries, status, cursor } = reply; const records = entries.map((entry: RecordsQueryReplyEntry) => { + const recordOptions = { /** * Extract the `author` DID from the record entry since records may be signed by the diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index e162d0cd4..5613553b0 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -23,6 +23,8 @@ export type RecordOptions = RecordsWriteMessage & { connectedDid: string; encodedData?: string | Blob; data?: Readable | ReadableStream; + initialWrite?: RecordsWriteMessage; + protocolRole?: string; remoteOrigin?: string; }; @@ -33,9 +35,10 @@ export type RecordOptions = RecordsWriteMessage & { * @beta */ export type RecordModel = RecordsWriteDescriptor - & Omit + & Omit & { author: string; + protocolRole?: RecordOptions['protocolRole']; recordId?: string; } @@ -51,6 +54,7 @@ export type RecordUpdateOptions = { dateModified?: RecordsWriteDescriptor['messageTimestamp']; datePublished?: RecordsWriteDescriptor['datePublished']; published?: RecordsWriteDescriptor['published']; + protocolRole?: RecordOptions['protocolRole']; } /** @@ -66,6 +70,20 @@ export type RecordUpdateOptions = { * * @beta */ + +function removeUndefinedProperties(obj: any): any { + if (typeof obj !== 'object' || obj === null) return; + Object.keys(obj).forEach(key => { + const val = obj[key]; + if (val === undefined) { + delete obj[key]; + } else if (typeof val === 'object') { + removeUndefinedProperties(val); + } + }); + return obj; +} + export class Record implements RecordModel { // Record instance metadata. private _agent: Web5Agent; @@ -77,16 +95,21 @@ export class Record implements RecordModel { // Private variables for DWN `RecordsWrite` message properties. private _author: string; private _attestation?: RecordsWriteMessage['attestation']; + private _authorization?: RecordsWriteMessage['authorization']; private _contextId?: string; private _descriptor: RecordsWriteDescriptor; private _encryption?: RecordsWriteMessage['encryption']; + private _initialWrite: RecordOptions['initialWrite']; private _recordId: string; - + private _protocolRole: RecordOptions['protocolRole']; // Getters for immutable DWN Record properties. /** Record's signatures attestation */ get attestation(): RecordsWriteMessage['attestation'] { return this._attestation; } + /** Record's signatures attestation */ + get authorization(): RecordsWriteMessage['authorization'] { return this._authorization; } + /** DID that signed the record. */ get author(): string { return this._author; } @@ -102,6 +125,8 @@ export class Record implements RecordModel { /** Record's encryption */ get encryption(): RecordsWriteMessage['encryption'] { return this._encryption; } + get initialWrite(): RecordOptions['initialWrite'] { return this._initialWrite; } + /** Record's ID */ get id() { return this._recordId; } @@ -120,6 +145,9 @@ export class Record implements RecordModel { /** Record's protocol path */ get protocolPath() { return this._descriptor.protocolPath; } + /** Role under which the author is writting the record */ + get protocolRole() { return this._protocolRole; } + /** Record's recipient */ get recipient() { return this._descriptor.recipient; } @@ -147,6 +175,7 @@ export class Record implements RecordModel { get published() { return this._descriptor.published; } constructor(agent: Web5Agent, options: RecordOptions) { + this._agent = agent; /** Store the author DID that originally signed the message as a convenience for developers, so @@ -165,10 +194,13 @@ export class Record implements RecordModel { // RecordsWriteMessage properties. this._attestation = options.attestation; + this._authorization = options.authorization; this._contextId = options.contextId; this._descriptor = options.descriptor; this._encryption = options.encryption; + this._initialWrite = options.initialWrite; this._recordId = options.recordId; + this._protocolRole = options.protocolRole; if (options.encodedData) { // If `encodedData` is set, then it is expected that: @@ -295,6 +327,90 @@ export class Record implements RecordModel { return dataObj; } + private async _processMessage(options: any){ + + const { store = true } = options; + + const request = { + messageType : DwnInterfaceName.Records + DwnMethodName.Write, + author : this._connectedDid, + target : this._connectedDid, + import : !!options.import, + store + } as any; + + if (options.rawMessage) { + removeUndefinedProperties(options.rawMessage); + request.rawMessage = options.rawMessage as Partial; + } + else { + request.messageOptions = options.messageOptions; + } + + if (options.dataStream) { + request.dataStream = options.dataStream; + } + + return this._agent.processDwnRequest(request); + } + + + async store(options: any = {}): Promise { + + // Add check to bail if already imported + + const { store = true, import: _import = false, storeAll = false } = options; + + const initialWrite = this._initialWrite; + + if (initialWrite && storeAll) { + + let agentResponse = await this._processMessage({ + import : _import, + store : store, + rawMessage : { + contextId: this._contextId, + ...initialWrite + } + }); + + const { message, reply: { status } } = agentResponse; + const responseMessage = message as RecordsWriteMessage; + + if (200 <= status.code && status.code <= 299) { + if (_import) initialWrite.authorization = responseMessage.authorization; + } + + } + + let agentResponse = await this._processMessage({ + import : !initialWrite && _import, + store : store, + dataStream : await this.data.blob(), + rawMessage : { + contextId : this._contextId, + recordId : this._recordId, + descriptor : this._descriptor, + attestation : this._attestation, + authorization : this._authorization, + encryption : this._encryption, + } + }); + + const { message, reply: { status } } = agentResponse; + const responseMessage = message as RecordsWriteMessage; + + if (_import) this._authorization = responseMessage.authorization; + + return status; + + } + + async import(options?: any): Promise { + // Add check to bail if already imported + return this.store({ store: options?.store !== false, storeAll: true, import: true }); + } + /** * Send the current record to a remote DWN by specifying their DID * (vs waiting for the regular DWN sync) @@ -304,14 +420,53 @@ export class Record implements RecordModel { * * @beta */ - async send(target: string): Promise { - const { reply: { status } } = await this._agent.sendDwnRequest({ - messageType : DwnInterfaceName.Records + DwnMethodName.Write, - author : this._connectedDid, - dataStream : await this.data.blob(), - target : target, - messageOptions : this.toJSON(), - }); + async send(_options?: string | { [key: string]: any }): Promise { + + const initialWrite = this._initialWrite; + const options = !_options ? { target: this._connectedDid } : typeof _options === 'string' ? { target: _options } : _options; + + if (!options.target){ + options.target = this._connectedDid; + } + + if (options.sendAll && initialWrite){ + + const initialState = { + messageType : DwnInterfaceName.Records + DwnMethodName.Write, + author : this._connectedDid, + target : options.target, + rawMessage : removeUndefinedProperties({ + contextId: this._contextId, + ...initialWrite + }) + } as any; + + await this._agent.sendDwnRequest(initialState); + + } + + const latestState = { + messageType : DwnInterfaceName.Records + DwnMethodName.Write, + author : this._connectedDid, + dataStream : await this.data.blob(), + target : options.target + } as any; + + if (this._authorization) { + latestState.rawMessage = removeUndefinedProperties({ + contextId : this._contextId, + recordId : this._recordId, + descriptor : this._descriptor, + attestation : this._attestation, + authorization : this._authorization, + encryption : this._encryption, + }); + } + else { + latestState.messageOptions = this.toJSON(); + } + + const { reply: { status } } = await this._agent.sendDwnRequest(latestState); return { status }; } @@ -324,6 +479,7 @@ export class Record implements RecordModel { return { attestation : this.attestation, author : this.author, + authorization : this.authorization, contextId : this.contextId, dataCid : this.dataCid, dataFormat : this.dataFormat, @@ -337,6 +493,7 @@ export class Record implements RecordModel { parentId : this.parentId, protocol : this.protocol, protocolPath : this.protocolPath, + protocolRole : this.protocolRole, published : this.published, recipient : this.recipient, recordId : this.id, @@ -427,7 +584,19 @@ export class Record implements RecordModel { const responseMessage = message as RecordsWriteMessage; if (200 <= status.code && status.code <= 299) { + if (!this._initialWrite) { + console.log('post update', responseMessage); + this._initialWrite = removeUndefinedProperties({ + contextId : this._contextId, + recordId : this._recordId, + descriptor : this._descriptor, + attestation : this._attestation, + authorization : this._authorization, + encryption : this._encryption, + }); + } // Only update the local Record instance mutable properties if the record was successfully (over)written. + this._authorization = responseMessage.authorization; mutableDescriptorProperties.forEach(property => { this._descriptor[property] = responseMessage.descriptor[property]; }); From 22e7fbd1217073222cdde58d0c5f943018212e1f Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Mon, 22 Jan 2024 11:19:55 -0600 Subject: [PATCH 02/35] Updates from branch --- .../fixtures/protocol-definitions/email.json | 22 ++++ packages/api/tests/record.spec.ts | 121 ++++++++++++++++++ 2 files changed, 143 insertions(+) diff --git a/packages/api/tests/fixtures/protocol-definitions/email.json b/packages/api/tests/fixtures/protocol-definitions/email.json index 1e23bf5d2..a7b20623b 100644 --- a/packages/api/tests/fixtures/protocol-definitions/email.json +++ b/packages/api/tests/fixtures/protocol-definitions/email.json @@ -2,12 +2,34 @@ "protocol": "http://email-protocol.xyz", "published": false, "types": { + "thread": { + "schema": "http://email-protocol.xyz/schema/thread", + "dataFormats": ["text/plain"] + }, "email": { "schema": "http://email-protocol.xyz/schema/email", "dataFormats": ["text/plain"] } }, "structure": { + "thread": { + "$actions": [ + { + "who": "recipient", + "of": "thread", + "can": "read" + }, + { + "who": "author", + "of": "thread", + "can": "write" + }, + { + "who": "anyone", + "can": "update" + } + ] + }, "email": { "$actions": [ { diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index 1fdfc0a32..07ce60fb0 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -97,6 +97,127 @@ describe('Record', () => { await testAgent.closeStorage(); }); + // FIRST PASS AT IMPORT + + it.only('imports a record that another user wrote', async () => { + + // Install the email protocol for Alice's local DWN. + let { protocol: aliceProtocol, status: aliceStatus } = await dwnAlice.protocols.configure({ + message: { + definition: emailProtocolDefinition + } + }); + expect(aliceStatus.code).to.equal(202); + expect(aliceProtocol).to.exist; + + // Install the email protocol for Alice's remote DWN. + const { status: alicePushStatus } = await aliceProtocol!.send(aliceDid.did); + expect(alicePushStatus.code).to.equal(202); + + // Install the email protocol for Bob's local DWN. + const { protocol: bobProtocol, status: bobStatus } = await dwnBob.protocols.configure({ + message: { + definition: emailProtocolDefinition + } + }); + + expect(bobStatus.code).to.equal(202); + expect(bobProtocol).to.exist; + + // Install the email protocol for Bob's remote DWN. + const { status: bobPushStatus } = await bobProtocol!.send(bobDid.did); + expect(bobPushStatus.code).to.equal(202); + + // Alice creates a new large record and stores it + const { status: aliceEmailStatus, record: aliceEmailRecord } = await dwnAlice.records.write({ + data : TestDataGenerator.randomString(DwnConstant.maxDataSizeAllowedToBeEncoded + 1000), + message : { + recipient : bobDid.did, + protocol : emailProtocolDefinition.protocol, + protocolPath : 'thread', + schema : 'http://email-protocol.xyz/schema/thread', + } + }); + expect(aliceEmailStatus.code).to.equal(202); + const { status: sendStatus } = await aliceEmailRecord!.send(aliceDid.did); + expect(sendStatus.code).to.equal(202); + + // Alice queries for the record that was just created on her remote DWN. + const { records: queryRecords, status: queryRecordStatus } = await dwnBob.records.query({ + from : aliceDid.did, + message : { + filter: { + protocol : emailProtocolDefinition.protocol, + protocolPath : 'thread', + } + } + }); + expect(queryRecordStatus.code).to.equal(200); + const importRecord = queryRecords[0]; + const importRecordStatus = await importRecord.import(); + expect(importRecordStatus.code).to.equal(202); + + const { status: importSendStatus } = await importRecord!.send({ sendAll: true }); + expect(importSendStatus.code).to.equal(202); + + // Alice updates her record + let { status: aliceEmailStatusUpdated } = await aliceEmailRecord.update({ + data: TestDataGenerator.randomString(DwnConstant.maxDataSizeAllowedToBeEncoded + 1000) + }); + expect(aliceEmailStatusUpdated.code).to.equal(202); + + const { status: sentToSelfStatus } = await aliceEmailRecord!.send(); + expect(sentToSelfStatus.code).to.equal(202); + + const { status: sentToBobStatus } = await aliceEmailRecord!.send(bobDid.did); + expect(sentToBobStatus.code).to.equal(202); + + // Alice updates her record + + const updatedText = TestDataGenerator.randomString(DwnConstant.maxDataSizeAllowedToBeEncoded + 1000); + let { status: aliceEmailStatusUpdatedAgain } = await aliceEmailRecord.update({ + data: updatedText + }); + expect(aliceEmailStatusUpdatedAgain.code).to.equal(202); + + // Sends it to her own remote DWN again + const { status: sentToSelfAgainStatus } = await aliceEmailRecord!.send(); + expect(sentToSelfAgainStatus.code).to.equal(202); + + const { records: updatedRecords, status: updatedRecordsStatus } = await dwnBob.records.query({ + from : aliceDid.did, + message : { + filter: { + protocol : emailProtocolDefinition.protocol, + protocolPath : 'thread', + } + } + }); + expect(updatedRecordsStatus.code).to.equal(200); + + const updatedRecord = updatedRecords[0]; + const updatedRecordStoredStatus = await updatedRecord.store(); + expect(updatedRecordStoredStatus.code).to.equal(202); + + expect(await updatedRecord.data.text() === updatedText).to.equal(true); + + const { status: updatedRecordToSelfStatus } = await updatedRecord!.send(); + expect(updatedRecordToSelfStatus.code).to.equal(202); + + // Confirm Bob can query his own remote DWN for the created record. + // const bobQueryResult = await dwnBob.records.query({ + // from : bobDid.did, + // message : { + // filter: { + // schema: 'http://email-protocol.xyz/schema/email' + // } + // } + // }); + // expect(bobQueryResult.status.code).to.equal(200); + // expect(bobQueryResult.records).to.exist; + // expect(bobQueryResult.records!.length).to.equal(1); + }); + it('should retain all defined properties', async () => { // RecordOptions properties const author = aliceDid.did; From c277199c95156c18bc5d9f13fba46728d059ee45 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Mon, 22 Jan 2024 16:22:15 -0600 Subject: [PATCH 03/35] Removing logging --- packages/agent/src/dwn-manager.ts | 4 ---- packages/api/src/record.ts | 5 ++--- packages/api/tests/record.spec.ts | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/agent/src/dwn-manager.ts b/packages/agent/src/dwn-manager.ts index 201caf54c..aa1a6bc05 100644 --- a/packages/agent/src/dwn-manager.ts +++ b/packages/agent/src/dwn-manager.ts @@ -196,9 +196,7 @@ export class DwnManager { messageData = data; } else { - console.log('start constructDwnMessage'); const { message } = await this.constructDwnMessage({ request }); - console.log('passed constructDwnMessage'); dwnRpcRequest.message = message; messageData = request.dataStream; } @@ -258,8 +256,6 @@ export class DwnManager { const { request } = options; const rawMessage = (request as any).rawMessage; - console.log('259', rawMessage); - let readableStream: Readable | undefined; // TODO: Consider refactoring to move data transformations imposed by fetch() limitations to the HTTP transport-related methods. diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 5613553b0..de898ffae 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -367,7 +367,7 @@ export class Record implements RecordModel { let agentResponse = await this._processMessage({ import : _import, - store : store, + store : true, rawMessage : { contextId: this._contextId, ...initialWrite @@ -385,7 +385,7 @@ export class Record implements RecordModel { let agentResponse = await this._processMessage({ import : !initialWrite && _import, - store : store, + store : store || storeAll, dataStream : await this.data.blob(), rawMessage : { contextId : this._contextId, @@ -585,7 +585,6 @@ export class Record implements RecordModel { if (200 <= status.code && status.code <= 299) { if (!this._initialWrite) { - console.log('post update', responseMessage); this._initialWrite = removeUndefinedProperties({ contextId : this._contextId, recordId : this._recordId, diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index 07ce60fb0..ca7d7e1b6 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -99,7 +99,7 @@ describe('Record', () => { // FIRST PASS AT IMPORT - it.only('imports a record that another user wrote', async () => { + it('imports a record that another user wrote', async () => { // Install the email protocol for Alice's local DWN. let { protocol: aliceProtocol, status: aliceStatus } = await dwnAlice.protocols.configure({ From 8676306ecc6b233d50d3ffce0a330d7769c5acc7 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Mon, 22 Jan 2024 17:12:20 -0600 Subject: [PATCH 04/35] Add docs --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 7dd0dfd37..9709d6f7b 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,10 @@ Each `Record` instance has the following instance methods: - **`text`** - _`function`_: returns the data as a string. - **`send`** - _`function`_: sends the record the instance represents to the DWeb Node endpoints of a provided DID. - **`update`** - _`function`_: takes in a new request object matching the expected method signature of a `write` and overwrites the record. This is a convenience method that allows you to easily overwrite records with less verbosity. +- **`store`** - _`function`_: stores the record in the local DWN instance, offering the following options: + - `storeAll`: stores the initial record state in addition to the latest state. This is required if your local DWN instance does not already contain this record. +- **`import`** - _`function`_: signs a record with an owner override to import the record into the local DWN instance: + - `store` - _`boolean`_: when false is passed, the record will only be signed with an owner override, not stored in the local DWN instance. Defaults to `true`. ### **`web5.dwn.records.query(request)`** From fd78372458b0f5af6433c4945b6826ba7dd32818 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Wed, 24 Jan 2024 09:51:24 -0600 Subject: [PATCH 05/35] Update based on PR feedback --- packages/agent/src/dwn-manager.ts | 8 -------- packages/api/src/record.ts | 18 ++++++++++-------- packages/api/tests/record.spec.ts | 2 +- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/packages/agent/src/dwn-manager.ts b/packages/agent/src/dwn-manager.ts index aa1a6bc05..ccbfe9370 100644 --- a/packages/agent/src/dwn-manager.ts +++ b/packages/agent/src/dwn-manager.ts @@ -75,14 +75,6 @@ const dwnMessageConstructors = { [DwnInterfaceName.Protocols + DwnMethodName.Configure] : ProtocolsConfigure, }; -(RecordsWrite.prototype as any).forceSet = function(prop: string, value: unknown){ - this[prop] = value; -}; - -(RecordsWrite.prototype as any).forceGet = function(prop: string){ - return this[prop]; -}; - export type DwnManagerOptions = { agent?: Web5ManagedAgent; dwn: Dwn; diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index de898ffae..c20e11ea2 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -10,7 +10,6 @@ import { Convert, NodeStream, Stream } from '@web5/common'; import { DwnInterfaceName, DwnMethodName } from '@tbd54566975/dwn-sdk-js'; import type { ResponseStatus } from './dwn-api.js'; - import { dataToBlob } from './utils.js'; /** @@ -100,6 +99,7 @@ export class Record implements RecordModel { private _descriptor: RecordsWriteDescriptor; private _encryption?: RecordsWriteMessage['encryption']; private _initialWrite: RecordOptions['initialWrite']; + private _initialWriteStored: boolean; private _recordId: string; private _protocolRole: RecordOptions['protocolRole']; // Getters for immutable DWN Record properties. @@ -357,13 +357,11 @@ export class Record implements RecordModel { async store(options: any = {}): Promise { - // Add check to bail if already imported - - const { store = true, import: _import = false, storeAll = false } = options; + const { store = true, import: _import = false } = options; const initialWrite = this._initialWrite; - if (initialWrite && storeAll) { + if (initialWrite && !this._initialWriteStored) { let agentResponse = await this._processMessage({ import : _import, @@ -378,6 +376,7 @@ export class Record implements RecordModel { const responseMessage = message as RecordsWriteMessage; if (200 <= status.code && status.code <= 299) { + this._initialWriteStored = true; if (_import) initialWrite.authorization = responseMessage.authorization; } @@ -385,7 +384,7 @@ export class Record implements RecordModel { let agentResponse = await this._processMessage({ import : !initialWrite && _import, - store : store || storeAll, + store : store, dataStream : await this.data.blob(), rawMessage : { contextId : this._contextId, @@ -400,7 +399,9 @@ export class Record implements RecordModel { const { message, reply: { status } } = agentResponse; const responseMessage = message as RecordsWriteMessage; - if (_import) this._authorization = responseMessage.authorization; + if (200 <= status.code && status.code <= 299) { + if (_import) this._authorization = responseMessage.authorization; + } return status; @@ -408,7 +409,7 @@ export class Record implements RecordModel { async import(options?: any): Promise { // Add check to bail if already imported - return this.store({ store: options?.store !== false, storeAll: true, import: true }); + return this.store({ store: options?.store !== false, import: true }); } /** @@ -596,6 +597,7 @@ export class Record implements RecordModel { } // Only update the local Record instance mutable properties if the record was successfully (over)written. this._authorization = responseMessage.authorization; + this._protocolRole = messageOptions.protocolRole; mutableDescriptorProperties.forEach(property => { this._descriptor[property] = responseMessage.descriptor[property]; }); diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index ca7d7e1b6..07ce60fb0 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -99,7 +99,7 @@ describe('Record', () => { // FIRST PASS AT IMPORT - it('imports a record that another user wrote', async () => { + it.only('imports a record that another user wrote', async () => { // Install the email protocol for Alice's local DWN. let { protocol: aliceProtocol, status: aliceStatus } = await dwnAlice.protocols.configure({ From 74348f29d800d1ab908199c5503dbce6658dbf0f Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Wed, 24 Jan 2024 09:58:56 -0600 Subject: [PATCH 06/35] update readme --- README.md | 2 +- packages/api/tests/record.spec.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9709d6f7b..01c5359ab 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ Each `Record` instance has the following instance methods: - **`send`** - _`function`_: sends the record the instance represents to the DWeb Node endpoints of a provided DID. - **`update`** - _`function`_: takes in a new request object matching the expected method signature of a `write` and overwrites the record. This is a convenience method that allows you to easily overwrite records with less verbosity. - **`store`** - _`function`_: stores the record in the local DWN instance, offering the following options: - - `storeAll`: stores the initial record state in addition to the latest state. This is required if your local DWN instance does not already contain this record. + - `import`: imports the record as with an owner-signed override (still subject to Protocol rules, when a record is Protocol-based) - **`import`** - _`function`_: signs a record with an owner override to import the record into the local DWN instance: - `store` - _`boolean`_: when false is passed, the record will only be signed with an owner override, not stored in the local DWN instance. Defaults to `true`. diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index 07ce60fb0..ca7d7e1b6 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -99,7 +99,7 @@ describe('Record', () => { // FIRST PASS AT IMPORT - it.only('imports a record that another user wrote', async () => { + it('imports a record that another user wrote', async () => { // Install the email protocol for Alice's local DWN. let { protocol: aliceProtocol, status: aliceStatus } = await dwnAlice.protocols.configure({ From a94d9d81464a717239af6dfeb9f7089c5ff22c41 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Mon, 29 Jan 2024 16:14:31 -0600 Subject: [PATCH 07/35] Add role-based record tests, fix for cloned obj issue --- packages/agent/src/dwn-manager.ts | 3 +- packages/api/src/record.ts | 31 ++++++++- packages/api/tests/dwn-api.spec.ts | 68 ++++++++++++++++++- .../fixtures/protocol-definitions/photos.json | 51 ++++++++++++++ packages/api/tests/record.spec.ts | 13 ++-- 5 files changed, 151 insertions(+), 15 deletions(-) create mode 100644 packages/api/tests/fixtures/protocol-definitions/photos.json diff --git a/packages/agent/src/dwn-manager.ts b/packages/agent/src/dwn-manager.ts index ccbfe9370..7d29f2150 100644 --- a/packages/agent/src/dwn-manager.ts +++ b/packages/agent/src/dwn-manager.ts @@ -63,13 +63,12 @@ type DwnMessage = { data?: Blob; } -const recordsWriteType = DwnInterfaceName.Records + DwnMethodName.Write; const dwnMessageConstructors = { [DwnInterfaceName.Events + DwnMethodName.Get] : EventsGet, [DwnInterfaceName.Messages + DwnMethodName.Get] : MessagesGet, [DwnInterfaceName.Records + DwnMethodName.Read] : RecordsRead, [DwnInterfaceName.Records + DwnMethodName.Query] : RecordsQuery, - [recordsWriteType] : RecordsWrite, + [DwnInterfaceName.Records + DwnMethodName.Write] : RecordsWrite, [DwnInterfaceName.Records + DwnMethodName.Delete] : RecordsDelete, [DwnInterfaceName.Protocols + DwnMethodName.Query] : ProtocolsQuery, [DwnInterfaceName.Protocols + DwnMethodName.Configure] : ProtocolsConfigure, diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index c20e11ea2..0487004f1 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -104,6 +104,29 @@ export class Record implements RecordModel { private _protocolRole: RecordOptions['protocolRole']; // Getters for immutable DWN Record properties. + static sendCache = new Map(); + static sendCacheLimit = 100; + static setSendCache(recordId, target){ + const recordCache = Record.sendCache; + let targetCache = recordCache.get(recordId) || new Set(); + recordCache.delete(recordId); + recordCache.set(recordId, targetCache); + if (recordCache.size > Record.sendCacheLimit) { + const firstRecord = recordCache.keys().next().value; + recordCache.delete(firstRecord); + } + targetCache.delete(target); + targetCache.add(target); + if (targetCache.size > Record.sendCacheLimit) { + const firstTarget = targetCache.keys().next().value; + targetCache.delete(firstTarget); + } + } + static checkSendCache(recordId, target){ + let targetCache = Record.sendCache.get(recordId); + return target && targetCache ? targetCache.has(target) : targetCache; + } + /** Record's signatures attestation */ get attestation(): RecordsWriteMessage['attestation'] { return this._attestation; } @@ -430,7 +453,7 @@ export class Record implements RecordModel { options.target = this._connectedDid; } - if (options.sendAll && initialWrite){ + if (initialWrite && !Record.checkSendCache(this._recordId, options.target)){ const initialState = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, @@ -444,6 +467,8 @@ export class Record implements RecordModel { await this._agent.sendDwnRequest(initialState); + Record.setSendCache(this._recordId, options.target); + } const latestState = { @@ -586,14 +611,14 @@ export class Record implements RecordModel { if (200 <= status.code && status.code <= 299) { if (!this._initialWrite) { - this._initialWrite = removeUndefinedProperties({ + this._initialWrite = JSON.parse(JSON.stringify(removeUndefinedProperties({ contextId : this._contextId, recordId : this._recordId, descriptor : this._descriptor, attestation : this._attestation, authorization : this._authorization, encryption : this._encryption, - }); + }))); } // Only update the local Record instance mutable properties if the record was successfully (over)written. this._authorization = responseMessage.authorization; diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index d04705f20..abd792b65 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -9,6 +9,7 @@ import { DwnApi } from '../src/dwn-api.js'; import { testDwnUrl } from './utils/test-config.js'; import { TestUserAgent } from './utils/test-user-agent.js'; import emailProtocolDefinition from './fixtures/protocol-definitions/email.json' assert { type: 'json' }; +import photosProtocolDefinition from './fixtures/protocol-definitions/photos.json' assert { type: 'json' }; let testDwnUrls: string[] = [testDwnUrl]; @@ -261,6 +262,72 @@ describe('DwnApi', () => { expect(result.record).to.exist; expect(await result.record?.data.json()).to.deep.equal(dataJson); }); + + it('creates a role record for another user that they can use to create role-based records', async () => { + /** + * WHAT IS BEING TESTED? + * + * We are testing whether role records can be created for outbound participants + * so they can use them to create records corresponding to the roles they are granted. + * + * TEST SETUP STEPS: + * 1. Configure the email protocol on Bob's local DWN. + */ + + // Configure the email protocol on Alice and Bob's local and remote DWNs. + const { status: bobProtocolStatus, protocol: bobProtocol } = await dwnBob.protocols.configure({ + message: { + definition: photosProtocolDefinition + } + }); + expect(bobProtocolStatus.code).to.equal(202); + const { status: bobRemoteProtocolStatus } = await bobProtocol.send(bobDid.did); + expect(bobRemoteProtocolStatus.code).to.equal(202); + + const { status: aliceProtocolStatus, protocol: aliceProtocol } = await dwnAlice.protocols.configure({ + message: { + definition: photosProtocolDefinition + } + }); + expect(aliceProtocolStatus.code).to.equal(202); + const { status: aliceRemoteProtocolStatus } = await aliceProtocol.send(aliceDid.did); + expect(aliceRemoteProtocolStatus.code).to.equal(202); + + // Alice creates a role-based 'friend' record, and sends it to her remote + const { status: friendCreateStatus, record: friendRecord} = await dwnAlice.records.create({ + data : 'test', + message : { + recipient : bobDid.did, + protocol : photosProtocolDefinition.protocol, + protocolPath : 'friend', + schema : photosProtocolDefinition.types.friend.schema, + dataFormat : 'text/plain' + } + }); + expect(friendCreateStatus.code).to.equal(202); + const { status: friendRecordUpdateStatus } = await friendRecord.update({ data: 'update' }); + expect(friendRecordUpdateStatus.code).to.equal(202); + const { status: friendSendStatus } = await friendRecord.send(aliceDid.did); + expect(friendSendStatus.code).to.equal(202); + + // Bob creates a thread record using the role 'friend' and sends it to Alice + const { status: albumCreateStatus, record: albumRecord} = await dwnBob.records.create({ + data : 'test', + message : { + recipient : aliceDid.did, + protocol : photosProtocolDefinition.protocol, + protocolPath : 'album', + protocolRole : 'friend', + schema : photosProtocolDefinition.types.album.schema, + dataFormat : 'text/plain' + } + }); + expect(albumCreateStatus.code).to.equal(202); + const { status: bobSendStatus } = await albumRecord.send(bobDid.did); + expect(bobSendStatus.code).to.equal(202); + const { status: aliceSendStatus } = await albumRecord.send(aliceDid.did); + expect(aliceSendStatus.code).to.equal(202); + }); }); describe('agent store: false', () => { @@ -705,7 +772,6 @@ describe('DwnApi', () => { } } }); - // Confirm that the record does not currently exist on Bob's DWN. expect(result.status.code).to.equal(200); expect(result.records).to.exist; diff --git a/packages/api/tests/fixtures/protocol-definitions/photos.json b/packages/api/tests/fixtures/protocol-definitions/photos.json new file mode 100644 index 000000000..9a002dff7 --- /dev/null +++ b/packages/api/tests/fixtures/protocol-definitions/photos.json @@ -0,0 +1,51 @@ +{ + "protocol": "http://photo-protocol.xyz", + "published": true, + "types": { + "album": { + "schema": "http://photo-protocol.xyz/schema/album", + "dataFormats": ["text/plain"] + }, + "photo": { + "schema": "http://photo-protocol.xyz/schema/photo", + "dataFormats": ["text/plain"] + }, + "friend": { + "schema": "http://photo-protocol.xyz/schema/friend", + "dataFormats": ["text/plain"] + }, + "participant": { + "schema": "http://photo-protocol.xyz/schema/participant", + "dataFormats": ["text/plain"] + } + }, + "structure": { + "friend": { + "$globalRole": true + }, + "album": { + "$actions": [ + { + "role": "friend", + "can": "write" + } + ], + "participant": { + "$contextRole": true + }, + "photo": { + "$actions": [ + { + "role": "album/participant", + "can": "write" + }, + { + "who": "author", + "of": "thread", + "can": "write" + } + ] + } + } + } +} diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index ca7d7e1b6..e54e4ec89 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -157,7 +157,7 @@ describe('Record', () => { const importRecordStatus = await importRecord.import(); expect(importRecordStatus.code).to.equal(202); - const { status: importSendStatus } = await importRecord!.send({ sendAll: true }); + const { status: importSendStatus } = await importRecord!.send(); expect(importSendStatus.code).to.equal(202); // Alice updates her record @@ -1306,8 +1306,7 @@ describe('Record', () => { * 4. Validate that Bob is able to write the record to Alice's remote DWN. */ const { status: sendStatusToAlice } = await queryRecordsFrom[0]!.send(aliceDid.did); - expect(sendStatusToAlice.code).to.equal(401); - expect(sendStatusToAlice.detail).to.equal(`Cannot read properties of undefined (reading 'authorization')`); + expect(sendStatusToAlice.code).to.equal(202); }); }); }); @@ -1466,7 +1465,7 @@ describe('Record', () => { }); // TODO: Fix after changes are made to dwn-sdk-js to include the initial write in every query/read response. - it('fails to write updated records to a remote DWN that is missing the initial write', async () => { + it('automatically sends the initial write and update of a record to a remote DWN', async () => { // Alice writes a message to her agent connected DWN. const { status, record } = await dwnAlice.records.write({ data : 'Hello, world!', @@ -1483,11 +1482,7 @@ describe('Record', () => { // Write the updated record to Alice's remote DWN a second time. const sendResult = await record!.send(aliceDid.did); - expect(sendResult.status.code).to.equal(400); - expect(sendResult.status.detail).to.equal('RecordsWriteGetInitialWriteNotFound: initial write is not found'); - - // TODO: Uncomment the following line after changes are made to dwn-sdk-js to include the initial write in every query/read response. - // expect(sendResult.status.code).to.equal(202); + expect(sendResult.status.code).to.equal(202); }); it('writes large records to remote DWNs that were initially queried from a remote DWN', async () => { From bd67e0c0439a767dfc1377855d2f721c28354f94 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Mon, 29 Jan 2024 22:46:59 -0600 Subject: [PATCH 08/35] Updated protocol config, added elaborate role tests --- packages/api/tests/dwn-api.spec.ts | 77 +++++++++++++++++-- .../fixtures/protocol-definitions/photos.json | 11 ++- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index abd792b65..6f81e82d2 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -307,8 +307,8 @@ describe('DwnApi', () => { expect(friendCreateStatus.code).to.equal(202); const { status: friendRecordUpdateStatus } = await friendRecord.update({ data: 'update' }); expect(friendRecordUpdateStatus.code).to.equal(202); - const { status: friendSendStatus } = await friendRecord.send(aliceDid.did); - expect(friendSendStatus.code).to.equal(202); + const { status: aliceFriendSendStatus } = await friendRecord.send(aliceDid.did); + expect(aliceFriendSendStatus.code).to.equal(202); // Bob creates a thread record using the role 'friend' and sends it to Alice const { status: albumCreateStatus, record: albumRecord} = await dwnBob.records.create({ @@ -323,10 +323,75 @@ describe('DwnApi', () => { } }); expect(albumCreateStatus.code).to.equal(202); - const { status: bobSendStatus } = await albumRecord.send(bobDid.did); - expect(bobSendStatus.code).to.equal(202); - const { status: aliceSendStatus } = await albumRecord.send(aliceDid.did); - expect(aliceSendStatus.code).to.equal(202); + const { status: bobAlbumSendStatus } = await albumRecord.send(bobDid.did); + expect(bobAlbumSendStatus.code).to.equal(202); + const { status: aliceAlbumSendStatus } = await albumRecord.send(aliceDid.did); + expect(aliceAlbumSendStatus.code).to.equal(202); + + // Alice fetches the album record Bob created using his friend role + const aliceAlbumReadResult = await dwnAlice.records.read({ + from : aliceDid.did, + message : { + filter: { + recordId: albumRecord.id + } + } + }); + expect(aliceAlbumReadResult.status.code).to.equal(200); + expect(aliceAlbumReadResult.record).to.exist; + const aliceAlbumReadStoreStatus = await aliceAlbumReadResult.record.store(); + expect(aliceAlbumReadStoreStatus.code).to.equal(202); + + // Bob makes Alice a participant + const { status: participantCreateStatus, record: participantRecord} = await dwnBob.records.create({ + data : 'test', + message : { + contextId : albumRecord.id, + parentId : albumRecord.id, + recipient : aliceDid.did, + protocol : photosProtocolDefinition.protocol, + protocolPath : 'album/participant', + schema : photosProtocolDefinition.types.participant.schema, + dataFormat : 'text/plain' + } + }); + expect(participantCreateStatus.code).to.equal(202); + const { status: bobParticipantSendStatus } = await participantRecord.send(bobDid.did); + expect(bobParticipantSendStatus.code).to.equal(202); + const { status: aliceParticipantSendStatus } = await participantRecord.send(aliceDid.did); + expect(aliceParticipantSendStatus.code).to.equal(202); + + const aliceParticipantReadResult = await dwnAlice.records.read({ + from : aliceDid.did, + message : { + filter: { + recordId: participantRecord.id + } + } + }); + expect(aliceParticipantReadResult.status.code).to.equal(200); + expect(aliceParticipantReadResult.record).to.exist; + const aliceParticipantReadStoreStatus = await aliceParticipantReadResult.record.store(); + expect(aliceParticipantReadStoreStatus.code).to.equal(202); + + // Alice creates a photo using her participant role + const { status: photoCreateStatus, record: photoRecord} = await dwnAlice.records.create({ + data : 'test', + message : { + contextId : albumRecord.id, + parentId : albumRecord.id, + protocol : photosProtocolDefinition.protocol, + protocolPath : 'album/photo', + protocolRole : 'album/participant', + schema : photosProtocolDefinition.types.photo.schema, + dataFormat : 'text/plain' + } + }); + expect(photoCreateStatus.code).to.equal(202); + const { status:alicePhotoSendStatus } = await photoRecord.send(aliceDid.did); + expect(alicePhotoSendStatus.code).to.equal(202); + const { status: bobPhotoSendStatus } = await photoRecord.send(bobDid.did); + expect(bobPhotoSendStatus.code).to.equal(202); }); }); diff --git a/packages/api/tests/fixtures/protocol-definitions/photos.json b/packages/api/tests/fixtures/protocol-definitions/photos.json index 9a002dff7..6e5c1a4b9 100644 --- a/packages/api/tests/fixtures/protocol-definitions/photos.json +++ b/packages/api/tests/fixtures/protocol-definitions/photos.json @@ -31,7 +31,14 @@ } ], "participant": { - "$contextRole": true + "$contextRole": true, + "$actions": [ + { + "who": "author", + "of": "album", + "can": "write" + } + ] }, "photo": { "$actions": [ @@ -41,7 +48,7 @@ }, { "who": "author", - "of": "thread", + "of": "album", "can": "write" } ] From 2f046b158584aa952bea0b67a69b669b927edf3a Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 09:21:23 -0600 Subject: [PATCH 09/35] Pass option to initial write processing --- packages/api/src/record.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 0487004f1..d318ae8b8 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -384,11 +384,12 @@ export class Record implements RecordModel { const initialWrite = this._initialWrite; + // Is there an initial write? Have we already stored this record? if (initialWrite && !this._initialWriteStored) { let agentResponse = await this._processMessage({ import : _import, - store : true, + store : store, rawMessage : { contextId: this._contextId, ...initialWrite @@ -398,6 +399,7 @@ export class Record implements RecordModel { const { message, reply: { status } } = agentResponse; const responseMessage = message as RecordsWriteMessage; + // If we are importing, make sure to update the initial write's authorization, because now it will have the owner's signature on it if (200 <= status.code && status.code <= 299) { this._initialWriteStored = true; if (_import) initialWrite.authorization = responseMessage.authorization; From 9cdc8afdb7e20d78209a4aa354b863f019c91cf5 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 10:57:44 -0600 Subject: [PATCH 10/35] Add new test for updating with existing role in play --- packages/api/tests/dwn-api.spec.ts | 22 ++++++++++++++++++- .../fixtures/protocol-definitions/photos.json | 5 +++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 6f81e82d2..5d16b4f88 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -263,7 +263,7 @@ describe('DwnApi', () => { expect(await result.record?.data.json()).to.deep.equal(dataJson); }); - it('creates a role record for another user that they can use to create role-based records', async () => { + it.only('creates a role record for another user that they can use to create role-based records', async () => { /** * WHAT IS BEING TESTED? * @@ -392,6 +392,26 @@ describe('DwnApi', () => { expect(alicePhotoSendStatus.code).to.equal(202); const { status: bobPhotoSendStatus } = await photoRecord.send(bobDid.did); expect(bobPhotoSendStatus.code).to.equal(202); + + const { status: photoUpdateStatus, record: photoUpdateRecord} = await dwnBob.records.write({ + data : 'test again', + store : false, + message : { + contextId : albumRecord.id, + parentId : albumRecord.id, + recordId : photoRecord.id, + dateCreated : photoRecord.dateCreated, + protocol : photosProtocolDefinition.protocol, + protocolPath : 'album/photo', + schema : photosProtocolDefinition.types.photo.schema, + dataFormat : 'text/plain' + } + }); + expect(photoUpdateStatus.code).to.equal(202); + const { status:alicePhotoUpdateSendStatus } = await photoUpdateRecord.send(aliceDid.did); + expect(alicePhotoUpdateSendStatus.code).to.equal(202); + const { status: bobPhotoUpdateSendStatus } = await photoUpdateRecord.send(bobDid.did); + expect(bobPhotoUpdateSendStatus.code).to.equal(202); }); }); diff --git a/packages/api/tests/fixtures/protocol-definitions/photos.json b/packages/api/tests/fixtures/protocol-definitions/photos.json index 6e5c1a4b9..4430ae2de 100644 --- a/packages/api/tests/fixtures/protocol-definitions/photos.json +++ b/packages/api/tests/fixtures/protocol-definitions/photos.json @@ -50,6 +50,11 @@ "who": "author", "of": "album", "can": "write" + }, + { + "who": "author", + "of": "album", + "can": "update" } ] } From ff8ddc58bd89e60600962bf288a908ab473ac8c9 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 30 Jan 2024 11:00:29 -0500 Subject: [PATCH 11/35] Store/Import Typing Updates (#388) * optional target for send * guard store options --- packages/api/src/record.ts | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index d318ae8b8..f445ef25c 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -377,8 +377,7 @@ export class Record implements RecordModel { return this._agent.processDwnRequest(request); } - - async store(options: any = {}): Promise { + async _processRecord(options: { store: boolean, import: boolean }): Promise { const { store = true, import: _import = false } = options; @@ -432,35 +431,40 @@ export class Record implements RecordModel { } - async import(options?: any): Promise { + async store(options?: { import: boolean }): Promise { + return this._processRecord({ ...options, store: true }); + } + + async import(options?: { store: boolean }): Promise { // Add check to bail if already imported - return this.store({ store: options?.store !== false, import: true }); + return this._processRecord({ store: options?.store !== false, import: true }); } + + /** * Send the current record to a remote DWN by specifying their DID + * If no DID is specified, the target is assumed to be the owner (connectedDID). * (vs waiting for the regular DWN sync) - * @param target - the DID to send the record to + * @param target - the optional DID to send the record to, if none is set it is sent to the connectedDid * @returns the status of the send record request * @throws `Error` if the record has already been deleted. * * @beta */ - async send(_options?: string | { [key: string]: any }): Promise { + async send(target?: string): Promise { const initialWrite = this._initialWrite; - const options = !_options ? { target: this._connectedDid } : typeof _options === 'string' ? { target: _options } : _options; - - if (!options.target){ - options.target = this._connectedDid; + if (!target) { + target = this._connectedDid; } - if (initialWrite && !Record.checkSendCache(this._recordId, options.target)){ + if (initialWrite && !Record.checkSendCache(this._recordId, target)){ const initialState = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, author : this._connectedDid, - target : options.target, + target : target, rawMessage : removeUndefinedProperties({ contextId: this._contextId, ...initialWrite @@ -469,7 +473,7 @@ export class Record implements RecordModel { await this._agent.sendDwnRequest(initialState); - Record.setSendCache(this._recordId, options.target); + Record.setSendCache(this._recordId, target); } @@ -477,7 +481,7 @@ export class Record implements RecordModel { messageType : DwnInterfaceName.Records + DwnMethodName.Write, author : this._connectedDid, dataStream : await this.data.blob(), - target : options.target + target : target } as any; if (this._authorization) { From 418adc7c925acaecca896d612435fb14d4fd525d Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 30 Jan 2024 11:04:25 -0500 Subject: [PATCH 12/35] fix lint --- packages/api/src/record.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index f445ef25c..e9b8648b5 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -440,7 +440,6 @@ export class Record implements RecordModel { return this._processRecord({ store: options?.store !== false, import: true }); } - /** * Send the current record to a remote DWN by specifying their DID From 9f86f94affcd811a1d60ea8d1807911ef845ee9f Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 10:58:34 -0600 Subject: [PATCH 13/35] remove errant only() in tests --- packages/api/tests/dwn-api.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 5d16b4f88..683dfc4f5 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -263,7 +263,7 @@ describe('DwnApi', () => { expect(await result.record?.data.json()).to.deep.equal(dataJson); }); - it.only('creates a role record for another user that they can use to create role-based records', async () => { + it('creates a role record for another user that they can use to create role-based records', async () => { /** * WHAT IS BEING TESTED? * From 3271fd5817e56ddeb1fcb52ce0f2e90a2d944fd0 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 12:15:59 -0600 Subject: [PATCH 14/35] Add test for changing the protocol role that a record is being updated with --- packages/api/tests/dwn-api.spec.ts | 25 +++++++++++++++++-- .../fixtures/protocol-definitions/photos.json | 20 ++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 683dfc4f5..41e2b38e7 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -263,7 +263,7 @@ describe('DwnApi', () => { expect(await result.record?.data.json()).to.deep.equal(dataJson); }); - it('creates a role record for another user that they can use to create role-based records', async () => { + it.only('creates a role record for another user that they can use to create role-based records', async () => { /** * WHAT IS BEING TESTED? * @@ -342,7 +342,7 @@ describe('DwnApi', () => { const aliceAlbumReadStoreStatus = await aliceAlbumReadResult.record.store(); expect(aliceAlbumReadStoreStatus.code).to.equal(202); - // Bob makes Alice a participant + // Bob makes Alice a `participant` const { status: participantCreateStatus, record: participantRecord} = await dwnBob.records.create({ data : 'test', message : { @@ -374,6 +374,26 @@ describe('DwnApi', () => { const aliceParticipantReadStoreStatus = await aliceParticipantReadResult.record.store(); expect(aliceParticipantReadStoreStatus.code).to.equal(202); + // Alice makes Bob an `updater` + const { status: updaterCreateStatus, record: updaterRecord} = await dwnAlice.records.create({ + data : 'test', + message : { + contextId : albumRecord.id, + parentId : albumRecord.id, + recipient : bobDid.did, + protocol : photosProtocolDefinition.protocol, + protocolPath : 'album/updater', + protocolRole : 'album/participant', + schema : photosProtocolDefinition.types.updater.schema, + dataFormat : 'text/plain' + } + }); + expect(updaterCreateStatus.code).to.equal(202); + const { status: bobUpdaterSendStatus } = await updaterRecord.send(bobDid.did); + expect(bobUpdaterSendStatus.code).to.equal(202); + const { status: aliceUpdaterSendStatus } = await updaterRecord.send(aliceDid.did); + expect(aliceUpdaterSendStatus.code).to.equal(202); + // Alice creates a photo using her participant role const { status: photoCreateStatus, record: photoRecord} = await dwnAlice.records.create({ data : 'test', @@ -403,6 +423,7 @@ describe('DwnApi', () => { dateCreated : photoRecord.dateCreated, protocol : photosProtocolDefinition.protocol, protocolPath : 'album/photo', + protocolRole : 'album/updater', schema : photosProtocolDefinition.types.photo.schema, dataFormat : 'text/plain' } diff --git a/packages/api/tests/fixtures/protocol-definitions/photos.json b/packages/api/tests/fixtures/protocol-definitions/photos.json index 4430ae2de..1bf1db06a 100644 --- a/packages/api/tests/fixtures/protocol-definitions/photos.json +++ b/packages/api/tests/fixtures/protocol-definitions/photos.json @@ -17,6 +17,10 @@ "participant": { "schema": "http://photo-protocol.xyz/schema/participant", "dataFormats": ["text/plain"] + }, + "updater": { + "schema": "http://photo-protocol.xyz/schema/updater", + "dataFormats": ["text/plain"] } }, "structure": { @@ -40,6 +44,15 @@ } ] }, + "updater": { + "$contextRole": true, + "$actions": [ + { + "role": "album/participant", + "can": "write" + } + ] + }, "photo": { "$actions": [ { @@ -47,14 +60,13 @@ "can": "write" }, { - "who": "author", - "of": "album", - "can": "write" + "role": "album/updater", + "can": "update" }, { "who": "author", "of": "album", - "can": "update" + "can": "write" } ] } From a2742c2adb0ee966d71373544e0c1149c484d7da Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 12:16:17 -0600 Subject: [PATCH 15/35] Remove only() from test --- packages/api/tests/dwn-api.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 41e2b38e7..78881b152 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -263,7 +263,7 @@ describe('DwnApi', () => { expect(await result.record?.data.json()).to.deep.equal(dataJson); }); - it.only('creates a role record for another user that they can use to create role-based records', async () => { + it('creates a role record for another user that they can use to create role-based records', async () => { /** * WHAT IS BEING TESTED? * From 443c615b2c9fc8d5aaddd767dca5bb97abde239a Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 30 Jan 2024 13:20:28 -0500 Subject: [PATCH 16/35] method dyping more consistant --- packages/api/src/record.ts | 46 ++++++++++++++++-------------- packages/api/tests/dwn-api.spec.ts | 4 +-- packages/api/tests/record.spec.ts | 4 +-- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index e9b8648b5..0ab7c3add 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -1,4 +1,4 @@ -import type { Web5Agent } from '@web5/agent'; +import type { ProcessDwnRequest, Web5Agent } from '@web5/agent'; import type { Readable } from '@web5/common'; import type { RecordsWriteMessage, @@ -12,6 +12,14 @@ import { DwnInterfaceName, DwnMethodName } from '@tbd54566975/dwn-sdk-js'; import type { ResponseStatus } from './dwn-api.js'; import { dataToBlob } from './utils.js'; +export type ProcessRecordRequest = { + dataStream?: Blob | ReadableStream | Readable; + rawMessage?: Partial; + messageOptions?: unknown; + store: boolean; + import: boolean; +}; + /** * Options that are passed to Record constructor. * @@ -350,17 +358,14 @@ export class Record implements RecordModel { return dataObj; } - private async _processMessage(options: any){ - - const { store = true } = options; - - const request = { + private _prepareMessage(options: ProcessRecordRequest): ProcessDwnRequest { + const request: ProcessDwnRequest = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, author : this._connectedDid, target : this._connectedDid, - import : !!options.import, - store - } as any; + import : options.import, + store : options.store, + }; if (options.rawMessage) { removeUndefinedProperties(options.rawMessage); @@ -374,19 +379,17 @@ export class Record implements RecordModel { request.dataStream = options.dataStream; } - return this._agent.processDwnRequest(request); + return request; } - async _processRecord(options: { store: boolean, import: boolean }): Promise { + private async _processRecord(options: { store: boolean, import: boolean }): Promise { const { store = true, import: _import = false } = options; - const initialWrite = this._initialWrite; // Is there an initial write? Have we already stored this record? if (initialWrite && !this._initialWriteStored) { - - let agentResponse = await this._processMessage({ + const requestOptions = this._prepareMessage({ import : _import, store : store, rawMessage : { @@ -395,6 +398,7 @@ export class Record implements RecordModel { } }); + const agentResponse = await this._agent.processDwnRequest(requestOptions); const { message, reply: { status } } = agentResponse; const responseMessage = message as RecordsWriteMessage; @@ -403,10 +407,9 @@ export class Record implements RecordModel { this._initialWriteStored = true; if (_import) initialWrite.authorization = responseMessage.authorization; } - } - let agentResponse = await this._processMessage({ + const requestOptions = this._prepareMessage({ import : !initialWrite && _import, store : store, dataStream : await this.data.blob(), @@ -420,6 +423,7 @@ export class Record implements RecordModel { } }); + const agentResponse = await this._agent.processDwnRequest(requestOptions); const { message, reply: { status } } = agentResponse; const responseMessage = message as RecordsWriteMessage; @@ -427,16 +431,16 @@ export class Record implements RecordModel { if (_import) this._authorization = responseMessage.authorization; } - return status; - + return { status }; } - async store(options?: { import: boolean }): Promise { + async store(options?: { import: boolean }): Promise { + // process the record and always set store to true return this._processRecord({ ...options, store: true }); } - async import(options?: { store: boolean }): Promise { - // Add check to bail if already imported + async import(options?: { store: boolean }): Promise { + // process the record and always set import to true, only skip storage if explicitly set to false return this._processRecord({ store: options?.store !== false, import: true }); } diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 78881b152..58bfbe631 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -339,7 +339,7 @@ describe('DwnApi', () => { }); expect(aliceAlbumReadResult.status.code).to.equal(200); expect(aliceAlbumReadResult.record).to.exist; - const aliceAlbumReadStoreStatus = await aliceAlbumReadResult.record.store(); + const { status: aliceAlbumReadStoreStatus } = await aliceAlbumReadResult.record.store(); expect(aliceAlbumReadStoreStatus.code).to.equal(202); // Bob makes Alice a `participant` @@ -371,7 +371,7 @@ describe('DwnApi', () => { }); expect(aliceParticipantReadResult.status.code).to.equal(200); expect(aliceParticipantReadResult.record).to.exist; - const aliceParticipantReadStoreStatus = await aliceParticipantReadResult.record.store(); + const { status: aliceParticipantReadStoreStatus } = await aliceParticipantReadResult.record.store(); expect(aliceParticipantReadStoreStatus.code).to.equal(202); // Alice makes Bob an `updater` diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index e54e4ec89..120a7dc23 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -154,7 +154,7 @@ describe('Record', () => { }); expect(queryRecordStatus.code).to.equal(200); const importRecord = queryRecords[0]; - const importRecordStatus = await importRecord.import(); + const { status: importRecordStatus } = await importRecord.import(); expect(importRecordStatus.code).to.equal(202); const { status: importSendStatus } = await importRecord!.send(); @@ -196,7 +196,7 @@ describe('Record', () => { expect(updatedRecordsStatus.code).to.equal(200); const updatedRecord = updatedRecords[0]; - const updatedRecordStoredStatus = await updatedRecord.store(); + const { status: updatedRecordStoredStatus } = await updatedRecord.store(); expect(updatedRecordStoredStatus.code).to.equal(202); expect(await updatedRecord.data.text() === updatedText).to.equal(true); From fb66a7b4ef861ffc1e617530bbc2b8fc1033fd14 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 13:39:23 -0600 Subject: [PATCH 17/35] Added comments and removed TODO --- packages/api/src/record.ts | 4 +++- packages/api/tests/record.spec.ts | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 0ab7c3add..571aa1ace 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -112,9 +112,11 @@ export class Record implements RecordModel { private _protocolRole: RecordOptions['protocolRole']; // Getters for immutable DWN Record properties. + // Cache to minimize the amount of redundant two-phase commits we do in store() and send() + // Retains awareness of the last 100 records stored/sent for up to 100 target DIDs each. static sendCache = new Map(); static sendCacheLimit = 100; - static setSendCache(recordId, target){ + static setSendCache(recordId, target) { const recordCache = Record.sendCache; let targetCache = recordCache.get(recordId) || new Set(); recordCache.delete(recordId); diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index 120a7dc23..c6b443ea5 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -1464,7 +1464,6 @@ describe('Record', () => { expect(sendResult.status.code).to.equal(202); }); - // TODO: Fix after changes are made to dwn-sdk-js to include the initial write in every query/read response. it('automatically sends the initial write and update of a record to a remote DWN', async () => { // Alice writes a message to her agent connected DWN. const { status, record } = await dwnAlice.records.write({ From 54a12cac6b37bfa4896808bf566bec29bc609ed7 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 16:04:14 -0600 Subject: [PATCH 18/35] Add comment about send()'s new features --- packages/api/src/record.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 571aa1ace..83193d6b0 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -450,6 +450,7 @@ export class Record implements RecordModel { /** * Send the current record to a remote DWN by specifying their DID * If no DID is specified, the target is assumed to be the owner (connectedDID). + * If an initial write is present and the Record class send cache has no awareness of it, the initial write is sent first * (vs waiting for the regular DWN sync) * @param target - the optional DID to send the record to, if none is set it is sent to the connectedDid * @returns the status of the send record request From 380085caadd1c1b389cc18fcd686351f76bc7912 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 30 Jan 2024 15:53:32 -0500 Subject: [PATCH 19/35] clean up types, use web5/common removeUndefinedProperties --- packages/agent/src/dwn-manager.ts | 7 ++-- packages/api/src/record.ts | 65 ++++++++++++------------------- 2 files changed, 28 insertions(+), 44 deletions(-) diff --git a/packages/agent/src/dwn-manager.ts b/packages/agent/src/dwn-manager.ts index 7d29f2150..87a8d72e9 100644 --- a/packages/agent/src/dwn-manager.ts +++ b/packages/agent/src/dwn-manager.ts @@ -242,11 +242,10 @@ export class DwnManager { } private async constructDwnMessage(options: { - request : ProcessDwnRequest + request: ProcessDwnRequest }) { const { request } = options; - const rawMessage = (request as any).rawMessage; - + const rawMessage = request.rawMessage as any; let readableStream: Readable | undefined; // TODO: Consider refactoring to move data transformations imposed by fetch() limitations to the HTTP transport-related methods. @@ -289,7 +288,7 @@ export class DwnManager { } } - return { message: (dwnMessage as any).message, dataStream: readableStream }; + return { message: dwnMessage.message, dataStream: readableStream }; } private async getAuthorSigningKeyId(options: { diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 83193d6b0..8953131ab 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -1,4 +1,4 @@ -import type { ProcessDwnRequest, Web5Agent } from '@web5/agent'; +import type { ProcessDwnRequest, SendDwnRequest, Web5Agent } from '@web5/agent'; import type { Readable } from '@web5/common'; import type { RecordsWriteMessage, @@ -6,13 +6,13 @@ import type { RecordsWriteDescriptor, } from '@tbd54566975/dwn-sdk-js'; -import { Convert, NodeStream, Stream } from '@web5/common'; +import { Convert, NodeStream, removeUndefinedProperties, Stream } from '@web5/common'; import { DwnInterfaceName, DwnMethodName } from '@tbd54566975/dwn-sdk-js'; import type { ResponseStatus } from './dwn-api.js'; import { dataToBlob } from './utils.js'; -export type ProcessRecordRequest = { +type ProcessRecordRequest = { dataStream?: Blob | ReadableStream | Readable; rawMessage?: Partial; messageOptions?: unknown; @@ -77,20 +77,6 @@ export type RecordUpdateOptions = { * * @beta */ - -function removeUndefinedProperties(obj: any): any { - if (typeof obj !== 'object' || obj === null) return; - Object.keys(obj).forEach(key => { - const val = obj[key]; - if (val === undefined) { - delete obj[key]; - } else if (typeof val === 'object') { - removeUndefinedProperties(val); - } - }); - return obj; -} - export class Record implements RecordModel { // Record instance metadata. private _agent: Web5Agent; @@ -459,54 +445,51 @@ export class Record implements RecordModel { * @beta */ async send(target?: string): Promise { - const initialWrite = this._initialWrite; - if (!target) { - target = this._connectedDid; - } + target??= this._connectedDid; if (initialWrite && !Record.checkSendCache(this._recordId, target)){ + const rawMessage = { + contextId: this._contextId, + ...initialWrite + }; + removeUndefinedProperties(rawMessage); - const initialState = { + const initialState: SendDwnRequest = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, author : this._connectedDid, target : target, - rawMessage : removeUndefinedProperties({ - contextId: this._contextId, - ...initialWrite - }) - } as any; - + rawMessage + }; await this._agent.sendDwnRequest(initialState); Record.setSendCache(this._recordId, target); - } - const latestState = { + const latestState: SendDwnRequest = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, author : this._connectedDid, dataStream : await this.data.blob(), target : target - } as any; + }; if (this._authorization) { - latestState.rawMessage = removeUndefinedProperties({ + const rawMessage = { contextId : this._contextId, recordId : this._recordId, descriptor : this._descriptor, attestation : this._attestation, authorization : this._authorization, encryption : this._encryption, - }); - } - else { + }; + removeUndefinedProperties(rawMessage); + latestState.rawMessage = rawMessage; + } else { latestState.messageOptions = this.toJSON(); } - const { reply: { status } } = await this._agent.sendDwnRequest(latestState); - - return { status }; + const { reply } = await this._agent.sendDwnRequest(latestState); + return reply; } /** @@ -623,14 +606,16 @@ export class Record implements RecordModel { if (200 <= status.code && status.code <= 299) { if (!this._initialWrite) { - this._initialWrite = JSON.parse(JSON.stringify(removeUndefinedProperties({ + const initialWrite: RecordsWriteMessage = { contextId : this._contextId, recordId : this._recordId, descriptor : this._descriptor, attestation : this._attestation, authorization : this._authorization, encryption : this._encryption, - }))); + }; + removeUndefinedProperties(initialWrite); + this._initialWrite = JSON.parse(JSON.stringify(initialWrite)); } // Only update the local Record instance mutable properties if the record was successfully (over)written. this._authorization = responseMessage.authorization; From 9f52b833aec9aa5b8c48400b3ca715f92e57ce87 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 16:05:40 -0600 Subject: [PATCH 20/35] Delete unused test --- packages/api/tests/record.spec.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index c6b443ea5..fbae234dc 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -204,18 +204,6 @@ describe('Record', () => { const { status: updatedRecordToSelfStatus } = await updatedRecord!.send(); expect(updatedRecordToSelfStatus.code).to.equal(202); - // Confirm Bob can query his own remote DWN for the created record. - // const bobQueryResult = await dwnBob.records.query({ - // from : bobDid.did, - // message : { - // filter: { - // schema: 'http://email-protocol.xyz/schema/email' - // } - // } - // }); - // expect(bobQueryResult.status.code).to.equal(200); - // expect(bobQueryResult.records).to.exist; - // expect(bobQueryResult.records!.length).to.equal(1); }); it('should retain all defined properties', async () => { From 8ce30acf6a4df52a0816af5604bf5aa6e5c2e21b Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 16:10:44 -0600 Subject: [PATCH 21/35] Add comments to store/import and other functions added to Record class --- packages/api/src/record.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 8953131ab..597593d29 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -370,6 +370,8 @@ export class Record implements RecordModel { return request; } + // Handles the various conditions around there being an initial write, whether to store initial/current state, + // and whether to add an owner signature to the initial write to enable storage when protocol rules require it. private async _processRecord(options: { store: boolean, import: boolean }): Promise { const { store = true, import: _import = false } = options; @@ -422,11 +424,13 @@ export class Record implements RecordModel { return { status }; } + // Uses _processRecord to manifest the storage-centric features of committing a foreign record to the local DWN async store(options?: { import: boolean }): Promise { // process the record and always set store to true return this._processRecord({ ...options, store: true }); } + // Uses _processRecord to manifest the import-centric features of ingesting and signing a foreign record async import(options?: { store: boolean }): Promise { // process the record and always set import to true, only skip storage if explicitly set to false return this._processRecord({ store: options?.store !== false, import: true }); From 8b28ac31915bacf00c399eba292b0263cd62e578 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 16:21:00 -0600 Subject: [PATCH 22/35] add code comments to make Frank smile --- packages/api/src/record.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 597593d29..31a785f3a 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -377,8 +377,9 @@ export class Record implements RecordModel { const { store = true, import: _import = false } = options; const initialWrite = this._initialWrite; - // Is there an initial write? Have we already stored this record? + // Is there an initial write? Have we already stored it? if (initialWrite && !this._initialWriteStored) { + // There is an initial write, and we have not dealt with it before, so prepare to do so const requestOptions = this._prepareMessage({ import : _import, store : store, @@ -388,6 +389,7 @@ export class Record implements RecordModel { } }); + // Process the prepared initial write, with the options set for storing and/or importing with an owner sig. const agentResponse = await this._agent.processDwnRequest(requestOptions); const { message, reply: { status } } = agentResponse; const responseMessage = message as RecordsWriteMessage; @@ -400,7 +402,7 @@ export class Record implements RecordModel { } const requestOptions = this._prepareMessage({ - import : !initialWrite && _import, + import : !initialWrite && _import, // if there is no initial write, this is the initial write (or having a forced import sig), so sign it. store : store, dataStream : await this.data.blob(), rawMessage : { @@ -418,6 +420,7 @@ export class Record implements RecordModel { const responseMessage = message as RecordsWriteMessage; if (200 <= status.code && status.code <= 299) { + // If we are importing, make sure to update the current record state's authorization, because now it will have the owner's signature on it. if (_import) this._authorization = responseMessage.authorization; } @@ -452,7 +455,9 @@ export class Record implements RecordModel { const initialWrite = this._initialWrite; target??= this._connectedDid; + // Is there an initial write? Do we know if we've already sent it to this target? if (initialWrite && !Record.checkSendCache(this._recordId, target)){ + // We do have an initial write, so prepare it for sending to the target. const rawMessage = { contextId: this._contextId, ...initialWrite @@ -467,9 +472,11 @@ export class Record implements RecordModel { }; await this._agent.sendDwnRequest(initialState); + // Set the cache to maintain awareness that we don't need to send the initial write next time. Record.setSendCache(this._recordId, target); } + // Prepare the current state for sending to the target const latestState: SendDwnRequest = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, author : this._connectedDid, @@ -477,6 +484,7 @@ export class Record implements RecordModel { target : target }; + // if there is already an authz payload, just pass along the record if (this._authorization) { const rawMessage = { contextId : this._contextId, @@ -489,6 +497,7 @@ export class Record implements RecordModel { removeUndefinedProperties(rawMessage); latestState.rawMessage = rawMessage; } else { + // if there is no authz, pass options so the DWN SDK can construct and sign the record latestState.messageOptions = this.toJSON(); } From 5122571f457dcd93d4fc13cb3553531ffee681a8 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 09:08:45 -0500 Subject: [PATCH 23/35] Some more clean up for import/store (#391) * simplification of records class * rename record options to reflect signing/storing --- packages/agent/src/dwn-manager.ts | 2 +- packages/agent/src/types/agent.ts | 2 +- packages/api/src/record.ts | 274 ++++++++++++++---------------- 3 files changed, 130 insertions(+), 148 deletions(-) diff --git a/packages/agent/src/dwn-manager.ts b/packages/agent/src/dwn-manager.ts index 87a8d72e9..c10c3053b 100644 --- a/packages/agent/src/dwn-manager.ts +++ b/packages/agent/src/dwn-manager.ts @@ -283,7 +283,7 @@ export class DwnManager { }); if (dwnMessageConstructor === RecordsWrite){ - if (request.import) { + if (request.signAsOwner) { await (dwnMessage as RecordsWrite).signAsOwner(dwnSigner); } } diff --git a/packages/agent/src/types/agent.ts b/packages/agent/src/types/agent.ts index f10a07b93..69520063d 100644 --- a/packages/agent/src/types/agent.ts +++ b/packages/agent/src/types/agent.ts @@ -78,7 +78,7 @@ export type ProcessDwnRequest = DwnRequest & { rawMessage?: unknown; messageOptions?: unknown; store?: boolean; - import?: boolean; + signAsOwner?: boolean; }; export type SendDwnRequest = DwnRequest & (ProcessDwnRequest | { messageCid: string }) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 31a785f3a..0f176564d 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -12,13 +12,31 @@ import { DwnInterfaceName, DwnMethodName } from '@tbd54566975/dwn-sdk-js'; import type { ResponseStatus } from './dwn-api.js'; import { dataToBlob } from './utils.js'; -type ProcessRecordRequest = { - dataStream?: Blob | ReadableStream | Readable; - rawMessage?: Partial; - messageOptions?: unknown; - store: boolean; - import: boolean; -}; +class SendCache { + private static cache = new Map(); + static sendCacheLimit = 100; + + static set(id: string, target: string) { + let targetCache = SendCache.cache.get(id) || new Set(); + SendCache.cache.delete(id); + SendCache.cache.set(id, targetCache); + if (this.cache.size > SendCache.sendCacheLimit) { + const firstRecord = SendCache.cache.keys().next().value; + SendCache.cache.delete(firstRecord); + } + targetCache.delete(target); + targetCache.add(target); + if (targetCache.size > SendCache.sendCacheLimit) { + const firstTarget = targetCache.keys().next().value; + targetCache.delete(firstTarget); + } + } + + static check(id: string, target: string){ + let targetCache = SendCache.cache.get(id); + return target && targetCache ? targetCache.has(target) : targetCache; + } +} /** * Options that are passed to Record constructor. @@ -78,6 +96,10 @@ export type RecordUpdateOptions = { * @beta */ export class Record implements RecordModel { + // Cache to minimize the amount of redundant two-phase commits we do in store() and send() + // Retains awareness of the last 100 records stored/sent for up to 100 target DIDs each. + private static _sendCache = SendCache; + // Record instance metadata. private _agent: Web5Agent; private _connectedDid: string; @@ -93,36 +115,11 @@ export class Record implements RecordModel { private _descriptor: RecordsWriteDescriptor; private _encryption?: RecordsWriteMessage['encryption']; private _initialWrite: RecordOptions['initialWrite']; - private _initialWriteStored: boolean; + private _initialWriteProcessed: boolean; private _recordId: string; private _protocolRole: RecordOptions['protocolRole']; // Getters for immutable DWN Record properties. - // Cache to minimize the amount of redundant two-phase commits we do in store() and send() - // Retains awareness of the last 100 records stored/sent for up to 100 target DIDs each. - static sendCache = new Map(); - static sendCacheLimit = 100; - static setSendCache(recordId, target) { - const recordCache = Record.sendCache; - let targetCache = recordCache.get(recordId) || new Set(); - recordCache.delete(recordId); - recordCache.set(recordId, targetCache); - if (recordCache.size > Record.sendCacheLimit) { - const firstRecord = recordCache.keys().next().value; - recordCache.delete(firstRecord); - } - targetCache.delete(target); - targetCache.add(target); - if (targetCache.size > Record.sendCacheLimit) { - const firstTarget = targetCache.keys().next().value; - targetCache.delete(firstTarget); - } - } - static checkSendCache(recordId, target){ - let targetCache = Record.sendCache.get(recordId); - return target && targetCache ? targetCache.has(target) : targetCache; - } - /** Record's signatures attestation */ get attestation(): RecordsWriteMessage['attestation'] { return this._attestation; } @@ -193,6 +190,23 @@ export class Record implements RecordModel { /** Record's published status (true/false) */ get published() { return this._descriptor.published; } + /** + * Returns a copy of the raw `RecordsWriteMessage` that was used to create the current `Record` instance. + */ + private get rawMessage(): RecordsWriteMessage { + const message = JSON.parse(JSON.stringify({ + contextId : this._contextId, + recordId : this._recordId, + descriptor : this._descriptor, + attestation : this._attestation, + authorization : this._authorization, + encryption : this._encryption, + })); + + removeUndefinedProperties(message); + return message; + } + constructor(agent: Web5Agent, options: RecordOptions) { this._agent = agent; @@ -346,100 +360,32 @@ export class Record implements RecordModel { return dataObj; } - private _prepareMessage(options: ProcessRecordRequest): ProcessDwnRequest { - const request: ProcessDwnRequest = { - messageType : DwnInterfaceName.Records + DwnMethodName.Write, - author : this._connectedDid, - target : this._connectedDid, - import : options.import, - store : options.store, - }; - - if (options.rawMessage) { - removeUndefinedProperties(options.rawMessage); - request.rawMessage = options.rawMessage as Partial; - } - else { - request.messageOptions = options.messageOptions; - } - - if (options.dataStream) { - request.dataStream = options.dataStream; - } - - return request; - } - - // Handles the various conditions around there being an initial write, whether to store initial/current state, - // and whether to add an owner signature to the initial write to enable storage when protocol rules require it. - private async _processRecord(options: { store: boolean, import: boolean }): Promise { - - const { store = true, import: _import = false } = options; - const initialWrite = this._initialWrite; - - // Is there an initial write? Have we already stored it? - if (initialWrite && !this._initialWriteStored) { - // There is an initial write, and we have not dealt with it before, so prepare to do so - const requestOptions = this._prepareMessage({ - import : _import, - store : store, - rawMessage : { - contextId: this._contextId, - ...initialWrite - } - }); - - // Process the prepared initial write, with the options set for storing and/or importing with an owner sig. - const agentResponse = await this._agent.processDwnRequest(requestOptions); - const { message, reply: { status } } = agentResponse; - const responseMessage = message as RecordsWriteMessage; - - // If we are importing, make sure to update the initial write's authorization, because now it will have the owner's signature on it - if (200 <= status.code && status.code <= 299) { - this._initialWriteStored = true; - if (_import) initialWrite.authorization = responseMessage.authorization; - } - } - - const requestOptions = this._prepareMessage({ - import : !initialWrite && _import, // if there is no initial write, this is the initial write (or having a forced import sig), so sign it. - store : store, - dataStream : await this.data.blob(), - rawMessage : { - contextId : this._contextId, - recordId : this._recordId, - descriptor : this._descriptor, - attestation : this._attestation, - authorization : this._authorization, - encryption : this._encryption, - } - }); - - const agentResponse = await this._agent.processDwnRequest(requestOptions); - const { message, reply: { status } } = agentResponse; - const responseMessage = message as RecordsWriteMessage; - - if (200 <= status.code && status.code <= 299) { - // If we are importing, make sure to update the current record state's authorization, because now it will have the owner's signature on it. - if (_import) this._authorization = responseMessage.authorization; - } - - return { status }; - } - - // Uses _processRecord to manifest the storage-centric features of committing a foreign record to the local DWN - async store(options?: { import: boolean }): Promise { - // process the record and always set store to true - return this._processRecord({ ...options, store: true }); + /** + * Stores the current record state as well as any initial write to the owner's DWN. + * + * @param importRecord - if true, the record will signed by the owner before storing it to the owner's DWN. Defaults to true. + * @returns the status of the store request + * + * @beta + */ + async store(importRecord: boolean = true): Promise { + // if we are importing the record we sign it as the owner + return this.processRecord({ signAsOwner: importRecord, store: true }); } - // Uses _processRecord to manifest the import-centric features of ingesting and signing a foreign record - async import(options?: { store: boolean }): Promise { - // process the record and always set import to true, only skip storage if explicitly set to false - return this._processRecord({ store: options?.store !== false, import: true }); + /** + * Signs the current record state as well as any initial write and optionally stores it to the owner's DWN. + * This is useful when importing a record that was signed by someone else int your own DWN. + * + * @param store - if true, the record will be stored to the owner's DWN after signing. Defaults to true. + * @returns the status of the import request + * + * @beta + */ + async import(store: boolean = true): Promise { + return this.processRecord({ store, signAsOwner: true }); } - /** * Send the current record to a remote DWN by specifying their DID * If no DID is specified, the target is assumed to be the owner (connectedDID). @@ -456,10 +402,9 @@ export class Record implements RecordModel { target??= this._connectedDid; // Is there an initial write? Do we know if we've already sent it to this target? - if (initialWrite && !Record.checkSendCache(this._recordId, target)){ + if (initialWrite && !Record._sendCache.check(this._recordId, target)){ // We do have an initial write, so prepare it for sending to the target. const rawMessage = { - contextId: this._contextId, ...initialWrite }; removeUndefinedProperties(rawMessage); @@ -473,7 +418,7 @@ export class Record implements RecordModel { await this._agent.sendDwnRequest(initialState); // Set the cache to maintain awareness that we don't need to send the initial write next time. - Record.setSendCache(this._recordId, target); + Record._sendCache.set(this._recordId, target); } // Prepare the current state for sending to the target @@ -486,16 +431,7 @@ export class Record implements RecordModel { // if there is already an authz payload, just pass along the record if (this._authorization) { - const rawMessage = { - contextId : this._contextId, - recordId : this._recordId, - descriptor : this._descriptor, - attestation : this._attestation, - authorization : this._authorization, - encryption : this._encryption, - }; - removeUndefinedProperties(rawMessage); - latestState.rawMessage = rawMessage; + latestState.rawMessage = { ...this.rawMessage }; } else { // if there is no authz, pass options so the DWN SDK can construct and sign the record latestState.messageOptions = this.toJSON(); @@ -618,24 +554,18 @@ export class Record implements RecordModel { const responseMessage = message as RecordsWriteMessage; if (200 <= status.code && status.code <= 299) { + // copy the original raw message to the initial write before we update the values. if (!this._initialWrite) { - const initialWrite: RecordsWriteMessage = { - contextId : this._contextId, - recordId : this._recordId, - descriptor : this._descriptor, - attestation : this._attestation, - authorization : this._authorization, - encryption : this._encryption, - }; - removeUndefinedProperties(initialWrite); - this._initialWrite = JSON.parse(JSON.stringify(initialWrite)); + this._initialWrite = { ...this.rawMessage }; } + // Only update the local Record instance mutable properties if the record was successfully (over)written. this._authorization = responseMessage.authorization; this._protocolRole = messageOptions.protocolRole; mutableDescriptorProperties.forEach(property => { this._descriptor[property] = responseMessage.descriptor[property]; }); + // Cache data. if (options.data !== undefined) { this._encodedData = dataBlob; @@ -645,6 +575,58 @@ export class Record implements RecordModel { return { status }; } + // Handles the various conditions around there being an initial write, whether to store initial/current state, + // and whether to add an owner signature to the initial write to enable storage when protocol rules require it. + private async processRecord(options: { store: boolean, signAsOwner: boolean }): Promise { + const { store, signAsOwner } = options; + + // if there is an initial write and we haven't already processed it, we first process it and marked it as such. + if (this._initialWrite && !this._initialWriteProcessed) { + const initialWriteRequest: ProcessDwnRequest = { + messageType : DwnInterfaceName.Records + DwnMethodName.Write, + rawMessage : this.initialWrite, + author : this._connectedDid, + target : this._connectedDid, + signAsOwner : signAsOwner, + store, + }; + + // Process the prepared initial write, with the options set for storing and/or signing as the owner. + const agentResponse = await this._agent.processDwnRequest(initialWriteRequest); + const { message, reply: { status } } = agentResponse; + const responseMessage = message as RecordsWriteMessage; + + // If we are signing as owner, make sure to update the initial write's authorization, because now it will have the owner's signature on it + // set it to processed so that we don't repeat this process again + if (200 <= status.code && status.code <= 299) { + this._initialWriteProcessed = true; + if (signAsOwner) this.initialWrite.authorization = responseMessage.authorization; + } + } + + // Now that we've processed a potential initial write, we can process the current record state. + const requestOptions: ProcessDwnRequest = { + messageType : DwnInterfaceName.Records + DwnMethodName.Write, + rawMessage : this.rawMessage, + author : this._connectedDid, + target : this._connectedDid, + dataStream : await this.data.blob(), + signAsOwner : !this.initialWrite && signAsOwner, // we only need to sign this record if it is the initial write and is marked for signing + store, + }; + + const agentResponse = await this._agent.processDwnRequest(requestOptions); + const { message, reply: { status } } = agentResponse; + const responseMessage = message as RecordsWriteMessage; + + if (200 <= status.code && status.code <= 299) { + // If we are signing as the owner, make sure to update the current record state's authorization, because now it will have the owner's signature on it. + if (signAsOwner) this._authorization = responseMessage.authorization; + } + + return { status }; + } + /** * Fetches the record's data from the specified DWN. * From 9e4af87a578127179c79874aadf0be3ce3edbf95 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 13:47:23 -0500 Subject: [PATCH 24/35] update test cases and coverage --- packages/api/src/record.ts | 14 +-- packages/api/tests/record.spec.ts | 199 ++++++++++++++++++++++++++---- 2 files changed, 183 insertions(+), 30 deletions(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 0f176564d..92c6d6ebf 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -12,11 +12,11 @@ import { DwnInterfaceName, DwnMethodName } from '@tbd54566975/dwn-sdk-js'; import type { ResponseStatus } from './dwn-api.js'; import { dataToBlob } from './utils.js'; -class SendCache { - private static cache = new Map(); +export class SendCache { + private static cache = new Map>(); static sendCacheLimit = 100; - static set(id: string, target: string) { + static set(id: string, target: string): void { let targetCache = SendCache.cache.get(id) || new Set(); SendCache.cache.delete(id); SendCache.cache.set(id, targetCache); @@ -32,9 +32,9 @@ class SendCache { } } - static check(id: string, target: string){ + static check(id: string, target: string): boolean { let targetCache = SendCache.cache.get(id); - return target && targetCache ? targetCache.has(target) : targetCache; + return targetCache ? targetCache.has(target) : false; } } @@ -587,7 +587,7 @@ export class Record implements RecordModel { rawMessage : this.initialWrite, author : this._connectedDid, target : this._connectedDid, - signAsOwner : signAsOwner, + signAsOwner, store, }; @@ -611,7 +611,7 @@ export class Record implements RecordModel { author : this._connectedDid, target : this._connectedDid, dataStream : await this.data.blob(), - signAsOwner : !this.initialWrite && signAsOwner, // we only need to sign this record if it is the initial write and is marked for signing + signAsOwner, store, }; diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index fbae234dc..2c9d1f475 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -22,7 +22,7 @@ import { KeyDerivationScheme, } from '@tbd54566975/dwn-sdk-js'; -import { Record } from '../src/record.js'; +import { Record, SendCache } from '../src/record.js'; import { DwnApi } from '../src/dwn-api.js'; import { dataToBlob } from '../src/utils.js'; import { testDwnUrl } from './utils/test-config.js'; @@ -97,8 +97,6 @@ describe('Record', () => { await testAgent.closeStorage(); }); - // FIRST PASS AT IMPORT - it('imports a record that another user wrote', async () => { // Install the email protocol for Alice's local DWN. @@ -128,7 +126,7 @@ describe('Record', () => { const { status: bobPushStatus } = await bobProtocol!.send(bobDid.did); expect(bobPushStatus.code).to.equal(202); - // Alice creates a new large record and stores it + // Alice creates a new large record and stores it on her own dwn const { status: aliceEmailStatus, record: aliceEmailRecord } = await dwnAlice.records.write({ data : TestDataGenerator.randomString(DwnConstant.maxDataSizeAllowedToBeEncoded + 1000), message : { @@ -142,8 +140,22 @@ describe('Record', () => { const { status: sendStatus } = await aliceEmailRecord!.send(aliceDid.did); expect(sendStatus.code).to.equal(202); - // Alice queries for the record that was just created on her remote DWN. - const { records: queryRecords, status: queryRecordStatus } = await dwnBob.records.query({ + + // Bob queries for the record on his own DWN (should not find it) + let bobQueryBobDwn = await dwnBob.records.query({ + from : bobDid.did, + message : { + filter: { + protocol : emailProtocolDefinition.protocol, + protocolPath : 'thread', + } + } + }); + expect(bobQueryBobDwn.status.code).to.equal(200); + expect(bobQueryBobDwn.records.length).to.equal(0); // no results + + // Bob queries for the record that was just created on Alice's remote DWN. + let bobQueryAliceDwn = await dwnBob.records.query({ from : aliceDid.did, message : { filter: { @@ -152,14 +164,32 @@ describe('Record', () => { } } }); - expect(queryRecordStatus.code).to.equal(200); - const importRecord = queryRecords[0]; + expect(bobQueryAliceDwn.status.code).to.equal(200); + expect(bobQueryAliceDwn.records.length).to.equal(1); + + // bob imports the record + const importRecord = bobQueryAliceDwn.records[0]; const { status: importRecordStatus } = await importRecord.import(); expect(importRecordStatus.code).to.equal(202); + // bob sends the record to his remote dwn const { status: importSendStatus } = await importRecord!.send(); expect(importSendStatus.code).to.equal(202); + // Bob queries for the record on his own DWN (should now return it) + bobQueryBobDwn = await dwnBob.records.query({ + from : bobDid.did, + message : { + filter: { + protocol : emailProtocolDefinition.protocol, + protocolPath : 'thread', + } + } + }); + expect(bobQueryBobDwn.status.code).to.equal(200); + expect(bobQueryBobDwn.records.length).to.equal(1); + expect(bobQueryBobDwn.records[0].id).to.equal(importRecord.id); + // Alice updates her record let { status: aliceEmailStatusUpdated } = await aliceEmailRecord.update({ data: TestDataGenerator.randomString(DwnConstant.maxDataSizeAllowedToBeEncoded + 1000) @@ -172,19 +202,17 @@ describe('Record', () => { const { status: sentToBobStatus } = await aliceEmailRecord!.send(bobDid.did); expect(sentToBobStatus.code).to.equal(202); - // Alice updates her record - + // Alice updates her record and sends it to her own DWN again const updatedText = TestDataGenerator.randomString(DwnConstant.maxDataSizeAllowedToBeEncoded + 1000); let { status: aliceEmailStatusUpdatedAgain } = await aliceEmailRecord.update({ data: updatedText }); expect(aliceEmailStatusUpdatedAgain.code).to.equal(202); - - // Sends it to her own remote DWN again const { status: sentToSelfAgainStatus } = await aliceEmailRecord!.send(); expect(sentToSelfAgainStatus.code).to.equal(202); - const { records: updatedRecords, status: updatedRecordsStatus } = await dwnBob.records.query({ + // Bob queries for the updated record on alice's DWN + bobQueryAliceDwn = await dwnBob.records.query({ from : aliceDid.did, message : { filter: { @@ -193,17 +221,33 @@ describe('Record', () => { } } }); - expect(updatedRecordsStatus.code).to.equal(200); + expect(bobQueryAliceDwn.status.code).to.equal(200); + expect(bobQueryAliceDwn.records.length).to.equal(1); + const updatedRecord = bobQueryAliceDwn.records[0]; - const updatedRecord = updatedRecords[0]; + // stores the record on his own DWN const { status: updatedRecordStoredStatus } = await updatedRecord.store(); expect(updatedRecordStoredStatus.code).to.equal(202); + expect(await updatedRecord.data.text()).to.equal(updatedText); - expect(await updatedRecord.data.text() === updatedText).to.equal(true); - + // sends the record to his own DWN const { status: updatedRecordToSelfStatus } = await updatedRecord!.send(); expect(updatedRecordToSelfStatus.code).to.equal(202); + // Bob queries for the updated record on his own DWN + bobQueryBobDwn = await dwnBob.records.query({ + from : bobDid.did, + message : { + filter: { + protocol : emailProtocolDefinition.protocol, + protocolPath : 'thread', + } + } + }); + expect(bobQueryBobDwn.status.code).to.equal(200); + expect(bobQueryBobDwn.records.length).to.equal(1); + expect(bobQueryBobDwn.records[0].id).to.equal(importRecord.id); + expect(await bobQueryBobDwn.records[0].data.text()).to.equal(updatedText); }); it('should retain all defined properties', async () => { @@ -265,7 +309,7 @@ describe('Record', () => { }); // Create a parent record to reference in the RecordsWriteMessage used for validation - const parentRecorsWrite = await RecordsWrite.create({ + const parentRecordsWrite = await RecordsWrite.create({ data : new Uint8Array(await dataBlob.arrayBuffer()), dataFormat, protocol, @@ -280,7 +324,7 @@ describe('Record', () => { data : new Uint8Array(await dataBlob.arrayBuffer()), dataFormat, encryptionInput, - parentId : parentRecorsWrite.recordId, + parentId : parentRecordsWrite.recordId, protocol, protocolPath, published, @@ -314,7 +358,7 @@ describe('Record', () => { expect(record.protocolPath).to.equal(protocolPath); expect(record.recipient).to.equal(recipient); expect(record.schema).to.equal(schema); - expect(record.parentId).to.equal(parentRecorsWrite.recordId); + expect(record.parentId).to.equal(parentRecordsWrite.recordId); expect(record.dataCid).to.equal(recordsWrite.message.descriptor.dataCid); expect(record.dataSize).to.equal(recordsWrite.message.descriptor.dataSize); expect(record.dateCreated).to.equal(recordsWrite.message.descriptor.dateCreated); @@ -1933,7 +1977,7 @@ describe('Record', () => { }); // Create a parent record to reference in the RecordsWriteMessage used for validation - const parentRecorsWrite = await RecordsWrite.create({ + const parentRecordsWrite = await RecordsWrite.create({ data : new Uint8Array(await dataBlob.arrayBuffer()), dataFormat, protocol, @@ -1948,7 +1992,7 @@ describe('Record', () => { data : new Uint8Array(await dataBlob.arrayBuffer()), dataFormat, encryptionInput, - parentId : parentRecorsWrite.recordId, + parentId : parentRecordsWrite.recordId, protocol, protocolPath, published, @@ -1987,7 +2031,7 @@ describe('Record', () => { expect(recordJson.protocolPath).to.equal(protocolPath); expect(recordJson.recipient).to.equal(recipient); expect(recordJson.schema).to.equal(schema); - expect(recordJson.parentId).to.equal(parentRecorsWrite.recordId); + expect(recordJson.parentId).to.equal(parentRecordsWrite.recordId); expect(recordJson.dataCid).to.equal(recordsWrite.message.descriptor.dataCid); expect(recordJson.dataSize).to.equal(recordsWrite.message.descriptor.dataSize); expect(recordJson.dateCreated).to.equal(recordsWrite.message.descriptor.dateCreated); @@ -2255,4 +2299,113 @@ describe('Record', () => { ).to.eventually.be.rejectedWith('is an immutable property. Its value cannot be changed.'); }); }); + + describe('store()', () => { + it('stores an updated record to the local DWN', async () => { + // Scenario: Alice creates a record and then updates it. + // Bob queries for the record from Alice's DWN and then stores the updated record locally. + + // Alice creates a public record then sends it to her remote DWN. + const { status, record } = await dwnAlice.records.write({ + data : 'Hello, world!', + message : { + published : true, + schema : 'foo/bar', + dataFormat : 'text/plain' + } + }); + expect(status.code).to.equal(202, status.detail); + let sendResponse = await record.send(); + expect(sendResponse.status.code).to.equal(202, sendResponse.status.detail); + + // Alice updates the record and once again does not store it. + const updatedText = 'updated text'; + const updateResult = await record!.update({ data: updatedText }); + expect(updateResult.status.code).to.equal(202, updateResult.status.detail); + sendResponse = await record.send(); + expect(sendResponse.status.code).to.equal(202, sendResponse.status.detail); + + // Bob queries for the record from his own node, should not return any results + let queryResult = await dwnBob.records.query({ + message : { + filter: { + recordId: record!.id + } + } + }); + expect(queryResult.status.code).to.equal(200); + expect(queryResult.records.length).to.equal(0); + + // Bob queries for the record from Alice's remote DWN + const queryResultFromAlice = await dwnBob.records.query({ + from : aliceDid.did, + message : { + filter: { + recordId: record!.id + } + } + }); + expect(queryResultFromAlice.status.code).to.equal(200); + expect(queryResultFromAlice.records.length).to.equal(1); + const queriedRecord = queryResultFromAlice.records[0]; + expect(await queriedRecord.data.text()).to.equal(updatedText) + + // stores the record in Bob's DWN, since import is defaulted to true, it will sign the record as bob + const { status: storeRecordStatus } = await queriedRecord.store(); + expect(storeRecordStatus.code).to.equal(202, storeRecordStatus.detail); + + // The record should now exist on bob's node + queryResult = await dwnBob.records.query({ + message : { + filter: { + recordId: record!.id + } + } + }); + expect(queryResult.status.code).to.equal(200); + expect(queryResult.records.length).to.equal(1); + const storedRecord = queryResult.records[0]; + expect(storedRecord.id).to.equal(record!.id); + expect(await storedRecord.data.text()).to.equal(updatedText); + }); + }); +}); + +describe('SendCache', () => { + it('sets and checks an item in the cache', async () => { + // checks for 'id' and 'target', returns false because we have not set them yet + expect(SendCache.check('id', 'target')).to.equal(false); + + // set 'id' and 'target, and then check + SendCache.set('id', 'target'); + expect(SendCache.check('id', 'target')).to.equal(true); + + // check for 'id' with a different target + expect(SendCache.check('id', 'target2')).to.equal(false); + }); + + it('purges the first item in teh cache when the cache is full (100 items)', async () => { + const recordId = 'id'; + // set 100 items in the cache to the same id + for (let i = 0; i < 100; i++) { + SendCache.set(recordId, `target-${i}`); + } + + // check that the first item is in the cache + expect(SendCache.check(recordId, 'target-0')).to.equal(true); + + // set another item in the cache + SendCache.set(recordId, 'target-new'); + + // check that the first item is no longer in the cache but the one after it is as well as the new one. + expect(SendCache.check(recordId, 'target-0')).to.equal(false); + expect(SendCache.check(recordId, 'target-1')).to.equal(true); + expect(SendCache.check(recordId, 'target-new')).to.equal(true); + + // add another item + SendCache.set(recordId, 'target-new2'); + expect(SendCache.check(recordId, 'target-1')).to.equal(false); + expect(SendCache.check(recordId, 'target-2')).to.equal(true); + expect(SendCache.check(recordId, 'target-new2')).to.equal(true); + }); }); \ No newline at end of file From 8bd889ea469650b88cb13374f674680b919ab685 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 13:49:03 -0500 Subject: [PATCH 25/35] linting errors --- packages/api/tests/record.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index 2c9d1f475..fe4285ada 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -142,7 +142,7 @@ describe('Record', () => { // Bob queries for the record on his own DWN (should not find it) - let bobQueryBobDwn = await dwnBob.records.query({ + let bobQueryBobDwn = await dwnBob.records.query({ from : bobDid.did, message : { filter: { @@ -2327,7 +2327,7 @@ describe('Record', () => { // Bob queries for the record from his own node, should not return any results let queryResult = await dwnBob.records.query({ - message : { + message: { filter: { recordId: record!.id } @@ -2348,7 +2348,7 @@ describe('Record', () => { expect(queryResultFromAlice.status.code).to.equal(200); expect(queryResultFromAlice.records.length).to.equal(1); const queriedRecord = queryResultFromAlice.records[0]; - expect(await queriedRecord.data.text()).to.equal(updatedText) + expect(await queriedRecord.data.text()).to.equal(updatedText); // stores the record in Bob's DWN, since import is defaulted to true, it will sign the record as bob const { status: storeRecordStatus } = await queriedRecord.store(); @@ -2356,7 +2356,7 @@ describe('Record', () => { // The record should now exist on bob's node queryResult = await dwnBob.records.query({ - message : { + message: { filter: { recordId: record!.id } From ff8816d5b8bebee2467032863b2c90882957e702 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 13:58:47 -0500 Subject: [PATCH 26/35] move SendCache class into it's own file, not exported in index.ts --- packages/api/src/record.ts | 27 +--------------- packages/api/src/send-cache.ts | 25 +++++++++++++++ packages/api/tests/record.spec.ts | 41 +----------------------- packages/api/tests/send-cache.spec.ts | 45 +++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 66 deletions(-) create mode 100644 packages/api/src/send-cache.ts create mode 100644 packages/api/tests/send-cache.spec.ts diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 92c6d6ebf..ee0a1fd1a 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -11,32 +11,7 @@ import { DwnInterfaceName, DwnMethodName } from '@tbd54566975/dwn-sdk-js'; import type { ResponseStatus } from './dwn-api.js'; import { dataToBlob } from './utils.js'; - -export class SendCache { - private static cache = new Map>(); - static sendCacheLimit = 100; - - static set(id: string, target: string): void { - let targetCache = SendCache.cache.get(id) || new Set(); - SendCache.cache.delete(id); - SendCache.cache.set(id, targetCache); - if (this.cache.size > SendCache.sendCacheLimit) { - const firstRecord = SendCache.cache.keys().next().value; - SendCache.cache.delete(firstRecord); - } - targetCache.delete(target); - targetCache.add(target); - if (targetCache.size > SendCache.sendCacheLimit) { - const firstTarget = targetCache.keys().next().value; - targetCache.delete(firstTarget); - } - } - - static check(id: string, target: string): boolean { - let targetCache = SendCache.cache.get(id); - return targetCache ? targetCache.has(target) : false; - } -} +import { SendCache } from './send-cache.js'; /** * Options that are passed to Record constructor. diff --git a/packages/api/src/send-cache.ts b/packages/api/src/send-cache.ts new file mode 100644 index 000000000..a8bc761bd --- /dev/null +++ b/packages/api/src/send-cache.ts @@ -0,0 +1,25 @@ +export class SendCache { + private static cache = new Map>(); + static sendCacheLimit = 100; + + static set(id: string, target: string): void { + let targetCache = SendCache.cache.get(id) || new Set(); + SendCache.cache.delete(id); + SendCache.cache.set(id, targetCache); + if (this.cache.size > SendCache.sendCacheLimit) { + const firstRecord = SendCache.cache.keys().next().value; + SendCache.cache.delete(firstRecord); + } + targetCache.delete(target); + targetCache.add(target); + if (targetCache.size > SendCache.sendCacheLimit) { + const firstTarget = targetCache.keys().next().value; + targetCache.delete(firstTarget); + } + } + + static check(id: string, target: string): boolean { + let targetCache = SendCache.cache.get(id); + return targetCache ? targetCache.has(target) : false; + } +} \ No newline at end of file diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index fe4285ada..e9c9f37b2 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -22,7 +22,7 @@ import { KeyDerivationScheme, } from '@tbd54566975/dwn-sdk-js'; -import { Record, SendCache } from '../src/record.js'; +import { Record } from '../src/record.js'; import { DwnApi } from '../src/dwn-api.js'; import { dataToBlob } from '../src/utils.js'; import { testDwnUrl } from './utils/test-config.js'; @@ -2369,43 +2369,4 @@ describe('Record', () => { expect(await storedRecord.data.text()).to.equal(updatedText); }); }); -}); - -describe('SendCache', () => { - it('sets and checks an item in the cache', async () => { - // checks for 'id' and 'target', returns false because we have not set them yet - expect(SendCache.check('id', 'target')).to.equal(false); - - // set 'id' and 'target, and then check - SendCache.set('id', 'target'); - expect(SendCache.check('id', 'target')).to.equal(true); - - // check for 'id' with a different target - expect(SendCache.check('id', 'target2')).to.equal(false); - }); - - it('purges the first item in teh cache when the cache is full (100 items)', async () => { - const recordId = 'id'; - // set 100 items in the cache to the same id - for (let i = 0; i < 100; i++) { - SendCache.set(recordId, `target-${i}`); - } - - // check that the first item is in the cache - expect(SendCache.check(recordId, 'target-0')).to.equal(true); - - // set another item in the cache - SendCache.set(recordId, 'target-new'); - - // check that the first item is no longer in the cache but the one after it is as well as the new one. - expect(SendCache.check(recordId, 'target-0')).to.equal(false); - expect(SendCache.check(recordId, 'target-1')).to.equal(true); - expect(SendCache.check(recordId, 'target-new')).to.equal(true); - - // add another item - SendCache.set(recordId, 'target-new2'); - expect(SendCache.check(recordId, 'target-1')).to.equal(false); - expect(SendCache.check(recordId, 'target-2')).to.equal(true); - expect(SendCache.check(recordId, 'target-new2')).to.equal(true); - }); }); \ No newline at end of file diff --git a/packages/api/tests/send-cache.spec.ts b/packages/api/tests/send-cache.spec.ts new file mode 100644 index 000000000..d2645d950 --- /dev/null +++ b/packages/api/tests/send-cache.spec.ts @@ -0,0 +1,45 @@ +import chai, { expect } from 'chai'; +import chaiAsPromised from 'chai-as-promised'; + +import { SendCache } from '../src/send-cache.js'; + +chai.use(chaiAsPromised); + +describe('SendCache', () => { + it('sets and checks an item in the cache', async () => { + // checks for 'id' and 'target', returns false because we have not set them yet + expect(SendCache.check('id', 'target')).to.equal(false); + + // set 'id' and 'target, and then check + SendCache.set('id', 'target'); + expect(SendCache.check('id', 'target')).to.equal(true); + + // check for 'id' with a different target + expect(SendCache.check('id', 'target2')).to.equal(false); + }); + + it('purges the first item in teh cache when the cache is full (100 items)', async () => { + const recordId = 'id'; + // set 100 items in the cache to the same id + for (let i = 0; i < 100; i++) { + SendCache.set(recordId, `target-${i}`); + } + + // check that the first item is in the cache + expect(SendCache.check(recordId, 'target-0')).to.equal(true); + + // set another item in the cache + SendCache.set(recordId, 'target-new'); + + // check that the first item is no longer in the cache but the one after it is as well as the new one. + expect(SendCache.check(recordId, 'target-0')).to.equal(false); + expect(SendCache.check(recordId, 'target-1')).to.equal(true); + expect(SendCache.check(recordId, 'target-new')).to.equal(true); + + // add another item + SendCache.set(recordId, 'target-new2'); + expect(SendCache.check(recordId, 'target-1')).to.equal(false); + expect(SendCache.check(recordId, 'target-2')).to.equal(true); + expect(SendCache.check(recordId, 'target-new2')).to.equal(true); + }); +}); \ No newline at end of file From 5df757457e0ff05d3f1c7de006242b6067e5c2b5 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 14:06:33 -0500 Subject: [PATCH 27/35] js doc updates --- packages/api/src/record.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index ee0a1fd1a..e29587ed9 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -116,6 +116,7 @@ export class Record implements RecordModel { /** Record's encryption */ get encryption(): RecordsWriteMessage['encryption'] { return this._encryption; } + /** Record's initial write if the record has been updated */ get initialWrite(): RecordOptions['initialWrite'] { return this._initialWrite; } /** Record's ID */ @@ -136,7 +137,7 @@ export class Record implements RecordModel { /** Record's protocol path */ get protocolPath() { return this._descriptor.protocolPath; } - /** Role under which the author is writting the record */ + /** Role under which the author is writing the record */ get protocolRole() { return this._protocolRole; } /** Record's recipient */ @@ -552,9 +553,7 @@ export class Record implements RecordModel { // Handles the various conditions around there being an initial write, whether to store initial/current state, // and whether to add an owner signature to the initial write to enable storage when protocol rules require it. - private async processRecord(options: { store: boolean, signAsOwner: boolean }): Promise { - const { store, signAsOwner } = options; - + private async processRecord({ store, signAsOwner }:{ store: boolean, signAsOwner: boolean }): Promise { // if there is an initial write and we haven't already processed it, we first process it and marked it as such. if (this._initialWrite && !this._initialWriteProcessed) { const initialWriteRequest: ProcessDwnRequest = { From 420efb1e369d0871e71d99c45b9171f8f5846cfd Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 15:59:16 -0500 Subject: [PATCH 28/35] add test coverage to dwn manager --- packages/agent/tests/dwn-manager.spec.ts | 143 ++++++++++++++++++----- 1 file changed, 115 insertions(+), 28 deletions(-) diff --git a/packages/agent/tests/dwn-manager.spec.ts b/packages/agent/tests/dwn-manager.spec.ts index c3d050ff9..0dccf227e 100644 --- a/packages/agent/tests/dwn-manager.spec.ts +++ b/packages/agent/tests/dwn-manager.spec.ts @@ -95,17 +95,24 @@ describe('DwnManager', () => { }); describe('processRequest()', () => { - let identity: ManagedIdentity; + let alice: ManagedIdentity; + let bob: ManagedIdentity; beforeEach(async () => { await testAgent.clearStorage(); await testAgent.createAgentDid(); // Creates a new Identity to author the DWN messages. - identity = await testAgent.agent.identityManager.create({ + alice = await testAgent.agent.identityManager.create({ name : 'Alice', didMethod : 'key', kms : 'local' }); + + bob = await testAgent.agent.identityManager.create({ + name : 'Bob', + didMethod : 'key', + kms : 'local' + }); }); it('handles EventsGet', async () => { @@ -113,8 +120,8 @@ describe('DwnManager', () => { // Attempt to process the EventsGet. let eventsGetResponse = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'EventsGet', messageOptions : { cursor: testCursor, @@ -140,8 +147,8 @@ describe('DwnManager', () => { // Write a record to use for the MessagesGet test. let { message, reply: { status: writeStatus } } = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'RecordsWrite', messageOptions : { dataFormat : 'text/plain', @@ -157,8 +164,8 @@ describe('DwnManager', () => { // Attempt to process the MessagesGet. let messagesGetResponse = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'MessagesGet', messageOptions : { messageCids: [messageCid] @@ -187,8 +194,8 @@ describe('DwnManager', () => { it('handles ProtocolsConfigure', async () => { let protocolsConfigureResponse = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'ProtocolsConfigure', messageOptions : { definition: emailProtocolDefinition @@ -211,8 +218,8 @@ describe('DwnManager', () => { it('handles ProtocolsQuery', async () => { // Configure a protocol to use for the ProtocolsQuery test. let protocolsConfigureResponse = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'ProtocolsConfigure', messageOptions : { definition: emailProtocolDefinition @@ -222,8 +229,8 @@ describe('DwnManager', () => { // Attempt to query for the protocol that was just configured. let protocolsQueryResponse = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'ProtocolsQuery', messageOptions : { filter: { protocol: emailProtocolDefinition.protocol }, @@ -252,8 +259,8 @@ describe('DwnManager', () => { // Write a record that can be deleted. let { message, reply: { status: writeStatus } } = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'RecordsWrite', messageOptions : { dataFormat : 'text/plain', @@ -266,8 +273,8 @@ describe('DwnManager', () => { // Attempt to process the RecordsRead. const deleteResponse = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'RecordsDelete', messageOptions : { recordId: writeMessage.recordId @@ -294,8 +301,8 @@ describe('DwnManager', () => { // Write a record that can be queried for. let { message, reply: { status: writeStatus } } = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'RecordsWrite', messageOptions : { dataFormat : 'text/plain', @@ -308,8 +315,8 @@ describe('DwnManager', () => { // Attempt to process the RecordsQuery. const queryResponse = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'RecordsQuery', messageOptions : { filter: { @@ -343,8 +350,8 @@ describe('DwnManager', () => { // Write a record that can be read. let { message, reply: { status: writeStatus } } = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'RecordsWrite', messageOptions : { dataFormat : 'text/plain', @@ -357,8 +364,8 @@ describe('DwnManager', () => { // Attempt to process the RecordsRead. const readResponse = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'RecordsRead', messageOptions : { filter: { @@ -391,8 +398,8 @@ describe('DwnManager', () => { // Attempt to process the RecordsWrite let writeResponse = await testAgent.agent.dwnManager.processRequest({ - author : identity.did, - target : identity.did, + author : alice.did, + target : alice.did, messageType : 'RecordsWrite', messageOptions : { dataFormat: 'text/plain' @@ -414,6 +421,86 @@ describe('DwnManager', () => { expect(writeReply).to.have.property('status'); expect(writeReply.status.code).to.equal(202); }); + + xit('handles RecordsWrite messages to sign as owner', async () => { + // bob authors a public record to his dwn + const bobWrite = await testAgent.agent.dwnManager.processRequest({ + author : bob.did, + target : bob.did, + messageType : 'RecordsWrite', + messageOptions : { + published : true, + dataFormat : 'text/plain' + }, + dataStream: new Blob([ Convert.string('Hello, world!').toUint8Array() ]) + }); + expect(bobWrite.reply.status.code).to.equal(202); + const message = bobWrite.message as RecordsWriteMessage; + + // alice queries bob's DWN for the record + const queryBobResponse = await testAgent.agent.dwnManager.processRequest({ + messageType : 'RecordsQuery', + author : alice.did, + target : bob.did, + messageOptions : { + filter: { + recordId: message.recordId + } + } + }); + let reply = queryBobResponse.reply as RecordsQueryReply; + expect(reply.status.code).to.equal(200); + expect(reply.entries!.length).to.equal(1); + expect(reply.entries![0].recordId).to.equal(message.recordId); + + + // alice attempts to process the rawMessage as is without signing it, should fail + let aliceWrite = await testAgent.agent.dwnManager.processRequest({ + messageType : 'RecordsWrite', + author : alice.did, + target : alice.did, + rawMessage : message, + }); + expect(aliceWrite.reply.status.code).to.equal(401); + + // alice queries to make sure the record is not saved on her dwn + let queryAliceResponse = await testAgent.agent.dwnManager.processRequest({ + messageType : 'RecordsQuery', + author : alice.did, + target : alice.did, + messageOptions : { + filter: { + recordId: message.recordId + } + } + }); + expect(queryAliceResponse.reply.status.code).to.equal(200); + expect(queryAliceResponse.reply.entries!.length).to.equal(0); + + // alice attempts to process the rawMessage again this time marking it to be signed as owner + aliceWrite = await testAgent.agent.dwnManager.processRequest({ + messageType : 'RecordsWrite', + author : alice.did, + target : alice.did, + rawMessage : message, + signAsOwner : true, + }); + expect(aliceWrite.reply.status.code).to.equal(202); + + // alice now queries for the record, it should be there + queryAliceResponse = await testAgent.agent.dwnManager.processRequest({ + messageType : 'RecordsQuery', + author : alice.did, + target : alice.did, + messageOptions : { + filter: { + recordId: message.recordId + } + } + }); + expect(queryAliceResponse.reply.status.code).to.equal(200); + expect(queryAliceResponse.reply.entries!.length).to.equal(1); + }); }); describe('sendDwnRequest()', () => { From f5b3d558170d07ac06fb97361a73a07a92b6e7d9 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 18:04:34 -0500 Subject: [PATCH 29/35] test coverage for store/import scenarios --- packages/api/src/record.ts | 18 +- packages/api/tests/record.spec.ts | 321 +++++++++++++++++++++++++++++- 2 files changed, 321 insertions(+), 18 deletions(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index e29587ed9..629d5b308 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -90,7 +90,8 @@ export class Record implements RecordModel { private _descriptor: RecordsWriteDescriptor; private _encryption?: RecordsWriteMessage['encryption']; private _initialWrite: RecordOptions['initialWrite']; - private _initialWriteProcessed: boolean; + private _initialWriteStored: boolean; + private _initialWriteSigned: boolean; private _recordId: string; private _protocolRole: RecordOptions['protocolRole']; // Getters for immutable DWN Record properties. @@ -339,12 +340,12 @@ export class Record implements RecordModel { /** * Stores the current record state as well as any initial write to the owner's DWN. * - * @param importRecord - if true, the record will signed by the owner before storing it to the owner's DWN. Defaults to true. + * @param importRecord - if true, the record will signed by the owner before storing it to the owner's DWN. Defaults to false. * @returns the status of the store request * * @beta */ - async store(importRecord: boolean = true): Promise { + async store(importRecord: boolean = false): Promise { // if we are importing the record we sign it as the owner return this.processRecord({ signAsOwner: importRecord, store: true }); } @@ -555,7 +556,7 @@ export class Record implements RecordModel { // and whether to add an owner signature to the initial write to enable storage when protocol rules require it. private async processRecord({ store, signAsOwner }:{ store: boolean, signAsOwner: boolean }): Promise { // if there is an initial write and we haven't already processed it, we first process it and marked it as such. - if (this._initialWrite && !this._initialWriteProcessed) { + if (this._initialWrite && ((signAsOwner && !this._initialWriteSigned) || (store && !this._initialWriteStored))) { const initialWriteRequest: ProcessDwnRequest = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, rawMessage : this.initialWrite, @@ -571,10 +572,13 @@ export class Record implements RecordModel { const responseMessage = message as RecordsWriteMessage; // If we are signing as owner, make sure to update the initial write's authorization, because now it will have the owner's signature on it - // set it to processed so that we don't repeat this process again + // set the stored or signed status to true so we don't process it again. if (200 <= status.code && status.code <= 299) { - this._initialWriteProcessed = true; - if (signAsOwner) this.initialWrite.authorization = responseMessage.authorization; + if (store) this._initialWriteStored = true; + if (signAsOwner) { + this._initialWriteSigned = true; + this.initialWrite.authorization = responseMessage.authorization; + } } } diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index e9c9f37b2..40c785561 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -2301,11 +2301,11 @@ describe('Record', () => { }); describe('store()', () => { - it('stores an updated record to the local DWN', async () => { - // Scenario: Alice creates a record and then updates it. - // Bob queries for the record from Alice's DWN and then stores the updated record locally. + it('should store an external record if it has been imported by the dwn owner', async () => { + // Scenario: Alice creates a record. + // Bob queries for the record from Alice's DWN and then stores it to their own DWN. - // Alice creates a public record then sends it to her remote DWN. + // alice creates a record and sends it to their DWN const { status, record } = await dwnAlice.records.write({ data : 'Hello, world!', message : { @@ -2318,18 +2318,78 @@ describe('Record', () => { let sendResponse = await record.send(); expect(sendResponse.status.code).to.equal(202, sendResponse.status.detail); - // Alice updates the record and once again does not store it. + // bob queries alice's DWN for the record + const aliceQueryResult = await dwnBob.records.query({ + from : aliceDid.did, + message : { + filter: { + recordId: record.id + } + } + }); + expect(aliceQueryResult.status.code).to.equal(200); + expect(aliceQueryResult.records.length).to.equal(1); + const queriedRecord = aliceQueryResult.records[0]; + + // bob queries their own DWN for the record, should not return any results + let bobQueryResult = await dwnBob.records.query({ + message: { + filter: { + recordId: record.id + } + } + }); + expect(bobQueryResult.status.code).to.equal(200); + expect(bobQueryResult.records.length).to.equal(0); + + // attempts to store the record without importing it, should fail + let { status: storeRecordStatus } = await queriedRecord.store(); + expect(storeRecordStatus.code).to.equal(401, storeRecordStatus.detail); + + // attempts to store the record flagging it for import + ({ status: storeRecordStatus } = await queriedRecord.store(true)); + expect(storeRecordStatus.code).to.equal(202, storeRecordStatus.detail); + + // bob queries their own DWN for the record, should return the record + bobQueryResult = await dwnBob.records.query({ + message: { + filter: { + recordId: record.id + } + } + }); + expect(bobQueryResult.status.code).to.equal(200); + expect(bobQueryResult.records.length).to.equal(1); + const storedRecord = bobQueryResult.records[0]; + expect(storedRecord.id).to.equal(record.id); + }); + + it('stores an updated record to the local DWN along with the initial write', async () => { + // Scenario: Alice creates a record and then updates it. + // Bob queries for the record from Alice's DWN and then stores the updated record along with it's initial write. + + // Alice creates a public record then sends it to her remote DWN. + const { status, record } = await dwnAlice.records.write({ + data : 'Hello, world!', + message : { + published : true, + schema : 'foo/bar', + dataFormat : 'text/plain' + } + }); + expect(status.code).to.equal(202, status.detail); const updatedText = 'updated text'; const updateResult = await record!.update({ data: updatedText }); expect(updateResult.status.code).to.equal(202, updateResult.status.detail); - sendResponse = await record.send(); + + const sendResponse = await record.send(); expect(sendResponse.status.code).to.equal(202, sendResponse.status.detail); // Bob queries for the record from his own node, should not return any results let queryResult = await dwnBob.records.query({ message: { filter: { - recordId: record!.id + recordId: record.id } } }); @@ -2341,7 +2401,7 @@ describe('Record', () => { from : aliceDid.did, message : { filter: { - recordId: record!.id + recordId: record.id } } }); @@ -2350,15 +2410,19 @@ describe('Record', () => { const queriedRecord = queryResultFromAlice.records[0]; expect(await queriedRecord.data.text()).to.equal(updatedText); - // stores the record in Bob's DWN, since import is defaulted to true, it will sign the record as bob - const { status: storeRecordStatus } = await queriedRecord.store(); + // attempts to store the record without signing it, should fail + let { status: storeRecordStatus } = await queriedRecord.store(); + expect(storeRecordStatus.code).to.equal(401, storeRecordStatus.detail); + + // stores the record in Bob's DWN, the importRecord parameter is set to true so that bob signs the record before storing it + ({ status: storeRecordStatus } = await queriedRecord.store(true)); expect(storeRecordStatus.code).to.equal(202, storeRecordStatus.detail); // The record should now exist on bob's node queryResult = await dwnBob.records.query({ message: { filter: { - recordId: record!.id + recordId: record.id } } }); @@ -2369,4 +2433,239 @@ describe('Record', () => { expect(await storedRecord.data.text()).to.equal(updatedText); }); }); + + describe('import()', () => { + it('should import an external record without storing it', async () => { + // Scenario: Alice creates a record. + // Bob queries for the record from Alice's DWN and then imports it without storing + // Bob then .stores() it without specifying import explicitly as it's already been imported. + + // alice creates a record and sends it to her DWN + const { status, record } = await dwnAlice.records.write({ + data : 'Hello, world!', + message : { + published : true, + schema : 'foo/bar', + dataFormat : 'text/plain' + } + }); + expect(status.code).to.equal(202, status.detail); + let sendResponse = await record.send(); + expect(sendResponse.status.code).to.equal(202, sendResponse.status.detail); + + // bob queries alice's DWN for the record + const aliceQueryResult = await dwnBob.records.query({ + from : aliceDid.did, + message : { + filter: { + recordId: record.id + } + } + }); + expect(aliceQueryResult.status.code).to.equal(200); + expect(aliceQueryResult.records.length).to.equal(1); + const queriedRecord = aliceQueryResult.records[0]; + + // imports the record without storing it + let { status: importRecordStatus } = await queriedRecord.import(); + expect(importRecordStatus.code).to.equal(202, importRecordStatus.detail); + + // bob queries their own DWN for the record, should return the record + const bobQueryResult = await dwnBob.records.query({ + message: { + filter: { + recordId: record.id + } + } + }); + expect(bobQueryResult.status.code).to.equal(200); + expect(bobQueryResult.records.length).to.equal(1); + const storedRecord = bobQueryResult.records[0]; + expect(storedRecord.id).to.equal(record.id); + }); + + it('import an external record along with the initial write', async () => { + // Scenario: Alice creates a record and then updates it. + // Bob queries for the record from Alice's DWN and then stores the updated record along with it's initial write. + + // Alice creates a public record then sends it to her remote DWN. + const { status, record } = await dwnAlice.records.write({ + data : 'Hello, world!', + message : { + published : true, + schema : 'foo/bar', + dataFormat : 'text/plain' + } + }); + expect(status.code).to.equal(202, status.detail); + const updatedText = 'updated text'; + const updateResult = await record!.update({ data: updatedText }); + expect(updateResult.status.code).to.equal(202, updateResult.status.detail); + const sendResponse = await record.send(); + expect(sendResponse.status.code).to.equal(202, sendResponse.status.detail); + + // bob queries alice's DWN for the record + const aliceQueryResult = await dwnBob.records.query({ + from : aliceDid.did, + message : { + filter: { + recordId: record.id + } + } + }); + expect(aliceQueryResult.status.code).to.equal(200); + expect(aliceQueryResult.records.length).to.equal(1); + const queriedRecord = aliceQueryResult.records[0]; + + // imports the record without storing it + let { status: importRecordStatus } = await queriedRecord.import(); + expect(importRecordStatus.code).to.equal(202, importRecordStatus.detail); + + // bob queries their own DWN for the record, should return the record + const bobQueryResult = await dwnBob.records.query({ + message: { + filter: { + recordId: record.id + } + } + }); + expect(bobQueryResult.status.code).to.equal(200); + expect(bobQueryResult.records.length).to.equal(1); + const storedRecord = bobQueryResult.records[0]; + expect(storedRecord.id).to.equal(record.id); + }); + + describe('store: false', () => { + it('should import an external record without storing it', async () => { + // Scenario: Alice creates a record. + // Bob queries for the record from Alice's DWN and then imports it without storing + // Bob then .stores() it without specifying import explicitly as it's already been imported. + + // alice creates a record and sends it to her DWN + const { status, record } = await dwnAlice.records.write({ + data : 'Hello, world!', + message : { + published : true, + schema : 'foo/bar', + dataFormat : 'text/plain' + } + }); + expect(status.code).to.equal(202, status.detail); + let sendResponse = await record.send(); + expect(sendResponse.status.code).to.equal(202, sendResponse.status.detail); + + // bob queries alice's DWN for the record + const aliceQueryResult = await dwnBob.records.query({ + from : aliceDid.did, + message : { + filter: { + recordId: record.id + } + } + }); + expect(aliceQueryResult.status.code).to.equal(200); + expect(aliceQueryResult.records.length).to.equal(1); + const queriedRecord = aliceQueryResult.records[0]; + + // imports the record without storing it + let { status: importRecordStatus } = await queriedRecord.import(false); + expect(importRecordStatus.code).to.equal(202, importRecordStatus.detail); + + // queries for the record from bob's DWN, should not return any results + let bobQueryResult = await dwnBob.records.query({ + message: { + filter: { + recordId: record.id + } + } + }); + expect(bobQueryResult.status.code).to.equal(200); + expect(bobQueryResult.records.length).to.equal(0); + + // attempts to store the record without explicitly marking it for import as it's already been imported + ({ status: importRecordStatus } = await queriedRecord.store()); + expect(importRecordStatus.code).to.equal(202, importRecordStatus.detail); + + // bob queries their own DWN for the record, should return the record + bobQueryResult = await dwnBob.records.query({ + message: { + filter: { + recordId: record.id + } + } + }); + expect(bobQueryResult.status.code).to.equal(200); + expect(bobQueryResult.records.length).to.equal(1); + const storedRecord = bobQueryResult.records[0]; + expect(storedRecord.id).to.equal(record.id); + }); + + it('import an external record along with the initial write', async () => { + // Scenario: Alice creates a record and then updates it. + // Bob queries for the record from Alice's DWN and then stores the updated record along with it's initial write. + + // Alice creates a public record then sends it to her remote DWN. + const { status, record } = await dwnAlice.records.write({ + data : 'Hello, world!', + message : { + published : true, + schema : 'foo/bar', + dataFormat : 'text/plain' + } + }); + expect(status.code).to.equal(202, status.detail); + const updatedText = 'updated text'; + const updateResult = await record.update({ data: updatedText }); + expect(updateResult.status.code).to.equal(202, updateResult.status.detail); + const sendResponse = await record.send(); + expect(sendResponse.status.code).to.equal(202, sendResponse.status.detail); + + // bob queries alice's DWN for the record + const aliceQueryResult = await dwnBob.records.query({ + from : aliceDid.did, + message : { + filter: { + recordId: record.id + } + } + }); + expect(aliceQueryResult.status.code).to.equal(200); + expect(aliceQueryResult.records.length).to.equal(1); + const queriedRecord = aliceQueryResult.records[0]; + + // imports the record without storing it + let { status: importRecordStatus } = await queriedRecord.import(false); + expect(importRecordStatus.code).to.equal(202, importRecordStatus.detail); + + // queries for the record from bob's DWN, should not return any results + let bobQueryResult = await dwnBob.records.query({ + message: { + filter: { + recordId: record.id + } + } + }); + expect(bobQueryResult.status.code).to.equal(200); + expect(bobQueryResult.records.length).to.equal(0); + + // attempts to store the record without explicitly marking it for import as it's already been imported + ({ status: importRecordStatus } = await queriedRecord.store()); + console.log('attempt to store the record', importRecordStatus); + expect(importRecordStatus.code).to.equal(202, importRecordStatus.detail); + + // bob queries their own DWN for the record, should return the record + bobQueryResult = await dwnBob.records.query({ + message: { + filter: { + recordId: record.id + } + } + }); + expect(bobQueryResult.status.code).to.equal(200); + expect(bobQueryResult.records.length).to.equal(1); + const storedRecord = bobQueryResult.records[0]; + expect(storedRecord.id).to.equal(record.id); + }); + }); + }); }); \ No newline at end of file From 1790b9470da3a73b3fa1026fac3465288d9b7444 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 19:00:14 -0500 Subject: [PATCH 30/35] fix test case in dwn manager test --- packages/agent/tests/dwn-manager.spec.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/packages/agent/tests/dwn-manager.spec.ts b/packages/agent/tests/dwn-manager.spec.ts index 0dccf227e..d9ac8c1ef 100644 --- a/packages/agent/tests/dwn-manager.spec.ts +++ b/packages/agent/tests/dwn-manager.spec.ts @@ -15,6 +15,7 @@ import { RecordsWriteMessage, RecordsDeleteMessage, ProtocolsConfigureMessage, + RecordsWrite, } from '@tbd54566975/dwn-sdk-js'; import { testDwnUrl } from './utils/test-config.js'; @@ -422,17 +423,20 @@ describe('DwnManager', () => { expect(writeReply.status.code).to.equal(202); }); - xit('handles RecordsWrite messages to sign as owner', async () => { + it('handles RecordsWrite messages to sign as owner', async () => { // bob authors a public record to his dwn + const dataStream = new Blob([ Convert.string('Hello, world!').toUint8Array() ]); + const bobWrite = await testAgent.agent.dwnManager.processRequest({ author : bob.did, target : bob.did, messageType : 'RecordsWrite', messageOptions : { published : true, + schema : 'foo/bar', dataFormat : 'text/plain' }, - dataStream: new Blob([ Convert.string('Hello, world!').toUint8Array() ]) + dataStream, }); expect(bobWrite.reply.status.code).to.equal(202); const message = bobWrite.message as RecordsWriteMessage; @@ -453,13 +457,13 @@ describe('DwnManager', () => { expect(reply.entries!.length).to.equal(1); expect(reply.entries![0].recordId).to.equal(message.recordId); - // alice attempts to process the rawMessage as is without signing it, should fail let aliceWrite = await testAgent.agent.dwnManager.processRequest({ messageType : 'RecordsWrite', author : alice.did, target : alice.did, rawMessage : message, + dataStream, }); expect(aliceWrite.reply.status.code).to.equal(401); @@ -484,6 +488,7 @@ describe('DwnManager', () => { target : alice.did, rawMessage : message, signAsOwner : true, + dataStream, }); expect(aliceWrite.reply.status.code).to.equal(202); From f01bee1853ff4fde97e10244fdf5a1985fb0e0a8 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 19:02:33 -0500 Subject: [PATCH 31/35] remove unused import --- packages/agent/tests/dwn-manager.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/agent/tests/dwn-manager.spec.ts b/packages/agent/tests/dwn-manager.spec.ts index d9ac8c1ef..6573949be 100644 --- a/packages/agent/tests/dwn-manager.spec.ts +++ b/packages/agent/tests/dwn-manager.spec.ts @@ -15,7 +15,6 @@ import { RecordsWriteMessage, RecordsDeleteMessage, ProtocolsConfigureMessage, - RecordsWrite, } from '@tbd54566975/dwn-sdk-js'; import { testDwnUrl } from './utils/test-config.js'; From 7453b688e7f5f8f13ae38ec4da1096f3432b9f20 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 19:50:38 -0500 Subject: [PATCH 32/35] update test covreage for bug fix failures --- packages/api/tests/record.spec.ts | 154 +++++++++++++++++------------- 1 file changed, 85 insertions(+), 69 deletions(-) diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index 40c785561..304d4446a 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -1261,10 +1261,8 @@ describe('Record', () => { expect(recordData.size).to.equal(dataTextExceedingMaxSize.length); }); - it('fails to return large data payloads of records signed by another entity after remote dwn.records.query()', async () => { + it('returns large data payloads of records signed by another entity after remote dwn.records.query()', async () => { /** - * ! TODO: Fix this once the bug in `dwn-sdk-js` is resolved. - * * WHAT IS BEING TESTED? * * We are testing whether a large (> `DwnConstant.maxDataSizeAllowedToBeEncoded`) record @@ -1339,6 +1337,19 @@ describe('Record', () => { */ const { status: sendStatusToAlice } = await queryRecordsFrom[0]!.send(aliceDid.did); expect(sendStatusToAlice.code).to.equal(202); + /** + * 5. Alice queries her remote DWN for the record that Bob just wrote. + */ + const { records: queryRecordsTo, status: queryRecordStatusTo } = await dwnAlice.records.query({ + from : aliceDid.did, + message : { filter: { recordId: record!.id }} + }); + expect(queryRecordStatusTo.code).to.equal(200); + /** + * 6. Validate that Alice is able to access the data payload. + */ + const recordData = await queryRecordsTo[0].data.text(); + expect(recordData).to.deep.equal(dataTextExceedingMaxSize); }); }); }); @@ -2078,8 +2089,7 @@ describe('Record', () => { expect(updatedData).to.equal('bye'); }); - // TODO: Fix after changes are made to dwn-sdk-js to include the initial write in every query/read response. - it('fails to update a record locally that only written to a remote DWN', async () => { + it('updates a record locally that only written to a remote DWN', async () => { // Create a record but do not store it on the local DWN. const { status, record } = await dwnAlice.records.write({ store : false, @@ -2093,43 +2103,45 @@ describe('Record', () => { expect(record).to.not.be.undefined; // Store the data CID of the record before it is updated. - // const dataCidBeforeDataUpdate = record!.dataCid; + const dataCidBeforeDataUpdate = record!.dataCid; // Write the record to a remote DWN. const { status: sendStatus } = await record!.send(aliceDid.did); expect(sendStatus.code).to.equal(202); - /** Attempt to update the record, which should write the updated record the local DWN but - * instead fails due to a missing initial write. */ - const updateResult = await record!.update({ data: 'bye' }); + // fails because record has not been stored in the local dwn yet + let updateResult = await record!.update({ data: 'bye' }); expect(updateResult.status.code).to.equal(400); expect(updateResult.status.detail).to.equal('RecordsWriteGetInitialWriteNotFound: initial write is not found'); - // TODO: Uncomment these lines after the issue mentioned above is fixed. - // expect(updateResult.status.code).to.equal(202); + const { status: recordStoreStatus }= await record.store(); + expect(recordStoreStatus.code).to.equal(202); + + // now succeeds with the update + updateResult = await record!.update({ data: 'bye' }); + expect(updateResult.status.code).to.equal(202); // Confirm that the record was written to the local DWN. - // const readResult = await dwnAlice.records.read({ - // message: { - // filter: { - // recordId: record!.id - // } - // } - // }); - // expect(readResult.status.code).to.equal(200); - // expect(readResult.record).to.not.be.undefined; + const readResult = await dwnAlice.records.read({ + message: { + filter: { + recordId: record!.id + } + } + }); + expect(readResult.status.code).to.equal(200); + expect(readResult.record).to.not.be.undefined; // Confirm that the data CID of the record was updated. - // expect(readResult.record.dataCid).to.not.equal(dataCidBeforeDataUpdate); - // expect(readResult.record.dataCid).to.equal(record!.dataCid); + expect(readResult.record.dataCid).to.not.equal(dataCidBeforeDataUpdate); + expect(readResult.record.dataCid).to.equal(record!.dataCid); // Confirm that the data payload of the record was modified. - // const updatedData = await record!.data.text(); - // expect(updatedData).to.equal('bye'); + const updatedData = await record!.data.text(); + expect(updatedData).to.equal('bye'); }); - // TODO: Fix after changes are made to dwn-sdk-js to include the initial write in every query/read response. - it('fails to update a record locally that was initially read from a remote DWN', async () => { + it('allows to update a record locally that was initially read from a remote DWN if store() is issued', async () => { // Create a record but do not store it on the local DWN. const { status, record } = await dwnAlice.records.write({ store : false, @@ -2143,14 +2155,14 @@ describe('Record', () => { expect(record).to.not.be.undefined; // Store the data CID of the record before it is updated. - // const dataCidBeforeDataUpdate = record!.dataCid; + const dataCidBeforeDataUpdate = record!.dataCid; // Write the record to a remote DWN. const { status: sendStatus } = await record!.send(aliceDid.did); expect(sendStatus.code).to.equal(202); // Read the record from the remote DWN. - const readResult = await dwnAlice.records.read({ + let readResult = await dwnAlice.records.read({ from : aliceDid.did, message : { filter: { @@ -2161,36 +2173,37 @@ describe('Record', () => { expect(readResult.status.code).to.equal(200); expect(readResult.record).to.not.be.undefined; - // Attempt to update the record, which should write the updated record the local DWN. - const updateResult = await readResult.record!.update({ data: 'bye' }); + const readRecord = readResult.record; + + // Attempt to update the record without storing, should fail + let updateResult = await readRecord.update({ data: 'bye' }); expect(updateResult.status.code).to.equal(400); - expect(updateResult.status.detail).to.equal('RecordsWriteGetInitialWriteNotFound: initial write is not found'); - // TODO: Uncomment these lines after the issue mentioned above is fixed. - // expect(updateResult.status.code).to.equal(202); + // store the record locally + const { status: storeStatus } = await readRecord.store(); + expect(storeStatus.code).to.equal(202); + + // Attempt to update the record, which should write the updated record the local DWN. + updateResult = await readRecord.update({ data: 'bye' }); + expect(updateResult.status.code).to.equal(202); // Confirm that the record was written to the local DWN. - // const readResult = await dwnAlice.records.read({ - // message: { - // filter: { - // recordId: record!.id - // } - // } - // }); - // expect(readResult.status.code).to.equal(200); - // expect(readResult.record).to.not.be.undefined; + readResult = await dwnAlice.records.read({ + message: { + filter: { + recordId: record!.id + } + } + }); + expect(readResult.status.code).to.equal(200); + expect(readResult.record).to.not.be.undefined; // Confirm that the data CID of the record was updated. - // expect(readResult.record.dataCid).to.not.equal(dataCidBeforeDataUpdate); - // expect(readResult.record.dataCid).to.equal(record!.dataCid); - - // Confirm that the data payload of the record was modified. - // const updatedData = await record!.data.text(); - // expect(updatedData).to.equal('bye'); + expect(readResult.record.dataCid).to.not.equal(dataCidBeforeDataUpdate); + expect(readResult.record.dataCid).to.equal(readRecord.dataCid); }); - // TODO: Fix after changes are made to dwn-sdk-js to include the initial write in every query/read response. - it('fails to update a record locally that was initially queried from a remote DWN', async () => { + it('updates a record locally that was initially queried from a remote DWN', async () => { // Create a record but do not store it on the local DWN. const { status, record } = await dwnAlice.records.write({ store : false, @@ -2204,7 +2217,7 @@ describe('Record', () => { expect(record).to.not.be.undefined; // Store the data CID of the record before it is updated. - // const dataCidBeforeDataUpdate = record!.dataCid; + const dataCidBeforeDataUpdate = record!.dataCid; // Write the record to a remote DWN. const { status: sendStatus } = await record!.send(aliceDid.did); @@ -2223,33 +2236,37 @@ describe('Record', () => { expect(queryResult.records).to.not.be.undefined; expect(queryResult.records.length).to.equal(1); - // Attempt to update the queried record, which should write the updated record the local DWN. + // Attempt to update the queried record, which will fail because we haven't stored the queried record locally yet const [ queriedRecord ] = queryResult.records; - const updateResult = await queriedRecord!.update({ data: 'bye' }); + let updateResult = await queriedRecord!.update({ data: 'bye' }); expect(updateResult.status.code).to.equal(400); expect(updateResult.status.detail).to.equal('RecordsWriteGetInitialWriteNotFound: initial write is not found'); - // TODO: Uncomment these lines after the issue mentioned above is fixed. - // expect(updateResult.status.code).to.equal(202); + // store the queried record + const { status: queriedStoreStatus } = await queriedRecord.store(); + expect(queriedStoreStatus.code).to.equal(202); + + updateResult = await queriedRecord!.update({ data: 'bye' }); + expect(updateResult.status.code).to.equal(202); // Confirm that the record was written to the local DWN. - // const readResult = await dwnAlice.records.read({ - // message: { - // filter: { - // recordId: record!.id - // } - // } - // }); - // expect(readResult.status.code).to.equal(200); - // expect(readResult.record).to.not.be.undefined; + const readResult = await dwnAlice.records.read({ + message: { + filter: { + recordId: record!.id + } + } + }); + expect(readResult.status.code).to.equal(200); + expect(readResult.record).to.not.be.undefined; // Confirm that the data CID of the record was updated. - // expect(readResult.record.dataCid).to.not.equal(dataCidBeforeDataUpdate); - // expect(readResult.record.dataCid).to.equal(record!.dataCid); + expect(readResult.record.dataCid).to.not.equal(dataCidBeforeDataUpdate); + expect(readResult.record.dataCid).to.equal(queriedRecord!.dataCid); // Confirm that the data payload of the record was modified. - // const updatedData = await record!.data.text(); - // expect(updatedData).to.equal('bye'); + const updatedData = await queriedRecord!.data.text(); + expect(updatedData).to.equal('bye'); }); it('returns new dateModified after each update', async () => { @@ -2650,7 +2667,6 @@ describe('Record', () => { // attempts to store the record without explicitly marking it for import as it's already been imported ({ status: importRecordStatus } = await queriedRecord.store()); - console.log('attempt to store the record', importRecordStatus); expect(importRecordStatus.code).to.equal(202, importRecordStatus.detail); // bob queries their own DWN for the record, should return the record From 31136972a2cd9bf3354358aa1c62941f63413dbf Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 20:17:37 -0500 Subject: [PATCH 33/35] remove unreachable code path --- packages/api/src/record.ts | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 629d5b308..933aa9db2 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -406,14 +406,7 @@ export class Record implements RecordModel { target : target }; - // if there is already an authz payload, just pass along the record - if (this._authorization) { - latestState.rawMessage = { ...this.rawMessage }; - } else { - // if there is no authz, pass options so the DWN SDK can construct and sign the record - latestState.messageOptions = this.toJSON(); - } - + latestState.rawMessage = { ...this.rawMessage }; const { reply } = await this._agent.sendDwnRequest(latestState); return reply; } From e30a4669da6c8527f79a4394f7b326a8748b1c72 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 20:29:50 -0500 Subject: [PATCH 34/35] send cache full coverage --- packages/api/tests/send-cache.spec.ts | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/api/tests/send-cache.spec.ts b/packages/api/tests/send-cache.spec.ts index d2645d950..1d733e3a3 100644 --- a/packages/api/tests/send-cache.spec.ts +++ b/packages/api/tests/send-cache.spec.ts @@ -18,7 +18,7 @@ describe('SendCache', () => { expect(SendCache.check('id', 'target2')).to.equal(false); }); - it('purges the first item in teh cache when the cache is full (100 items)', async () => { + it('purges the first item in the cache when the target cache is full (100 items)', async () => { const recordId = 'id'; // set 100 items in the cache to the same id for (let i = 0; i < 100; i++) { @@ -42,4 +42,29 @@ describe('SendCache', () => { expect(SendCache.check(recordId, 'target-2')).to.equal(true); expect(SendCache.check(recordId, 'target-new2')).to.equal(true); }); + + it('purges the first item in the cache when the record cache is full (100 items)', async () => { + const target = 'target'; + // set 100 items in the cache to the same id + for (let i = 0; i < 100; i++) { + SendCache.set(`record-${i}`, target); + } + + // check that the first item is in the cache + expect(SendCache.check('record-0', target)).to.equal(true); + + // set another item in the cache + SendCache.set('record-new', target); + + // check that the first item is no longer in the cache but the one after it is as well as the new one. + expect(SendCache.check('record-0', target)).to.equal(false); + expect(SendCache.check('record-1', target)).to.equal(true); + expect(SendCache.check('record-new', target)).to.equal(true); + + // add another item + SendCache.set('record-new2', target); + expect(SendCache.check('record-1', target)).to.equal(false); + expect(SendCache.check('record-2', target)).to.equal(true); + expect(SendCache.check('record-new2', target)).to.equal(true); + }); }); \ No newline at end of file From f51f8965819f37e6c32d179aa6e3b6a55f5f3c30 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Fri, 2 Feb 2024 12:25:11 -0500 Subject: [PATCH 35/35] updated test to have a better description and comments --- packages/api/tests/dwn-api.spec.ts | 65 ++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 58bfbe631..9b0376d35 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -271,10 +271,17 @@ describe('DwnApi', () => { * so they can use them to create records corresponding to the roles they are granted. * * TEST SETUP STEPS: - * 1. Configure the email protocol on Bob's local DWN. + * 1. Configure the photos protocol on Bob and Alice's remote and local DWNs. + * 2. Alice creates a role-based 'friend' record for Bob, updates it, then sends it to her remote DWN. + * 3. Bob creates an album record using the role 'friend', adds Alice as a `participant` of the album and sends the records to Alice. + * 4. Alice fetches the album, and the `participant` record to store it on her local DWN. + * 5. Alice adds Bob as an `updater` of the album and sends the record to Bob and her own remote node. This allows bob to edit photos in the album. + * 6. Alice creates a photo using her participant role and sends it to her own DWN and Bob's DWN. + * 7. Bob updates the photo using his updater role and sends it to Alice and his own DWN. + * 8. Alice fetches the photo and stores it on her local DWN. */ - // Configure the email protocol on Alice and Bob's local and remote DWNs. + // Configure the photos protocol on Alice and Bob's local and remote DWNs. const { status: bobProtocolStatus, protocol: bobProtocol } = await dwnBob.protocols.configure({ message: { definition: photosProtocolDefinition @@ -293,7 +300,7 @@ describe('DwnApi', () => { const { status: aliceRemoteProtocolStatus } = await aliceProtocol.send(aliceDid.did); expect(aliceRemoteProtocolStatus.code).to.equal(202); - // Alice creates a role-based 'friend' record, and sends it to her remote + // Alice creates a role-based 'friend' record, updates it, then sends it to her remote DWN. const { status: friendCreateStatus, record: friendRecord} = await dwnAlice.records.create({ data : 'test', message : { @@ -310,7 +317,7 @@ describe('DwnApi', () => { const { status: aliceFriendSendStatus } = await friendRecord.send(aliceDid.did); expect(aliceFriendSendStatus.code).to.equal(202); - // Bob creates a thread record using the role 'friend' and sends it to Alice + // Bob creates an album record using the role 'friend' and sends it to Alice const { status: albumCreateStatus, record: albumRecord} = await dwnBob.records.create({ data : 'test', message : { @@ -328,21 +335,7 @@ describe('DwnApi', () => { const { status: aliceAlbumSendStatus } = await albumRecord.send(aliceDid.did); expect(aliceAlbumSendStatus.code).to.equal(202); - // Alice fetches the album record Bob created using his friend role - const aliceAlbumReadResult = await dwnAlice.records.read({ - from : aliceDid.did, - message : { - filter: { - recordId: albumRecord.id - } - } - }); - expect(aliceAlbumReadResult.status.code).to.equal(200); - expect(aliceAlbumReadResult.record).to.exist; - const { status: aliceAlbumReadStoreStatus } = await aliceAlbumReadResult.record.store(); - expect(aliceAlbumReadStoreStatus.code).to.equal(202); - - // Bob makes Alice a `participant` + // Bob makes Alice a `participant` and sends the record to her and his own remote node. const { status: participantCreateStatus, record: participantRecord} = await dwnBob.records.create({ data : 'test', message : { @@ -361,6 +354,20 @@ describe('DwnApi', () => { const { status: aliceParticipantSendStatus } = await participantRecord.send(aliceDid.did); expect(aliceParticipantSendStatus.code).to.equal(202); + // Alice fetches the album record as well as the participant record that Bob created and stores it on her local node. + const aliceAlbumReadResult = await dwnAlice.records.read({ + from : aliceDid.did, + message : { + filter: { + recordId: albumRecord.id + } + } + }); + expect(aliceAlbumReadResult.status.code).to.equal(200); + expect(aliceAlbumReadResult.record).to.exist; + const { status: aliceAlbumReadStoreStatus } = await aliceAlbumReadResult.record.store(); + expect(aliceAlbumReadStoreStatus.code).to.equal(202); + const aliceParticipantReadResult = await dwnAlice.records.read({ from : aliceDid.did, message : { @@ -374,7 +381,8 @@ describe('DwnApi', () => { const { status: aliceParticipantReadStoreStatus } = await aliceParticipantReadResult.record.store(); expect(aliceParticipantReadStoreStatus.code).to.equal(202); - // Alice makes Bob an `updater` + // Using the participant role, Alice can make Bob an `updater` and send the record to him and her own remote node. + // Only updater roles can update the photo record after it's been created. const { status: updaterCreateStatus, record: updaterRecord} = await dwnAlice.records.create({ data : 'test', message : { @@ -394,7 +402,7 @@ describe('DwnApi', () => { const { status: aliceUpdaterSendStatus } = await updaterRecord.send(aliceDid.did); expect(aliceUpdaterSendStatus.code).to.equal(202); - // Alice creates a photo using her participant role + // Alice creates a photo using her participant role and sends it to her own DWN and Bob's DWN. const { status: photoCreateStatus, record: photoRecord} = await dwnAlice.records.create({ data : 'test', message : { @@ -413,6 +421,7 @@ describe('DwnApi', () => { const { status: bobPhotoSendStatus } = await photoRecord.send(bobDid.did); expect(bobPhotoSendStatus.code).to.equal(202); + // Bob updates the photo using his updater role and sends it to Alice and his own DWN. const { status: photoUpdateStatus, record: photoUpdateRecord} = await dwnBob.records.write({ data : 'test again', store : false, @@ -433,6 +442,20 @@ describe('DwnApi', () => { expect(alicePhotoUpdateSendStatus.code).to.equal(202); const { status: bobPhotoUpdateSendStatus } = await photoUpdateRecord.send(bobDid.did); expect(bobPhotoUpdateSendStatus.code).to.equal(202); + + // Alice fetches the photo and stores it on her local DWN. + const alicePhotoReadResult = await dwnAlice.records.read({ + from : aliceDid.did, + message : { + filter: { + recordId: photoRecord.id + } + } + }); + expect(alicePhotoReadResult.status.code).to.equal(200); + expect(alicePhotoReadResult.record).to.exist; + const { status: alicePhotoReadStoreStatus } = await alicePhotoReadResult.record.store(); + expect(alicePhotoReadStoreStatus.code).to.equal(202); }); });