From b29a919235f2f0e163d8abed1bc31d3de2f30a4a Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Mon, 22 Jan 2024 11:10:03 -0600 Subject: [PATCH 01/23] Adding changes to src --- packages/agent/src/dwn-manager.ts | 47 ++++++-- packages/agent/src/types/agent.ts | 4 +- packages/api/src/dwn-api.ts | 1 + packages/api/src/record.ts | 189 ++++++++++++++++++++++++++++-- 4 files changed, 217 insertions(+), 24 deletions(-) diff --git a/packages/agent/src/dwn-manager.ts b/packages/agent/src/dwn-manager.ts index 44a6be195..201caf54c 100644 --- a/packages/agent/src/dwn-manager.ts +++ b/packages/agent/src/dwn-manager.ts @@ -63,17 +63,26 @@ type DwnMessage = { data?: Blob; } -const dwnMessageCreators = { +const recordsWriteType = DwnInterfaceName.Records + DwnMethodName.Write; +const dwnMessageConstructors = { [DwnInterfaceName.Events + DwnMethodName.Get] : EventsGet, [DwnInterfaceName.Messages + DwnMethodName.Get] : MessagesGet, [DwnInterfaceName.Records + DwnMethodName.Read] : RecordsRead, [DwnInterfaceName.Records + DwnMethodName.Query] : RecordsQuery, - [DwnInterfaceName.Records + DwnMethodName.Write] : RecordsWrite, + [recordsWriteType] : RecordsWrite, [DwnInterfaceName.Records + DwnMethodName.Delete] : RecordsDelete, [DwnInterfaceName.Protocols + DwnMethodName.Query] : ProtocolsQuery, [DwnInterfaceName.Protocols + DwnMethodName.Configure] : ProtocolsConfigure, }; +(RecordsWrite.prototype as any).forceSet = function(prop: string, value: unknown){ + this[prop] = value; +}; + +(RecordsWrite.prototype as any).forceGet = function(prop: string){ + return this[prop]; +}; + export type DwnManagerOptions = { agent?: Web5ManagedAgent; dwn: Dwn; @@ -187,7 +196,9 @@ export class DwnManager { messageData = data; } else { + console.log('start constructDwnMessage'); const { message } = await this.constructDwnMessage({ request }); + console.log('passed constructDwnMessage'); dwnRpcRequest.message = message; messageData = request.dataStream; } @@ -242,9 +253,12 @@ export class DwnManager { } private async constructDwnMessage(options: { - request: ProcessDwnRequest + request : ProcessDwnRequest }) { const { request } = options; + const rawMessage = (request as any).rawMessage; + + console.log('259', rawMessage); let readableStream: Readable | undefined; @@ -252,7 +266,7 @@ export class DwnManager { if (request.messageType === 'RecordsWrite') { const messageOptions = request.messageOptions as RecordsWriteOptions; - if (request.dataStream && !messageOptions.data) { + if (request.dataStream && !messageOptions?.data) { const { dataStream } = request; let isomorphicNodeReadable: Readable; @@ -266,22 +280,29 @@ export class DwnManager { readableStream = webReadableToIsomorphicNodeReadable(forProcessMessage); } - // @ts-ignore - messageOptions.dataCid = await Cid.computeDagPbCidFromStream(isomorphicNodeReadable); - // @ts-ignore - messageOptions.dataSize ??= isomorphicNodeReadable['bytesRead']; + if (!rawMessage) { + // @ts-ignore + messageOptions.dataCid = await Cid.computeDagPbCidFromStream(isomorphicNodeReadable); + // @ts-ignore + messageOptions.dataSize ??= isomorphicNodeReadable['bytesRead']; + } } } const dwnSigner = await this.constructDwnSigner(request.author); - - const messageCreator = dwnMessageCreators[request.messageType]; - const dwnMessage = await messageCreator.create({ + const dwnMessageConstructor = dwnMessageConstructors[request.messageType]; + const dwnMessage = rawMessage ? await dwnMessageConstructor.parse(rawMessage) : await dwnMessageConstructor.create({ ...request.messageOptions, signer: dwnSigner }); - return { message: dwnMessage.message, dataStream: readableStream }; + if (dwnMessageConstructor === RecordsWrite){ + if (request.import) { + await (dwnMessage as RecordsWrite).signAsOwner(dwnSigner); + } + } + + return { message: (dwnMessage as any).message, dataStream: readableStream }; } private async getAuthorSigningKeyId(options: { @@ -411,7 +432,7 @@ export class DwnManager { const dwnSigner = await this.constructDwnSigner(author); - const messageCreator = dwnMessageCreators[messageType]; + const messageCreator = dwnMessageConstructors[messageType]; const dwnMessage = await messageCreator.create({ ...messageOptions, diff --git a/packages/agent/src/types/agent.ts b/packages/agent/src/types/agent.ts index 97691f724..f10a07b93 100644 --- a/packages/agent/src/types/agent.ts +++ b/packages/agent/src/types/agent.ts @@ -75,8 +75,10 @@ export type DwnRequest = { */ export type ProcessDwnRequest = DwnRequest & { dataStream?: Blob | ReadableStream | Readable; - messageOptions: unknown; + rawMessage?: unknown; + messageOptions?: unknown; store?: boolean; + import?: boolean; }; export type SendDwnRequest = DwnRequest & (ProcessDwnRequest | { messageCid: string }) diff --git a/packages/api/src/dwn-api.ts b/packages/api/src/dwn-api.ts index bc387a4dc..b74e63c06 100644 --- a/packages/api/src/dwn-api.ts +++ b/packages/api/src/dwn-api.ts @@ -366,6 +366,7 @@ export class DwnApi { const { entries, status, cursor } = reply; const records = entries.map((entry: RecordsQueryReplyEntry) => { + const recordOptions = { /** * Extract the `author` DID from the record entry since records may be signed by the diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index e162d0cd4..5613553b0 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -23,6 +23,8 @@ export type RecordOptions = RecordsWriteMessage & { connectedDid: string; encodedData?: string | Blob; data?: Readable | ReadableStream; + initialWrite?: RecordsWriteMessage; + protocolRole?: string; remoteOrigin?: string; }; @@ -33,9 +35,10 @@ export type RecordOptions = RecordsWriteMessage & { * @beta */ export type RecordModel = RecordsWriteDescriptor - & Omit + & Omit & { author: string; + protocolRole?: RecordOptions['protocolRole']; recordId?: string; } @@ -51,6 +54,7 @@ export type RecordUpdateOptions = { dateModified?: RecordsWriteDescriptor['messageTimestamp']; datePublished?: RecordsWriteDescriptor['datePublished']; published?: RecordsWriteDescriptor['published']; + protocolRole?: RecordOptions['protocolRole']; } /** @@ -66,6 +70,20 @@ export type RecordUpdateOptions = { * * @beta */ + +function removeUndefinedProperties(obj: any): any { + if (typeof obj !== 'object' || obj === null) return; + Object.keys(obj).forEach(key => { + const val = obj[key]; + if (val === undefined) { + delete obj[key]; + } else if (typeof val === 'object') { + removeUndefinedProperties(val); + } + }); + return obj; +} + export class Record implements RecordModel { // Record instance metadata. private _agent: Web5Agent; @@ -77,16 +95,21 @@ export class Record implements RecordModel { // Private variables for DWN `RecordsWrite` message properties. private _author: string; private _attestation?: RecordsWriteMessage['attestation']; + private _authorization?: RecordsWriteMessage['authorization']; private _contextId?: string; private _descriptor: RecordsWriteDescriptor; private _encryption?: RecordsWriteMessage['encryption']; + private _initialWrite: RecordOptions['initialWrite']; private _recordId: string; - + private _protocolRole: RecordOptions['protocolRole']; // Getters for immutable DWN Record properties. /** Record's signatures attestation */ get attestation(): RecordsWriteMessage['attestation'] { return this._attestation; } + /** Record's signatures attestation */ + get authorization(): RecordsWriteMessage['authorization'] { return this._authorization; } + /** DID that signed the record. */ get author(): string { return this._author; } @@ -102,6 +125,8 @@ export class Record implements RecordModel { /** Record's encryption */ get encryption(): RecordsWriteMessage['encryption'] { return this._encryption; } + get initialWrite(): RecordOptions['initialWrite'] { return this._initialWrite; } + /** Record's ID */ get id() { return this._recordId; } @@ -120,6 +145,9 @@ export class Record implements RecordModel { /** Record's protocol path */ get protocolPath() { return this._descriptor.protocolPath; } + /** Role under which the author is writting the record */ + get protocolRole() { return this._protocolRole; } + /** Record's recipient */ get recipient() { return this._descriptor.recipient; } @@ -147,6 +175,7 @@ export class Record implements RecordModel { get published() { return this._descriptor.published; } constructor(agent: Web5Agent, options: RecordOptions) { + this._agent = agent; /** Store the author DID that originally signed the message as a convenience for developers, so @@ -165,10 +194,13 @@ export class Record implements RecordModel { // RecordsWriteMessage properties. this._attestation = options.attestation; + this._authorization = options.authorization; this._contextId = options.contextId; this._descriptor = options.descriptor; this._encryption = options.encryption; + this._initialWrite = options.initialWrite; this._recordId = options.recordId; + this._protocolRole = options.protocolRole; if (options.encodedData) { // If `encodedData` is set, then it is expected that: @@ -295,6 +327,90 @@ export class Record implements RecordModel { return dataObj; } + private async _processMessage(options: any){ + + const { store = true } = options; + + const request = { + messageType : DwnInterfaceName.Records + DwnMethodName.Write, + author : this._connectedDid, + target : this._connectedDid, + import : !!options.import, + store + } as any; + + if (options.rawMessage) { + removeUndefinedProperties(options.rawMessage); + request.rawMessage = options.rawMessage as Partial; + } + else { + request.messageOptions = options.messageOptions; + } + + if (options.dataStream) { + request.dataStream = options.dataStream; + } + + return this._agent.processDwnRequest(request); + } + + + async store(options: any = {}): Promise { + + // Add check to bail if already imported + + const { store = true, import: _import = false, storeAll = false } = options; + + const initialWrite = this._initialWrite; + + if (initialWrite && storeAll) { + + let agentResponse = await this._processMessage({ + import : _import, + store : store, + rawMessage : { + contextId: this._contextId, + ...initialWrite + } + }); + + const { message, reply: { status } } = agentResponse; + const responseMessage = message as RecordsWriteMessage; + + if (200 <= status.code && status.code <= 299) { + if (_import) initialWrite.authorization = responseMessage.authorization; + } + + } + + let agentResponse = await this._processMessage({ + import : !initialWrite && _import, + store : store, + dataStream : await this.data.blob(), + rawMessage : { + contextId : this._contextId, + recordId : this._recordId, + descriptor : this._descriptor, + attestation : this._attestation, + authorization : this._authorization, + encryption : this._encryption, + } + }); + + const { message, reply: { status } } = agentResponse; + const responseMessage = message as RecordsWriteMessage; + + if (_import) this._authorization = responseMessage.authorization; + + return status; + + } + + async import(options?: any): Promise { + // Add check to bail if already imported + return this.store({ store: options?.store !== false, storeAll: true, import: true }); + } + /** * Send the current record to a remote DWN by specifying their DID * (vs waiting for the regular DWN sync) @@ -304,14 +420,53 @@ export class Record implements RecordModel { * * @beta */ - async send(target: string): Promise { - const { reply: { status } } = await this._agent.sendDwnRequest({ - messageType : DwnInterfaceName.Records + DwnMethodName.Write, - author : this._connectedDid, - dataStream : await this.data.blob(), - target : target, - messageOptions : this.toJSON(), - }); + async send(_options?: string | { [key: string]: any }): Promise { + + const initialWrite = this._initialWrite; + const options = !_options ? { target: this._connectedDid } : typeof _options === 'string' ? { target: _options } : _options; + + if (!options.target){ + options.target = this._connectedDid; + } + + if (options.sendAll && initialWrite){ + + const initialState = { + messageType : DwnInterfaceName.Records + DwnMethodName.Write, + author : this._connectedDid, + target : options.target, + rawMessage : removeUndefinedProperties({ + contextId: this._contextId, + ...initialWrite + }) + } as any; + + await this._agent.sendDwnRequest(initialState); + + } + + const latestState = { + messageType : DwnInterfaceName.Records + DwnMethodName.Write, + author : this._connectedDid, + dataStream : await this.data.blob(), + target : options.target + } as any; + + if (this._authorization) { + latestState.rawMessage = removeUndefinedProperties({ + contextId : this._contextId, + recordId : this._recordId, + descriptor : this._descriptor, + attestation : this._attestation, + authorization : this._authorization, + encryption : this._encryption, + }); + } + else { + latestState.messageOptions = this.toJSON(); + } + + const { reply: { status } } = await this._agent.sendDwnRequest(latestState); return { status }; } @@ -324,6 +479,7 @@ export class Record implements RecordModel { return { attestation : this.attestation, author : this.author, + authorization : this.authorization, contextId : this.contextId, dataCid : this.dataCid, dataFormat : this.dataFormat, @@ -337,6 +493,7 @@ export class Record implements RecordModel { parentId : this.parentId, protocol : this.protocol, protocolPath : this.protocolPath, + protocolRole : this.protocolRole, published : this.published, recipient : this.recipient, recordId : this.id, @@ -427,7 +584,19 @@ export class Record implements RecordModel { const responseMessage = message as RecordsWriteMessage; if (200 <= status.code && status.code <= 299) { + if (!this._initialWrite) { + console.log('post update', responseMessage); + this._initialWrite = removeUndefinedProperties({ + contextId : this._contextId, + recordId : this._recordId, + descriptor : this._descriptor, + attestation : this._attestation, + authorization : this._authorization, + encryption : this._encryption, + }); + } // Only update the local Record instance mutable properties if the record was successfully (over)written. + this._authorization = responseMessage.authorization; mutableDescriptorProperties.forEach(property => { this._descriptor[property] = responseMessage.descriptor[property]; }); From a857c67319a4431cbae4eecd752a4a79548bf7cb Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Mon, 22 Jan 2024 11:19:55 -0600 Subject: [PATCH 02/23] Updates from branch --- .../fixtures/protocol-definitions/email.json | 22 ++++ packages/api/tests/record.spec.ts | 121 ++++++++++++++++++ 2 files changed, 143 insertions(+) diff --git a/packages/api/tests/fixtures/protocol-definitions/email.json b/packages/api/tests/fixtures/protocol-definitions/email.json index 1e23bf5d2..a7b20623b 100644 --- a/packages/api/tests/fixtures/protocol-definitions/email.json +++ b/packages/api/tests/fixtures/protocol-definitions/email.json @@ -2,12 +2,34 @@ "protocol": "http://email-protocol.xyz", "published": false, "types": { + "thread": { + "schema": "http://email-protocol.xyz/schema/thread", + "dataFormats": ["text/plain"] + }, "email": { "schema": "http://email-protocol.xyz/schema/email", "dataFormats": ["text/plain"] } }, "structure": { + "thread": { + "$actions": [ + { + "who": "recipient", + "of": "thread", + "can": "read" + }, + { + "who": "author", + "of": "thread", + "can": "write" + }, + { + "who": "anyone", + "can": "update" + } + ] + }, "email": { "$actions": [ { diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index 1fdfc0a32..07ce60fb0 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -97,6 +97,127 @@ describe('Record', () => { await testAgent.closeStorage(); }); + // FIRST PASS AT IMPORT + + it.only('imports a record that another user wrote', async () => { + + // Install the email protocol for Alice's local DWN. + let { protocol: aliceProtocol, status: aliceStatus } = await dwnAlice.protocols.configure({ + message: { + definition: emailProtocolDefinition + } + }); + expect(aliceStatus.code).to.equal(202); + expect(aliceProtocol).to.exist; + + // Install the email protocol for Alice's remote DWN. + const { status: alicePushStatus } = await aliceProtocol!.send(aliceDid.did); + expect(alicePushStatus.code).to.equal(202); + + // Install the email protocol for Bob's local DWN. + const { protocol: bobProtocol, status: bobStatus } = await dwnBob.protocols.configure({ + message: { + definition: emailProtocolDefinition + } + }); + + expect(bobStatus.code).to.equal(202); + expect(bobProtocol).to.exist; + + // Install the email protocol for Bob's remote DWN. + const { status: bobPushStatus } = await bobProtocol!.send(bobDid.did); + expect(bobPushStatus.code).to.equal(202); + + // Alice creates a new large record and stores it + const { status: aliceEmailStatus, record: aliceEmailRecord } = await dwnAlice.records.write({ + data : TestDataGenerator.randomString(DwnConstant.maxDataSizeAllowedToBeEncoded + 1000), + message : { + recipient : bobDid.did, + protocol : emailProtocolDefinition.protocol, + protocolPath : 'thread', + schema : 'http://email-protocol.xyz/schema/thread', + } + }); + expect(aliceEmailStatus.code).to.equal(202); + const { status: sendStatus } = await aliceEmailRecord!.send(aliceDid.did); + expect(sendStatus.code).to.equal(202); + + // Alice queries for the record that was just created on her remote DWN. + const { records: queryRecords, status: queryRecordStatus } = await dwnBob.records.query({ + from : aliceDid.did, + message : { + filter: { + protocol : emailProtocolDefinition.protocol, + protocolPath : 'thread', + } + } + }); + expect(queryRecordStatus.code).to.equal(200); + const importRecord = queryRecords[0]; + const importRecordStatus = await importRecord.import(); + expect(importRecordStatus.code).to.equal(202); + + const { status: importSendStatus } = await importRecord!.send({ sendAll: true }); + expect(importSendStatus.code).to.equal(202); + + // Alice updates her record + let { status: aliceEmailStatusUpdated } = await aliceEmailRecord.update({ + data: TestDataGenerator.randomString(DwnConstant.maxDataSizeAllowedToBeEncoded + 1000) + }); + expect(aliceEmailStatusUpdated.code).to.equal(202); + + const { status: sentToSelfStatus } = await aliceEmailRecord!.send(); + expect(sentToSelfStatus.code).to.equal(202); + + const { status: sentToBobStatus } = await aliceEmailRecord!.send(bobDid.did); + expect(sentToBobStatus.code).to.equal(202); + + // Alice updates her record + + const updatedText = TestDataGenerator.randomString(DwnConstant.maxDataSizeAllowedToBeEncoded + 1000); + let { status: aliceEmailStatusUpdatedAgain } = await aliceEmailRecord.update({ + data: updatedText + }); + expect(aliceEmailStatusUpdatedAgain.code).to.equal(202); + + // Sends it to her own remote DWN again + const { status: sentToSelfAgainStatus } = await aliceEmailRecord!.send(); + expect(sentToSelfAgainStatus.code).to.equal(202); + + const { records: updatedRecords, status: updatedRecordsStatus } = await dwnBob.records.query({ + from : aliceDid.did, + message : { + filter: { + protocol : emailProtocolDefinition.protocol, + protocolPath : 'thread', + } + } + }); + expect(updatedRecordsStatus.code).to.equal(200); + + const updatedRecord = updatedRecords[0]; + const updatedRecordStoredStatus = await updatedRecord.store(); + expect(updatedRecordStoredStatus.code).to.equal(202); + + expect(await updatedRecord.data.text() === updatedText).to.equal(true); + + const { status: updatedRecordToSelfStatus } = await updatedRecord!.send(); + expect(updatedRecordToSelfStatus.code).to.equal(202); + + // Confirm Bob can query his own remote DWN for the created record. + // const bobQueryResult = await dwnBob.records.query({ + // from : bobDid.did, + // message : { + // filter: { + // schema: 'http://email-protocol.xyz/schema/email' + // } + // } + // }); + // expect(bobQueryResult.status.code).to.equal(200); + // expect(bobQueryResult.records).to.exist; + // expect(bobQueryResult.records!.length).to.equal(1); + }); + it('should retain all defined properties', async () => { // RecordOptions properties const author = aliceDid.did; From df8317db312a24d230827ef24cd68a107678c217 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Mon, 22 Jan 2024 16:22:15 -0600 Subject: [PATCH 03/23] Removing logging --- packages/agent/src/dwn-manager.ts | 4 ---- packages/api/src/record.ts | 5 ++--- packages/api/tests/record.spec.ts | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/agent/src/dwn-manager.ts b/packages/agent/src/dwn-manager.ts index 201caf54c..aa1a6bc05 100644 --- a/packages/agent/src/dwn-manager.ts +++ b/packages/agent/src/dwn-manager.ts @@ -196,9 +196,7 @@ export class DwnManager { messageData = data; } else { - console.log('start constructDwnMessage'); const { message } = await this.constructDwnMessage({ request }); - console.log('passed constructDwnMessage'); dwnRpcRequest.message = message; messageData = request.dataStream; } @@ -258,8 +256,6 @@ export class DwnManager { const { request } = options; const rawMessage = (request as any).rawMessage; - console.log('259', rawMessage); - let readableStream: Readable | undefined; // TODO: Consider refactoring to move data transformations imposed by fetch() limitations to the HTTP transport-related methods. diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 5613553b0..de898ffae 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -367,7 +367,7 @@ export class Record implements RecordModel { let agentResponse = await this._processMessage({ import : _import, - store : store, + store : true, rawMessage : { contextId: this._contextId, ...initialWrite @@ -385,7 +385,7 @@ export class Record implements RecordModel { let agentResponse = await this._processMessage({ import : !initialWrite && _import, - store : store, + store : store || storeAll, dataStream : await this.data.blob(), rawMessage : { contextId : this._contextId, @@ -585,7 +585,6 @@ export class Record implements RecordModel { if (200 <= status.code && status.code <= 299) { if (!this._initialWrite) { - console.log('post update', responseMessage); this._initialWrite = removeUndefinedProperties({ contextId : this._contextId, recordId : this._recordId, diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index 07ce60fb0..ca7d7e1b6 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -99,7 +99,7 @@ describe('Record', () => { // FIRST PASS AT IMPORT - it.only('imports a record that another user wrote', async () => { + it('imports a record that another user wrote', async () => { // Install the email protocol for Alice's local DWN. let { protocol: aliceProtocol, status: aliceStatus } = await dwnAlice.protocols.configure({ From ccee2eeb0043f8a43b04993b4edc4597edd1077b Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Mon, 22 Jan 2024 17:12:20 -0600 Subject: [PATCH 04/23] Add docs --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 7dd0dfd37..9709d6f7b 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,10 @@ Each `Record` instance has the following instance methods: - **`text`** - _`function`_: returns the data as a string. - **`send`** - _`function`_: sends the record the instance represents to the DWeb Node endpoints of a provided DID. - **`update`** - _`function`_: takes in a new request object matching the expected method signature of a `write` and overwrites the record. This is a convenience method that allows you to easily overwrite records with less verbosity. +- **`store`** - _`function`_: stores the record in the local DWN instance, offering the following options: + - `storeAll`: stores the initial record state in addition to the latest state. This is required if your local DWN instance does not already contain this record. +- **`import`** - _`function`_: signs a record with an owner override to import the record into the local DWN instance: + - `store` - _`boolean`_: when false is passed, the record will only be signed with an owner override, not stored in the local DWN instance. Defaults to `true`. ### **`web5.dwn.records.query(request)`** From b0fcc569f041e771acf13478214739cc0120aa46 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Wed, 24 Jan 2024 09:51:24 -0600 Subject: [PATCH 05/23] Update based on PR feedback --- packages/agent/src/dwn-manager.ts | 8 -------- packages/api/src/record.ts | 18 ++++++++++-------- packages/api/tests/record.spec.ts | 2 +- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/packages/agent/src/dwn-manager.ts b/packages/agent/src/dwn-manager.ts index aa1a6bc05..ccbfe9370 100644 --- a/packages/agent/src/dwn-manager.ts +++ b/packages/agent/src/dwn-manager.ts @@ -75,14 +75,6 @@ const dwnMessageConstructors = { [DwnInterfaceName.Protocols + DwnMethodName.Configure] : ProtocolsConfigure, }; -(RecordsWrite.prototype as any).forceSet = function(prop: string, value: unknown){ - this[prop] = value; -}; - -(RecordsWrite.prototype as any).forceGet = function(prop: string){ - return this[prop]; -}; - export type DwnManagerOptions = { agent?: Web5ManagedAgent; dwn: Dwn; diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index de898ffae..c20e11ea2 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -10,7 +10,6 @@ import { Convert, NodeStream, Stream } from '@web5/common'; import { DwnInterfaceName, DwnMethodName } from '@tbd54566975/dwn-sdk-js'; import type { ResponseStatus } from './dwn-api.js'; - import { dataToBlob } from './utils.js'; /** @@ -100,6 +99,7 @@ export class Record implements RecordModel { private _descriptor: RecordsWriteDescriptor; private _encryption?: RecordsWriteMessage['encryption']; private _initialWrite: RecordOptions['initialWrite']; + private _initialWriteStored: boolean; private _recordId: string; private _protocolRole: RecordOptions['protocolRole']; // Getters for immutable DWN Record properties. @@ -357,13 +357,11 @@ export class Record implements RecordModel { async store(options: any = {}): Promise { - // Add check to bail if already imported - - const { store = true, import: _import = false, storeAll = false } = options; + const { store = true, import: _import = false } = options; const initialWrite = this._initialWrite; - if (initialWrite && storeAll) { + if (initialWrite && !this._initialWriteStored) { let agentResponse = await this._processMessage({ import : _import, @@ -378,6 +376,7 @@ export class Record implements RecordModel { const responseMessage = message as RecordsWriteMessage; if (200 <= status.code && status.code <= 299) { + this._initialWriteStored = true; if (_import) initialWrite.authorization = responseMessage.authorization; } @@ -385,7 +384,7 @@ export class Record implements RecordModel { let agentResponse = await this._processMessage({ import : !initialWrite && _import, - store : store || storeAll, + store : store, dataStream : await this.data.blob(), rawMessage : { contextId : this._contextId, @@ -400,7 +399,9 @@ export class Record implements RecordModel { const { message, reply: { status } } = agentResponse; const responseMessage = message as RecordsWriteMessage; - if (_import) this._authorization = responseMessage.authorization; + if (200 <= status.code && status.code <= 299) { + if (_import) this._authorization = responseMessage.authorization; + } return status; @@ -408,7 +409,7 @@ export class Record implements RecordModel { async import(options?: any): Promise { // Add check to bail if already imported - return this.store({ store: options?.store !== false, storeAll: true, import: true }); + return this.store({ store: options?.store !== false, import: true }); } /** @@ -596,6 +597,7 @@ export class Record implements RecordModel { } // Only update the local Record instance mutable properties if the record was successfully (over)written. this._authorization = responseMessage.authorization; + this._protocolRole = messageOptions.protocolRole; mutableDescriptorProperties.forEach(property => { this._descriptor[property] = responseMessage.descriptor[property]; }); diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index ca7d7e1b6..07ce60fb0 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -99,7 +99,7 @@ describe('Record', () => { // FIRST PASS AT IMPORT - it('imports a record that another user wrote', async () => { + it.only('imports a record that another user wrote', async () => { // Install the email protocol for Alice's local DWN. let { protocol: aliceProtocol, status: aliceStatus } = await dwnAlice.protocols.configure({ From f83e5b6978421869b0e84d5c002643f1419b1cfc Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Wed, 24 Jan 2024 09:58:56 -0600 Subject: [PATCH 06/23] update readme --- README.md | 2 +- packages/api/tests/record.spec.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9709d6f7b..01c5359ab 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ Each `Record` instance has the following instance methods: - **`send`** - _`function`_: sends the record the instance represents to the DWeb Node endpoints of a provided DID. - **`update`** - _`function`_: takes in a new request object matching the expected method signature of a `write` and overwrites the record. This is a convenience method that allows you to easily overwrite records with less verbosity. - **`store`** - _`function`_: stores the record in the local DWN instance, offering the following options: - - `storeAll`: stores the initial record state in addition to the latest state. This is required if your local DWN instance does not already contain this record. + - `import`: imports the record as with an owner-signed override (still subject to Protocol rules, when a record is Protocol-based) - **`import`** - _`function`_: signs a record with an owner override to import the record into the local DWN instance: - `store` - _`boolean`_: when false is passed, the record will only be signed with an owner override, not stored in the local DWN instance. Defaults to `true`. diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index 07ce60fb0..ca7d7e1b6 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -99,7 +99,7 @@ describe('Record', () => { // FIRST PASS AT IMPORT - it.only('imports a record that another user wrote', async () => { + it('imports a record that another user wrote', async () => { // Install the email protocol for Alice's local DWN. let { protocol: aliceProtocol, status: aliceStatus } = await dwnAlice.protocols.configure({ From 3c9de13e7aed70b8e6c5a86b843bae2e44d42d20 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Mon, 29 Jan 2024 16:14:31 -0600 Subject: [PATCH 07/23] Add role-based record tests, fix for cloned obj issue --- packages/agent/src/dwn-manager.ts | 3 +- packages/api/src/record.ts | 31 ++++++++- packages/api/tests/dwn-api.spec.ts | 68 ++++++++++++++++++- .../fixtures/protocol-definitions/photos.json | 51 ++++++++++++++ packages/api/tests/record.spec.ts | 13 ++-- 5 files changed, 151 insertions(+), 15 deletions(-) create mode 100644 packages/api/tests/fixtures/protocol-definitions/photos.json diff --git a/packages/agent/src/dwn-manager.ts b/packages/agent/src/dwn-manager.ts index ccbfe9370..7d29f2150 100644 --- a/packages/agent/src/dwn-manager.ts +++ b/packages/agent/src/dwn-manager.ts @@ -63,13 +63,12 @@ type DwnMessage = { data?: Blob; } -const recordsWriteType = DwnInterfaceName.Records + DwnMethodName.Write; const dwnMessageConstructors = { [DwnInterfaceName.Events + DwnMethodName.Get] : EventsGet, [DwnInterfaceName.Messages + DwnMethodName.Get] : MessagesGet, [DwnInterfaceName.Records + DwnMethodName.Read] : RecordsRead, [DwnInterfaceName.Records + DwnMethodName.Query] : RecordsQuery, - [recordsWriteType] : RecordsWrite, + [DwnInterfaceName.Records + DwnMethodName.Write] : RecordsWrite, [DwnInterfaceName.Records + DwnMethodName.Delete] : RecordsDelete, [DwnInterfaceName.Protocols + DwnMethodName.Query] : ProtocolsQuery, [DwnInterfaceName.Protocols + DwnMethodName.Configure] : ProtocolsConfigure, diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index c20e11ea2..0487004f1 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -104,6 +104,29 @@ export class Record implements RecordModel { private _protocolRole: RecordOptions['protocolRole']; // Getters for immutable DWN Record properties. + static sendCache = new Map(); + static sendCacheLimit = 100; + static setSendCache(recordId, target){ + const recordCache = Record.sendCache; + let targetCache = recordCache.get(recordId) || new Set(); + recordCache.delete(recordId); + recordCache.set(recordId, targetCache); + if (recordCache.size > Record.sendCacheLimit) { + const firstRecord = recordCache.keys().next().value; + recordCache.delete(firstRecord); + } + targetCache.delete(target); + targetCache.add(target); + if (targetCache.size > Record.sendCacheLimit) { + const firstTarget = targetCache.keys().next().value; + targetCache.delete(firstTarget); + } + } + static checkSendCache(recordId, target){ + let targetCache = Record.sendCache.get(recordId); + return target && targetCache ? targetCache.has(target) : targetCache; + } + /** Record's signatures attestation */ get attestation(): RecordsWriteMessage['attestation'] { return this._attestation; } @@ -430,7 +453,7 @@ export class Record implements RecordModel { options.target = this._connectedDid; } - if (options.sendAll && initialWrite){ + if (initialWrite && !Record.checkSendCache(this._recordId, options.target)){ const initialState = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, @@ -444,6 +467,8 @@ export class Record implements RecordModel { await this._agent.sendDwnRequest(initialState); + Record.setSendCache(this._recordId, options.target); + } const latestState = { @@ -586,14 +611,14 @@ export class Record implements RecordModel { if (200 <= status.code && status.code <= 299) { if (!this._initialWrite) { - this._initialWrite = removeUndefinedProperties({ + this._initialWrite = JSON.parse(JSON.stringify(removeUndefinedProperties({ contextId : this._contextId, recordId : this._recordId, descriptor : this._descriptor, attestation : this._attestation, authorization : this._authorization, encryption : this._encryption, - }); + }))); } // Only update the local Record instance mutable properties if the record was successfully (over)written. this._authorization = responseMessage.authorization; diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index d04705f20..abd792b65 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -9,6 +9,7 @@ import { DwnApi } from '../src/dwn-api.js'; import { testDwnUrl } from './utils/test-config.js'; import { TestUserAgent } from './utils/test-user-agent.js'; import emailProtocolDefinition from './fixtures/protocol-definitions/email.json' assert { type: 'json' }; +import photosProtocolDefinition from './fixtures/protocol-definitions/photos.json' assert { type: 'json' }; let testDwnUrls: string[] = [testDwnUrl]; @@ -261,6 +262,72 @@ describe('DwnApi', () => { expect(result.record).to.exist; expect(await result.record?.data.json()).to.deep.equal(dataJson); }); + + it('creates a role record for another user that they can use to create role-based records', async () => { + /** + * WHAT IS BEING TESTED? + * + * We are testing whether role records can be created for outbound participants + * so they can use them to create records corresponding to the roles they are granted. + * + * TEST SETUP STEPS: + * 1. Configure the email protocol on Bob's local DWN. + */ + + // Configure the email protocol on Alice and Bob's local and remote DWNs. + const { status: bobProtocolStatus, protocol: bobProtocol } = await dwnBob.protocols.configure({ + message: { + definition: photosProtocolDefinition + } + }); + expect(bobProtocolStatus.code).to.equal(202); + const { status: bobRemoteProtocolStatus } = await bobProtocol.send(bobDid.did); + expect(bobRemoteProtocolStatus.code).to.equal(202); + + const { status: aliceProtocolStatus, protocol: aliceProtocol } = await dwnAlice.protocols.configure({ + message: { + definition: photosProtocolDefinition + } + }); + expect(aliceProtocolStatus.code).to.equal(202); + const { status: aliceRemoteProtocolStatus } = await aliceProtocol.send(aliceDid.did); + expect(aliceRemoteProtocolStatus.code).to.equal(202); + + // Alice creates a role-based 'friend' record, and sends it to her remote + const { status: friendCreateStatus, record: friendRecord} = await dwnAlice.records.create({ + data : 'test', + message : { + recipient : bobDid.did, + protocol : photosProtocolDefinition.protocol, + protocolPath : 'friend', + schema : photosProtocolDefinition.types.friend.schema, + dataFormat : 'text/plain' + } + }); + expect(friendCreateStatus.code).to.equal(202); + const { status: friendRecordUpdateStatus } = await friendRecord.update({ data: 'update' }); + expect(friendRecordUpdateStatus.code).to.equal(202); + const { status: friendSendStatus } = await friendRecord.send(aliceDid.did); + expect(friendSendStatus.code).to.equal(202); + + // Bob creates a thread record using the role 'friend' and sends it to Alice + const { status: albumCreateStatus, record: albumRecord} = await dwnBob.records.create({ + data : 'test', + message : { + recipient : aliceDid.did, + protocol : photosProtocolDefinition.protocol, + protocolPath : 'album', + protocolRole : 'friend', + schema : photosProtocolDefinition.types.album.schema, + dataFormat : 'text/plain' + } + }); + expect(albumCreateStatus.code).to.equal(202); + const { status: bobSendStatus } = await albumRecord.send(bobDid.did); + expect(bobSendStatus.code).to.equal(202); + const { status: aliceSendStatus } = await albumRecord.send(aliceDid.did); + expect(aliceSendStatus.code).to.equal(202); + }); }); describe('agent store: false', () => { @@ -705,7 +772,6 @@ describe('DwnApi', () => { } } }); - // Confirm that the record does not currently exist on Bob's DWN. expect(result.status.code).to.equal(200); expect(result.records).to.exist; diff --git a/packages/api/tests/fixtures/protocol-definitions/photos.json b/packages/api/tests/fixtures/protocol-definitions/photos.json new file mode 100644 index 000000000..9a002dff7 --- /dev/null +++ b/packages/api/tests/fixtures/protocol-definitions/photos.json @@ -0,0 +1,51 @@ +{ + "protocol": "http://photo-protocol.xyz", + "published": true, + "types": { + "album": { + "schema": "http://photo-protocol.xyz/schema/album", + "dataFormats": ["text/plain"] + }, + "photo": { + "schema": "http://photo-protocol.xyz/schema/photo", + "dataFormats": ["text/plain"] + }, + "friend": { + "schema": "http://photo-protocol.xyz/schema/friend", + "dataFormats": ["text/plain"] + }, + "participant": { + "schema": "http://photo-protocol.xyz/schema/participant", + "dataFormats": ["text/plain"] + } + }, + "structure": { + "friend": { + "$globalRole": true + }, + "album": { + "$actions": [ + { + "role": "friend", + "can": "write" + } + ], + "participant": { + "$contextRole": true + }, + "photo": { + "$actions": [ + { + "role": "album/participant", + "can": "write" + }, + { + "who": "author", + "of": "thread", + "can": "write" + } + ] + } + } + } +} diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index ca7d7e1b6..e54e4ec89 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -157,7 +157,7 @@ describe('Record', () => { const importRecordStatus = await importRecord.import(); expect(importRecordStatus.code).to.equal(202); - const { status: importSendStatus } = await importRecord!.send({ sendAll: true }); + const { status: importSendStatus } = await importRecord!.send(); expect(importSendStatus.code).to.equal(202); // Alice updates her record @@ -1306,8 +1306,7 @@ describe('Record', () => { * 4. Validate that Bob is able to write the record to Alice's remote DWN. */ const { status: sendStatusToAlice } = await queryRecordsFrom[0]!.send(aliceDid.did); - expect(sendStatusToAlice.code).to.equal(401); - expect(sendStatusToAlice.detail).to.equal(`Cannot read properties of undefined (reading 'authorization')`); + expect(sendStatusToAlice.code).to.equal(202); }); }); }); @@ -1466,7 +1465,7 @@ describe('Record', () => { }); // TODO: Fix after changes are made to dwn-sdk-js to include the initial write in every query/read response. - it('fails to write updated records to a remote DWN that is missing the initial write', async () => { + it('automatically sends the initial write and update of a record to a remote DWN', async () => { // Alice writes a message to her agent connected DWN. const { status, record } = await dwnAlice.records.write({ data : 'Hello, world!', @@ -1483,11 +1482,7 @@ describe('Record', () => { // Write the updated record to Alice's remote DWN a second time. const sendResult = await record!.send(aliceDid.did); - expect(sendResult.status.code).to.equal(400); - expect(sendResult.status.detail).to.equal('RecordsWriteGetInitialWriteNotFound: initial write is not found'); - - // TODO: Uncomment the following line after changes are made to dwn-sdk-js to include the initial write in every query/read response. - // expect(sendResult.status.code).to.equal(202); + expect(sendResult.status.code).to.equal(202); }); it('writes large records to remote DWNs that were initially queried from a remote DWN', async () => { From 33cdf73b74a0f434ac6d6a43f889612ff27eb58d Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Mon, 29 Jan 2024 22:46:59 -0600 Subject: [PATCH 08/23] Updated protocol config, added elaborate role tests --- packages/api/tests/dwn-api.spec.ts | 77 +++++++++++++++++-- .../fixtures/protocol-definitions/photos.json | 11 ++- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index abd792b65..6f81e82d2 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -307,8 +307,8 @@ describe('DwnApi', () => { expect(friendCreateStatus.code).to.equal(202); const { status: friendRecordUpdateStatus } = await friendRecord.update({ data: 'update' }); expect(friendRecordUpdateStatus.code).to.equal(202); - const { status: friendSendStatus } = await friendRecord.send(aliceDid.did); - expect(friendSendStatus.code).to.equal(202); + const { status: aliceFriendSendStatus } = await friendRecord.send(aliceDid.did); + expect(aliceFriendSendStatus.code).to.equal(202); // Bob creates a thread record using the role 'friend' and sends it to Alice const { status: albumCreateStatus, record: albumRecord} = await dwnBob.records.create({ @@ -323,10 +323,75 @@ describe('DwnApi', () => { } }); expect(albumCreateStatus.code).to.equal(202); - const { status: bobSendStatus } = await albumRecord.send(bobDid.did); - expect(bobSendStatus.code).to.equal(202); - const { status: aliceSendStatus } = await albumRecord.send(aliceDid.did); - expect(aliceSendStatus.code).to.equal(202); + const { status: bobAlbumSendStatus } = await albumRecord.send(bobDid.did); + expect(bobAlbumSendStatus.code).to.equal(202); + const { status: aliceAlbumSendStatus } = await albumRecord.send(aliceDid.did); + expect(aliceAlbumSendStatus.code).to.equal(202); + + // Alice fetches the album record Bob created using his friend role + const aliceAlbumReadResult = await dwnAlice.records.read({ + from : aliceDid.did, + message : { + filter: { + recordId: albumRecord.id + } + } + }); + expect(aliceAlbumReadResult.status.code).to.equal(200); + expect(aliceAlbumReadResult.record).to.exist; + const aliceAlbumReadStoreStatus = await aliceAlbumReadResult.record.store(); + expect(aliceAlbumReadStoreStatus.code).to.equal(202); + + // Bob makes Alice a participant + const { status: participantCreateStatus, record: participantRecord} = await dwnBob.records.create({ + data : 'test', + message : { + contextId : albumRecord.id, + parentId : albumRecord.id, + recipient : aliceDid.did, + protocol : photosProtocolDefinition.protocol, + protocolPath : 'album/participant', + schema : photosProtocolDefinition.types.participant.schema, + dataFormat : 'text/plain' + } + }); + expect(participantCreateStatus.code).to.equal(202); + const { status: bobParticipantSendStatus } = await participantRecord.send(bobDid.did); + expect(bobParticipantSendStatus.code).to.equal(202); + const { status: aliceParticipantSendStatus } = await participantRecord.send(aliceDid.did); + expect(aliceParticipantSendStatus.code).to.equal(202); + + const aliceParticipantReadResult = await dwnAlice.records.read({ + from : aliceDid.did, + message : { + filter: { + recordId: participantRecord.id + } + } + }); + expect(aliceParticipantReadResult.status.code).to.equal(200); + expect(aliceParticipantReadResult.record).to.exist; + const aliceParticipantReadStoreStatus = await aliceParticipantReadResult.record.store(); + expect(aliceParticipantReadStoreStatus.code).to.equal(202); + + // Alice creates a photo using her participant role + const { status: photoCreateStatus, record: photoRecord} = await dwnAlice.records.create({ + data : 'test', + message : { + contextId : albumRecord.id, + parentId : albumRecord.id, + protocol : photosProtocolDefinition.protocol, + protocolPath : 'album/photo', + protocolRole : 'album/participant', + schema : photosProtocolDefinition.types.photo.schema, + dataFormat : 'text/plain' + } + }); + expect(photoCreateStatus.code).to.equal(202); + const { status:alicePhotoSendStatus } = await photoRecord.send(aliceDid.did); + expect(alicePhotoSendStatus.code).to.equal(202); + const { status: bobPhotoSendStatus } = await photoRecord.send(bobDid.did); + expect(bobPhotoSendStatus.code).to.equal(202); }); }); diff --git a/packages/api/tests/fixtures/protocol-definitions/photos.json b/packages/api/tests/fixtures/protocol-definitions/photos.json index 9a002dff7..6e5c1a4b9 100644 --- a/packages/api/tests/fixtures/protocol-definitions/photos.json +++ b/packages/api/tests/fixtures/protocol-definitions/photos.json @@ -31,7 +31,14 @@ } ], "participant": { - "$contextRole": true + "$contextRole": true, + "$actions": [ + { + "who": "author", + "of": "album", + "can": "write" + } + ] }, "photo": { "$actions": [ @@ -41,7 +48,7 @@ }, { "who": "author", - "of": "thread", + "of": "album", "can": "write" } ] From ae5ec5e426ed70b118253ad8c667db129515d122 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 09:21:23 -0600 Subject: [PATCH 09/23] Pass option to initial write processing --- packages/api/src/record.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 0487004f1..d318ae8b8 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -384,11 +384,12 @@ export class Record implements RecordModel { const initialWrite = this._initialWrite; + // Is there an initial write? Have we already stored this record? if (initialWrite && !this._initialWriteStored) { let agentResponse = await this._processMessage({ import : _import, - store : true, + store : store, rawMessage : { contextId: this._contextId, ...initialWrite @@ -398,6 +399,7 @@ export class Record implements RecordModel { const { message, reply: { status } } = agentResponse; const responseMessage = message as RecordsWriteMessage; + // If we are importing, make sure to update the initial write's authorization, because now it will have the owner's signature on it if (200 <= status.code && status.code <= 299) { this._initialWriteStored = true; if (_import) initialWrite.authorization = responseMessage.authorization; From f24300b6b4905adbc40a343d66421a7dfd564c29 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 30 Jan 2024 11:00:29 -0500 Subject: [PATCH 10/23] Store/Import Typing Updates (#388) * optional target for send * guard store options --- packages/api/src/record.ts | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index d318ae8b8..f445ef25c 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -377,8 +377,7 @@ export class Record implements RecordModel { return this._agent.processDwnRequest(request); } - - async store(options: any = {}): Promise { + async _processRecord(options: { store: boolean, import: boolean }): Promise { const { store = true, import: _import = false } = options; @@ -432,35 +431,40 @@ export class Record implements RecordModel { } - async import(options?: any): Promise { + async store(options?: { import: boolean }): Promise { + return this._processRecord({ ...options, store: true }); + } + + async import(options?: { store: boolean }): Promise { // Add check to bail if already imported - return this.store({ store: options?.store !== false, import: true }); + return this._processRecord({ store: options?.store !== false, import: true }); } + + /** * Send the current record to a remote DWN by specifying their DID + * If no DID is specified, the target is assumed to be the owner (connectedDID). * (vs waiting for the regular DWN sync) - * @param target - the DID to send the record to + * @param target - the optional DID to send the record to, if none is set it is sent to the connectedDid * @returns the status of the send record request * @throws `Error` if the record has already been deleted. * * @beta */ - async send(_options?: string | { [key: string]: any }): Promise { + async send(target?: string): Promise { const initialWrite = this._initialWrite; - const options = !_options ? { target: this._connectedDid } : typeof _options === 'string' ? { target: _options } : _options; - - if (!options.target){ - options.target = this._connectedDid; + if (!target) { + target = this._connectedDid; } - if (initialWrite && !Record.checkSendCache(this._recordId, options.target)){ + if (initialWrite && !Record.checkSendCache(this._recordId, target)){ const initialState = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, author : this._connectedDid, - target : options.target, + target : target, rawMessage : removeUndefinedProperties({ contextId: this._contextId, ...initialWrite @@ -469,7 +473,7 @@ export class Record implements RecordModel { await this._agent.sendDwnRequest(initialState); - Record.setSendCache(this._recordId, options.target); + Record.setSendCache(this._recordId, target); } @@ -477,7 +481,7 @@ export class Record implements RecordModel { messageType : DwnInterfaceName.Records + DwnMethodName.Write, author : this._connectedDid, dataStream : await this.data.blob(), - target : options.target + target : target } as any; if (this._authorization) { From b1626a9d226f6163a2efccadcfea07a2ab698c7b Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 30 Jan 2024 11:04:25 -0500 Subject: [PATCH 11/23] fix lint --- packages/api/src/record.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index f445ef25c..e9b8648b5 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -440,7 +440,6 @@ export class Record implements RecordModel { return this._processRecord({ store: options?.store !== false, import: true }); } - /** * Send the current record to a remote DWN by specifying their DID From 0af8bce88c86948af7d743cbd7bb20fdcff9bafd Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 10:57:44 -0600 Subject: [PATCH 12/23] Add new test for updating with existing role in play --- packages/api/tests/dwn-api.spec.ts | 22 ++++++++++++++++++- .../fixtures/protocol-definitions/photos.json | 5 +++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 6f81e82d2..5d16b4f88 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -263,7 +263,7 @@ describe('DwnApi', () => { expect(await result.record?.data.json()).to.deep.equal(dataJson); }); - it('creates a role record for another user that they can use to create role-based records', async () => { + it.only('creates a role record for another user that they can use to create role-based records', async () => { /** * WHAT IS BEING TESTED? * @@ -392,6 +392,26 @@ describe('DwnApi', () => { expect(alicePhotoSendStatus.code).to.equal(202); const { status: bobPhotoSendStatus } = await photoRecord.send(bobDid.did); expect(bobPhotoSendStatus.code).to.equal(202); + + const { status: photoUpdateStatus, record: photoUpdateRecord} = await dwnBob.records.write({ + data : 'test again', + store : false, + message : { + contextId : albumRecord.id, + parentId : albumRecord.id, + recordId : photoRecord.id, + dateCreated : photoRecord.dateCreated, + protocol : photosProtocolDefinition.protocol, + protocolPath : 'album/photo', + schema : photosProtocolDefinition.types.photo.schema, + dataFormat : 'text/plain' + } + }); + expect(photoUpdateStatus.code).to.equal(202); + const { status:alicePhotoUpdateSendStatus } = await photoUpdateRecord.send(aliceDid.did); + expect(alicePhotoUpdateSendStatus.code).to.equal(202); + const { status: bobPhotoUpdateSendStatus } = await photoUpdateRecord.send(bobDid.did); + expect(bobPhotoUpdateSendStatus.code).to.equal(202); }); }); diff --git a/packages/api/tests/fixtures/protocol-definitions/photos.json b/packages/api/tests/fixtures/protocol-definitions/photos.json index 6e5c1a4b9..4430ae2de 100644 --- a/packages/api/tests/fixtures/protocol-definitions/photos.json +++ b/packages/api/tests/fixtures/protocol-definitions/photos.json @@ -50,6 +50,11 @@ "who": "author", "of": "album", "can": "write" + }, + { + "who": "author", + "of": "album", + "can": "update" } ] } From 52dea08f6e79a4719d1443680f5e88e5f93d74d1 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 10:58:34 -0600 Subject: [PATCH 13/23] remove errant only() in tests --- packages/api/tests/dwn-api.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 5d16b4f88..683dfc4f5 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -263,7 +263,7 @@ describe('DwnApi', () => { expect(await result.record?.data.json()).to.deep.equal(dataJson); }); - it.only('creates a role record for another user that they can use to create role-based records', async () => { + it('creates a role record for another user that they can use to create role-based records', async () => { /** * WHAT IS BEING TESTED? * From a8ab38ef6249ae14a7e2d3aafb0b7fd895a9f9f3 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 12:15:59 -0600 Subject: [PATCH 14/23] Add test for changing the protocol role that a record is being updated with --- packages/api/tests/dwn-api.spec.ts | 25 +++++++++++++++++-- .../fixtures/protocol-definitions/photos.json | 20 ++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 683dfc4f5..41e2b38e7 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -263,7 +263,7 @@ describe('DwnApi', () => { expect(await result.record?.data.json()).to.deep.equal(dataJson); }); - it('creates a role record for another user that they can use to create role-based records', async () => { + it.only('creates a role record for another user that they can use to create role-based records', async () => { /** * WHAT IS BEING TESTED? * @@ -342,7 +342,7 @@ describe('DwnApi', () => { const aliceAlbumReadStoreStatus = await aliceAlbumReadResult.record.store(); expect(aliceAlbumReadStoreStatus.code).to.equal(202); - // Bob makes Alice a participant + // Bob makes Alice a `participant` const { status: participantCreateStatus, record: participantRecord} = await dwnBob.records.create({ data : 'test', message : { @@ -374,6 +374,26 @@ describe('DwnApi', () => { const aliceParticipantReadStoreStatus = await aliceParticipantReadResult.record.store(); expect(aliceParticipantReadStoreStatus.code).to.equal(202); + // Alice makes Bob an `updater` + const { status: updaterCreateStatus, record: updaterRecord} = await dwnAlice.records.create({ + data : 'test', + message : { + contextId : albumRecord.id, + parentId : albumRecord.id, + recipient : bobDid.did, + protocol : photosProtocolDefinition.protocol, + protocolPath : 'album/updater', + protocolRole : 'album/participant', + schema : photosProtocolDefinition.types.updater.schema, + dataFormat : 'text/plain' + } + }); + expect(updaterCreateStatus.code).to.equal(202); + const { status: bobUpdaterSendStatus } = await updaterRecord.send(bobDid.did); + expect(bobUpdaterSendStatus.code).to.equal(202); + const { status: aliceUpdaterSendStatus } = await updaterRecord.send(aliceDid.did); + expect(aliceUpdaterSendStatus.code).to.equal(202); + // Alice creates a photo using her participant role const { status: photoCreateStatus, record: photoRecord} = await dwnAlice.records.create({ data : 'test', @@ -403,6 +423,7 @@ describe('DwnApi', () => { dateCreated : photoRecord.dateCreated, protocol : photosProtocolDefinition.protocol, protocolPath : 'album/photo', + protocolRole : 'album/updater', schema : photosProtocolDefinition.types.photo.schema, dataFormat : 'text/plain' } diff --git a/packages/api/tests/fixtures/protocol-definitions/photos.json b/packages/api/tests/fixtures/protocol-definitions/photos.json index 4430ae2de..1bf1db06a 100644 --- a/packages/api/tests/fixtures/protocol-definitions/photos.json +++ b/packages/api/tests/fixtures/protocol-definitions/photos.json @@ -17,6 +17,10 @@ "participant": { "schema": "http://photo-protocol.xyz/schema/participant", "dataFormats": ["text/plain"] + }, + "updater": { + "schema": "http://photo-protocol.xyz/schema/updater", + "dataFormats": ["text/plain"] } }, "structure": { @@ -40,6 +44,15 @@ } ] }, + "updater": { + "$contextRole": true, + "$actions": [ + { + "role": "album/participant", + "can": "write" + } + ] + }, "photo": { "$actions": [ { @@ -47,14 +60,13 @@ "can": "write" }, { - "who": "author", - "of": "album", - "can": "write" + "role": "album/updater", + "can": "update" }, { "who": "author", "of": "album", - "can": "update" + "can": "write" } ] } From 2b4f38a1ba2a81bd0a3835d631ef88312e10d932 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 12:16:17 -0600 Subject: [PATCH 15/23] Remove only() from test --- packages/api/tests/dwn-api.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 41e2b38e7..78881b152 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -263,7 +263,7 @@ describe('DwnApi', () => { expect(await result.record?.data.json()).to.deep.equal(dataJson); }); - it.only('creates a role record for another user that they can use to create role-based records', async () => { + it('creates a role record for another user that they can use to create role-based records', async () => { /** * WHAT IS BEING TESTED? * From c0c8cae001e8b19790d775b21cf45a2e126cb148 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 30 Jan 2024 13:20:28 -0500 Subject: [PATCH 16/23] method dyping more consistant --- packages/api/src/record.ts | 46 ++++++++++++++++-------------- packages/api/tests/dwn-api.spec.ts | 4 +-- packages/api/tests/record.spec.ts | 4 +-- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index e9b8648b5..0ab7c3add 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -1,4 +1,4 @@ -import type { Web5Agent } from '@web5/agent'; +import type { ProcessDwnRequest, Web5Agent } from '@web5/agent'; import type { Readable } from '@web5/common'; import type { RecordsWriteMessage, @@ -12,6 +12,14 @@ import { DwnInterfaceName, DwnMethodName } from '@tbd54566975/dwn-sdk-js'; import type { ResponseStatus } from './dwn-api.js'; import { dataToBlob } from './utils.js'; +export type ProcessRecordRequest = { + dataStream?: Blob | ReadableStream | Readable; + rawMessage?: Partial; + messageOptions?: unknown; + store: boolean; + import: boolean; +}; + /** * Options that are passed to Record constructor. * @@ -350,17 +358,14 @@ export class Record implements RecordModel { return dataObj; } - private async _processMessage(options: any){ - - const { store = true } = options; - - const request = { + private _prepareMessage(options: ProcessRecordRequest): ProcessDwnRequest { + const request: ProcessDwnRequest = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, author : this._connectedDid, target : this._connectedDid, - import : !!options.import, - store - } as any; + import : options.import, + store : options.store, + }; if (options.rawMessage) { removeUndefinedProperties(options.rawMessage); @@ -374,19 +379,17 @@ export class Record implements RecordModel { request.dataStream = options.dataStream; } - return this._agent.processDwnRequest(request); + return request; } - async _processRecord(options: { store: boolean, import: boolean }): Promise { + private async _processRecord(options: { store: boolean, import: boolean }): Promise { const { store = true, import: _import = false } = options; - const initialWrite = this._initialWrite; // Is there an initial write? Have we already stored this record? if (initialWrite && !this._initialWriteStored) { - - let agentResponse = await this._processMessage({ + const requestOptions = this._prepareMessage({ import : _import, store : store, rawMessage : { @@ -395,6 +398,7 @@ export class Record implements RecordModel { } }); + const agentResponse = await this._agent.processDwnRequest(requestOptions); const { message, reply: { status } } = agentResponse; const responseMessage = message as RecordsWriteMessage; @@ -403,10 +407,9 @@ export class Record implements RecordModel { this._initialWriteStored = true; if (_import) initialWrite.authorization = responseMessage.authorization; } - } - let agentResponse = await this._processMessage({ + const requestOptions = this._prepareMessage({ import : !initialWrite && _import, store : store, dataStream : await this.data.blob(), @@ -420,6 +423,7 @@ export class Record implements RecordModel { } }); + const agentResponse = await this._agent.processDwnRequest(requestOptions); const { message, reply: { status } } = agentResponse; const responseMessage = message as RecordsWriteMessage; @@ -427,16 +431,16 @@ export class Record implements RecordModel { if (_import) this._authorization = responseMessage.authorization; } - return status; - + return { status }; } - async store(options?: { import: boolean }): Promise { + async store(options?: { import: boolean }): Promise { + // process the record and always set store to true return this._processRecord({ ...options, store: true }); } - async import(options?: { store: boolean }): Promise { - // Add check to bail if already imported + async import(options?: { store: boolean }): Promise { + // process the record and always set import to true, only skip storage if explicitly set to false return this._processRecord({ store: options?.store !== false, import: true }); } diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 78881b152..58bfbe631 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -339,7 +339,7 @@ describe('DwnApi', () => { }); expect(aliceAlbumReadResult.status.code).to.equal(200); expect(aliceAlbumReadResult.record).to.exist; - const aliceAlbumReadStoreStatus = await aliceAlbumReadResult.record.store(); + const { status: aliceAlbumReadStoreStatus } = await aliceAlbumReadResult.record.store(); expect(aliceAlbumReadStoreStatus.code).to.equal(202); // Bob makes Alice a `participant` @@ -371,7 +371,7 @@ describe('DwnApi', () => { }); expect(aliceParticipantReadResult.status.code).to.equal(200); expect(aliceParticipantReadResult.record).to.exist; - const aliceParticipantReadStoreStatus = await aliceParticipantReadResult.record.store(); + const { status: aliceParticipantReadStoreStatus } = await aliceParticipantReadResult.record.store(); expect(aliceParticipantReadStoreStatus.code).to.equal(202); // Alice makes Bob an `updater` diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index e54e4ec89..120a7dc23 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -154,7 +154,7 @@ describe('Record', () => { }); expect(queryRecordStatus.code).to.equal(200); const importRecord = queryRecords[0]; - const importRecordStatus = await importRecord.import(); + const { status: importRecordStatus } = await importRecord.import(); expect(importRecordStatus.code).to.equal(202); const { status: importSendStatus } = await importRecord!.send(); @@ -196,7 +196,7 @@ describe('Record', () => { expect(updatedRecordsStatus.code).to.equal(200); const updatedRecord = updatedRecords[0]; - const updatedRecordStoredStatus = await updatedRecord.store(); + const { status: updatedRecordStoredStatus } = await updatedRecord.store(); expect(updatedRecordStoredStatus.code).to.equal(202); expect(await updatedRecord.data.text() === updatedText).to.equal(true); From 67923741ff41f5f9eaeccaf4881ecf4fba285e13 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 13:39:23 -0600 Subject: [PATCH 17/23] Added comments and removed TODO --- packages/api/src/record.ts | 4 +++- packages/api/tests/record.spec.ts | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 0ab7c3add..571aa1ace 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -112,9 +112,11 @@ export class Record implements RecordModel { private _protocolRole: RecordOptions['protocolRole']; // Getters for immutable DWN Record properties. + // Cache to minimize the amount of redundant two-phase commits we do in store() and send() + // Retains awareness of the last 100 records stored/sent for up to 100 target DIDs each. static sendCache = new Map(); static sendCacheLimit = 100; - static setSendCache(recordId, target){ + static setSendCache(recordId, target) { const recordCache = Record.sendCache; let targetCache = recordCache.get(recordId) || new Set(); recordCache.delete(recordId); diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index 120a7dc23..c6b443ea5 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -1464,7 +1464,6 @@ describe('Record', () => { expect(sendResult.status.code).to.equal(202); }); - // TODO: Fix after changes are made to dwn-sdk-js to include the initial write in every query/read response. it('automatically sends the initial write and update of a record to a remote DWN', async () => { // Alice writes a message to her agent connected DWN. const { status, record } = await dwnAlice.records.write({ From 3b4c993e15031e0600821f41d8f957f04359f1b4 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 30 Jan 2024 15:53:32 -0500 Subject: [PATCH 18/23] clean up types, use web5/common removeUndefinedProperties --- packages/agent/src/dwn-manager.ts | 7 ++-- packages/api/src/record.ts | 65 ++++++++++++------------------- 2 files changed, 28 insertions(+), 44 deletions(-) diff --git a/packages/agent/src/dwn-manager.ts b/packages/agent/src/dwn-manager.ts index 7d29f2150..87a8d72e9 100644 --- a/packages/agent/src/dwn-manager.ts +++ b/packages/agent/src/dwn-manager.ts @@ -242,11 +242,10 @@ export class DwnManager { } private async constructDwnMessage(options: { - request : ProcessDwnRequest + request: ProcessDwnRequest }) { const { request } = options; - const rawMessage = (request as any).rawMessage; - + const rawMessage = request.rawMessage as any; let readableStream: Readable | undefined; // TODO: Consider refactoring to move data transformations imposed by fetch() limitations to the HTTP transport-related methods. @@ -289,7 +288,7 @@ export class DwnManager { } } - return { message: (dwnMessage as any).message, dataStream: readableStream }; + return { message: dwnMessage.message, dataStream: readableStream }; } private async getAuthorSigningKeyId(options: { diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 571aa1ace..9f769e866 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -1,4 +1,4 @@ -import type { ProcessDwnRequest, Web5Agent } from '@web5/agent'; +import type { ProcessDwnRequest, SendDwnRequest, Web5Agent } from '@web5/agent'; import type { Readable } from '@web5/common'; import type { RecordsWriteMessage, @@ -6,13 +6,13 @@ import type { RecordsWriteDescriptor, } from '@tbd54566975/dwn-sdk-js'; -import { Convert, NodeStream, Stream } from '@web5/common'; +import { Convert, NodeStream, removeUndefinedProperties, Stream } from '@web5/common'; import { DwnInterfaceName, DwnMethodName } from '@tbd54566975/dwn-sdk-js'; import type { ResponseStatus } from './dwn-api.js'; import { dataToBlob } from './utils.js'; -export type ProcessRecordRequest = { +type ProcessRecordRequest = { dataStream?: Blob | ReadableStream | Readable; rawMessage?: Partial; messageOptions?: unknown; @@ -77,20 +77,6 @@ export type RecordUpdateOptions = { * * @beta */ - -function removeUndefinedProperties(obj: any): any { - if (typeof obj !== 'object' || obj === null) return; - Object.keys(obj).forEach(key => { - const val = obj[key]; - if (val === undefined) { - delete obj[key]; - } else if (typeof val === 'object') { - removeUndefinedProperties(val); - } - }); - return obj; -} - export class Record implements RecordModel { // Record instance metadata. private _agent: Web5Agent; @@ -458,54 +444,51 @@ export class Record implements RecordModel { * @beta */ async send(target?: string): Promise { - const initialWrite = this._initialWrite; - if (!target) { - target = this._connectedDid; - } + target??= this._connectedDid; if (initialWrite && !Record.checkSendCache(this._recordId, target)){ + const rawMessage = { + contextId: this._contextId, + ...initialWrite + }; + removeUndefinedProperties(rawMessage); - const initialState = { + const initialState: SendDwnRequest = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, author : this._connectedDid, target : target, - rawMessage : removeUndefinedProperties({ - contextId: this._contextId, - ...initialWrite - }) - } as any; - + rawMessage + }; await this._agent.sendDwnRequest(initialState); Record.setSendCache(this._recordId, target); - } - const latestState = { + const latestState: SendDwnRequest = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, author : this._connectedDid, dataStream : await this.data.blob(), target : target - } as any; + }; if (this._authorization) { - latestState.rawMessage = removeUndefinedProperties({ + const rawMessage = { contextId : this._contextId, recordId : this._recordId, descriptor : this._descriptor, attestation : this._attestation, authorization : this._authorization, encryption : this._encryption, - }); - } - else { + }; + removeUndefinedProperties(rawMessage); + latestState.rawMessage = rawMessage; + } else { latestState.messageOptions = this.toJSON(); } - const { reply: { status } } = await this._agent.sendDwnRequest(latestState); - - return { status }; + const { reply } = await this._agent.sendDwnRequest(latestState); + return reply; } /** @@ -622,14 +605,16 @@ export class Record implements RecordModel { if (200 <= status.code && status.code <= 299) { if (!this._initialWrite) { - this._initialWrite = JSON.parse(JSON.stringify(removeUndefinedProperties({ + const initialWrite: RecordsWriteMessage = { contextId : this._contextId, recordId : this._recordId, descriptor : this._descriptor, attestation : this._attestation, authorization : this._authorization, encryption : this._encryption, - }))); + }; + removeUndefinedProperties(initialWrite); + this._initialWrite = JSON.parse(JSON.stringify(initialWrite)); } // Only update the local Record instance mutable properties if the record was successfully (over)written. this._authorization = responseMessage.authorization; From b9fa3af38e91925181b4462b897314cb118cf60e Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 16:04:14 -0600 Subject: [PATCH 19/23] Add comment about send()'s new features --- packages/api/src/record.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 571aa1ace..83193d6b0 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -450,6 +450,7 @@ export class Record implements RecordModel { /** * Send the current record to a remote DWN by specifying their DID * If no DID is specified, the target is assumed to be the owner (connectedDID). + * If an initial write is present and the Record class send cache has no awareness of it, the initial write is sent first * (vs waiting for the regular DWN sync) * @param target - the optional DID to send the record to, if none is set it is sent to the connectedDid * @returns the status of the send record request From 2e03452086d4ed8d53d5c2c8790faa9fc3981fa7 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 16:05:40 -0600 Subject: [PATCH 20/23] Delete unused test --- packages/api/tests/record.spec.ts | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index c6b443ea5..fbae234dc 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -204,18 +204,6 @@ describe('Record', () => { const { status: updatedRecordToSelfStatus } = await updatedRecord!.send(); expect(updatedRecordToSelfStatus.code).to.equal(202); - // Confirm Bob can query his own remote DWN for the created record. - // const bobQueryResult = await dwnBob.records.query({ - // from : bobDid.did, - // message : { - // filter: { - // schema: 'http://email-protocol.xyz/schema/email' - // } - // } - // }); - // expect(bobQueryResult.status.code).to.equal(200); - // expect(bobQueryResult.records).to.exist; - // expect(bobQueryResult.records!.length).to.equal(1); }); it('should retain all defined properties', async () => { From 4fafff6b7bede89fe717707eaf4fc4c3c4868410 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 16:10:44 -0600 Subject: [PATCH 21/23] Add comments to store/import and other functions added to Record class --- packages/api/src/record.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 8953131ab..597593d29 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -370,6 +370,8 @@ export class Record implements RecordModel { return request; } + // Handles the various conditions around there being an initial write, whether to store initial/current state, + // and whether to add an owner signature to the initial write to enable storage when protocol rules require it. private async _processRecord(options: { store: boolean, import: boolean }): Promise { const { store = true, import: _import = false } = options; @@ -422,11 +424,13 @@ export class Record implements RecordModel { return { status }; } + // Uses _processRecord to manifest the storage-centric features of committing a foreign record to the local DWN async store(options?: { import: boolean }): Promise { // process the record and always set store to true return this._processRecord({ ...options, store: true }); } + // Uses _processRecord to manifest the import-centric features of ingesting and signing a foreign record async import(options?: { store: boolean }): Promise { // process the record and always set import to true, only skip storage if explicitly set to false return this._processRecord({ store: options?.store !== false, import: true }); From e36b35ebf645ecb1c3b531ffbb82e880e50ed9b9 Mon Sep 17 00:00:00 2001 From: Daniel Buchner Date: Tue, 30 Jan 2024 16:21:00 -0600 Subject: [PATCH 22/23] add code comments to make Frank smile --- packages/api/src/record.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 597593d29..31a785f3a 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -377,8 +377,9 @@ export class Record implements RecordModel { const { store = true, import: _import = false } = options; const initialWrite = this._initialWrite; - // Is there an initial write? Have we already stored this record? + // Is there an initial write? Have we already stored it? if (initialWrite && !this._initialWriteStored) { + // There is an initial write, and we have not dealt with it before, so prepare to do so const requestOptions = this._prepareMessage({ import : _import, store : store, @@ -388,6 +389,7 @@ export class Record implements RecordModel { } }); + // Process the prepared initial write, with the options set for storing and/or importing with an owner sig. const agentResponse = await this._agent.processDwnRequest(requestOptions); const { message, reply: { status } } = agentResponse; const responseMessage = message as RecordsWriteMessage; @@ -400,7 +402,7 @@ export class Record implements RecordModel { } const requestOptions = this._prepareMessage({ - import : !initialWrite && _import, + import : !initialWrite && _import, // if there is no initial write, this is the initial write (or having a forced import sig), so sign it. store : store, dataStream : await this.data.blob(), rawMessage : { @@ -418,6 +420,7 @@ export class Record implements RecordModel { const responseMessage = message as RecordsWriteMessage; if (200 <= status.code && status.code <= 299) { + // If we are importing, make sure to update the current record state's authorization, because now it will have the owner's signature on it. if (_import) this._authorization = responseMessage.authorization; } @@ -452,7 +455,9 @@ export class Record implements RecordModel { const initialWrite = this._initialWrite; target??= this._connectedDid; + // Is there an initial write? Do we know if we've already sent it to this target? if (initialWrite && !Record.checkSendCache(this._recordId, target)){ + // We do have an initial write, so prepare it for sending to the target. const rawMessage = { contextId: this._contextId, ...initialWrite @@ -467,9 +472,11 @@ export class Record implements RecordModel { }; await this._agent.sendDwnRequest(initialState); + // Set the cache to maintain awareness that we don't need to send the initial write next time. Record.setSendCache(this._recordId, target); } + // Prepare the current state for sending to the target const latestState: SendDwnRequest = { messageType : DwnInterfaceName.Records + DwnMethodName.Write, author : this._connectedDid, @@ -477,6 +484,7 @@ export class Record implements RecordModel { target : target }; + // if there is already an authz payload, just pass along the record if (this._authorization) { const rawMessage = { contextId : this._contextId, @@ -489,6 +497,7 @@ export class Record implements RecordModel { removeUndefinedProperties(rawMessage); latestState.rawMessage = rawMessage; } else { + // if there is no authz, pass options so the DWN SDK can construct and sign the record latestState.messageOptions = this.toJSON(); } From 4d1ada1f31a02cd03ecba95a8ceeeb1afa71ca8b Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Wed, 31 Jan 2024 09:08:45 -0500 Subject: [PATCH 23/23] Some more clean up for import/store (#391) * simplification of records class * rename record options to reflect signing/storing --- packages/agent/src/dwn-manager.ts | 2 +- packages/agent/src/types/agent.ts | 2 +- packages/api/src/record.ts | 274 ++++++++++++++---------------- 3 files changed, 130 insertions(+), 148 deletions(-) diff --git a/packages/agent/src/dwn-manager.ts b/packages/agent/src/dwn-manager.ts index 87a8d72e9..c10c3053b 100644 --- a/packages/agent/src/dwn-manager.ts +++ b/packages/agent/src/dwn-manager.ts @@ -283,7 +283,7 @@ export class DwnManager { }); if (dwnMessageConstructor === RecordsWrite){ - if (request.import) { + if (request.signAsOwner) { await (dwnMessage as RecordsWrite).signAsOwner(dwnSigner); } } diff --git a/packages/agent/src/types/agent.ts b/packages/agent/src/types/agent.ts index f10a07b93..69520063d 100644 --- a/packages/agent/src/types/agent.ts +++ b/packages/agent/src/types/agent.ts @@ -78,7 +78,7 @@ export type ProcessDwnRequest = DwnRequest & { rawMessage?: unknown; messageOptions?: unknown; store?: boolean; - import?: boolean; + signAsOwner?: boolean; }; export type SendDwnRequest = DwnRequest & (ProcessDwnRequest | { messageCid: string }) diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index 31a785f3a..0f176564d 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -12,13 +12,31 @@ import { DwnInterfaceName, DwnMethodName } from '@tbd54566975/dwn-sdk-js'; import type { ResponseStatus } from './dwn-api.js'; import { dataToBlob } from './utils.js'; -type ProcessRecordRequest = { - dataStream?: Blob | ReadableStream | Readable; - rawMessage?: Partial; - messageOptions?: unknown; - store: boolean; - import: boolean; -}; +class SendCache { + private static cache = new Map(); + static sendCacheLimit = 100; + + static set(id: string, target: string) { + let targetCache = SendCache.cache.get(id) || new Set(); + SendCache.cache.delete(id); + SendCache.cache.set(id, targetCache); + if (this.cache.size > SendCache.sendCacheLimit) { + const firstRecord = SendCache.cache.keys().next().value; + SendCache.cache.delete(firstRecord); + } + targetCache.delete(target); + targetCache.add(target); + if (targetCache.size > SendCache.sendCacheLimit) { + const firstTarget = targetCache.keys().next().value; + targetCache.delete(firstTarget); + } + } + + static check(id: string, target: string){ + let targetCache = SendCache.cache.get(id); + return target && targetCache ? targetCache.has(target) : targetCache; + } +} /** * Options that are passed to Record constructor. @@ -78,6 +96,10 @@ export type RecordUpdateOptions = { * @beta */ export class Record implements RecordModel { + // Cache to minimize the amount of redundant two-phase commits we do in store() and send() + // Retains awareness of the last 100 records stored/sent for up to 100 target DIDs each. + private static _sendCache = SendCache; + // Record instance metadata. private _agent: Web5Agent; private _connectedDid: string; @@ -93,36 +115,11 @@ export class Record implements RecordModel { private _descriptor: RecordsWriteDescriptor; private _encryption?: RecordsWriteMessage['encryption']; private _initialWrite: RecordOptions['initialWrite']; - private _initialWriteStored: boolean; + private _initialWriteProcessed: boolean; private _recordId: string; private _protocolRole: RecordOptions['protocolRole']; // Getters for immutable DWN Record properties. - // Cache to minimize the amount of redundant two-phase commits we do in store() and send() - // Retains awareness of the last 100 records stored/sent for up to 100 target DIDs each. - static sendCache = new Map(); - static sendCacheLimit = 100; - static setSendCache(recordId, target) { - const recordCache = Record.sendCache; - let targetCache = recordCache.get(recordId) || new Set(); - recordCache.delete(recordId); - recordCache.set(recordId, targetCache); - if (recordCache.size > Record.sendCacheLimit) { - const firstRecord = recordCache.keys().next().value; - recordCache.delete(firstRecord); - } - targetCache.delete(target); - targetCache.add(target); - if (targetCache.size > Record.sendCacheLimit) { - const firstTarget = targetCache.keys().next().value; - targetCache.delete(firstTarget); - } - } - static checkSendCache(recordId, target){ - let targetCache = Record.sendCache.get(recordId); - return target && targetCache ? targetCache.has(target) : targetCache; - } - /** Record's signatures attestation */ get attestation(): RecordsWriteMessage['attestation'] { return this._attestation; } @@ -193,6 +190,23 @@ export class Record implements RecordModel { /** Record's published status (true/false) */ get published() { return this._descriptor.published; } + /** + * Returns a copy of the raw `RecordsWriteMessage` that was used to create the current `Record` instance. + */ + private get rawMessage(): RecordsWriteMessage { + const message = JSON.parse(JSON.stringify({ + contextId : this._contextId, + recordId : this._recordId, + descriptor : this._descriptor, + attestation : this._attestation, + authorization : this._authorization, + encryption : this._encryption, + })); + + removeUndefinedProperties(message); + return message; + } + constructor(agent: Web5Agent, options: RecordOptions) { this._agent = agent; @@ -346,100 +360,32 @@ export class Record implements RecordModel { return dataObj; } - private _prepareMessage(options: ProcessRecordRequest): ProcessDwnRequest { - const request: ProcessDwnRequest = { - messageType : DwnInterfaceName.Records + DwnMethodName.Write, - author : this._connectedDid, - target : this._connectedDid, - import : options.import, - store : options.store, - }; - - if (options.rawMessage) { - removeUndefinedProperties(options.rawMessage); - request.rawMessage = options.rawMessage as Partial; - } - else { - request.messageOptions = options.messageOptions; - } - - if (options.dataStream) { - request.dataStream = options.dataStream; - } - - return request; - } - - // Handles the various conditions around there being an initial write, whether to store initial/current state, - // and whether to add an owner signature to the initial write to enable storage when protocol rules require it. - private async _processRecord(options: { store: boolean, import: boolean }): Promise { - - const { store = true, import: _import = false } = options; - const initialWrite = this._initialWrite; - - // Is there an initial write? Have we already stored it? - if (initialWrite && !this._initialWriteStored) { - // There is an initial write, and we have not dealt with it before, so prepare to do so - const requestOptions = this._prepareMessage({ - import : _import, - store : store, - rawMessage : { - contextId: this._contextId, - ...initialWrite - } - }); - - // Process the prepared initial write, with the options set for storing and/or importing with an owner sig. - const agentResponse = await this._agent.processDwnRequest(requestOptions); - const { message, reply: { status } } = agentResponse; - const responseMessage = message as RecordsWriteMessage; - - // If we are importing, make sure to update the initial write's authorization, because now it will have the owner's signature on it - if (200 <= status.code && status.code <= 299) { - this._initialWriteStored = true; - if (_import) initialWrite.authorization = responseMessage.authorization; - } - } - - const requestOptions = this._prepareMessage({ - import : !initialWrite && _import, // if there is no initial write, this is the initial write (or having a forced import sig), so sign it. - store : store, - dataStream : await this.data.blob(), - rawMessage : { - contextId : this._contextId, - recordId : this._recordId, - descriptor : this._descriptor, - attestation : this._attestation, - authorization : this._authorization, - encryption : this._encryption, - } - }); - - const agentResponse = await this._agent.processDwnRequest(requestOptions); - const { message, reply: { status } } = agentResponse; - const responseMessage = message as RecordsWriteMessage; - - if (200 <= status.code && status.code <= 299) { - // If we are importing, make sure to update the current record state's authorization, because now it will have the owner's signature on it. - if (_import) this._authorization = responseMessage.authorization; - } - - return { status }; - } - - // Uses _processRecord to manifest the storage-centric features of committing a foreign record to the local DWN - async store(options?: { import: boolean }): Promise { - // process the record and always set store to true - return this._processRecord({ ...options, store: true }); + /** + * Stores the current record state as well as any initial write to the owner's DWN. + * + * @param importRecord - if true, the record will signed by the owner before storing it to the owner's DWN. Defaults to true. + * @returns the status of the store request + * + * @beta + */ + async store(importRecord: boolean = true): Promise { + // if we are importing the record we sign it as the owner + return this.processRecord({ signAsOwner: importRecord, store: true }); } - // Uses _processRecord to manifest the import-centric features of ingesting and signing a foreign record - async import(options?: { store: boolean }): Promise { - // process the record and always set import to true, only skip storage if explicitly set to false - return this._processRecord({ store: options?.store !== false, import: true }); + /** + * Signs the current record state as well as any initial write and optionally stores it to the owner's DWN. + * This is useful when importing a record that was signed by someone else int your own DWN. + * + * @param store - if true, the record will be stored to the owner's DWN after signing. Defaults to true. + * @returns the status of the import request + * + * @beta + */ + async import(store: boolean = true): Promise { + return this.processRecord({ store, signAsOwner: true }); } - /** * Send the current record to a remote DWN by specifying their DID * If no DID is specified, the target is assumed to be the owner (connectedDID). @@ -456,10 +402,9 @@ export class Record implements RecordModel { target??= this._connectedDid; // Is there an initial write? Do we know if we've already sent it to this target? - if (initialWrite && !Record.checkSendCache(this._recordId, target)){ + if (initialWrite && !Record._sendCache.check(this._recordId, target)){ // We do have an initial write, so prepare it for sending to the target. const rawMessage = { - contextId: this._contextId, ...initialWrite }; removeUndefinedProperties(rawMessage); @@ -473,7 +418,7 @@ export class Record implements RecordModel { await this._agent.sendDwnRequest(initialState); // Set the cache to maintain awareness that we don't need to send the initial write next time. - Record.setSendCache(this._recordId, target); + Record._sendCache.set(this._recordId, target); } // Prepare the current state for sending to the target @@ -486,16 +431,7 @@ export class Record implements RecordModel { // if there is already an authz payload, just pass along the record if (this._authorization) { - const rawMessage = { - contextId : this._contextId, - recordId : this._recordId, - descriptor : this._descriptor, - attestation : this._attestation, - authorization : this._authorization, - encryption : this._encryption, - }; - removeUndefinedProperties(rawMessage); - latestState.rawMessage = rawMessage; + latestState.rawMessage = { ...this.rawMessage }; } else { // if there is no authz, pass options so the DWN SDK can construct and sign the record latestState.messageOptions = this.toJSON(); @@ -618,24 +554,18 @@ export class Record implements RecordModel { const responseMessage = message as RecordsWriteMessage; if (200 <= status.code && status.code <= 299) { + // copy the original raw message to the initial write before we update the values. if (!this._initialWrite) { - const initialWrite: RecordsWriteMessage = { - contextId : this._contextId, - recordId : this._recordId, - descriptor : this._descriptor, - attestation : this._attestation, - authorization : this._authorization, - encryption : this._encryption, - }; - removeUndefinedProperties(initialWrite); - this._initialWrite = JSON.parse(JSON.stringify(initialWrite)); + this._initialWrite = { ...this.rawMessage }; } + // Only update the local Record instance mutable properties if the record was successfully (over)written. this._authorization = responseMessage.authorization; this._protocolRole = messageOptions.protocolRole; mutableDescriptorProperties.forEach(property => { this._descriptor[property] = responseMessage.descriptor[property]; }); + // Cache data. if (options.data !== undefined) { this._encodedData = dataBlob; @@ -645,6 +575,58 @@ export class Record implements RecordModel { return { status }; } + // Handles the various conditions around there being an initial write, whether to store initial/current state, + // and whether to add an owner signature to the initial write to enable storage when protocol rules require it. + private async processRecord(options: { store: boolean, signAsOwner: boolean }): Promise { + const { store, signAsOwner } = options; + + // if there is an initial write and we haven't already processed it, we first process it and marked it as such. + if (this._initialWrite && !this._initialWriteProcessed) { + const initialWriteRequest: ProcessDwnRequest = { + messageType : DwnInterfaceName.Records + DwnMethodName.Write, + rawMessage : this.initialWrite, + author : this._connectedDid, + target : this._connectedDid, + signAsOwner : signAsOwner, + store, + }; + + // Process the prepared initial write, with the options set for storing and/or signing as the owner. + const agentResponse = await this._agent.processDwnRequest(initialWriteRequest); + const { message, reply: { status } } = agentResponse; + const responseMessage = message as RecordsWriteMessage; + + // If we are signing as owner, make sure to update the initial write's authorization, because now it will have the owner's signature on it + // set it to processed so that we don't repeat this process again + if (200 <= status.code && status.code <= 299) { + this._initialWriteProcessed = true; + if (signAsOwner) this.initialWrite.authorization = responseMessage.authorization; + } + } + + // Now that we've processed a potential initial write, we can process the current record state. + const requestOptions: ProcessDwnRequest = { + messageType : DwnInterfaceName.Records + DwnMethodName.Write, + rawMessage : this.rawMessage, + author : this._connectedDid, + target : this._connectedDid, + dataStream : await this.data.blob(), + signAsOwner : !this.initialWrite && signAsOwner, // we only need to sign this record if it is the initial write and is marked for signing + store, + }; + + const agentResponse = await this._agent.processDwnRequest(requestOptions); + const { message, reply: { status } } = agentResponse; + const responseMessage = message as RecordsWriteMessage; + + if (200 <= status.code && status.code <= 299) { + // If we are signing as the owner, make sure to update the current record state's authorization, because now it will have the owner's signature on it. + if (signAsOwner) this._authorization = responseMessage.authorization; + } + + return { status }; + } + /** * Fetches the record's data from the specified DWN. *