From 5da184578d58ec8af54ccb99f4728da93621668d Mon Sep 17 00:00:00 2001 From: Henry Tsai Date: Mon, 4 Dec 2023 10:08:58 -0800 Subject: [PATCH] Included initial write to RecordsRead and RecordsQuery if they are not initial write (#637) * Included initial write to RecordsRead and RecordsQuery if they are not initial write --- src/handlers/messages-get.ts | 4 ++-- src/handlers/records-query.ts | 24 ++++++++++++++++++----- src/handlers/records-read.ts | 29 ++++++++++++++++++++-------- src/handlers/records-write.ts | 22 ++++++++++----------- src/store/storage-controller.ts | 4 ++-- src/types/message-types.ts | 6 ++---- src/types/records-types.ts | 21 +++++++++++++------- tests/core/message.spec.ts | 4 ++-- tests/handlers/records-query.spec.ts | 23 ++++++++++++++++++++++ tests/handlers/records-read.spec.ts | 22 +++++++++++++++++++++ tests/handlers/records-write.spec.ts | 14 +++++++------- 11 files changed, 125 insertions(+), 48 deletions(-) diff --git a/src/handlers/messages-get.ts b/src/handlers/messages-get.ts index 9eb62d22f..32792a180 100644 --- a/src/handlers/messages-get.ts +++ b/src/handlers/messages-get.ts @@ -2,7 +2,7 @@ import type { DataStore } from '../types/data-store.js'; import type { DidResolver } from '../did/did-resolver.js'; import type { MessageStore } from '../types/message-store.js'; import type { MethodHandler } from '../types/method-handler.js'; -import type { RecordsWriteMessageWithOptionalEncodedData } from '../types/records-types.js'; +import type { RecordsQueryReplyEntry } from '../types/records-types.js'; import type { MessagesGetMessage, MessagesGetReply, MessagesGetReplyEntry } from '../types/messages-types.js'; import { messageReplyFromError } from '../core/message-reply.js'; @@ -65,7 +65,7 @@ export class MessagesGetHandler implements MethodHandler { // RecordsWrite specific handling, if MessageStore has embedded `encodedData` return it with the entry. // we store `encodedData` along with the message if the data is below a certain threshold. - const recordsWrite = message as RecordsWriteMessageWithOptionalEncodedData; + const recordsWrite = message as RecordsQueryReplyEntry; if (recordsWrite.encodedData !== undefined) { entry.encodedData = recordsWrite.encodedData; delete recordsWrite.encodedData; diff --git a/src/handlers/records-query.ts b/src/handlers/records-query.ts index 9de192b90..ddae815af 100644 --- a/src/handlers/records-query.ts +++ b/src/handlers/records-query.ts @@ -4,7 +4,7 @@ import type { Filter } from '../types/query-types.js'; import type { MessageStore } from '../types//message-store.js'; import type { MethodHandler } from '../types/method-handler.js'; import type { GenericMessage, MessageSort } from '../types/message-types.js'; -import type { RecordsQueryMessage, RecordsQueryReply, RecordsWriteMessageWithOptionalEncodedData } from '../types/records-types.js'; +import type { RecordsQueryMessage, RecordsQueryReply, RecordsQueryReplyEntry } from '../types/records-types.js'; import { authenticate } from '../core/auth.js'; import { DateSort } from '../types/records-types.js'; @@ -12,6 +12,7 @@ import { messageReplyFromError } from '../core/message-reply.js'; import { ProtocolAuthorization } from '../core/protocol-authorization.js'; import { Records } from '../utils/records.js'; import { RecordsQuery } from '../interfaces/records-query.js'; +import { RecordsWrite } from '../interfaces/records-write.js'; import { SortDirection } from '../types/query-types.js'; import { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js'; @@ -30,12 +31,12 @@ export class RecordsQueryHandler implements MethodHandler { return messageReplyFromError(e, 400); } - let recordsWrites: RecordsWriteMessageWithOptionalEncodedData[]; + let recordsWrites: RecordsQueryReplyEntry[]; let cursor: string|undefined; // if this is an anonymous query and the filter supports published records, query only published records if (RecordsQueryHandler.filterIncludesPublishedRecords(recordsQuery) && recordsQuery.author === undefined) { const results = await this.fetchPublishedRecords(tenant, recordsQuery); - recordsWrites = results.messages as RecordsWriteMessageWithOptionalEncodedData[]; + recordsWrites = results.messages as RecordsQueryReplyEntry[]; cursor = results.cursor; } else { // authentication and authorization @@ -52,15 +53,28 @@ export class RecordsQueryHandler implements MethodHandler { if (recordsQuery.author === tenant) { const results = await this.fetchRecordsAsOwner(tenant, recordsQuery); - recordsWrites = results.messages as RecordsWriteMessageWithOptionalEncodedData[]; + recordsWrites = results.messages as RecordsQueryReplyEntry[]; cursor = results.cursor; } else { const results = await this.fetchRecordsAsNonOwner(tenant, recordsQuery); - recordsWrites = results.messages as RecordsWriteMessageWithOptionalEncodedData[]; + recordsWrites = results.messages as RecordsQueryReplyEntry[]; cursor = results.cursor; } } + // attach initial write if returned RecordsWrite is not initial write + for (const recordsWrite of recordsWrites) { + if (!await RecordsWrite.isInitialWrite(recordsWrite)) { + const initialWriteQueryResult = await this.messageStore.query( + tenant, + [{ recordId: recordsWrite.recordId, isLatestBaseState: false, method: DwnMethodName.Write }] + ); + const initialWrite = initialWriteQueryResult.messages[0] as RecordsQueryReplyEntry; + delete initialWrite.encodedData; + recordsWrite.initialWrite = initialWrite; + } + } + return { status : { code: 200, detail: 'OK' }, entries : recordsWrites, diff --git a/src/handlers/records-read.ts b/src/handlers/records-read.ts index 1f65f8c2d..b8f218f2f 100644 --- a/src/handlers/records-read.ts +++ b/src/handlers/records-read.ts @@ -3,11 +3,10 @@ import type { DidResolver } from '../did/did-resolver.js'; import type { Filter } from '../types/query-types.js'; import type { MessageStore } from '../types//message-store.js'; import type { MethodHandler } from '../types/method-handler.js'; -import type { RecordsReadMessage, RecordsReadReply, RecordsWriteMessageWithOptionalEncodedData } from '../types/records-types.js'; +import type { RecordsQueryReplyEntry, RecordsReadMessage, RecordsReadReply } from '../types/records-types.js'; import { authenticate } from '../core/auth.js'; import { DataStream } from '../utils/data-stream.js'; -import { DwnInterfaceName } from '../enums/dwn-interface-method.js'; import { Encoder } from '../utils/encoder.js'; import { GrantAuthorization } from '../core/grant-authorization.js'; import { Message } from '../core/message.js'; @@ -18,6 +17,7 @@ import { RecordsGrantAuthorization } from '../core/records-grant-authorization.j import { RecordsRead } from '../interfaces/records-read.js'; import { RecordsWrite } from '../interfaces/records-write.js'; import { DwnError, DwnErrorCode } from '../core/dwn-error.js'; +import { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js'; export class RecordsReadHandler implements MethodHandler { @@ -63,7 +63,7 @@ export class RecordsReadHandler implements MethodHandler { ), 400); } - const newestRecordsWrite = existingMessages[0] as RecordsWriteMessageWithOptionalEncodedData; + const newestRecordsWrite = existingMessages[0] as RecordsQueryReplyEntry; try { await RecordsReadHandler.authorizeRecordsRead(tenant, recordsRead, await RecordsWrite.parse(newestRecordsWrite), this.messageStore); } catch (error) { @@ -86,12 +86,25 @@ export class RecordsReadHandler implements MethodHandler { data = result.dataStream; } + const record = { + ...newestRecordsWrite, + data + }; + + // attach initial write if returned RecordsWrite is not initial write + if (!await RecordsWrite.isInitialWrite(record)) { + const initialWriteQueryResult = await this.messageStore.query( + tenant, + [{ recordId: record.recordId, isLatestBaseState: false, method: DwnMethodName.Write }] + ); + const initialWrite = initialWriteQueryResult.messages[0] as RecordsQueryReplyEntry; + delete initialWrite.encodedData; + record.initialWrite = initialWrite; + } + const messageReply: RecordsReadReply = { - status : { code: 200, detail: 'OK' }, - record : { - ...newestRecordsWrite, - data, - } + status: { code: 200, detail: 'OK' }, + record }; return messageReply; }; diff --git a/src/handlers/records-write.ts b/src/handlers/records-write.ts index 544412f10..33eb15b97 100644 --- a/src/handlers/records-write.ts +++ b/src/handlers/records-write.ts @@ -4,7 +4,7 @@ import type { EventLog } from '../types/event-log.js'; import type { GenericMessageReply } from '../core/message-reply.js'; import type { MessageStore } from '../types//message-store.js'; import type { MethodHandler } from '../types/method-handler.js'; -import type { RecordsWriteMessage, RecordsWriteMessageWithOptionalEncodedData } from '../types/records-types.js'; +import type { RecordsQueryReplyEntry, RecordsWriteMessage } from '../types/records-types.js'; import { authenticate } from '../core/auth.js'; import { Cid } from '../utils/cid.js'; @@ -94,7 +94,7 @@ export class RecordsWriteHandler implements MethodHandler { // thus preventing a user's attempt to gain authorized access to data by referencing the dataCid of a private data in their initial writes, // See: https://github.com/TBD54566975/dwn-sdk-js/issues/359 for more info let isLatestBaseState = false; - let messageWithOptionalEncodedData = message as RecordsWriteMessageWithOptionalEncodedData; + let messageWithOptionalEncodedData = message as RecordsQueryReplyEntry; if (dataStream !== undefined) { messageWithOptionalEncodedData = await this.processMessageWithDataStream(tenant, message, dataStream); @@ -114,7 +114,7 @@ export class RecordsWriteHandler implements MethodHandler { // if the incoming message is not an initial write, and no dataStream is provided, we would allow it provided it passes validation // processMessageWithoutDataStream() abstracts that logic if (!newMessageIsInitialWrite) { - const newestExistingWrite = newestExistingMessage as RecordsWriteMessageWithOptionalEncodedData; + const newestExistingWrite = newestExistingMessage as RecordsQueryReplyEntry; messageWithOptionalEncodedData = await this.processMessageWithoutDataStream(tenant, message, newestExistingWrite ); isLatestBaseState = true; } @@ -151,10 +151,10 @@ export class RecordsWriteHandler implements MethodHandler { }; /** - * Returns a `RecordsWriteMessageWithOptionalEncodedData` with a copy of the incoming message and the incoming data encoded to `Base64URL`. + * Returns a `RecordsQueryReplyEntry` with a copy of the incoming message and the incoming data encoded to `Base64URL`. */ - public async cloneAndAddEncodedData(message: RecordsWriteMessage, dataBytes: Uint8Array):Promise { - const recordsWrite: RecordsWriteMessageWithOptionalEncodedData = { ...message }; + public async cloneAndAddEncodedData(message: RecordsWriteMessage, dataBytes: Uint8Array):Promise { + const recordsWrite: RecordsQueryReplyEntry = { ...message }; recordsWrite.encodedData = Encoder.bytesToBase64Url(dataBytes); return recordsWrite; } @@ -163,8 +163,8 @@ export class RecordsWriteHandler implements MethodHandler { tenant: string, message: RecordsWriteMessage, dataStream: _Readable.Readable, - ):Promise { - let messageWithOptionalEncodedData: RecordsWriteMessageWithOptionalEncodedData = message; + ):Promise { + let messageWithOptionalEncodedData: RecordsQueryReplyEntry = message; // if data is below the threshold, we store it within MessageStore if (message.descriptor.dataSize <= DwnConstant.maxDataSizeAllowedToBeEncoded) { @@ -185,9 +185,9 @@ export class RecordsWriteHandler implements MethodHandler { private async processMessageWithoutDataStream( tenant: string, message: RecordsWriteMessage, - newestExistingWrite: RecordsWriteMessageWithOptionalEncodedData, - ):Promise { - const messageWithOptionalEncodedData: RecordsWriteMessageWithOptionalEncodedData = { ...message }; // clone + newestExistingWrite: RecordsQueryReplyEntry, + ):Promise { + const messageWithOptionalEncodedData: RecordsQueryReplyEntry = { ...message }; // clone const { dataCid, dataSize } = message.descriptor; // Since incoming message is not an initial write, and no dataStream is provided, we first check integrity against newest existing write. diff --git a/src/store/storage-controller.ts b/src/store/storage-controller.ts index 8a781ce69..14c9932ad 100644 --- a/src/store/storage-controller.ts +++ b/src/store/storage-controller.ts @@ -2,7 +2,7 @@ import type { DataStore } from '../types/data-store.js'; import type { EventLog } from '../types/event-log.js'; import type { GenericMessage } from '../types/message-types.js'; import type { MessageStore } from '../types/message-store.js'; -import type { RecordsWriteMessage, RecordsWriteMessageWithOptionalEncodedData } from '../types/records-types.js'; +import type { RecordsQueryReplyEntry, RecordsWriteMessage } from '../types/records-types.js'; import { DwnConstant } from '../core/dwn-constant.js'; import { DwnMethodName } from '../enums/dwn-interface-method.js'; @@ -65,7 +65,7 @@ export class StorageController { const existingRecordsWrite = await RecordsWrite.parse(message as RecordsWriteMessage); const isLatestBaseState = false; const indexes = await existingRecordsWrite.constructRecordsWriteIndexes(isLatestBaseState); - const writeMessage = message as RecordsWriteMessageWithOptionalEncodedData; + const writeMessage = message as RecordsQueryReplyEntry; delete writeMessage.encodedData; await messageStore.put(tenant, writeMessage, indexes); } else { diff --git a/src/types/message-types.ts b/src/types/message-types.ts index 947112c86..a338b3837 100644 --- a/src/types/message-types.ts +++ b/src/types/message-types.ts @@ -61,11 +61,9 @@ export type Descriptor = { /** * Message returned in a query result. * NOTE: the message structure is a modified version of the message received, the most notable differences are: - * 1. does not contain `authorization` - * 2. may include encoded data + * 1. May include encoded data */ -export type QueryResultEntry = { - descriptor: Descriptor; +export type QueryResultEntry = GenericMessage & { encodedData?: string; }; diff --git a/src/types/records-types.ts b/src/types/records-types.ts index 19d375ce8..fe36c7503 100644 --- a/src/types/records-types.ts +++ b/src/types/records-types.ts @@ -54,11 +54,6 @@ export type RecordsWriteMessage = GenericMessage & { encryption?: EncryptionProperty; }; -/** - * records with a data size below a threshold are stored within MessageStore with their data embedded - */ -export type RecordsWriteMessageWithOptionalEncodedData = RecordsWriteMessage & { encodedData?: string }; - export type EncryptionProperty = { algorithm: EncryptionAlgorithm; initializationVector: string; @@ -86,10 +81,18 @@ export type EncryptedKey = { /** * Data structure returned in a `RecordsQuery` reply entry. * NOTE: the message structure is a modified version of the message received, the most notable differences are: - * 1. does not contain `authorization` - * 2. may include encoded data + * 1. May include an initial RecordsWrite message + * 2. May include encoded data */ export type RecordsQueryReplyEntry = RecordsWriteMessage & { + /** + * The initial write of the record if the returned RecordsWrite message itself is not the initial write. + */ + initialWrite?: RecordsWriteMessage; + + /** + * The encoded data of the record if the data associated with the record is equal or smaller than `DwnConstant.maxDataSizeAllowedToBeEncoded`. + */ encodedData?: string; }; @@ -149,6 +152,10 @@ export type RecordsReadMessage = { export type RecordsReadReply = GenericMessageReply & { record?: RecordsWriteMessage & { + /** + * The initial write of the record if the returned RecordsWrite message itself is not the initial write. + */ + initialWrite?: RecordsWriteMessage; data: Readable; } }; diff --git a/tests/core/message.spec.ts b/tests/core/message.spec.ts index d0061d641..f27053821 100644 --- a/tests/core/message.spec.ts +++ b/tests/core/message.spec.ts @@ -1,4 +1,4 @@ -import type { RecordsWriteMessageWithOptionalEncodedData } from '../../src/types/records-types.js'; +import type { RecordsQueryReplyEntry } from '../../src/types/records-types.js'; import { expect } from 'chai'; import { Message } from '../../src/core/message.js'; @@ -77,7 +77,7 @@ describe('Message', () => { const { message } = await TestDataGenerator.generateRecordsWrite(); const cid1 = await Message.getCid(message); - const messageWithData: RecordsWriteMessageWithOptionalEncodedData = message; + const messageWithData: RecordsQueryReplyEntry = message; messageWithData.encodedData = TestDataGenerator.randomString(25); const cid2 = await Message.getCid(messageWithData); diff --git a/tests/handlers/records-query.spec.ts b/tests/handlers/records-query.spec.ts index bd5c44f96..a441d8227 100644 --- a/tests/handlers/records-query.spec.ts +++ b/tests/handlers/records-query.spec.ts @@ -183,6 +183,29 @@ export function testRecordsQueryHandler(): void { expect(reply.entries![0].encodedData).to.be.undefined; }); + it('should include `initialWrite` property if RecordsWrite is not initial write', async () => { + const alice = await DidKeyResolver.generate(); + const write = await TestDataGenerator.generateRecordsWrite({ author: alice, published: false }); + + const writeReply = await dwn.processMessage(alice.did, write.message, write.dataStream); + expect(writeReply.status.code).to.equal(202); + + // write an update to the record + const write2 = await RecordsWrite.createFrom({ recordsWriteMessage: write.message, published: true, signer: Jws.createSigner(alice) }); + const write2Reply = await dwn.processMessage(alice.did, write2.message); + expect(write2Reply.status.code).to.equal(202); + + // make sure result returned now has `initialWrite` property + const messageData = await TestDataGenerator.generateRecordsQuery({ author: alice, filter: { recordId: write.message.recordId } }); + const reply = await dwn.processMessage(alice.did, messageData.message); + + expect(reply.status.code).to.equal(200); + expect(reply.entries?.length).to.equal(1); + expect(reply.entries![0].initialWrite).to.exist; + expect(reply.entries![0].initialWrite?.recordId).to.equal(write.message.recordId); + + }); + it('should be able to query by attester', async () => { // scenario: 2 records authored by alice, 1st attested by alice, 2nd attested by bob const alice = await DidKeyResolver.generate(); diff --git a/tests/handlers/records-read.spec.ts b/tests/handlers/records-read.spec.ts index 268a6a568..2559ca10b 100644 --- a/tests/handlers/records-read.spec.ts +++ b/tests/handlers/records-read.spec.ts @@ -196,6 +196,28 @@ export function testRecordsReadHandler(): void { expect(ArrayUtility.byteArraysEqual(dataFetched, dataBytes!)).to.be.true; }); + it('should include `initialWrite` property if RecordsWrite is not initial write', async () => { + const alice = await DidKeyResolver.generate(); + const write = await TestDataGenerator.generateRecordsWrite({ author: alice, published: false }); + + const writeReply = await dwn.processMessage(alice.did, write.message, write.dataStream); + expect(writeReply.status.code).to.equal(202); + + // write an update to the record + const write2 = await RecordsWrite.createFrom({ recordsWriteMessage: write.message, published: true, signer: Jws.createSigner(alice) }); + const write2Reply = await dwn.processMessage(alice.did, write2.message); + expect(write2Reply.status.code).to.equal(202); + + // make sure result returned now has `initialWrite` property + const messageData = await RecordsRead.create({ filter: { recordId: write.message.recordId }, signer: Jws.createSigner(alice) }); + const reply = await dwn.processMessage(alice.did, messageData.message); + + expect(reply.status.code).to.equal(200); + expect(reply.record?.initialWrite).to.exist; + expect(reply.record?.initialWrite?.recordId).to.equal(write.message.recordId); + + }); + describe('protocol based reads', () => { it('should allow read with allow-anyone rule', async () => { // scenario: Alice writes an image to her DWN, then Bob reads the image because he is "anyone". diff --git a/tests/handlers/records-write.spec.ts b/tests/handlers/records-write.spec.ts index 30866d0b2..7b2f8530b 100644 --- a/tests/handlers/records-write.spec.ts +++ b/tests/handlers/records-write.spec.ts @@ -1,7 +1,7 @@ import type { EncryptionInput } from '../../src/interfaces/records-write.js'; import type { GenerateFromRecordsWriteOut } from '../utils/test-data-generator.js'; import type { ProtocolDefinition } from '../../src/types/protocols-types.js'; -import type { RecordsWriteMessageWithOptionalEncodedData } from '../../src/types/records-types.js'; +import type { RecordsQueryReplyEntry } from '../../src/types/records-types.js'; import type { DataStore, EventLog, GetResult, MessageStore } from '../../src/index.js'; import anyoneCollaborateProtocolDefinition from '../vectors/protocol-definitions/anyone-collaborate.json' assert { type: 'json' }; @@ -2333,7 +2333,7 @@ export function testRecordsWriteHandler(): void { }); }); - it('should allow overwriting records by the same author', async () => { + it('should allow updating records by the initial author', async () => { // scenario: Bob writes into Alice's DWN given Alice's "message" protocol allow-anyone rule, then modifies the message // write a protocol definition with an allow-anyone rule @@ -4109,7 +4109,7 @@ export function testRecordsWriteHandler(): void { const messageCid = await Message.getCid(message); const storedMessage = await messageStore.get(alice.did, messageCid); - expect((storedMessage as RecordsWriteMessageWithOptionalEncodedData).encodedData).to.exist.and.not.be.undefined; + expect((storedMessage as RecordsQueryReplyEntry).encodedData).to.exist.and.not.be.undefined; }); it('should not have encodedData field if dataSize greater than threshold', async () => { @@ -4122,7 +4122,7 @@ export function testRecordsWriteHandler(): void { const messageCid = await Message.getCid(message); const storedMessage = await messageStore.get(alice.did, messageCid); - expect((storedMessage as RecordsWriteMessageWithOptionalEncodedData).encodedData).to.not.exist; + expect((storedMessage as RecordsQueryReplyEntry).encodedData).to.not.exist; }); it('should retain original RecordsWrite message but without the encodedData if data is under threshold', async () => { @@ -4135,7 +4135,7 @@ export function testRecordsWriteHandler(): void { const messageCid = await Message.getCid(message); const storedMessage = await messageStore.get(alice.did, messageCid); - expect((storedMessage as RecordsWriteMessageWithOptionalEncodedData).encodedData).to.exist.and.not.be.undefined; + expect((storedMessage as RecordsQueryReplyEntry).encodedData).to.exist.and.not.be.undefined; const updatedDataBytes = TestDataGenerator.randomBytes(DwnConstant.maxDataSizeAllowedToBeEncoded); const newWrite = await RecordsWrite.createFrom({ @@ -4151,10 +4151,10 @@ export function testRecordsWriteHandler(): void { expect(writeMessage2.status.code).to.equal(202); const originalWrite = await messageStore.get(alice.did, messageCid); - expect((originalWrite as RecordsWriteMessageWithOptionalEncodedData).encodedData).to.not.exist; + expect((originalWrite as RecordsQueryReplyEntry).encodedData).to.not.exist; const newestWrite = await messageStore.get(alice.did, await Message.getCid(newWrite.message)); - expect((newestWrite as RecordsWriteMessageWithOptionalEncodedData).encodedData).to.exist.and.not.be.undefined; + expect((newestWrite as RecordsQueryReplyEntry).encodedData).to.exist.and.not.be.undefined; }); }); });