Included initial write to RecordsRead and RecordsQuery if they are no…

…t initial write (#637)

* Included initial write to RecordsRead and RecordsQuery if they are not initial write

src/handlers/messages-get.ts

@@ -2,7 +2,7 @@ import type { DataStore } from '../types/data-store.js';
 import type { DidResolver } from '../did/did-resolver.js';
 import type { MessageStore } from '../types/message-store.js';
 import type { MethodHandler } from '../types/method-handler.js';
-import type { RecordsWriteMessageWithOptionalEncodedData } from '../types/records-types.js';
+import type { RecordsQueryReplyEntry } from '../types/records-types.js';
 import type { MessagesGetMessage, MessagesGetReply, MessagesGetReplyEntry } from '../types/messages-types.js';
 
 import { messageReplyFromError } from '../core/message-reply.js';
@@ -65,7 +65,7 @@ export class MessagesGetHandler implements MethodHandler {
 
       // RecordsWrite specific handling, if MessageStore has embedded `encodedData` return it with the entry.
       // we store `encodedData` along with the message if the data is below a certain threshold.
-      const recordsWrite = message as RecordsWriteMessageWithOptionalEncodedData;
+      const recordsWrite = message as RecordsQueryReplyEntry;
       if (recordsWrite.encodedData !== undefined) {
         entry.encodedData = recordsWrite.encodedData;
         delete recordsWrite.encodedData;

src/handlers/records-query.ts

@@ -4,14 +4,15 @@ import type { Filter } from '../types/query-types.js';
 import type { MessageStore } from '../types//message-store.js';
 import type { MethodHandler } from '../types/method-handler.js';
 import type { GenericMessage, MessageSort } from '../types/message-types.js';
-import type { RecordsQueryMessage, RecordsQueryReply, RecordsWriteMessageWithOptionalEncodedData } from '../types/records-types.js';
+import type { RecordsQueryMessage, RecordsQueryReply, RecordsQueryReplyEntry } from '../types/records-types.js';
 
 import { authenticate } from '../core/auth.js';
 import { DateSort } from '../types/records-types.js';
 import { messageReplyFromError } from '../core/message-reply.js';
 import { ProtocolAuthorization } from '../core/protocol-authorization.js';
 import { Records } from '../utils/records.js';
 import { RecordsQuery } from '../interfaces/records-query.js';
+import { RecordsWrite } from '../interfaces/records-write.js';
 import { SortDirection } from '../types/query-types.js';
 import { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
 
@@ -30,12 +31,12 @@ export class RecordsQueryHandler implements MethodHandler {
       return messageReplyFromError(e, 400);
     }
 
-    let recordsWrites: RecordsWriteMessageWithOptionalEncodedData[];
+    let recordsWrites: RecordsQueryReplyEntry[];
     let cursor: string|undefined;
     // if this is an anonymous query and the filter supports published records, query only published records
     if (RecordsQueryHandler.filterIncludesPublishedRecords(recordsQuery) && recordsQuery.author === undefined) {
       const results = await this.fetchPublishedRecords(tenant, recordsQuery);
-      recordsWrites = results.messages as RecordsWriteMessageWithOptionalEncodedData[];
+      recordsWrites = results.messages as RecordsQueryReplyEntry[];
       cursor = results.cursor;
     } else {
       // authentication and authorization
@@ -52,15 +53,28 @@ export class RecordsQueryHandler implements MethodHandler {
 
       if (recordsQuery.author === tenant) {
         const results = await this.fetchRecordsAsOwner(tenant, recordsQuery);
-        recordsWrites = results.messages as RecordsWriteMessageWithOptionalEncodedData[];
+        recordsWrites = results.messages as RecordsQueryReplyEntry[];
         cursor = results.cursor;
       } else {
         const results = await this.fetchRecordsAsNonOwner(tenant, recordsQuery);
-        recordsWrites = results.messages as RecordsWriteMessageWithOptionalEncodedData[];
+        recordsWrites = results.messages as RecordsQueryReplyEntry[];
         cursor = results.cursor;
       }
     }
 
+    // attach initial write if returned RecordsWrite is not initial write
+    for (const recordsWrite of recordsWrites) {
+      if (!await RecordsWrite.isInitialWrite(recordsWrite)) {
+        const initialWriteQueryResult = await this.messageStore.query(
+          tenant,
+          [{ recordId: recordsWrite.recordId, isLatestBaseState: false, method: DwnMethodName.Write }]
+        );
+        const initialWrite = initialWriteQueryResult.messages[0] as RecordsQueryReplyEntry;
+        delete initialWrite.encodedData;
+        recordsWrite.initialWrite = initialWrite;
+      }
+    }
+
     return {
       status  : { code: 200, detail: 'OK' },
       entries : recordsWrites,

src/handlers/records-read.ts

@@ -3,11 +3,10 @@ import type { DidResolver } from '../did/did-resolver.js';
 import type { Filter } from '../types/query-types.js';
 import type { MessageStore } from '../types//message-store.js';
 import type { MethodHandler } from '../types/method-handler.js';
-import type { RecordsReadMessage, RecordsReadReply, RecordsWriteMessageWithOptionalEncodedData } from '../types/records-types.js';
+import type { RecordsQueryReplyEntry, RecordsReadMessage, RecordsReadReply } from '../types/records-types.js';
 
 import { authenticate } from '../core/auth.js';
 import { DataStream } from '../utils/data-stream.js';
-import { DwnInterfaceName } from '../enums/dwn-interface-method.js';
 import { Encoder } from '../utils/encoder.js';
 import { GrantAuthorization } from '../core/grant-authorization.js';
 import { Message } from '../core/message.js';
@@ -18,6 +17,7 @@ import { RecordsGrantAuthorization } from '../core/records-grant-authorization.j
 import { RecordsRead } from '../interfaces/records-read.js';
 import { RecordsWrite } from '../interfaces/records-write.js';
 import { DwnError, DwnErrorCode } from '../core/dwn-error.js';
+import { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
 
 export class RecordsReadHandler implements MethodHandler {
 
@@ -63,7 +63,7 @@ export class RecordsReadHandler implements MethodHandler {
       ), 400);
     }
 
-    const newestRecordsWrite = existingMessages[0] as RecordsWriteMessageWithOptionalEncodedData;
+    const newestRecordsWrite = existingMessages[0] as RecordsQueryReplyEntry;
     try {
       await RecordsReadHandler.authorizeRecordsRead(tenant, recordsRead, await RecordsWrite.parse(newestRecordsWrite), this.messageStore);
     } catch (error) {
@@ -86,12 +86,25 @@ export class RecordsReadHandler implements MethodHandler {
       data = result.dataStream;
     }
 
+    const record = {
+      ...newestRecordsWrite,
+      data
+    };
+
+    // attach initial write if returned RecordsWrite is not initial write
+    if (!await RecordsWrite.isInitialWrite(record)) {
+      const initialWriteQueryResult = await this.messageStore.query(
+        tenant,
+        [{ recordId: record.recordId, isLatestBaseState: false, method: DwnMethodName.Write }]
+      );
+      const initialWrite = initialWriteQueryResult.messages[0] as RecordsQueryReplyEntry;
+      delete initialWrite.encodedData;
+      record.initialWrite = initialWrite;
+    }
+
     const messageReply: RecordsReadReply = {
-      status : { code: 200, detail: 'OK' },
-      record : {
-        ...newestRecordsWrite,
-        data,
-      }
+      status: { code: 200, detail: 'OK' },
+      record
     };
     return messageReply;
   };

src/handlers/records-write.ts

@@ -4,7 +4,7 @@ import type { EventLog } from '../types/event-log.js';
 import type { GenericMessageReply } from '../core/message-reply.js';
 import type { MessageStore } from '../types//message-store.js';
 import type { MethodHandler } from '../types/method-handler.js';
-import type { RecordsWriteMessage, RecordsWriteMessageWithOptionalEncodedData } from '../types/records-types.js';
+import type { RecordsQueryReplyEntry, RecordsWriteMessage } from '../types/records-types.js';
 
 import { authenticate } from '../core/auth.js';
 import { Cid } from '../utils/cid.js';
@@ -94,7 +94,7 @@ export class RecordsWriteHandler implements MethodHandler {
       // thus preventing a user's attempt to gain authorized access to data by referencing the dataCid of a private data in their initial writes,
       // See: https://github.com/TBD54566975/dwn-sdk-js/issues/359 for more info
       let isLatestBaseState = false;
-      let messageWithOptionalEncodedData = message as RecordsWriteMessageWithOptionalEncodedData;
+      let messageWithOptionalEncodedData = message as RecordsQueryReplyEntry;
 
       if (dataStream !== undefined) {
         messageWithOptionalEncodedData = await this.processMessageWithDataStream(tenant, message, dataStream);
@@ -114,7 +114,7 @@ export class RecordsWriteHandler implements MethodHandler {
         // if the incoming message is not an initial write, and no dataStream is provided, we would allow it provided it passes validation
         // processMessageWithoutDataStream() abstracts that logic
         if (!newMessageIsInitialWrite) {
-          const newestExistingWrite = newestExistingMessage as RecordsWriteMessageWithOptionalEncodedData;
+          const newestExistingWrite = newestExistingMessage as RecordsQueryReplyEntry;
           messageWithOptionalEncodedData = await this.processMessageWithoutDataStream(tenant, message, newestExistingWrite );
           isLatestBaseState = true;
         }
@@ -151,10 +151,10 @@ export class RecordsWriteHandler implements MethodHandler {
   };
 
   /**
-   * Returns a `RecordsWriteMessageWithOptionalEncodedData` with a copy of the incoming message and the incoming data encoded to `Base64URL`.
+   * Returns a `RecordsQueryReplyEntry` with a copy of the incoming message and the incoming data encoded to `Base64URL`.
    */
-  public async cloneAndAddEncodedData(message: RecordsWriteMessage, dataBytes: Uint8Array):Promise<RecordsWriteMessageWithOptionalEncodedData> {
-    const recordsWrite: RecordsWriteMessageWithOptionalEncodedData = { ...message };
+  public async cloneAndAddEncodedData(message: RecordsWriteMessage, dataBytes: Uint8Array):Promise<RecordsQueryReplyEntry> {
+    const recordsWrite: RecordsQueryReplyEntry = { ...message };
     recordsWrite.encodedData = Encoder.bytesToBase64Url(dataBytes);
     return recordsWrite;
   }
@@ -163,8 +163,8 @@ export class RecordsWriteHandler implements MethodHandler {
     tenant: string,
     message: RecordsWriteMessage,
     dataStream: _Readable.Readable,
-  ):Promise<RecordsWriteMessageWithOptionalEncodedData> {
-    let messageWithOptionalEncodedData: RecordsWriteMessageWithOptionalEncodedData = message;
+  ):Promise<RecordsQueryReplyEntry> {
+    let messageWithOptionalEncodedData: RecordsQueryReplyEntry = message;
 
     // if data is below the threshold, we store it within MessageStore
     if (message.descriptor.dataSize <= DwnConstant.maxDataSizeAllowedToBeEncoded) {
@@ -185,9 +185,9 @@ export class RecordsWriteHandler implements MethodHandler {
   private async processMessageWithoutDataStream(
     tenant: string,
     message: RecordsWriteMessage,
-    newestExistingWrite: RecordsWriteMessageWithOptionalEncodedData,
-  ):Promise<RecordsWriteMessageWithOptionalEncodedData> {
-    const messageWithOptionalEncodedData: RecordsWriteMessageWithOptionalEncodedData = { ...message }; // clone
+    newestExistingWrite: RecordsQueryReplyEntry,
+  ):Promise<RecordsQueryReplyEntry> {
+    const messageWithOptionalEncodedData: RecordsQueryReplyEntry = { ...message }; // clone
     const { dataCid, dataSize } = message.descriptor;
 
     // Since incoming message is not an initial write, and no dataStream is provided, we first check integrity against newest existing write.

src/store/storage-controller.ts

@@ -2,7 +2,7 @@ import type { DataStore } from '../types/data-store.js';
 import type { EventLog } from '../types/event-log.js';
 import type { GenericMessage } from '../types/message-types.js';
 import type { MessageStore } from '../types/message-store.js';
-import type { RecordsWriteMessage, RecordsWriteMessageWithOptionalEncodedData } from '../types/records-types.js';
+import type { RecordsQueryReplyEntry, RecordsWriteMessage } from '../types/records-types.js';
 
 import { DwnConstant } from '../core/dwn-constant.js';
 import { DwnMethodName } from '../enums/dwn-interface-method.js';
@@ -65,7 +65,7 @@ export class StorageController {
           const existingRecordsWrite = await RecordsWrite.parse(message as RecordsWriteMessage);
           const isLatestBaseState = false;
           const indexes = await existingRecordsWrite.constructRecordsWriteIndexes(isLatestBaseState);
-          const writeMessage = message as RecordsWriteMessageWithOptionalEncodedData;
+          const writeMessage = message as RecordsQueryReplyEntry;
           delete writeMessage.encodedData;
           await messageStore.put(tenant, writeMessage, indexes);
         } else {

src/types/message-types.ts

@@ -61,11 +61,9 @@ export type Descriptor = {
 /**
  * Message returned in a query result.
  * NOTE: the message structure is a modified version of the message received, the most notable differences are:
- * 1. does not contain `authorization`
- * 2. may include encoded data
+ * 1. May include encoded data
  */
-export type QueryResultEntry = {
-  descriptor: Descriptor;
+export type QueryResultEntry = GenericMessage & {
   encodedData?: string;
 };
 

src/types/records-types.ts

@@ -54,11 +54,6 @@ export type RecordsWriteMessage = GenericMessage & {
   encryption?: EncryptionProperty;
 };
 
-/**
- * records with a data size below a threshold are stored within MessageStore with their data embedded
- */
-export type RecordsWriteMessageWithOptionalEncodedData = RecordsWriteMessage & { encodedData?: string };
-
 export type EncryptionProperty = {
   algorithm: EncryptionAlgorithm;
   initializationVector: string;
@@ -86,10 +81,18 @@ export type EncryptedKey = {
 /**
  * Data structure returned in a `RecordsQuery` reply entry.
  * NOTE: the message structure is a modified version of the message received, the most notable differences are:
- * 1. does not contain `authorization`
- * 2. may include encoded data
+ * 1. May include an initial RecordsWrite message
+ * 2. May include encoded data
  */
 export type RecordsQueryReplyEntry = RecordsWriteMessage & {
+  /**
+   * The initial write of the record if the returned RecordsWrite message itself is not the initial write.
+   */
+  initialWrite?: RecordsWriteMessage;
+
+  /**
+   * The encoded data of the record if the data associated with the record is equal or smaller than `DwnConstant.maxDataSizeAllowedToBeEncoded`.
+   */
   encodedData?: string;
 };
 
@@ -149,6 +152,10 @@ export type RecordsReadMessage = {
 
 export type RecordsReadReply = GenericMessageReply & {
   record?: RecordsWriteMessage & {
+    /**
+     * The initial write of the record if the returned RecordsWrite message itself is not the initial write.
+     */
+    initialWrite?: RecordsWriteMessage;
     data: Readable;
   }
 };

tests/core/message.spec.ts

@@ -1,4 +1,4 @@
-import type { RecordsWriteMessageWithOptionalEncodedData } from '../../src/types/records-types.js';
+import type { RecordsQueryReplyEntry } from '../../src/types/records-types.js';
 
 import { expect } from 'chai';
 import { Message } from '../../src/core/message.js';
@@ -77,7 +77,7 @@ describe('Message', () => {
       const { message } = await TestDataGenerator.generateRecordsWrite();
       const cid1 = await Message.getCid(message);
 
-      const messageWithData: RecordsWriteMessageWithOptionalEncodedData = message;
+      const messageWithData: RecordsQueryReplyEntry = message;
       messageWithData.encodedData = TestDataGenerator.randomString(25);
 
       const cid2 = await Message.getCid(messageWithData);

tests/handlers/records-query.spec.ts

@@ -183,6 +183,29 @@ export function testRecordsQueryHandler(): void {
         expect(reply.entries![0].encodedData).to.be.undefined;
       });
 
+      it('should include `initialWrite` property if RecordsWrite is not initial write', async () => {
+        const alice = await DidKeyResolver.generate();
+        const write = await TestDataGenerator.generateRecordsWrite({ author: alice, published: false });
+
+        const writeReply = await dwn.processMessage(alice.did, write.message, write.dataStream);
+        expect(writeReply.status.code).to.equal(202);
+
+        // write an update to the record
+        const write2 = await RecordsWrite.createFrom({ recordsWriteMessage: write.message, published: true, signer: Jws.createSigner(alice) });
+        const write2Reply = await dwn.processMessage(alice.did, write2.message);
+        expect(write2Reply.status.code).to.equal(202);
+
+        // make sure result returned now has `initialWrite` property
+        const messageData = await TestDataGenerator.generateRecordsQuery({ author: alice, filter: { recordId: write.message.recordId } });
+        const reply = await dwn.processMessage(alice.did, messageData.message);
+
+        expect(reply.status.code).to.equal(200);
+        expect(reply.entries?.length).to.equal(1);
+        expect(reply.entries![0].initialWrite).to.exist;
+        expect(reply.entries![0].initialWrite?.recordId).to.equal(write.message.recordId);
+
+      });
+
       it('should be able to query by attester', async () => {
       // scenario: 2 records authored by alice, 1st attested by alice, 2nd attested by bob
         const alice = await DidKeyResolver.generate();

tests/handlers/records-read.spec.ts

@@ -196,6 +196,28 @@ export function testRecordsReadHandler(): void {
         expect(ArrayUtility.byteArraysEqual(dataFetched, dataBytes!)).to.be.true;
       });
 
+      it('should include `initialWrite` property if RecordsWrite is not initial write', async () => {
+        const alice = await DidKeyResolver.generate();
+        const write = await TestDataGenerator.generateRecordsWrite({ author: alice, published: false });
+
+        const writeReply = await dwn.processMessage(alice.did, write.message, write.dataStream);
+        expect(writeReply.status.code).to.equal(202);
+
+        // write an update to the record
+        const write2 = await RecordsWrite.createFrom({ recordsWriteMessage: write.message, published: true, signer: Jws.createSigner(alice) });
+        const write2Reply = await dwn.processMessage(alice.did, write2.message);
+        expect(write2Reply.status.code).to.equal(202);
+
+        // make sure result returned now has `initialWrite` property
+        const messageData = await RecordsRead.create({ filter: { recordId: write.message.recordId }, signer: Jws.createSigner(alice) });
+        const reply = await dwn.processMessage(alice.did, messageData.message);
+
+        expect(reply.status.code).to.equal(200);
+        expect(reply.record?.initialWrite).to.exist;
+        expect(reply.record?.initialWrite?.recordId).to.equal(write.message.recordId);
+
+      });
+
       describe('protocol based reads', () => {
         it('should allow read with allow-anyone rule', async () => {
         // scenario: Alice writes an image to her DWN, then Bob reads the image because he is "anyone".