#198 - added size check when returning encodedData

* added size check when returning encodedData
* added typing for entries returned in query results

README.md

@@ -3,7 +3,7 @@
 # Decentralized Web Node (DWN) SDK
 
 Code Coverage
-![Statements](https://img.shields.io/badge/statements-94.91%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.93%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-93.27%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-94.91%25-brightgreen.svg?style=flat)
+![Statements](https://img.shields.io/badge/statements-94.93%25-brightgreen.svg?style=flat) ![Branches](https://img.shields.io/badge/branches-92.92%25-brightgreen.svg?style=flat) ![Functions](https://img.shields.io/badge/functions-93.27%25-brightgreen.svg?style=flat) ![Lines](https://img.shields.io/badge/lines-94.93%25-brightgreen.svg?style=flat)
 
 ## Introduction
 

src/core/dwn-constant.ts

@@ -0,0 +1,6 @@
+export class DwnConstant {
+  /**
+   * The maximum size of raw data that will be returned as `encodedData`.
+   */
+  public static readonly maxDataSizeAllowedToBeEncoded = 10_000;
+}

src/core/message-reply.ts

@@ -1,4 +1,5 @@
-import { Descriptor } from './types.js';
+import type { QueryResultEntry } from './types.js';
+
 import { Readable } from 'readable-stream';
 
 type Status = {
@@ -8,7 +9,7 @@ type Status = {
 
 type MessageReplyOptions = {
   status: Status,
-  entries?: { descriptor: Descriptor }[];
+  entries?: QueryResultEntry[];
   data? : Readable;
 };
 
@@ -20,7 +21,7 @@ export class MessageReply {
    * e.g. the resulting messages from a RecordsQuery
    * Mutually exclusive with `data`.
    */
-  entries?: { descriptor: Descriptor }[];
+  entries?: QueryResultEntry[];
 
   /**
    * Data corresponding to the message received if applicable (e.g. RecordsRead).

src/core/message.ts

@@ -71,15 +71,11 @@ export abstract class Message {
    * Gets the CID of the given message.
    */
   public static async getCid(message: BaseMessage): Promise<string> {
-    const messageCopy = { ...message };
-
-    // TODO: Once #219 (https://github.com/TBD54566975/dwn-sdk-js/issues/219) is implemented,
-    // `encodedData` will likely not exist directly as part of the message
-    if (messageCopy['encodedData'] !== undefined) {
-      delete (messageCopy as any).encodedData;
-    }
-
-    const cid = await computeCid(messageCopy);
+    // NOTE: we wrap the `computeCid()` here in case that
+    // the message will contain properties that should not be part of the CID computation
+    // and we need to strip them out (like `encodedData` that we historically had for a long time),
+    // but we can remove this method entirely if the code becomes stable and it is apparent that the wrapper is not needed
+    const cid = await computeCid(message);
     return cid;
   }
 

src/core/types.ts

@@ -35,22 +35,34 @@ export type TimestampedMessage = BaseMessage & {
 };
 
 /**
- * Message that references `dataCid`.
+ * Message returned in a query result.
+ * NOTE: the message structure is a modified version of the message received, the most notable differences are:
+ * 1. does not contain `authorization`
+ * 2. may include encoded data
  */
-export type DataReferencingMessage = {
-  descriptor: {
-    dataCid: string;
-  };
-
-  encodedData: string;
+export type QueryResultEntry = {
+  descriptor: Descriptor;
+  encodedData?: string;
 };
 
 export type EqualFilter = string | number | boolean;
 
 export type OneOfFilter = EqualFilter[];
 
-type GT = ({ gt: string } & { gte?: never }) | ({ gt?: never } & { gte: string });
-type LT = ({ lt: string } & { lte?: never }) | ({ lt?: never } & { lte: string });
+
+/**
+ * "greater than" or "greater than or equal to" range condition. `gt` and `gte` are mutually exclusive.
+ */
+export type GT = ({ gt: string } & { gte?: never }) | ({ gt?: never } & { gte: string });
+
+/**
+ * "less than" or "less than or equal to" range condition. `lt`, `lte` are mutually exclusive.
+ */
+export type LT = ({ lt: string } & { lte?: never }) | ({ lt?: never } & { lte: string });
+
+/**
+ * Ranger filter. 1 condition is required.
+ */
 export type RangeFilter = (GT | LT) & Partial<GT> & Partial<LT>;
 
 export type Filter = {

src/index.ts

@@ -24,6 +24,7 @@ export { DidKeyResolver } from './did/did-key-resolver.js';
 export { DidIonResolver } from './did/did-ion-resolver.js';
 export { DidResolver, DidMethodResolver } from './did/did-resolver.js';
 export { Dwn } from './dwn.js';
+export { DwnConstant } from './core/dwn-constant.js';
 export { DwnInterfaceName, DwnMethodName } from './core/message.js';
 export { Encoder } from './utils/encoder.js';
 export { HooksWrite, HooksWriteOptions } from './interfaces/hooks/messages/hooks-write.js';

src/interfaces/protocols/handlers/protocols-query.ts

@@ -1,5 +1,6 @@
 import type { MethodHandler } from '../../types.js';
 import type { ProtocolsQueryMessage } from '../types.js';
+import type { QueryResultEntry } from '../../../core/types.js';
 
 import { canonicalAuth } from '../../../core/auth.js';
 import { MessageReply } from '../../../core/message-reply.js';
@@ -43,7 +44,14 @@ export class ProtocolsQueryHandler implements MethodHandler {
     };
     removeUndefinedProperties(query);
 
-    const entries = await this.messageStore.query(tenant, query);
+    const records = await this.messageStore.query(tenant, query);
+
+    // strip away `authorization` property for each record before responding
+    const entries: QueryResultEntry[] = [];
+    for (const record of records) {
+      const { authorization: _, ...objectWithRemainingProperties } = record; // a trick to stripping away `authorization`
+      entries.push(objectWithRemainingProperties);
+    }
 
     return new MessageReply({
       status: { code: 200, detail: 'OK' },

src/interfaces/records/handlers/records-query.ts

@@ -1,8 +1,8 @@
 import type { MethodHandler } from '../../types.js';
+import type { BaseMessage, QueryResultEntry } from '../../../core/types.js';
 import type { RecordsQueryMessage, RecordsWriteMessage } from '../types.js';
 
 import { authenticate } from '../../../core/auth.js';
-import { BaseMessage } from '../../../core/types.js';
 import { lexicographicalCompare } from '../../../utils/string.js';
 import { MessageReply } from '../../../core/message-reply.js';
 import { StorageController } from '../../../store/storage-controller.js';
@@ -50,11 +50,10 @@ export class RecordsQueryHandler implements MethodHandler {
     }
 
     // strip away `authorization` property for each record before responding
-    const entries = [];
+    const entries: QueryResultEntry[] = [];
     for (const record of records) {
-      const recordDuplicate = { ...record };
-      delete recordDuplicate.authorization;
-      entries.push(recordDuplicate);
+      const { authorization: _, ...objectWithRemainingProperties } = record; // a trick to stripping away `authorization`
+      entries.push(objectWithRemainingProperties);
     }
 
     return new MessageReply({

src/store/blockstore-level.ts

@@ -11,6 +11,11 @@ import { sleep } from '../utils/time.js';
 // FreeBSD, including any future Node.js and Electron release thanks to Node-API, including ARM
 // platforms like Raspberry Pi and Android, as well as in Chrome, Firefox, Edge, Safari, iOS Safari
 //  and Chrome for Android.
+
+/**
+ * Blockstore implementation using LevelDB for storing the actual messages (in the case of MessageStore)
+ * or the data associated with messages (in the case of a DataStore).
+ */
 export class BlockstoreLevel implements Blockstore {
   config: BlockstoreLevelConfig;
 

src/store/index-level.ts

@@ -15,6 +15,9 @@ export interface IndexLevelOptions {
   signal?: AbortSignal;
 }
 
+/**
+ * A LevelDB implementation for indexing the messages stored in the DWN.
+ */
 export class IndexLevel {
   config: IndexLevelConfig;
 

src/store/storage-controller.ts

@@ -1,8 +1,9 @@
 import { DataStore } from './data-store.js';
+import { DwnConstant } from '../core/dwn-constant.js';
 import { MessageStore } from './message-store.js';
 import { Readable } from 'readable-stream';
-import { BaseMessage, Filter } from '../core/types.js';
 
+import { BaseMessage, Filter } from '../core/types.js';
 import { DataStream, Encoder } from '../index.js';
 import { DwnError, DwnErrorCode } from '../core/dwn-error.js';
 
@@ -84,12 +85,11 @@ export class StorageController {
 
     const messages = await messageStore.query(tenant, filter);
 
+    // for every message, only include the data as `encodedData` if the data size is equal or smaller than the size threshold
     for (const message of messages) {
       const dataCid = message.descriptor.dataCid;
-      if (dataCid !== undefined) {
-        // TODO: #219 (https://github.com/TBD54566975/dwn-sdk-js/issues/219)
-        // temporary placeholder for keeping status-quo of returning data in `encodedData`
-        // once #219 is implemented, `encodedData` may or may not exist directly as part of the returned message here
+      const dataSize = message.descriptor.dataSize;
+      if (dataCid !== undefined && dataSize! <= DwnConstant.maxDataSizeAllowedToBeEncoded) {
         const readableStream = await dataStore.get(tenant, 'not used yet', dataCid);
         const dataBytes = await DataStream.toBytes(readableStream);
 

tests/interfaces/records/handlers/records-delete.spec.ts

@@ -118,7 +118,7 @@ describe('RecordsDeleteHandler.handle()', () => {
       const reply = await dwn.processMessage(alice.did, queryData.message);
       expect(reply.status.code).to.equal(200);
       expect(reply.entries?.length).to.equal(1);
-      expect((reply.entries[0] as any).encodedData).to.equal(expectedEncodedData);
+      expect(reply.entries[0].encodedData).to.equal(expectedEncodedData);
     });
   });
 

tests/interfaces/records/handlers/records-query.spec.ts

@@ -4,6 +4,7 @@ import chai, { expect } from 'chai';
 
 import { DataStoreLevel } from '../../../../src/store/data-store-level.js';
 import { DidKeyResolver } from '../../../../src/did/did-key-resolver.js';
+import { DwnConstant } from '../../../../src/core/dwn-constant.js';
 import { Encoder } from '../../../../src/utils/encoder.js';
 import { Jws } from '../../../../src/utils/jws.js';
 import { MessageStoreLevel } from '../../../../src/store/message-store-level.js';
@@ -98,6 +99,38 @@ describe('RecordsQueryHandler.handle()', () => {
       expect(reply2.entries?.length).to.equal(1); // only 1 entry should match the query
     });
 
+    it('should return `encodedData` if data size is within the spec threshold', async () => {
+      const data = TestDataGenerator.randomBytes(DwnConstant.maxDataSizeAllowedToBeEncoded); // within/on threshold
+      const alice = await DidKeyResolver.generate();
+      const write= await TestDataGenerator.generateRecordsWrite({ requester: alice, data });
+
+      const writeReply = await dwn.processMessage(alice.did, write.message, write.dataStream);
+      expect(writeReply.status.code).to.equal(202);
+
+      const messageData = await TestDataGenerator.generateRecordsQuery({ requester: alice, filter: { recordId: write.message.recordId } });
+      const reply = await dwn.processMessage(alice.did, messageData.message);
+
+      expect(reply.status.code).to.equal(200);
+      expect(reply.entries?.length).to.equal(1);
+      expect(reply.entries[0].encodedData).to.equal(Encoder.bytesToBase64Url(data));
+    });
+
+    it('should not return `encodedData` if data size is greater then spec threshold', async () => {
+      const data = TestDataGenerator.randomBytes(DwnConstant.maxDataSizeAllowedToBeEncoded + 1); // exceeding threshold
+      const alice = await DidKeyResolver.generate();
+      const write= await TestDataGenerator.generateRecordsWrite({ requester: alice, data });
+
+      const writeReply = await dwn.processMessage(alice.did, write.message, write.dataStream);
+      expect(writeReply.status.code).to.equal(202);
+
+      const messageData = await TestDataGenerator.generateRecordsQuery({ requester: alice, filter: { recordId: write.message.recordId } });
+      const reply = await dwn.processMessage(alice.did, messageData.message);
+
+      expect(reply.status.code).to.equal(200);
+      expect(reply.entries?.length).to.equal(1);
+      expect(reply.entries[0].encodedData).to.be.undefined;
+    });
+
     it('should be able to query by attester', async () => {
       // scenario: 2 records authored by alice, 1st attested by alice, 2nd attested by bob
       const alice = await DidKeyResolver.generate();
@@ -162,8 +195,8 @@ describe('RecordsQueryHandler.handle()', () => {
       });
       const reply1 = await dwn.processMessage(alice.did, recordsQuery1.message);
       expect(reply1.entries?.length).to.equal(2);
-      expect((reply1.entries[0] as any).encodedData).to.equal(Encoder.bytesToBase64Url(write2.dataBytes));
-      expect((reply1.entries[1] as any).encodedData).to.equal(Encoder.bytesToBase64Url(write3.dataBytes));
+      expect(reply1.entries[0].encodedData).to.equal(Encoder.bytesToBase64Url(write2.dataBytes));
+      expect(reply1.entries[1].encodedData).to.equal(Encoder.bytesToBase64Url(write3.dataBytes));
 
       // testing `to` range
       const lastDayOf2022 = Temporal.PlainDateTime.from({ year: 2022, month: 12, day: 31 }).toString({ smallestUnit: 'microseconds' });
@@ -174,8 +207,8 @@ describe('RecordsQueryHandler.handle()', () => {
       });
       const reply2 = await dwn.processMessage(alice.did, recordsQuery2.message);
       expect(reply2.entries?.length).to.equal(2);
-      expect((reply2.entries[0] as any).encodedData).to.equal(Encoder.bytesToBase64Url(write1.dataBytes));
-      expect((reply2.entries[1] as any).encodedData).to.equal(Encoder.bytesToBase64Url(write2.dataBytes));
+      expect(reply2.entries[0].encodedData).to.equal(Encoder.bytesToBase64Url(write1.dataBytes));
+      expect(reply2.entries[1].encodedData).to.equal(Encoder.bytesToBase64Url(write2.dataBytes));
 
       // testing `from` and `to` range
       const lastDayOf2023 = Temporal.PlainDateTime.from({ year: 2023, month: 12, day: 31 }).toString({ smallestUnit: 'microseconds' });
@@ -186,7 +219,7 @@ describe('RecordsQueryHandler.handle()', () => {
       });
       const reply3 = await dwn.processMessage(alice.did, recordsQuery3.message);
       expect(reply3.entries?.length).to.equal(1);
-      expect((reply3.entries[0] as any).encodedData).to.equal(Encoder.bytesToBase64Url(write3.dataBytes));
+      expect(reply3.entries[0].encodedData).to.equal(Encoder.bytesToBase64Url(write3.dataBytes));
 
       // testing edge case where value equals `from` and `to`
       const recordsQuery4 = await TestDataGenerator.generateRecordsQuery({
@@ -196,7 +229,7 @@ describe('RecordsQueryHandler.handle()', () => {
       });
       const reply4 = await dwn.processMessage(alice.did, recordsQuery4.message);
       expect(reply4.entries?.length).to.equal(1);
-      expect((reply4.entries[0] as any).encodedData).to.equal(Encoder.bytesToBase64Url(write2.dataBytes));
+      expect(reply4.entries[0].encodedData).to.equal(Encoder.bytesToBase64Url(write2.dataBytes));
     });
 
     it('should be able use range and exact match queries at the same time', async () => {
@@ -237,7 +270,7 @@ describe('RecordsQueryHandler.handle()', () => {
       });
       const reply = await dwn.processMessage(alice.did, recordsQuery5.message);
       expect(reply.entries?.length).to.equal(1);
-      expect((reply.entries[0] as any).encodedData).to.equal(Encoder.bytesToBase64Url(write2.dataBytes));
+      expect(reply.entries[0].encodedData).to.equal(Encoder.bytesToBase64Url(write2.dataBytes));
     });
 
     it('should not include `authorization` in returned records', async () => {
@@ -441,9 +474,9 @@ describe('RecordsQueryHandler.handle()', () => {
       expect(replyToBob.status.code).to.equal(200);
       expect(replyToBob.entries?.length).to.equal(3); // expect 3 records
 
-      const privateRecordsForBob = replyToBob.entries.filter(message => (message as any).encodedData === Encoder.stringToBase64Url('2'));
-      const privateRecordsFromBob = replyToBob.entries.filter(message => (message as any).encodedData === Encoder.stringToBase64Url('3'));
-      const publicRecords = replyToBob.entries.filter(message => (message as any).encodedData === Encoder.stringToBase64Url('4'));
+      const privateRecordsForBob = replyToBob.entries.filter(message => message.encodedData === Encoder.stringToBase64Url('2'));
+      const privateRecordsFromBob = replyToBob.entries.filter(message => message.encodedData === Encoder.stringToBase64Url('3'));
+      const publicRecords = replyToBob.entries.filter(message => message.encodedData === Encoder.stringToBase64Url('4'));
       expect(privateRecordsForBob.length).to.equal(1);
       expect(privateRecordsFromBob.length).to.equal(1);
       expect(publicRecords.length).to.equal(1);