From 82fe049234423bd08a4b3c7e6cf48bdd5556d5a7 Mon Sep 17 00:00:00 2001 From: Liran Cohen Date: Tue, 7 May 2024 23:57:23 -0400 Subject: [PATCH] Add a helper method for generating a PaginationCursor (#513) Currently the only way to get a pagination cursor is from a query response that has more than the number of records you've queried for. However there could be some cases where the user wants to keep track of the last record they have retrieved and retrieve subsequent records. Added some helper methods to `agent` in order to build/return the data needed, and a convenience method to the `Record` class in `api` that utilizes them. --- .changeset/odd-eels-rest.md | 9 ++++ packages/agent/src/utils.ts | 32 +++++++++++- packages/agent/tests/utils.spec.ts | 80 ++++++++++++++++++++++++++++++ packages/api/package.json | 4 +- packages/api/src/record.ts | 15 +++++- packages/api/tests/record.spec.ts | 69 +++++++++++++++++++++++++- pnpm-lock.yaml | 42 ++-------------- 7 files changed, 207 insertions(+), 44 deletions(-) create mode 100644 .changeset/odd-eels-rest.md create mode 100644 packages/agent/tests/utils.spec.ts diff --git a/.changeset/odd-eels-rest.md b/.changeset/odd-eels-rest.md new file mode 100644 index 000000000..5f8f68737 --- /dev/null +++ b/.changeset/odd-eels-rest.md @@ -0,0 +1,9 @@ +--- +"@web5/identity-agent": patch +"@web5/proxy-agent": patch +"@web5/user-agent": patch +"@web5/agent": patch +"@web5/api": patch +--- + +Add a helper methods for generating a PaginationCursor from `api` without importing `dwn-sdk-js` directly diff --git a/packages/agent/src/utils.ts b/packages/agent/src/utils.ts index d09bcc242..845c2d0ed 100644 --- a/packages/agent/src/utils.ts +++ b/packages/agent/src/utils.ts @@ -1,10 +1,11 @@ import type { DidUrlDereferencer } from '@web5/dids'; -import type { RecordsWriteMessage } from '@tbd54566975/dwn-sdk-js'; +import type { PaginationCursor, RecordsWriteMessage } from '@tbd54566975/dwn-sdk-js'; +import { DateSort } from '@tbd54566975/dwn-sdk-js'; import { Readable } from '@web5/common'; import { utils as didUtils } from '@web5/dids'; import { ReadableWebToNodeStream } from 'readable-web-to-node-stream'; -import { DwnInterfaceName, DwnMethodName, RecordsWrite } from '@tbd54566975/dwn-sdk-js'; +import { DwnInterfaceName, DwnMethodName, Message, RecordsWrite } from '@tbd54566975/dwn-sdk-js'; export function blobToIsomorphicNodeReadable(blob: Blob): Readable { return webReadableToIsomorphicNodeReadable(blob.stream() as ReadableStream); @@ -55,6 +56,33 @@ export function isRecordsWrite(obj: unknown): obj is RecordsWrite { ); } +/** + * Get the CID of the given RecordsWriteMessage. + */ +export function getRecordMessageCid(message: RecordsWriteMessage): Promise { + return Message.getCid(message); +} + +/** + * Get the pagination cursor for the given RecordsWriteMessage and DateSort. + * + * @param message The RecordsWriteMessage for which to get the pagination cursor. + * @param dateSort The date sort that will be used in the query or subscription to which the cursor will be applied. + */ +export async function getPaginationCursor(message: RecordsWriteMessage, dateSort: DateSort): Promise { + const value = dateSort === DateSort.CreatedAscending || dateSort === DateSort.CreatedDescending ? + message.descriptor.dateCreated : message.descriptor.datePublished; + + if (value === undefined) { + throw new Error('The dateCreated or datePublished property is missing from the record descriptor.'); + } + + return { + messageCid: await getRecordMessageCid(message), + value + }; +} + export function webReadableToIsomorphicNodeReadable(webReadable: ReadableStream) { return new ReadableWebToNodeStream(webReadable); } \ No newline at end of file diff --git a/packages/agent/tests/utils.spec.ts b/packages/agent/tests/utils.spec.ts new file mode 100644 index 000000000..bcccace0f --- /dev/null +++ b/packages/agent/tests/utils.spec.ts @@ -0,0 +1,80 @@ +import { expect } from 'chai'; + +import { DateSort, Message, TestDataGenerator } from '@tbd54566975/dwn-sdk-js'; +import { getPaginationCursor, getRecordAuthor, getRecordMessageCid } from '../src/utils.js'; + +describe('Utils', () => { + describe('getPaginationCursor', () => { + it('should return a PaginationCursor object', async () => { + // create a RecordWriteMessage object which is published + const { message } = await TestDataGenerator.generateRecordsWrite({ + published: true, + }); + + const messageCid = await Message.getCid(message); + + // Published Ascending DateSort will get the datePublished as the cursor value + const datePublishedAscendingCursor = await getPaginationCursor(message, DateSort.PublishedAscending); + expect(datePublishedAscendingCursor).to.deep.equal({ + value: message.descriptor.datePublished, + messageCid, + }); + + // Published Descending DateSort will get the datePublished as the cursor value + const datePublishedDescendingCursor = await getPaginationCursor(message, DateSort.PublishedDescending); + expect(datePublishedDescendingCursor).to.deep.equal({ + value: message.descriptor.datePublished, + messageCid, + }); + + // Created Ascending DateSort will get the dateCreated as the cursor value + const dateCreatedAscendingCursor = await getPaginationCursor(message, DateSort.CreatedAscending); + expect(dateCreatedAscendingCursor).to.deep.equal({ + value: message.descriptor.dateCreated, + messageCid, + }); + + // Created Descending DateSort will get the dateCreated as the cursor value + const dateCreatedDescendingCursor = await getPaginationCursor(message, DateSort.CreatedDescending); + expect(dateCreatedDescendingCursor).to.deep.equal({ + value: message.descriptor.dateCreated, + messageCid, + }); + }); + + it('should fail for DateSort with PublishedAscending or PublishedDescending if the record is not published', async () => { + // create a RecordWriteMessage object which is not published + const { message } = await TestDataGenerator.generateRecordsWrite(); + + // Published Ascending DateSort will get the datePublished as the cursor value + try { + await getPaginationCursor(message, DateSort.PublishedAscending); + expect.fail('Expected getPaginationCursor to throw an error'); + } catch(error: any) { + expect(error.message).to.include('The dateCreated or datePublished property is missing from the record descriptor.'); + } + }); + }); + + describe('getRecordMessageCid', () => { + it('should get the CID of a RecordsWriteMessage', async () => { + // create a RecordWriteMessage object + const { message } = await TestDataGenerator.generateRecordsWrite(); + const messageCid = await Message.getCid(message); + + const messageCidFromFunction = await getRecordMessageCid(message); + expect(messageCidFromFunction).to.equal(messageCid); + }); + }); + + describe('getRecordAuthor', () => { + it('should get the author of a RecordsWriteMessage', async () => { + // create a RecordWriteMessage object + const { message, author } = await TestDataGenerator.generateRecordsWrite(); + + const authorFromFunction = getRecordAuthor(message); + expect(authorFromFunction).to.not.be.undefined; + expect(authorFromFunction!).to.equal(author.did); + }); + }); +}); \ No newline at end of file diff --git a/packages/api/package.json b/packages/api/package.json index a766a124f..9fe32a290 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -77,11 +77,11 @@ "node": ">=18.0.0" }, "dependencies": { - "@web5/agent": "0.3.2", + "@web5/agent": "0.3.4", "@web5/common": "1.0.0", "@web5/crypto": "1.0.0", "@web5/dids": "1.0.1", - "@web5/user-agent": "0.3.2" + "@web5/user-agent": "0.3.4" }, "devDependencies": { "@playwright/test": "1.40.1", diff --git a/packages/api/src/record.ts b/packages/api/src/record.ts index e3a1ff256..c4d744c68 100644 --- a/packages/api/src/record.ts +++ b/packages/api/src/record.ts @@ -5,13 +5,16 @@ /// import type { Readable } from '@web5/common'; -import type { +import { Web5Agent, DwnMessage, DwnMessageParams, DwnResponseStatus, ProcessDwnRequest, DwnMessageDescriptor, + getPaginationCursor, + DwnDateSort, + DwnPaginationCursor } from '@web5/agent'; import { DwnInterface } from '@web5/agent'; @@ -576,6 +579,16 @@ export class Record implements RecordModel { return str; } + /** + * Returns a pagination cursor for the current record given a sort order. + * + * @param sort the sort order to use for the pagination cursor. + * @returns A promise that resolves to a pagination cursor for the current record. + */ + async paginationCursor(sort: DwnDateSort): Promise { + return getPaginationCursor(this.rawMessage, sort); + } + /** * Update the current record on the DWN. * @param params - Parameters to update the record. diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index 148313b6b..3ad12f4f6 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -5,7 +5,7 @@ import { expect } from 'chai'; import { NodeStream } from '@web5/common'; import { utils as didUtils } from '@web5/dids'; import { Web5UserAgent } from '@web5/user-agent'; -import { DwnConstant, DwnEncryptionAlgorithm, DwnInterface, DwnKeyDerivationScheme, dwnMessageConstructors, PlatformAgentTestHarness } from '@web5/agent'; +import { DwnConstant, DwnDateSort, DwnEncryptionAlgorithm, DwnInterface, DwnKeyDerivationScheme, dwnMessageConstructors, PlatformAgentTestHarness } from '@web5/agent'; import { Record } from '../src/record.js'; import { DwnApi } from '../src/dwn-api.js'; import { dataToBlob } from '../src/utils.js'; @@ -16,6 +16,7 @@ import emailProtocolDefinition from './fixtures/protocol-definitions/email.json' // NOTE: @noble/secp256k1 requires globalThis.crypto polyfill for node.js <=18: https://github.com/paulmillr/noble-secp256k1/blob/main/README.md#usage // Remove when we move off of node.js v18 to v20, earliest possible time would be Oct 2023: https://github.com/nodejs/release#release-schedule import { webcrypto } from 'node:crypto'; +import { Message } from '@tbd54566975/dwn-sdk-js'; // @ts-ignore if (!globalThis.crypto) globalThis.crypto = webcrypto; @@ -2825,4 +2826,70 @@ describe('Record', () => { }); }); }); + + describe('paginationCursor', () => { + it('should return a cursor for pagination', async () => { + // Create a record that is not published. + const { status, record } = await dwnAlice.records.write({ + data : 'Hello, world!', + message : { + schema : 'foo/bar', + dataFormat : 'text/plain' + } + }); + + expect(status.code).to.equal(202); + const messageCid = await Message.getCid(record['rawMessage']); + + const paginationCursorCreatedAscending = await record.paginationCursor(DwnDateSort.CreatedAscending); + expect(paginationCursorCreatedAscending).to.be.deep.equal({ + messageCid, + value: record.dateCreated, + }); + + const paginationCursorCreatedDescending = await record.paginationCursor(DwnDateSort.CreatedDescending); + expect(paginationCursorCreatedDescending).to.be.deep.equal({ + messageCid, + value: record.dateCreated, + }); + }); + + it('should return a cursor for pagination for a published record', async () => { + // Create a record that is not published. + const { status, record } = await dwnAlice.records.write({ + data : 'Hello, world!', + message : { + published : true, + schema : 'foo/bar', + dataFormat : 'text/plain' + } + }); + expect(status.code).to.equal(202); + const messageCid = await Message.getCid(record['rawMessage']); + + const paginationCursorCreatedAscending = await record.paginationCursor(DwnDateSort.CreatedAscending); + expect(paginationCursorCreatedAscending).to.be.deep.equal({ + messageCid, + value: record.dateCreated, + }); + + const paginationCursorCreatedDescending = await record.paginationCursor(DwnDateSort.CreatedDescending); + expect(paginationCursorCreatedDescending).to.be.deep.equal({ + messageCid, + value: record.dateCreated, + }); + + const paginationCursorPublishedAscending = await record.paginationCursor(DwnDateSort.PublishedAscending); + expect(paginationCursorPublishedAscending).to.be.deep.equal({ + messageCid, + value: record.datePublished, + }); + + const paginationCursorPublishedDescending = await record.paginationCursor(DwnDateSort.PublishedDescending); + expect(paginationCursorPublishedDescending).to.be.deep.equal({ + messageCid, + value: record.datePublished, + }); + }); + }); }); \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 36db81cb7..778620a88 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -151,8 +151,8 @@ importers: packages/api: dependencies: '@web5/agent': - specifier: 0.3.2 - version: 0.3.2 + specifier: 0.3.4 + version: link:../agent '@web5/common': specifier: 1.0.0 version: link:../common @@ -163,8 +163,8 @@ importers: specifier: 1.0.1 version: 1.0.1 '@web5/user-agent': - specifier: 0.3.2 - version: 0.3.2 + specifier: 0.3.4 + version: link:../user-agent devDependencies: '@playwright/test': specifier: 1.40.1 @@ -3435,27 +3435,6 @@ packages: - utf-8-validate dev: true - /@web5/agent@0.3.2: - resolution: {integrity: sha512-oTObOgvY+sz7nfTIHofe8TulZ5AJspQXN2LzRx3ZZxCuNNepfvtofWVPwcRJmsa50Ko7GDlRhiVrkx6fqh6k9Q==} - engines: {node: '>=18.0.0'} - dependencies: - '@noble/ciphers': 0.4.1 - '@scure/bip39': 1.2.2 - '@tbd54566975/dwn-sdk-js': 0.3.1 - '@web5/common': 1.0.0 - '@web5/crypto': 1.0.0 - '@web5/dids': 1.0.1 - abstract-level: 1.0.4 - ed25519-keygen: 0.4.11 - level: 8.0.0 - ms: 2.1.3 - readable-web-to-node-stream: 3.0.2 - ulidx: 2.1.0 - transitivePeerDependencies: - - encoding - - supports-color - dev: false - /@web5/common@0.2.4: resolution: {integrity: sha512-h7I+FbAzTbkQWnv5RZCtE1lFsOzwssvcr4aQuQvO1mx8qTjzl5gGXhFIpl6JB9aSE2FayNUV6kQBID8Sb5HXTQ==} engines: {node: '>=18.0.0'} @@ -3530,19 +3509,6 @@ packages: - utf-8-validate dev: true - /@web5/user-agent@0.3.2: - resolution: {integrity: sha512-g/GJRYT7PBQpuCeA/I+7v2AKxDzEmyFUzSVMqEf92h7cIU2S6kBGmLwxN7WXqvoEHz/w1aM2JD4TPUloAX0nwg==} - engines: {node: '>=18.0.0'} - dependencies: - '@web5/agent': 0.3.2 - '@web5/common': 1.0.0 - '@web5/crypto': 1.0.0 - '@web5/dids': 1.0.1 - transitivePeerDependencies: - - encoding - - supports-color - dev: false - /@webassemblyjs/ast@1.11.6: resolution: {integrity: sha512-IN1xI7PwOvLPgjcf180gC1bqn3q/QaOCwYUahIOhbYUu8KA/3tw2RT/T0Gidi1l7Hhj5D/INhJxiICObqpMu4Q==} dependencies: