diff --git a/.github/workflows/tests-ci.yml b/.github/workflows/tests-ci.yml index cf98b6b5f..76a999111 100644 --- a/.github/workflows/tests-ci.yml +++ b/.github/workflows/tests-ci.yml @@ -59,7 +59,7 @@ jobs: - name: Run tests for all packages run: npm run test:node --ws -- --color env: - TEST_DWN_URL: http://localhost:3000 + TEST_DWN_URLS: http://localhost:3000 - name: Upload test coverage to Codecov uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # v3.1.4 @@ -106,4 +106,4 @@ jobs: - name: Run tests for all packages run: npm run test:browser --ws -- --color env: - TEST_DWN_URL: http://localhost:3000 + TEST_DWN_URLS: http://localhost:3000 diff --git a/packages/agent/karma.conf.cjs b/packages/agent/karma.conf.cjs index 6b7407b2c..6d5eeb080 100644 --- a/packages/agent/karma.conf.cjs +++ b/packages/agent/karma.conf.cjs @@ -36,7 +36,7 @@ module.exports = function (config) { timeout: 10000 // 10 seconds }, // If an environment variable is defined, override the default test DWN URL. - testDwnUrl: process.env.TEST_DWN_URL, + testDwnUrls: process.env.TEST_DWN_URLS, }, // list of files / patterns to load in the browser diff --git a/packages/agent/src/sync-manager.ts b/packages/agent/src/sync-manager.ts index 2f6dc830b..b13ad6ae3 100644 --- a/packages/agent/src/sync-manager.ts +++ b/packages/agent/src/sync-manager.ts @@ -1,5 +1,6 @@ import type { BatchOperation } from 'level'; import type { + Event, EventsGetReply, GenericMessage, MessagesGetReply, @@ -16,13 +17,28 @@ import type { Web5ManagedAgent } from './types/agent.js'; import { webReadableToIsomorphicNodeReadable } from './utils.js'; +// arbitrary number for now, but we should enforce some sane minimum +export const MIN_SYNC_INTERVAL = 3000; + +type SyncDirection = 'pull' | 'push'; + +interface SyncOptions { + interval?: number + direction?: SyncDirection +} + export interface SyncManager { agent: Web5ManagedAgent; registerIdentity(options: { did: string }): Promise; - startSync(options: { interval: number }): Promise; + + // sync will run the sync operation once. + // if a direction is passed, it will only sync in that direction. + sync(direction?: SyncDirection): Promise; + + // startSync will run sync on an interval + // if a direction is provided, it will only sync in that direction. + startSync(options?: SyncOptions): Promise; stopSync(): void; - push(): Promise; - pull(): Promise; } export type SyncManagerOptions = { @@ -31,12 +47,11 @@ export type SyncManagerOptions = { db?: Level; }; -type SyncDirection = 'push' | 'pull'; - type SyncState = { did: string; dwnUrl: string; - watermark: string | undefined; + pullWatermark?: string; + pushWatermark?: string; } type DwnMessage = { @@ -93,10 +108,7 @@ export class SyncManagerLevel implements SyncManager { await this._db.clear(); } - public async pull(): Promise { - const syncPeerState = await this.getSyncPeerState({ syncDirection: 'pull' }); - await this.enqueueOperations({ syncDirection: 'pull', syncPeerState }); - + private async pull(): Promise { const pullQueue = this.getPullQueue(); const pullJobs = await pullQueue.iterator().all(); @@ -171,7 +183,9 @@ export class SyncManagerLevel implements SyncManager { author : did, messageType : 'RecordsRead', messageOptions : { - recordId: message['recordId'] + filter: { + recordId: message['recordId'] + } } }); @@ -224,10 +238,7 @@ export class SyncManagerLevel implements SyncManager { await pullQueue.batch(deleteOperations as any); } - public async push(): Promise { - const syncPeerState = await this.getSyncPeerState({ syncDirection: 'push' }); - await this.enqueueOperations({ syncDirection: 'push', syncPeerState }); - + private async push(): Promise { const pushQueue = this.getPushQueue(); const pushJobs = await pushQueue.iterator().all(); @@ -293,11 +304,8 @@ export class SyncManagerLevel implements SyncManager { await registeredIdentities.put(did, ''); } - public startSync(options: { - interval: number - }): Promise { - const { interval = 120_000 } = options; - + public startSync(options: SyncOptions = {}): Promise { + const { interval = MIN_SYNC_INTERVAL, direction } = options; return new Promise((resolve, reject) => { if (this._syncIntervalId) { clearInterval(this._syncIntervalId); @@ -305,13 +313,12 @@ export class SyncManagerLevel implements SyncManager { this._syncIntervalId = setInterval(async () => { try { - await this.push(); - await this.pull(); + await this.sync(direction); } catch (error) { this.stopSync(); reject(error); } - }, interval); + }, interval >= MIN_SYNC_INTERVAL ? interval : MIN_SYNC_INTERVAL); }); } @@ -322,97 +329,104 @@ export class SyncManagerLevel implements SyncManager { } } - private async enqueueOperations(options: { - syncDirection: SyncDirection, - syncPeerState: SyncState[] - }) { - const { syncDirection, syncPeerState } = options; - - for (let syncState of syncPeerState) { - // Get the event log from the remote DWN if pull sync, or local DWN if push sync. - const eventLog = await this.getDwnEventLog({ - did : syncState.did, - dwnUrl : syncState.dwnUrl, - syncDirection, - watermark : syncState.watermark - }); + public async sync(direction?: SyncDirection): Promise { + await this.enqueueOperations(direction); + // enqueue operations handles the direction logic. + // we can just run both operations and only enqueued events will sync. + await Promise.all([ + this.push(), this.pull() + ]); + } - const syncOperations: DbBatchOperation[] = []; + private createOperationKey(did: string, dwnUrl: string, watermark: string, messageCid: string): string { + return [did, dwnUrl, watermark, messageCid].join('~'); + } - for (let event of eventLog) { - /** Use "did~dwnUrl~watermark~messageCid" as the key in the sync queue. - * Note: It is critical that `watermark` precedes `messageCid` to - * ensure that when the sync jobs are pulled off the queue, they - * are lexographically sorted oldest to newest. */ - const operationKey = [ - syncState.did, - syncState.dwnUrl, - event.watermark, - event.messageCid - ].join('~'); + private dbBatchOperationPut(did: string, dwnUrl: string, watermark: string, messageCid: string): DbBatchOperation { + const key = this.createOperationKey(did, dwnUrl, watermark, messageCid); + return { type: 'put', key, value: '' }; + } - const operation: DbBatchOperation = { type: 'put', key: operationKey, value: '' }; + /** + * Enqueues the operations needed for sync based on the supplied direction. + * + * @param direction the optional direction in which you would like to enqueue sync events for. + * If no direction is supplied it will sync in both directions. + */ + async enqueueOperations(direction?: SyncDirection) { + const syncPeerState = await this.getSyncPeerState(); - syncOperations.push(operation); + for (let syncState of syncPeerState) { + const batchPromises = []; + if (direction === undefined || direction === 'push') { + const localEventsPromise = this.getLocalDwnEvents({ + did : syncState.did, + watermark : syncState.pushWatermark, + }); + batchPromises.push(this.batchOperations('push', localEventsPromise, syncState)); } - if (syncOperations.length > 0) { - const syncQueue = (syncDirection === 'pull') - ? this.getPullQueue() - : this.getPushQueue(); - await syncQueue.batch(syncOperations as any); + if(direction === undefined || direction === 'pull') { + const remoteEventsPromise = this.getRemoteEvents({ + did : syncState.did, + dwnUrl : syncState.dwnUrl, + watermark : syncState.pullWatermark, + }); + batchPromises.push(this.batchOperations('pull', remoteEventsPromise, syncState)); } + await Promise.all(batchPromises); } } - private async getDwnEventLog(options: { - did: string, - dwnUrl: string, - syncDirection: SyncDirection, - watermark?: string - }) { - const { did, dwnUrl, syncDirection, watermark } = options; + private async batchOperations(direction: SyncDirection, eventsPromise: Promise, syncState: SyncState): Promise { + const { did, dwnUrl } = syncState; + const operations: DbBatchOperation[] = []; + (await eventsPromise).forEach(e => operations.push(this.dbBatchOperationPut(did, dwnUrl, e.watermark, e.messageCid))); + return direction === 'pull' ? this.getPullQueue().batch(operations as any) : this.getPushQueue().batch(operations as any); + } + private async getLocalDwnEvents(options:{ did: string, watermark?: string }) { + const { did, watermark } = options; let eventsReply = {} as EventsGetReply; + ({ reply: eventsReply } = await this.agent.dwnManager.processRequest({ + author : did, + target : did, + messageType : 'EventsGet', + messageOptions : { watermark } + })); + + return eventsReply.events ?? []; + } - if (syncDirection === 'pull') { - // When sync is a pull, get the event log from the remote DWN. - const eventsGetMessage = await this.agent.dwnManager.createMessage({ - author : did, - messageType : 'EventsGet', - messageOptions : { watermark } - }); + private async getRemoteEvents(options: { did: string, dwnUrl: string, watermark?: string }) { + const { did, dwnUrl, watermark } = options; - try { - eventsReply = await this.agent.rpcClient.sendDwnRequest({ - dwnUrl : dwnUrl, - targetDid : did, - message : eventsGetMessage - }); - } catch { - // If a particular DWN service endpoint is unreachable, silently ignore. - } + let eventsReply = {} as EventsGetReply; - } else if (syncDirection === 'push') { - // When sync is a push, get the event log from the local DWN. - ({ reply: eventsReply } = await this.agent.dwnManager.processRequest({ - author : did, - target : did, - messageType : 'EventsGet', - messageOptions : { watermark } - })); - } + const eventsGetMessage = await this.agent.dwnManager.createMessage({ + author : did, + messageType : 'EventsGet', + messageOptions : { watermark } + }); - const eventLog = eventsReply.events ?? []; + try { + eventsReply = await this.agent.rpcClient.sendDwnRequest({ + dwnUrl : dwnUrl, + targetDid : did, + message : eventsGetMessage + }); + } catch { + // If a particular DWN service endpoint is unreachable, silently ignore. + } - return eventLog; + return eventsReply.events ?? []; } private async getDwnMessage( author: string, messageCid: string ): Promise { - let messagesGetResponse = await this.agent.dwnManager.processRequest({ + const messagesGetResponse = await this.agent.dwnManager.processRequest({ author : author, target : author, messageType : 'MessagesGet', @@ -455,7 +469,9 @@ export class SyncManagerLevel implements SyncManager { target : author, messageType : 'RecordsRead', messageOptions : { - recordId: writeMessage.recordId + filter: { + recordId: writeMessage.recordId + } } }); const reply = readResponse.reply as RecordsReadReply; @@ -482,11 +498,7 @@ export class SyncManagerLevel implements SyncManager { return dwnMessage; } - private async getSyncPeerState(options: { - syncDirection: SyncDirection - }): Promise { - const { syncDirection } = options; - + private async getSyncPeerState(): Promise { // Get a list of the DIDs of all registered identities. const registeredIdentities = await this._db.sublevel('registeredIdentities').keys().all(); @@ -521,18 +533,21 @@ export class SyncManagerLevel implements SyncManager { /** Get the watermark (or undefined) for each (DID, DWN service endpoint, sync direction) * combination and add it to the sync peer state array. */ for (let dwnUrl of service.serviceEndpoint.nodes) { - const watermark = await this.getWatermark(did, dwnUrl, syncDirection); - syncPeerState.push({ did, dwnUrl, watermark }); + try { + const syncState = await this.getSyncState(did, dwnUrl); + syncPeerState.push(syncState); + } catch(error) { + // go onto next peer if this fails + } } } return syncPeerState; } - private async getWatermark(did: string, dwnUrl: string, direction: SyncDirection) { + private async getWatermark(did: string, dwnUrl: string, direction: SyncDirection): Promise { const wmKey = `${did}~${dwnUrl}~${direction}`; const watermarkStore = this.getWatermarkStore(); - try { return await watermarkStore.get(wmKey); } catch(error: any) { @@ -540,16 +555,22 @@ export class SyncManagerLevel implements SyncManager { if (error.notFound) { return undefined; } + throw new Error('SyncManager: invalid watermark store'); } } private async setWatermark(did: string, dwnUrl: string, direction: SyncDirection, watermark: string) { const wmKey = `${did}~${dwnUrl}~${direction}`; const watermarkStore = this.getWatermarkStore(); - await watermarkStore.put(wmKey, watermark); } + private async getSyncState(did: string, dwnUrl: string): Promise { + const pullWatermark = await this.getWatermark(did, dwnUrl, 'pull'); + const pushWatermark = await this.getWatermark(did, dwnUrl, 'push'); + return { did, dwnUrl, pullWatermark, pushWatermark }; + } + /** * The message store is used to prevent "echoes" that occur during a sync pull operation. * After a message is confirmed to already be synchronized on the local DWN, its CID is added diff --git a/packages/agent/tests/dwn-manager.spec.ts b/packages/agent/tests/dwn-manager.spec.ts index 105110058..6d8149710 100644 --- a/packages/agent/tests/dwn-manager.spec.ts +++ b/packages/agent/tests/dwn-manager.spec.ts @@ -18,7 +18,7 @@ import { ProtocolsConfigureMessage, } from '@tbd54566975/dwn-sdk-js'; -import { testDwnUrl } from './test-config.js'; +import { testDwnUrls } from './test-config.js'; import { TestAgent } from './utils/test-agent.js'; import { DwnManager } from '../src/dwn-manager.js'; import { ManagedIdentity } from '../src/identity-manager.js'; @@ -34,8 +34,6 @@ if (!globalThis.crypto) globalThis.crypto = webcrypto; chai.use(chaiAsPromised); -let testDwnUrls: string[] = [testDwnUrl]; - describe('DwnManager', () => { describe('constructor', () => { diff --git a/packages/agent/tests/sync-manager.spec.ts b/packages/agent/tests/sync-manager.spec.ts index 2a6272534..20e997afc 100644 --- a/packages/agent/tests/sync-manager.spec.ts +++ b/packages/agent/tests/sync-manager.spec.ts @@ -1,18 +1,59 @@ import type { PortableDid } from '@web5/dids'; import { expect } from 'chai'; -import * as sinon from 'sinon'; +import sinon from 'sinon'; import type { ManagedIdentity } from '../src/identity-manager.js'; -import { testDwnUrl } from './test-config.js'; -import { TestAgent } from './utils/test-agent.js'; -import { SyncManagerLevel } from '../src/sync-manager.js'; +import { testDwnUrls } from './test-config.js'; +import { TestAgent, randomString } from './utils/test-agent.js'; +import { MIN_SYNC_INTERVAL, SyncManagerLevel } from '../src/sync-manager.js'; import { TestManagedAgent } from '../src/test-managed-agent.js'; -import { RecordsQueryReply, RecordsWriteMessage } from '@tbd54566975/dwn-sdk-js'; - -let testDwnUrls: string[] = [testDwnUrl]; +import { ProcessDwnRequest } from '../src/index.js'; +import { DataStream, RecordsQueryReply, RecordsReadReply, RecordsWriteMessage } from '@tbd54566975/dwn-sdk-js'; +import { Readable } from 'readable-stream'; + +/** + * Generates a `RecordsWrite` ProcessDwnRequest for testing. + */ +export function TestRecordsWriteMessage(target: string, author: string, dataStream: Blob | ReadableStream | Readable ): ProcessDwnRequest { + return { + author : author, + target : target, + messageType : 'RecordsWrite', + messageOptions : { + schema : 'testSchema', + dataFormat : 'text/plain' + }, + dataStream, + }; +} + +async function streamToText(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder('utf-8'); // You can specify other encodings if necessary + let result = ''; + + try { + // eslint-disable-next-line + while (true) { + const { done, value } = await reader.read(); + + if (done) { + return result; + } + + result += decoder.decode(value, { stream: true }); + } + } catch (error) { + console.error('Error while reading stream:', error); + } finally { + reader.releaseLock(); + } + + return result; +} describe('SyncManagerLevel', () => { describe('get agent', () => { @@ -74,160 +115,102 @@ describe('SyncManagerLevel', () => { await testAgent.closeStorage(); }); - describe('pull()', () => { - it('takes no action if no identities are registered', async () => { - const didResolveSpy = sinon.spy(testAgent.agent.didResolver, 'resolve'); - const sendDwnRequestSpy = sinon.spy(testAgent.agent.rpcClient, 'sendDwnRequest'); + describe('startSync()', () => { + let clock: sinon.SinonFakeTimers; - await testAgent.agent.syncManager.pull(); - - // Verify DID resolution and DWN requests did not occur. - expect(didResolveSpy.notCalled).to.be.true; - expect(sendDwnRequestSpy.notCalled).to.be.true; + beforeEach(() => { + clock = sinon.useFakeTimers(); + }); - didResolveSpy.restore(); - sendDwnRequestSpy.restore(); + afterEach(() => { + clock.restore(); }); - it('synchronizes records for 1 identity from remove DWN to local DWN', async () => { - // Write a test record to Alice's remote DWN. - let writeResponse = await testAgent.agent.dwnManager.sendRequest({ - author : alice.did, - target : alice.did, - messageType : 'RecordsWrite', - messageOptions : { - dataFormat: 'text/plain' - }, - dataStream: new Blob(['Hello, world!']) + it('check sync interval input', async () => { + const syncSpy = sinon.spy(testAgent.agent.syncManager, 'sync'); + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did }); + testAgent.agent.syncManager.startSync({ interval: 5000 }); - // Get the record ID of the test record. - const testRecordId = (writeResponse.message as RecordsWriteMessage).recordId; + clock.tick(3 * 5000); - // Confirm the record does NOT exist on Alice's local DWN. - let queryResponse = await testAgent.agent.dwnManager.processRequest({ - author : alice.did, - target : alice.did, - messageType : 'RecordsQuery', - messageOptions : { filter: { recordId: testRecordId } } - }); - let localDwnQueryReply = queryResponse.reply as RecordsQueryReply; - expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. - expect(localDwnQueryReply.entries).to.have.length(0); // Record doesn't exist on local DWN. + expect(syncSpy.callCount).to.equal(3); + syncSpy.restore(); + }); - // Register Alice's DID to be synchronized. + it('sync interval below minimum allowed threshold will sync at the minimum interval', async () => { + const syncSpy = sinon.spy(testAgent.agent.syncManager, 'sync'); await testAgent.agent.syncManager.registerIdentity({ did: alice.did }); + testAgent.agent.syncManager.startSync({ interval: 100 }); - // Execute Sync to pull all records from Alice's remote DWN to Alice's local DWN. - await testAgent.agent.syncManager.pull(); + clock.tick(3 * MIN_SYNC_INTERVAL); - // Confirm the record now DOES exist on Alice's local DWN. - queryResponse = await testAgent.agent.dwnManager.processRequest({ - author : alice.did, - target : alice.did, - messageType : 'RecordsQuery', - messageOptions : { filter: { recordId: testRecordId } } - }); - localDwnQueryReply = queryResponse.reply as RecordsQueryReply; - expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. - expect(localDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN. + expect(syncSpy.callCount).to.equal(3); + syncSpy.restore(); }); - - it('synchronizes records for multiple identities from remote DWN to local DWN', async () => { - // Create a second Identity to author the DWN messages. - const { did: bobDid } = await testAgent.createIdentity({ testDwnUrls }); - const bob = await testAgent.agent.identityManager.import({ - did : bobDid, - identity : { name: 'Bob', did: bobDid.did }, - kms : 'local' + it('subsequent startSync should cancel the old sync and start a new sync interval', async () => { + const syncSpy = sinon.spy(testAgent.agent.syncManager, 'sync'); + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did }); - // Write a test record to Alice's remote DWN. - let writeResponse = await testAgent.agent.dwnManager.sendRequest({ - author : alice.did, - target : alice.did, - messageType : 'RecordsWrite', - messageOptions : { - dataFormat: 'text/plain' - }, - dataStream: new Blob(['Hello, Bob!']) - }); + // start sync with default timeout + testAgent.agent.syncManager.startSync(); - // Get the record ID of Alice's test record. - const testRecordIdAlice = (writeResponse.message as RecordsWriteMessage).recordId; + // go through 3 intervals + clock.tick(3 * MIN_SYNC_INTERVAL); - // Write a test record to Bob's remote DWN. - writeResponse = await testAgent.agent.dwnManager.sendRequest({ - author : bob.did, - target : bob.did, - messageType : 'RecordsWrite', - messageOptions : { - dataFormat: 'text/plain' - }, - dataStream: new Blob(['Hello, Alice!']) - }); + expect(syncSpy.callCount).to.equal(3); - // Get the record ID of Bob's test record. - const testRecordIdBob = (writeResponse.message as RecordsWriteMessage).recordId; + // start sync with a higher interval. Should cancel the old sync and set a new interval. + testAgent.agent.syncManager.startSync({ interval: 10_000 }); - // Register Alice's DID to be synchronized. - await testAgent.agent.syncManager.registerIdentity({ - did: alice.did - }); + // go through 3 intervals with the new timeout + clock.tick( 3 * 10_000); + + // should be called a total of 6 times. + expect(syncSpy.callCount).to.equal(6); - // Register Bob's DID to be synchronized. + syncSpy.restore(); + }); + + it('check sync default value passed', async () => { + const setIntervalSpy = sinon.spy(global, 'setInterval'); await testAgent.agent.syncManager.registerIdentity({ - did: bob.did + did: alice.did }); + testAgent.agent.syncManager.startSync(); - // Execute Sync to pull all records from Alice's and Bob's remove DWNs to their local DWNs. - await testAgent.agent.syncManager.pull(); + clock.tick( 1 * MIN_SYNC_INTERVAL); - // Confirm the Alice test record exist on Alice's local DWN. - let queryResponse = await testAgent.agent.dwnManager.processRequest({ - author : alice.did, - target : alice.did, - messageType : 'RecordsQuery', - messageOptions : { filter: { recordId: testRecordIdAlice } } - }); - let localDwnQueryReply = queryResponse.reply as RecordsQueryReply; - expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. - expect(localDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN. - - // Confirm the Bob test record exist on Bob's local DWN. - queryResponse = await testAgent.agent.dwnManager.sendRequest({ - author : bob.did, - target : bob.did, - messageType : 'RecordsQuery', - messageOptions : { filter: { recordId: testRecordIdBob } } - }); - localDwnQueryReply = queryResponse.reply as RecordsQueryReply; - expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. - expect(localDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN. - }).timeout(5000); + expect(setIntervalSpy.calledOnce).to.be.true; + expect(setIntervalSpy.getCall(0).args.at(1)).to.equal(MIN_SYNC_INTERVAL); + setIntervalSpy.restore(); + }); }); - describe('push()', () => { + describe('sync()', () => { it('takes no action if no identities are registered', async () => { const didResolveSpy = sinon.spy(testAgent.agent.didResolver, 'resolve'); - const processRequestSpy = sinon.spy(testAgent.agent.dwnManager, 'processRequest'); + const sendDwnRequestSpy = sinon.spy(testAgent.agent.rpcClient, 'sendDwnRequest'); - await testAgent.agent.syncManager.push(); + await testAgent.agent.syncManager.sync(); // Verify DID resolution and DWN requests did not occur. expect(didResolveSpy.notCalled).to.be.true; - expect(processRequestSpy.notCalled).to.be.true; + expect(sendDwnRequestSpy.notCalled).to.be.true; didResolveSpy.restore(); - processRequestSpy.restore(); + sendDwnRequestSpy.restore(); }); - it('synchronizes records for 1 identity from local DWN to remote DWN', async () => { - // Write a record that we can use for this test. - let writeResponse = await testAgent.agent.dwnManager.processRequest({ + it('silently ignore when a particular DWN service endpoint is unreachable', async () => { + // Write a test record to Alice's remote DWN. + let writeResponse = await testAgent.agent.dwnManager.sendRequest({ author : alice.did, target : alice.did, messageType : 'RecordsWrite', @@ -236,113 +219,504 @@ describe('SyncManagerLevel', () => { }, dataStream: new Blob(['Hello, world!']) }); + expect(writeResponse.reply.status.code).to.equal(202); - // Get the record ID of the test record. - const testRecordId = (writeResponse.message as RecordsWriteMessage).recordId; - - // Confirm the record does NOT exist on Alice's remote DWN. - let queryResponse = await testAgent.agent.dwnManager.sendRequest({ - author : alice.did, - target : alice.did, - messageType : 'RecordsQuery', - messageOptions : { filter: { recordId: testRecordId } } - }); - let remoteDwnQueryReply = queryResponse.reply as RecordsQueryReply; - expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. - expect(remoteDwnQueryReply.entries).to.have.length(0); // Record doesn't exist on remote DWN. + const getRemoteEventsSpy = sinon.spy(testAgent.agent.syncManager as any, 'getRemoteEvents'); + const sendDwnRequestStub = sinon.stub(testAgent.agent.rpcClient, 'sendDwnRequest').rejects('some failure'); // Register Alice's DID to be synchronized. await testAgent.agent.syncManager.registerIdentity({ did: alice.did }); - // Execute Sync to push all records from Alice's local DWN to Alice's remote DWN. - await testAgent.agent.syncManager.push(); + // Execute Sync to pull all records from Alice's remote DWN to Alice's local DWN. + await testAgent.agent.syncManager.sync(); - // Confirm the record now DOES exist on Alice's remote DWN. - queryResponse = await testAgent.agent.dwnManager.sendRequest({ - author : alice.did, - target : alice.did, - messageType : 'RecordsQuery', - messageOptions : { filter: { recordId: testRecordId } } - }); - remoteDwnQueryReply = queryResponse.reply as RecordsQueryReply; - expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. - expect(remoteDwnQueryReply.entries).to.have.length(1); // Record does exist on remote DWN. + //restore sinon stubs and spys + getRemoteEventsSpy.restore(); + sendDwnRequestStub.restore(); + + expect(getRemoteEventsSpy.called).to.be.true; + expect(getRemoteEventsSpy.threw()).to.be.false; }); - it('synchronizes records for multiple identities from local DWN to remote DWN', async () => { - // Create a second Identity to author the DWN messages. - const { did: bobDid } = await testAgent.createIdentity({ testDwnUrls }); - const bob = await testAgent.agent.identityManager.import({ - did : bobDid, - identity : { name: 'Bob', did: bobDid.did }, - kms : 'local' + it('synchronizes data in both directions for a single identity', async () => { + + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did }); - // Write a test record to Alice's local DWN. - let writeResponse = await testAgent.agent.dwnManager.processRequest({ - author : alice.did, - target : alice.did, - messageType : 'RecordsWrite', - messageOptions : { - dataFormat: 'text/plain' - }, - dataStream: new Blob(['Hello, Bob!']) + const everythingQuery = (): ProcessDwnRequest => { + return { + author : alice.did, + target : alice.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { schema: 'testSchema' } } + }; + }; + + const localRecords = (await Promise.all(Array(3).fill({}).map(_ => testAgent.agent.dwnManager.processRequest(TestRecordsWriteMessage( + alice.did, + alice.did, + new Blob([randomString(256)]), + ))))).map(r => (r.message as RecordsWriteMessage).recordId); + + const remoteRecords = (await Promise.all(Array(3).fill({}).map(_ => testAgent.agent.dwnManager.sendRequest(TestRecordsWriteMessage( + alice.did, + alice.did, + new Blob([randomString(256)]), + ))))).map(r => (r.message as RecordsWriteMessage).recordId); + + const { reply: localReply } = await testAgent.agent.dwnManager.processRequest(everythingQuery()); + expect(localReply.status.code).to.equal(200); + expect(localReply.entries?.length).to.equal(localRecords.length); + expect(localReply.entries?.every(e => localRecords.includes((e as RecordsWriteMessage).recordId))).to.be.true; + + const { reply: remoteReply } = await testAgent.agent.dwnManager.sendRequest(everythingQuery()); + expect(remoteReply.status.code).to.equal(200); + expect(remoteReply.entries?.length).to.equal(remoteRecords.length); + expect(remoteReply.entries?.every(e => remoteRecords.includes((e as RecordsWriteMessage).recordId))).to.be.true; + + await testAgent.agent.syncManager.sync(); + + const records = new Set([...remoteRecords, ...localRecords]); + const { reply: allRemoteReply } = await testAgent.agent.dwnManager.sendRequest(everythingQuery()); + expect(allRemoteReply.status.code).to.equal(200); + expect(allRemoteReply.entries?.length).to.equal(records.size); + expect(allRemoteReply.entries?.every(e => records.has((e as RecordsWriteMessage).recordId))).to.be.true; + + const { reply: allLocalReply } = await testAgent.agent.dwnManager.sendRequest(everythingQuery()); + expect(allLocalReply.status.code).to.equal(200); + expect(allLocalReply.entries?.length).to.equal(records.size); + expect(allLocalReply.entries?.every(e => records.has((e as RecordsWriteMessage).recordId))).to.be.true; + + }).timeout(5_000); + + it('should skip dwn if there a failure getting syncState', async () => { + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did }); - // Get the record ID of Alice's test record. - const testRecordIdAlice = (writeResponse.message as RecordsWriteMessage).recordId; + const getWatermarkStub = sinon.stub(testAgent.agent.syncManager as any, 'getSyncState').rejects('rejected'); + const getSyncPeerState = sinon.spy(testAgent.agent.syncManager as any, 'getSyncPeerState'); - // Write a test record to Bob's local DWN. - writeResponse = await testAgent.agent.dwnManager.processRequest({ - author : bob.did, - target : bob.did, - messageType : 'RecordsWrite', - messageOptions : { - dataFormat: 'text/plain' - }, - dataStream: new Blob(['Hello, Alice!']) + await testAgent.agent.syncManager.sync(); + getWatermarkStub.restore(); + getSyncPeerState.restore(); + + expect(getSyncPeerState.called).to.be.true; + expect(getWatermarkStub.called).to.be.true; + }); + + describe('batchOperations()', () => { + it('should only call once per remote DWN if pull direction is passed', async () => { + const batchOperationsSpy = sinon.spy(testAgent.agent.syncManager as any, 'batchOperations'); + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did + }); + await testAgent.agent.syncManager.sync('pull'); + batchOperationsSpy.restore(); // restore before assertions to avoid failures in other tests + expect(batchOperationsSpy.callCount).to.equal(testDwnUrls.length, 'pull direction is passed'); + expect(batchOperationsSpy.args.filter(arg => arg.includes('pull')).length).to.equal(testDwnUrls.length, `args must include pull ${batchOperationsSpy.args[0]}`); }); - // Get the record ID of Bob's test record. - const testRecordIdBob = (writeResponse.message as RecordsWriteMessage).recordId; + it('should only call once per remote DWN if push direction is passed', async () => { + const batchOperationsSpy = sinon.spy(testAgent.agent.syncManager as any, 'batchOperations'); + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did + }); + await testAgent.agent.syncManager.sync('push'); + batchOperationsSpy.restore(); // restore before assertions to avoid failures in other tests + expect(batchOperationsSpy.callCount).to.equal(testDwnUrls.length, 'push direction is passed'); + expect(batchOperationsSpy.args.filter(arg => arg.includes('push')).length).to.equal(testDwnUrls.length, `args must include push ${batchOperationsSpy.args[0]}`); + }); - // Register Alice's DID to be synchronized. - await testAgent.agent.syncManager.registerIdentity({ - did: alice.did + it('should be called twice per remote DWN if no direction is passed', async () => { + const batchOperationsSpy = sinon.spy(testAgent.agent.syncManager as any, 'batchOperations'); + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did + }); + await testAgent.agent.syncManager.sync(); + batchOperationsSpy.restore(); // restore before assertions to avoid failures in other tests + expect(batchOperationsSpy.callCount).to.equal((2 * testDwnUrls.length), 'no direction is passed'); + expect(batchOperationsSpy.args.filter(arg => arg.includes('pull')).length).to.equal(testDwnUrls.length, `args must include one pull ${batchOperationsSpy.args}`); + expect(batchOperationsSpy.args.filter(arg => arg.includes('push')).length).to.equal(testDwnUrls.length, `args must include one push ${batchOperationsSpy.args}`); }); + }); - // Register Bob's DID to be synchronized. - await testAgent.agent.syncManager.registerIdentity({ - did: bob.did + describe('pull', () => { + it('synchronizes records for 1 identity from remote DWN to local DWN', async () => { + // Write a test record to Alice's remote DWN. + let writeResponse = await testAgent.agent.dwnManager.sendRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsWrite', + messageOptions : { + dataFormat: 'text/plain' + }, + dataStream: new Blob(['Hello, world!']) + }); + + // Get the record ID of the test record. + const testRecordId = (writeResponse.message as RecordsWriteMessage).recordId; + + // Confirm the record does NOT exist on Alice's local DWN. + let queryResponse = await testAgent.agent.dwnManager.processRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { recordId: testRecordId } } + }); + let localDwnQueryReply = queryResponse.reply as RecordsQueryReply; + expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(localDwnQueryReply.entries).to.have.length(0); // Record doesn't exist on local DWN. + + // Register Alice's DID to be synchronized. + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did + }); + + // Execute Sync to pull all records from Alice's remote DWN to Alice's local DWN. + await testAgent.agent.syncManager.sync('pull'); + + // Confirm the record now DOES exist on Alice's local DWN. + queryResponse = await testAgent.agent.dwnManager.processRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { recordId: testRecordId } } + }); + localDwnQueryReply = queryResponse.reply as RecordsQueryReply; + expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(localDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN. }); - // Execute Sync to push all records from Alice's and Bob's local DWNs to their remote DWNs. - await testAgent.agent.syncManager.push(); + it('synchronizes records for multiple identities from remote DWN to local DWN', async () => { + // Create a second Identity to author the DWN messages. + const { did: bobDid } = await testAgent.createIdentity({ testDwnUrls }); + const bob = await testAgent.agent.identityManager.import({ + did : bobDid, + identity : { name: 'Bob', did: bobDid.did }, + kms : 'local' + }); + + // Write a test record to Alice's remote DWN. + let writeResponse = await testAgent.agent.dwnManager.sendRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsWrite', + messageOptions : { + dataFormat: 'text/plain' + }, + dataStream: new Blob(['Hello, Bob!']) + }); + + // Get the record ID of Alice's test record. + const testRecordIdAlice = (writeResponse.message as RecordsWriteMessage).recordId; + + // Write a test record to Bob's remote DWN. + writeResponse = await testAgent.agent.dwnManager.sendRequest({ + author : bob.did, + target : bob.did, + messageType : 'RecordsWrite', + messageOptions : { + dataFormat: 'text/plain' + }, + dataStream: new Blob(['Hello, Alice!']) + }); + + // Get the record ID of Bob's test record. + const testRecordIdBob = (writeResponse.message as RecordsWriteMessage).recordId; + + // Register Alice's DID to be synchronized. + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did + }); + + // Register Bob's DID to be synchronized. + await testAgent.agent.syncManager.registerIdentity({ + did: bob.did + }); + + // Execute Sync to pull all records from Alice's and Bob's remote DWNs to their local DWNs. + await testAgent.agent.syncManager.sync('pull'); + + // Confirm the Alice test record exist on Alice's local DWN. + let queryResponse = await testAgent.agent.dwnManager.processRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { recordId: testRecordIdAlice } } + }); + let localDwnQueryReply = queryResponse.reply as RecordsQueryReply; + expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(localDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN. + + // Confirm the Bob test record exist on Bob's local DWN. + queryResponse = await testAgent.agent.dwnManager.sendRequest({ + author : bob.did, + target : bob.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { recordId: testRecordIdBob } } + }); + localDwnQueryReply = queryResponse.reply as RecordsQueryReply; + expect(localDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(localDwnQueryReply.entries).to.have.length(1); // Record does exist on local DWN. + }).timeout(5_000); + + it('synchronizes records with data larger than the `encodedData` limit within the `RecordsQuery` response', async () => { + // larger than the size of data returned in a RecordsQuery + const LARGE_DATA_SIZE = 70_000; + + //register alice + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did + }); + + const dataString = randomString(LARGE_DATA_SIZE); + + // create remote record + const record = await testAgent.agent.dwnManager.processRequest(TestRecordsWriteMessage( + alice.did, + alice.did, + new Blob([ dataString ]), + )); + + // check that records don't exist locally + const { reply: localReply } = await testAgent.agent.dwnManager.processRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { recordId: (record.message as RecordsWriteMessage).recordId }} + }); + + expect(localReply.status.code).to.equal(200); + expect(localReply.entries?.length).to.equal(1); + + // initiate sync + await testAgent.agent.syncManager.sync(); + + // query for local records + const { reply: localReply2 } = await testAgent.agent.dwnManager.processRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { recordId: (record.message as RecordsWriteMessage).recordId }} + }); + + expect(localReply2.status.code).to.equal(200); + expect(localReply2.entries?.length).to.equal(1); + const entry = localReply2.entries![0]; + // check for response encodedData if it doesn't exist issue a RecordsRead + if (entry.encodedData === undefined) { + const recordId = (entry as RecordsWriteMessage).recordId; + // get individual records without encodedData to check that data exists + const record = await testAgent.agent.dwnManager.processRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsRead', + messageOptions : { filter: { recordId } } + }); + const reply = record.reply as RecordsReadReply; + expect(reply.status.code).to.equal(200); + expect(reply.record).to.not.be.undefined; + expect(reply.record!.data).to.not.be.undefined; + const data = await DataStream.toBytes(reply.record!.data); + const replyDataString = new TextDecoder().decode(data); + expect(replyDataString.length).to.equal(dataString.length); + expect(replyDataString).to.equal(dataString); + } + }); + }); - // Confirm the Alice test record exist on Alice's remote DWN. - let queryResponse = await testAgent.agent.dwnManager.sendRequest({ - author : alice.did, - target : alice.did, - messageType : 'RecordsQuery', - messageOptions : { filter: { recordId: testRecordIdAlice } } + describe('push', () => { + it('synchronizes records for 1 identity from local DWN to remote DWN', async () => { + // Write a record that we can use for this test. + let writeResponse = await testAgent.agent.dwnManager.processRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsWrite', + messageOptions : { + dataFormat: 'text/plain' + }, + dataStream: new Blob(['Hello, world!']) + }); + + // Get the record ID of the test record. + const testRecordId = (writeResponse.message as RecordsWriteMessage).recordId; + + // Confirm the record does NOT exist on Alice's remote DWN. + let queryResponse = await testAgent.agent.dwnManager.sendRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { recordId: testRecordId } } + }); + let remoteDwnQueryReply = queryResponse.reply as RecordsQueryReply; + expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(remoteDwnQueryReply.entries).to.have.length(0); // Record doesn't exist on remote DWN. + + // Register Alice's DID to be synchronized. + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did + }); + + // Execute Sync to push all records from Alice's local DWN to Alice's remote DWN. + await testAgent.agent.syncManager.sync('push'); + + // Confirm the record now DOES exist on Alice's remote DWN. + queryResponse = await testAgent.agent.dwnManager.sendRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { recordId: testRecordId } } + }); + remoteDwnQueryReply = queryResponse.reply as RecordsQueryReply; + expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(remoteDwnQueryReply.entries).to.have.length(1); // Record does exist on remote DWN. }); - let remoteDwnQueryReply = queryResponse.reply as RecordsQueryReply; - expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. - expect(remoteDwnQueryReply.entries).to.have.length(1); // Record does exist on remote DWN. - - // Confirm the Bob test record exist on Bob's remote DWN. - queryResponse = await testAgent.agent.dwnManager.sendRequest({ - author : bob.did, - target : bob.did, - messageType : 'RecordsQuery', - messageOptions : { filter: { recordId: testRecordIdBob } } + + it('synchronizes records for multiple identities from local DWN to remote DWN', async () => { + // Create a second Identity to author the DWN messages. + const { did: bobDid } = await testAgent.createIdentity({ testDwnUrls }); + const bob = await testAgent.agent.identityManager.import({ + did : bobDid, + identity : { name: 'Bob', did: bobDid.did }, + kms : 'local' + }); + + // Write a test record to Alice's local DWN. + let writeResponse = await testAgent.agent.dwnManager.processRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsWrite', + messageOptions : { + dataFormat: 'text/plain' + }, + dataStream: new Blob(['Hello, Bob!']) + }); + + // Get the record ID of Alice's test record. + const testRecordIdAlice = (writeResponse.message as RecordsWriteMessage).recordId; + + // Write a test record to Bob's local DWN. + writeResponse = await testAgent.agent.dwnManager.processRequest({ + author : bob.did, + target : bob.did, + messageType : 'RecordsWrite', + messageOptions : { + dataFormat: 'text/plain' + }, + dataStream: new Blob(['Hello, Alice!']) + }); + + // Get the record ID of Bob's test record. + const testRecordIdBob = (writeResponse.message as RecordsWriteMessage).recordId; + + // Register Alice's DID to be synchronized. + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did + }); + + // Register Bob's DID to be synchronized. + await testAgent.agent.syncManager.registerIdentity({ + did: bob.did + }); + + // Execute Sync to push all records from Alice's and Bob's local DWNs to their remote DWNs. + await testAgent.agent.syncManager.sync('push'); + + // Confirm the Alice test record exist on Alice's remote DWN. + let queryResponse = await testAgent.agent.dwnManager.sendRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { recordId: testRecordIdAlice } } + }); + let remoteDwnQueryReply = queryResponse.reply as RecordsQueryReply; + expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(remoteDwnQueryReply.entries).to.have.length(1); // Record does exist on remote DWN. + + // Confirm the Bob test record exist on Bob's remote DWN. + queryResponse = await testAgent.agent.dwnManager.sendRequest({ + author : bob.did, + target : bob.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { recordId: testRecordIdBob } } + }); + remoteDwnQueryReply = queryResponse.reply as RecordsQueryReply; + expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. + expect(remoteDwnQueryReply.entries).to.have.length(1); // Record does exist on remote DWN. + }).timeout(5_000); + + it('synchronizes records with data larger than the `encodedData` limit within the `RecordsQuery` response', async () => { + // larger than the size of data returned in a RecordsQuery + const LARGE_DATA_SIZE = 70_000; + + //register alice + await testAgent.agent.syncManager.registerIdentity({ + did: alice.did + }); + + const dataString = randomString(LARGE_DATA_SIZE); + + // create remote local record to sync to remote + const remoteRecord = await testAgent.agent.dwnManager.processRequest(TestRecordsWriteMessage( + alice.did, + alice.did, + new Blob([ dataString ]) + )); + expect(remoteRecord.reply.status.code).to.equal(202); + + // check that records don't exist on remote + const { reply: remoteReply } = await testAgent.agent.dwnManager.sendRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { schema: 'testSchema' } } + }); + expect(remoteReply.status.code).to.equal(200); + expect(remoteReply.entries?.length).to.equal(0); + + // initiate sync + await testAgent.agent.syncManager.sync(); + + // query for for remote records that now exist + const { reply: remoteReply2 } = await testAgent.agent.dwnManager.sendRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsQuery', + messageOptions : { filter: { schema: 'testSchema' } } + }); + expect(remoteReply2.status.code).to.equal(200); + expect(remoteReply2.entries?.length).to.equal(1); + + const entry = remoteReply2.entries![0]; + + // check for response encodedData if it doesn't exist issue a RecordsRead + if (entry.encodedData === undefined) { + const recordId = (entry as RecordsWriteMessage).recordId; + // get individual records without encodedData to check that data exists + const record = await testAgent.agent.dwnManager.sendRequest({ + author : alice.did, + target : alice.did, + messageType : 'RecordsRead', + messageOptions : { filter: { recordId } } + }); + const reply = record.reply as RecordsReadReply; + expect(reply.status.code).to.equal(200); + expect(reply.record).to.not.be.undefined; + expect(reply.record!.data).to.not.be.undefined; + + const replyRecord = reply.record as unknown as RecordsWriteMessage & { data: ReadableStream }; + expect(replyRecord.data).to.exist; + expect(replyRecord.data instanceof ReadableStream).to.be.true; + const replyDataString = await streamToText(replyRecord.data); + expect(replyDataString.length).to.eq(dataString.length); + expect(replyDataString).to.equal(dataString); + } }); - remoteDwnQueryReply = queryResponse.reply as RecordsQueryReply; - expect(remoteDwnQueryReply.status.code).to.equal(200); // Query was successfully executed. - expect(remoteDwnQueryReply.entries).to.have.length(1); // Record does exist on remote DWN. - }).timeout(5000); + }); }); }); }); \ No newline at end of file diff --git a/packages/agent/tests/test-config.ts b/packages/agent/tests/test-config.ts index 213604404..6ce799757 100644 --- a/packages/agent/tests/test-config.ts +++ b/packages/agent/tests/test-config.ts @@ -1,10 +1,10 @@ -declare const __karma__: { config?: { testDwnUrl?: string; } }; +declare const __karma__: { config?: { testDwnUrls?: string; } }; -const DEFAULT_TEST_DWN_URL = 'https://dwn.tbddev.org/dwn0'; +const DEFAULT_TEST_DWN_URLS = 'https://dwn.tbddev.org/dwn0'; -function getTestDwnUrl(): string { +function getTestDwnUrls(): string[] { // Check to see if we're running in a Karma browser test environment. - const browserTestEnvironment = typeof __karma__ !== 'undefined' && __karma__?.config?.testDwnUrl !== undefined; + const browserTestEnvironment = typeof __karma__ !== 'undefined' && __karma__?.config?.testDwnUrls !== undefined; // Check to see if we're running in a Node environment. const nodeTestEnvironment = process && process?.env !== undefined; @@ -12,13 +12,13 @@ function getTestDwnUrl(): string { // Attempt to use DWN URL defined in Karma config, if running a browser test. // Otherwise, attempt to use the Node environment variable. const envTestDwnUrl = (browserTestEnvironment) - ? __karma__.config!.testDwnUrl + ? __karma__.config!.testDwnUrls : (nodeTestEnvironment) - ? process.env.TEST_DWN_URL + ? process.env.TEST_DWN_URLS : undefined; // If defined, return the test environment DWN URL. Otherwise, return the default. - return envTestDwnUrl || DEFAULT_TEST_DWN_URL; + return (envTestDwnUrl || DEFAULT_TEST_DWN_URLS).split(','); } -export const testDwnUrl = getTestDwnUrl(); \ No newline at end of file +export const testDwnUrls = getTestDwnUrls(); \ No newline at end of file diff --git a/packages/agent/tests/utils/test-agent.ts b/packages/agent/tests/utils/test-agent.ts index 730c9fa0b..a2be007a4 100644 --- a/packages/agent/tests/utils/test-agent.ts +++ b/packages/agent/tests/utils/test-agent.ts @@ -203,4 +203,31 @@ export class TestAgent implements Web5ManagedAgent { async start(_options: { passphrase: string; }): Promise { throw new Error('Not implemented'); } +} + +/** + * Generates a random byte array of given length. + */ +export function randomBytes(length: number): Uint8Array { + const randomBytes = new Uint8Array(length); + for (let i = 0; i < length; i++) { + randomBytes[i] = Math.floor(Math.random() * 256); + } + + return randomBytes; +} + +/** + * Generates a random alpha-numeric string. + */ +export function randomString(length: number): string { + const charset = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + + // pick characters randomly + let randomString = ''; + for (let i = 0; i < length; i++) { + randomString += charset.charAt(Math.floor(Math.random() * charset.length)); + } + + return randomString; } \ No newline at end of file diff --git a/packages/api/karma.conf.cjs b/packages/api/karma.conf.cjs index 6b7407b2c..79b45d1b5 100644 --- a/packages/api/karma.conf.cjs +++ b/packages/api/karma.conf.cjs @@ -36,7 +36,7 @@ module.exports = function (config) { timeout: 10000 // 10 seconds }, // If an environment variable is defined, override the default test DWN URL. - testDwnUrl: process.env.TEST_DWN_URL, + testDwnUrls: process.env.TEST_DWN_URLS }, // list of files / patterns to load in the browser diff --git a/packages/api/tests/dwn-api.spec.ts b/packages/api/tests/dwn-api.spec.ts index 2374a7899..09ea469d7 100644 --- a/packages/api/tests/dwn-api.spec.ts +++ b/packages/api/tests/dwn-api.spec.ts @@ -5,12 +5,10 @@ import { expect } from 'chai'; import { TestManagedAgent } from '@web5/agent'; import { DwnApi } from '../src/dwn-api.js'; -import { testDwnUrl } from './test-config.js'; +import { testDwnUrls } from './test-config.js'; import { TestUserAgent } from './utils/test-user-agent.js'; import emailProtocolDefinition from './fixtures/protocol-definitions/email.json' assert { type: 'json' }; -let testDwnUrls: string[] = [testDwnUrl]; - describe('DwnApi', () => { let alice: ManagedIdentity; let aliceDid: PortableDid; diff --git a/packages/api/tests/protocol.spec.ts b/packages/api/tests/protocol.spec.ts index ea1cd1061..da2c8a716 100644 --- a/packages/api/tests/protocol.spec.ts +++ b/packages/api/tests/protocol.spec.ts @@ -6,7 +6,7 @@ import chaiAsPromised from 'chai-as-promised'; import { TestManagedAgent } from '@web5/agent'; import { DwnApi } from '../src/dwn-api.js'; -import { testDwnUrl } from './test-config.js'; +import { testDwnUrls } from './test-config.js'; import { TestUserAgent } from './utils/test-user-agent.js'; import emailProtocolDefinition from './fixtures/protocol-definitions/email.json' assert { type: 'json' }; @@ -21,8 +21,6 @@ import { webcrypto } from 'node:crypto'; if (!globalThis.crypto) globalThis.crypto = webcrypto; // TODO: Come up with a better way of resolving the TS errors. -let testDwnUrls: string[] = [testDwnUrl]; - describe('Protocol', () => { let dwn: DwnApi; let alice: ManagedIdentity; diff --git a/packages/api/tests/record.spec.ts b/packages/api/tests/record.spec.ts index ba3e881d8..f49087204 100644 --- a/packages/api/tests/record.spec.ts +++ b/packages/api/tests/record.spec.ts @@ -22,7 +22,7 @@ import { import { Record } from '../src/record.js'; import { DwnApi } from '../src/dwn-api.js'; import { dataToBlob } from '../src/utils.js'; -import { testDwnUrl } from './test-config.js'; +import { testDwnUrls } from './test-config.js'; import { TestUserAgent } from './utils/test-user-agent.js'; import { TestDataGenerator } from './utils/test-data-generator.js'; import emailProtocolDefinition from './fixtures/protocol-definitions/email.json' assert { type: 'json' }; @@ -41,8 +41,6 @@ if (!globalThis.crypto) globalThis.crypto = webcrypto; // TODO: Come up with a better way of resolving the TS errors. type RecordsWriteTest = RecordsWrite & RecordsWriteMessage; -let testDwnUrls: string[] = [testDwnUrl]; - describe('Record', () => { let dataText: string; let dataBlob: Blob; diff --git a/packages/api/tests/test-config.ts b/packages/api/tests/test-config.ts index 213604404..56374a65e 100644 --- a/packages/api/tests/test-config.ts +++ b/packages/api/tests/test-config.ts @@ -1,8 +1,8 @@ declare const __karma__: { config?: { testDwnUrl?: string; } }; -const DEFAULT_TEST_DWN_URL = 'https://dwn.tbddev.org/dwn0'; +const DEFAULT_TEST_DWN_URLS = 'https://dwn.tbddev.org/dwn0'; -function getTestDwnUrl(): string { +function getTestDwnUrl(): string[] { // Check to see if we're running in a Karma browser test environment. const browserTestEnvironment = typeof __karma__ !== 'undefined' && __karma__?.config?.testDwnUrl !== undefined; @@ -14,11 +14,11 @@ function getTestDwnUrl(): string { const envTestDwnUrl = (browserTestEnvironment) ? __karma__.config!.testDwnUrl : (nodeTestEnvironment) - ? process.env.TEST_DWN_URL + ? process.env.TEST_DWN_URLS : undefined; // If defined, return the test environment DWN URL. Otherwise, return the default. - return envTestDwnUrl || DEFAULT_TEST_DWN_URL; + return (envTestDwnUrl || DEFAULT_TEST_DWN_URLS).split(','); } -export const testDwnUrl = getTestDwnUrl(); \ No newline at end of file +export const testDwnUrls = getTestDwnUrl(); \ No newline at end of file diff --git a/packages/common/karma.conf.cjs b/packages/common/karma.conf.cjs index 6b7407b2c..79b45d1b5 100644 --- a/packages/common/karma.conf.cjs +++ b/packages/common/karma.conf.cjs @@ -36,7 +36,7 @@ module.exports = function (config) { timeout: 10000 // 10 seconds }, // If an environment variable is defined, override the default test DWN URL. - testDwnUrl: process.env.TEST_DWN_URL, + testDwnUrls: process.env.TEST_DWN_URLS }, // list of files / patterns to load in the browser diff --git a/packages/credentials/karma.conf.cjs b/packages/credentials/karma.conf.cjs index 6b7407b2c..79b45d1b5 100644 --- a/packages/credentials/karma.conf.cjs +++ b/packages/credentials/karma.conf.cjs @@ -36,7 +36,7 @@ module.exports = function (config) { timeout: 10000 // 10 seconds }, // If an environment variable is defined, override the default test DWN URL. - testDwnUrl: process.env.TEST_DWN_URL, + testDwnUrls: process.env.TEST_DWN_URLS }, // list of files / patterns to load in the browser diff --git a/packages/crypto/karma.conf.cjs b/packages/crypto/karma.conf.cjs index 6b7407b2c..79b45d1b5 100644 --- a/packages/crypto/karma.conf.cjs +++ b/packages/crypto/karma.conf.cjs @@ -36,7 +36,7 @@ module.exports = function (config) { timeout: 10000 // 10 seconds }, // If an environment variable is defined, override the default test DWN URL. - testDwnUrl: process.env.TEST_DWN_URL, + testDwnUrls: process.env.TEST_DWN_URLS }, // list of files / patterns to load in the browser diff --git a/packages/dids/karma.conf.cjs b/packages/dids/karma.conf.cjs index 6b7407b2c..79b45d1b5 100644 --- a/packages/dids/karma.conf.cjs +++ b/packages/dids/karma.conf.cjs @@ -36,7 +36,7 @@ module.exports = function (config) { timeout: 10000 // 10 seconds }, // If an environment variable is defined, override the default test DWN URL. - testDwnUrl: process.env.TEST_DWN_URL, + testDwnUrls: process.env.TEST_DWN_URLS }, // list of files / patterns to load in the browser diff --git a/packages/identity-agent/karma.conf.cjs b/packages/identity-agent/karma.conf.cjs index 6b7407b2c..79b45d1b5 100644 --- a/packages/identity-agent/karma.conf.cjs +++ b/packages/identity-agent/karma.conf.cjs @@ -36,7 +36,7 @@ module.exports = function (config) { timeout: 10000 // 10 seconds }, // If an environment variable is defined, override the default test DWN URL. - testDwnUrl: process.env.TEST_DWN_URL, + testDwnUrls: process.env.TEST_DWN_URLS }, // list of files / patterns to load in the browser diff --git a/packages/proxy-agent/karma.conf.cjs b/packages/proxy-agent/karma.conf.cjs index 6b7407b2c..79b45d1b5 100644 --- a/packages/proxy-agent/karma.conf.cjs +++ b/packages/proxy-agent/karma.conf.cjs @@ -36,7 +36,7 @@ module.exports = function (config) { timeout: 10000 // 10 seconds }, // If an environment variable is defined, override the default test DWN URL. - testDwnUrl: process.env.TEST_DWN_URL, + testDwnUrls: process.env.TEST_DWN_URLS }, // list of files / patterns to load in the browser diff --git a/packages/user-agent/karma.conf.cjs b/packages/user-agent/karma.conf.cjs index 6b7407b2c..79b45d1b5 100644 --- a/packages/user-agent/karma.conf.cjs +++ b/packages/user-agent/karma.conf.cjs @@ -36,7 +36,7 @@ module.exports = function (config) { timeout: 10000 // 10 seconds }, // If an environment variable is defined, override the default test DWN URL. - testDwnUrl: process.env.TEST_DWN_URL, + testDwnUrls: process.env.TEST_DWN_URLS }, // list of files / patterns to load in the browser