From d0906df36be28c135c59a1d992bfb7daeba4b0f6 Mon Sep 17 00:00:00 2001 From: Evan Hahn Date: Tue, 29 Oct 2024 18:46:24 +0000 Subject: [PATCH] test: basic test that uses IPC CoMapeo Mobile doesn't use `MapeoManager` directly; it uses an IPC bridge. This adds a test that creates a manager in a separate process and uses `@comapeo/ipc` to test it. This is something we've been discussing for awhile, and this finally implements it. I think this is useful on its own but also makes a future change easier. --- package-lock.json | 257 ++++++++++++++++++++++++++++++++++++- package.json | 3 +- test-e2e/forked-process.js | 85 ++++++++++++ test-e2e/ipc-basic.js | 22 ++++ test-e2e/utils.js | 34 +++++ 5 files changed, 397 insertions(+), 4 deletions(-) create mode 100644 test-e2e/forked-process.js create mode 100644 test-e2e/ipc-basic.js diff --git a/package-lock.json b/package-lock.json index 79587be4..4d52c51e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@hyperswarm/secret-stream": "^6.6.3", "@mapeo/crypto": "1.0.0-alpha.10", "@mapeo/sqlite-indexer": "1.0.0-alpha.9", - "@sinclair/typebox": "^0.29.6", + "@sinclair/typebox": "^0.33.17", "b4a": "^1.6.3", "bcp-47": "^2.1.0", "better-sqlite3": "^8.7.0", @@ -61,6 +61,7 @@ "devDependencies": { "@bufbuild/buf": "^1.26.1", "@comapeo/core2.0.1": "npm:@comapeo/core@2.0.1", + "@comapeo/ipc": "^2.0.2", "@mapeo/default-config": "5.0.0", "@mapeo/mock-data": "^2.1.1", "@sinonjs/fake-timers": "^10.0.2", @@ -244,6 +245,81 @@ "@bufbuild/buf-win32-x64": "1.26.1" } }, + "node_modules/@comapeo/core": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@comapeo/core/-/core-2.0.1.tgz", + "integrity": "sha512-6ZUb5umzitYKZx28bYFMxQfpE2B6pdC2ukCd9rcM1l+EgNUAcL+1NWeo/YRlXPnbzhltZ7kr0LvmZtXoMb/ing==", + "dev": true, + "peer": true, + "dependencies": { + "@comapeo/fallback-smp": "^1.0.0", + "@comapeo/schema": "1.0.0", + "@digidem/types": "^2.3.0", + "@fastify/error": "^3.4.1", + "@fastify/type-provider-typebox": "^4.1.0", + "@hyperswarm/secret-stream": "^6.6.3", + "@mapeo/crypto": "1.0.0-alpha.10", + "@mapeo/sqlite-indexer": "1.0.0-alpha.9", + "@sinclair/typebox": "^0.29.6", + "b4a": "^1.6.3", + "bcp-47": "^2.1.0", + "better-sqlite3": "^8.7.0", + "big-sparse-array": "^1.0.3", + "bogon": "^1.1.0", + "compact-encoding": "^2.12.0", + "corestore": "6.8.4", + "debug": "^4.3.4", + "dot-prop": "^9.0.0", + "drizzle-orm": "^0.30.8", + "fastify": ">= 4", + "fastify-plugin": "^4.5.1", + "hyperblobs": "2.3.0", + "hypercore": "10.17.0", + "hypercore-crypto": "3.4.2", + "hyperdrive": "11.5.3", + "json-stable-stringify": "^1.1.1", + "magic-bytes.js": "^1.10.0", + "map-obj": "^5.0.2", + "mime": "^4.0.3", + "multi-core-indexer": "^1.0.0-alpha.10", + "p-defer": "^4.0.0", + "p-event": "^6.0.1", + "p-timeout": "^6.1.2", + "protobufjs": "^7.2.3", + "protomux": "^3.4.1", + "quickbit-universal": "^2.2.0", + "sodium-universal": "^4.0.0", + "start-stop-state-machine": "^1.2.0", + "streamx": "^2.19.0", + "styled-map-package": "^2.0.0", + "sub-encoder": "^2.1.1", + "throttle-debounce": "^5.0.0", + "tiny-typed-emitter": "^2.1.0", + "type-fest": "^4.5.0", + "undici": "^6.13.0", + "varint": "^6.0.0", + "yauzl-promise": "^4.0.0" + } + }, + "node_modules/@comapeo/core/node_modules/@comapeo/schema": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@comapeo/schema/-/schema-1.0.0.tgz", + "integrity": "sha512-dK227I+0yg9D2y5/O5NGywx50tgeNYyUkl1uYnSmNAPlbv+r2KX9aaC9m4dEjIja2aR2VFnYn6z537ERZiahqQ==", + "dev": true, + "peer": true, + "dependencies": { + "compact-encoding": "^2.12.0", + "protobufjs": "^7.2.5", + "type-fest": "^4.26.0" + } + }, + "node_modules/@comapeo/core/node_modules/@sinclair/typebox": { + "version": "0.29.6", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.29.6.tgz", + "integrity": "sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==", + "dev": true, + "peer": true + }, "node_modules/@comapeo/core2.0.1": { "name": "@comapeo/core", "version": "2.0.1", @@ -311,6 +387,12 @@ "type-fest": "^4.26.0" } }, + "node_modules/@comapeo/core2.0.1/node_modules/@sinclair/typebox": { + "version": "0.29.6", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.29.6.tgz", + "integrity": "sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==", + "dev": true + }, "node_modules/@comapeo/fallback-smp": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@comapeo/fallback-smp/-/fallback-smp-1.0.0.tgz", @@ -324,6 +406,23 @@ "protobufjs": "^7.4.0" } }, + "node_modules/@comapeo/ipc": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@comapeo/ipc/-/ipc-2.0.2.tgz", + "integrity": "sha512-nOm2MzUjnN9GNYW8cAXzfAyW3URJz+lWm9nyKm8vmTTxiwBt6ZMEd2DCU28dl1KNfasqw79501jWY+fELFt/wg==", + "dev": true, + "dependencies": { + "eventemitter3": "^5.0.1", + "p-defer": "^4.0.0", + "rpc-reflector": "^1.3.11" + }, + "engines": { + "node": ">=18.17.1" + }, + "peerDependencies": { + "@comapeo/core": "^2.0.0" + } + }, "node_modules/@comapeo/schema": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@comapeo/schema/-/schema-1.2.0.tgz", @@ -914,6 +1013,15 @@ "gl-style-validate": "dist/gl-style-validate.mjs" } }, + "node_modules/@msgpack/msgpack": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-1.12.2.tgz", + "integrity": "sha512-Vwhc3ObxmDZmA5hY8mfsau2rJ4vGPvzbj20QSZ2/E1GDPF61QVyjLfNHak9xmel6pW4heRt3v1fHa6np9Ehfeg==", + "dev": true, + "engines": { + "node": ">= 10" + } + }, "node_modules/@napi-rs/wasm-runtime": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.4.tgz", @@ -1307,8 +1415,9 @@ "dev": true }, "node_modules/@sinclair/typebox": { - "version": "0.29.6", - "license": "MIT" + "version": "0.33.17", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", + "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==" }, "node_modules/@sinonjs/commons": { "version": "2.0.0", @@ -2692,6 +2801,12 @@ "dev": true, "license": "MIT" }, + "node_modules/const-max-uint32": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/const-max-uint32/-/const-max-uint32-1.0.2.tgz", + "integrity": "sha512-T8/9bffg5RThuejasJWrwqxs3Q0fsJvyl7/33IB6svroD8JC93E7X60AuuOnDE8RlP6Jlb5FxmlrVDpl9KiU2Q==", + "dev": true + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -3450,6 +3565,18 @@ } } }, + "node_modules/duplexify": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", + "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.4.1", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "stream-shift": "^1.0.2" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "license": "MIT" @@ -5582,6 +5709,23 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/length-prefixed-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/length-prefixed-stream/-/length-prefixed-stream-2.0.0.tgz", + "integrity": "sha512-dvjTuWTKWe0oEznQcG6a9osfiYknCs7DEFJMP88n9Y581IFhYh1sZIgAFcuDOojKB0G7ftPreKhh4D0kh/VPjQ==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "varint": "^5.0.0" + } + }, + "node_modules/length-prefixed-stream/node_modules/varint": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", + "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==", + "dev": true + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -8101,6 +8245,46 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/rpc-reflector": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/rpc-reflector/-/rpc-reflector-1.3.11.tgz", + "integrity": "sha512-TIf/RHJy11q/xmNBj0Xj2Z4GVPP1aLeaZXSRtthcMnXNuK+tv7SpZldB5Jk6RFHzA9TgxhcWvLHdHrdlEDKH0w==", + "dev": true, + "dependencies": { + "@msgpack/msgpack": "^1.12.1", + "@types/node": "^18.16.19", + "duplexify": "^4.1.2", + "eventemitter3": "^5.0.1", + "is-stream": "^2.0.1", + "length-prefixed-stream": "^2.0.0", + "p-timeout": "^4.1.0", + "pump": "^3.0.0", + "serialize-error": "^8.1.0", + "through2": "^4.0.2", + "validate.io-array-like": "^1.0.2" + } + }, + "node_modules/rpc-reflector/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/rpc-reflector/node_modules/p-timeout": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-4.1.0.tgz", + "integrity": "sha512-+/wmHtzJuWii1sXn3HCuH/FTwGhrp4tmJTxSKJbfS+vkipci6osxXM5mY0jUiRzWKMTgUT8l7HFbeSwZAynqHw==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/run-applescript": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.0.0.tgz", @@ -8215,6 +8399,33 @@ "version": "2.7.0", "license": "BSD-3-Clause" }, + "node_modules/serialize-error": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", + "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", + "dev": true, + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/serialize-error/node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/set-cookie-parser": { "version": "2.6.0", "license": "MIT" @@ -8716,6 +8927,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/stream-shift": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", + "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==", + "dev": true + }, "node_modules/streamx": { "version": "2.19.0", "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.19.0.tgz", @@ -9181,6 +9398,15 @@ "node": ">=12.22" } }, + "node_modules/through2": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "dev": true, + "dependencies": { + "readable-stream": "3" + } + }, "node_modules/timeout-refresh": { "version": "2.0.1", "license": "MIT" @@ -9641,6 +9867,31 @@ "spdx-expression-parse": "^3.0.0" } }, + "node_modules/validate.io-array-like": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/validate.io-array-like/-/validate.io-array-like-1.0.2.tgz", + "integrity": "sha512-rGLiN0cvY9OWzQcWP+RtqZR/MK9RUz3gKDTCcRLtEQ/BvlanMF5PyqtVIN+CgrIBCv/ypfme9v7r4yMJPYpbNA==", + "dev": true, + "dependencies": { + "const-max-uint32": "^1.0.2", + "validate.io-integer-primitive": "^1.0.0" + } + }, + "node_modules/validate.io-integer-primitive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/validate.io-integer-primitive/-/validate.io-integer-primitive-1.0.0.tgz", + "integrity": "sha512-4ARGKA4FImVWJgrgttLYsYJmDGwxlhLfDCdq09gyVgohLKKRUfD3VAo1L2vTRCLt6hDhDtFKdZiuYUTWyBggwg==", + "dev": true, + "dependencies": { + "validate.io-number-primitive": "^1.0.0" + } + }, + "node_modules/validate.io-number-primitive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/validate.io-number-primitive/-/validate.io-number-primitive-1.0.0.tgz", + "integrity": "sha512-8rlCe7N0TRTd50dwk4WNoMXNbX/4+RdtqE3TO6Bk0GJvAgbQlfL5DGr/Pl9ZLbWR6CutMjE2cu+yOoCnFWk+Qw==", + "dev": true + }, "node_modules/varint": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/varint/-/varint-6.0.0.tgz", diff --git a/package.json b/package.json index d254d668..8de7b6e9 100644 --- a/package.json +++ b/package.json @@ -109,6 +109,7 @@ "devDependencies": { "@bufbuild/buf": "^1.26.1", "@comapeo/core2.0.1": "npm:@comapeo/core@2.0.1", + "@comapeo/ipc": "^2.0.2", "@mapeo/default-config": "5.0.0", "@mapeo/mock-data": "^2.1.1", "@sinonjs/fake-timers": "^10.0.2", @@ -161,7 +162,7 @@ "@hyperswarm/secret-stream": "^6.6.3", "@mapeo/crypto": "1.0.0-alpha.10", "@mapeo/sqlite-indexer": "1.0.0-alpha.9", - "@sinclair/typebox": "^0.29.6", + "@sinclair/typebox": "^0.33.17", "b4a": "^1.6.3", "bcp-47": "^2.1.0", "better-sqlite3": "^8.7.0", diff --git a/test-e2e/forked-process.js b/test-e2e/forked-process.js new file mode 100644 index 00000000..c570cdf8 --- /dev/null +++ b/test-e2e/forked-process.js @@ -0,0 +1,85 @@ +import { createMapeoServer } from '@comapeo/ipc' +import { Type } from '@sinclair/typebox' +import { Value } from '@sinclair/typebox/value' +import Fastify from 'fastify' +import assert from 'node:assert/strict' +import { randomBytes } from 'node:crypto' +import * as process from 'node:process' +import { isMainThread, MessagePort, workerData } from 'node:worker_threads' +import RAM from 'random-access-memory' +import { MapeoManager } from '../src/mapeo-manager.js' + +/** + * @internal + * @typedef {ConstructorParameters[0]} MapeoManagerConstructorOptions + */ + +const mapeoManagerConstructorOptionsType = Type.Partial( + Type.Object({ + rootKey: Type.Uint8Array(), + dbFolder: Type.String(), + coreStorage: Type.String(), + projectMigrationsFolder: Type.String(), + clientMigrationsFolder: Type.String(), + defaultConfigPath: Type.String(), + customMapPath: Type.String(), + fallbackMapPath: Type.String(), + defaultOnlineStyleUrl: Type.String(), + }) +) + +/** + * @internal + * @typedef {object} ParsedWorkerData + * @prop {MessagePort} childPort + * @prop {Partial} managerConstructorOverrides + */ + +/** + * @param {unknown} workerData + * @return {ParsedWorkerData} + */ +function parseWorkerData(workerData) { + assert( + workerData && + typeof workerData === 'object' && + 'childPort' in workerData && + workerData.childPort instanceof MessagePort && + 'managerConstructorOverrides' in workerData + ) + return { + childPort: workerData.childPort, + managerConstructorOverrides: Value.Parse( + mapeoManagerConstructorOptionsType, + workerData.managerConstructorOverrides + ), + } +} + +async function main() { + assert(!isMainThread, 'this should not be run as the main thread') + + const { managerConstructorOverrides, childPort } = parseWorkerData(workerData) + + const manager = new MapeoManager({ + rootKey: randomBytes(16), + projectMigrationsFolder: new URL('../drizzle/project', import.meta.url) + .pathname, + clientMigrationsFolder: new URL('../drizzle/client', import.meta.url) + .pathname, + dbFolder: ':memory:', + coreStorage: () => new RAM(), + fastify: Fastify(), + ...managerConstructorOverrides, + }) + + // This `any` cast is needed because `@comapeo/ipc` expects a `MapeoManager` + // from the `@comapeo/core` install from npm, not our local development one. + createMapeoServer(/** @type {any} */ (manager), childPort) + + childPort.start() +} + +main().catch(() => { + process.exit(1) +}) diff --git a/test-e2e/ipc-basic.js b/test-e2e/ipc-basic.js new file mode 100644 index 00000000..2407ad1e --- /dev/null +++ b/test-e2e/ipc-basic.js @@ -0,0 +1,22 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createIpcManager } from './utils.js' +import { valueOf } from '@comapeo/schema' +import { generate } from '@mapeo/mock-data' + +test('basic functionality of a manager in a separate process', async (t) => { + const manager = await createIpcManager('manager', t) + await manager.setDeviceInfo({ name: 'manager', deviceType: 'mobile' }) + + const projectId = await manager.createProject() + const project = await manager.getProject(projectId) + + const { docId } = await project.observation.create( + valueOf(generate('observation')[0]) + ) + + assert( + await project.observation.getByDocId(docId), + 'can retrieve an observation we just created' + ) +}) diff --git a/test-e2e/utils.js b/test-e2e/utils.js index d07fdf8d..0ab9621d 100644 --- a/test-e2e/utils.js +++ b/test-e2e/utils.js @@ -9,6 +9,8 @@ import { createRequire } from 'node:module' import { fileURLToPath } from 'node:url' import * as v8 from 'node:v8' import { pEvent } from 'p-event' +import { createMapeoClient } from '@comapeo/ipc' +import { Worker, MessageChannel } from 'node:worker_threads' import { MapeoManager as MapeoManager_2_0_1 } from '@comapeo/core2.0.1' import { MapeoManager, roles } from '../src/index.js' @@ -267,6 +269,7 @@ export function createManager(seed, t, overrides = {}) { ...overrides, }) } + /** * @param {string} seed * @param {Partial[0]>} [overrides] @@ -293,6 +296,37 @@ export async function createOldManagerOnVersion2_0_1(seed, overrides = {}) { }) } +/** + * @param {string} seed + * @param {import('node:test').TestContext} t + * @param {Partial[0]>} [overrides] + * @returns {Promise>} + */ +export async function createIpcManager(seed, t, overrides = {}) { + const { port1: parentPort, port2: childPort } = new MessageChannel() + + const forkedProcessPath = fileURLToPath( + new URL('./forked-process.js', import.meta.url) + ) + const worker = new Worker(forkedProcessPath, { + workerData: { + managerConstructorOverrides: { rootKey: getRootKey(seed), ...overrides }, + childPort, + }, + transferList: [childPort], + }) + + t.after(() => worker.terminate()) + + // As an optimization, we can prevent the worker from keeping the test + // process alive. This isn't necessary for correctness (the call to + // `worker.terminate()` should be enough), but it can speed up tests by + // letting the process end early. + worker.unref() + + return createMapeoClient(parentPort) +} + /** * `ManagerCustodian` helps you test the creation of multiple managers accessing * the same underlying files.