diff --git a/package-lock.json b/package-lock.json index 02032776..efaa0f51 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,7 +17,7 @@ "@hyperswarm/secret-stream": "^6.6.3", "@mapeo/crypto": "1.0.0-alpha.10", "@mapeo/sqlite-indexer": "1.0.0-alpha.9", - "@sinclair/typebox": "^0.29.6", + "@sinclair/typebox": "^0.33.17", "@sindresorhus/merge-streams": "^4.0.0", "b4a": "^1.6.3", "bcp-47": "^2.1.0", @@ -65,6 +65,7 @@ "@bufbuild/buf": "^1.26.1", "@comapeo/cloud": "^0.1.0", "@comapeo/core2.0.1": "npm:@comapeo/core@2.0.1", + "@comapeo/ipc": "^2.1.0", "@mapeo/default-config": "5.0.0", "@mapeo/mock-data": "^2.1.1", "@sinonjs/fake-timers": "^10.0.2", @@ -268,21 +269,15 @@ "ws": "^8.18.0" } }, - "node_modules/@comapeo/cloud/node_modules/@sinclair/typebox": { - "version": "0.33.17", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.17.tgz", - "integrity": "sha512-75232GRx3wp3P7NP+yc4nRK3XUAnaQShxTAzapgmQrgs0QvSq0/mOJGoZXRpH15cFCKyys+4laCPbBselqJ5Ag==", - "dev": true, - "license": "MIT" - }, "node_modules/@comapeo/core": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@comapeo/core/-/core-2.1.0.tgz", - "integrity": "sha512-Fvi/EO1RJIQfpmKFUs4QApM2TsV8JrKw3HbNZ3hmlXiPl1oVVvIce0KkfdPJOoYHbEznTc9dIN1A2vkaoi431A==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@comapeo/core/-/core-2.3.0.tgz", + "integrity": "sha512-yH/GOKMGSRNbdo4mKf+XXw7TGzdK+e4Ze9QfojhKs2Gdmq6uXMGp74XbUBUyzLKxiUnYCyOM0dPIyQR+i08yqg==", "dev": true, + "license": "MIT", "dependencies": { "@comapeo/fallback-smp": "^1.0.0", - "@comapeo/schema": "1.2.0", + "@comapeo/schema": "1.3.0", "@digidem/types": "^2.3.0", "@fastify/error": "^3.4.1", "@fastify/type-provider-typebox": "^4.1.0", @@ -290,6 +285,7 @@ "@mapeo/crypto": "1.0.0-alpha.10", "@mapeo/sqlite-indexer": "1.0.0-alpha.9", "@sinclair/typebox": "^0.29.6", + "@sindresorhus/merge-streams": "^4.0.0", "b4a": "^1.6.3", "bcp-47": "^2.1.0", "better-sqlite3": "^8.7.0", @@ -325,25 +321,20 @@ "sub-encoder": "^2.1.1", "throttle-debounce": "^5.0.0", "tiny-typed-emitter": "^2.1.0", - "type-fest": "^4.5.0", + "type-fest": "^4.30.0", "undici": "^6.13.0", + "unix-path-resolve": "^1.0.2", "varint": "^6.0.0", "ws": "^8.18.0", "yauzl-promise": "^4.0.0" } }, - "node_modules/@comapeo/core/node_modules/@comapeo/schema": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@comapeo/schema/-/schema-1.2.0.tgz", - "integrity": "sha512-LWrUSqtXmrEmE/B9V/zffKBbJmMo37AlvjXczvGx1+BbCAjOYCPDX6GCtnSKNsvtnNS2KQZDm9apg3mp92tFGA==", + "node_modules/@comapeo/core/node_modules/@sinclair/typebox": { + "version": "0.29.6", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.29.6.tgz", + "integrity": "sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==", "dev": true, - "license": "MIT", - "dependencies": { - "@comapeo/geometry": "^1.0.2", - "compact-encoding": "^2.12.0", - "protobufjs": "^7.2.5", - "type-fest": "^4.26.0" - } + "license": "MIT" }, "node_modules/@comapeo/core2.0.1": { "name": "@comapeo/core", @@ -412,6 +403,13 @@ "type-fest": "^4.26.0" } }, + "node_modules/@comapeo/core2.0.1/node_modules/@sinclair/typebox": { + "version": "0.29.6", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.29.6.tgz", + "integrity": "sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@comapeo/fallback-smp": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@comapeo/fallback-smp/-/fallback-smp-1.0.0.tgz", @@ -426,6 +424,24 @@ "protobufjs": "^7.4.0" } }, + "node_modules/@comapeo/ipc": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@comapeo/ipc/-/ipc-2.1.0.tgz", + "integrity": "sha512-awrVjwI25wJVX1OoE4yBBdWP7rLO/lHeYfjne2wk7ecuuO5abekKUQ0QFoKLpvo94LNnK+K61jRrEN5A4FY9lw==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.1", + "p-defer": "^4.0.0", + "rpc-reflector": "^1.3.11" + }, + "engines": { + "node": ">=18.17.1" + }, + "peerDependencies": { + "@comapeo/core": "^2.2.0" + } + }, "node_modules/@comapeo/schema": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@comapeo/schema/-/schema-1.3.0.tgz", @@ -983,6 +999,16 @@ "gl-style-validate": "dist/gl-style-validate.mjs" } }, + "node_modules/@msgpack/msgpack": { + "version": "1.12.2", + "resolved": "https://registry.npmjs.org/@msgpack/msgpack/-/msgpack-1.12.2.tgz", + "integrity": "sha512-Vwhc3ObxmDZmA5hY8mfsau2rJ4vGPvzbj20QSZ2/E1GDPF61QVyjLfNHak9xmel6pW4heRt3v1fHa6np9Ehfeg==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">= 10" + } + }, "node_modules/@node-rs/crc32": { "version": "1.10.3", "resolved": "https://registry.npmjs.org/@node-rs/crc32/-/crc32-1.10.3.tgz", @@ -1192,9 +1218,10 @@ "dev": true }, "node_modules/@sinclair/typebox": { - "version": "0.29.6", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.29.6.tgz", - "integrity": "sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==" + "version": "0.33.22", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.33.22.tgz", + "integrity": "sha512-auUj4k+f4pyrIVf4GW5UKquSZFHJWri06QgARy9C0t9ZTjJLIuNIrr1yl9bWcJWJ1Gz1vOvYN1D+QPaIlNMVkQ==", + "license": "MIT" }, "node_modules/@sindresorhus/merge-streams": { "version": "4.0.0", @@ -2597,6 +2624,13 @@ "dev": true, "license": "MIT" }, + "node_modules/const-max-uint32": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/const-max-uint32/-/const-max-uint32-1.0.2.tgz", + "integrity": "sha512-T8/9bffg5RThuejasJWrwqxs3Q0fsJvyl7/33IB6svroD8JC93E7X60AuuOnDE8RlP6Jlb5FxmlrVDpl9KiU2Q==", + "dev": true, + "license": "MIT" + }, "node_modules/content-disposition": { "version": "0.5.4", "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", @@ -5679,6 +5713,25 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/length-prefixed-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/length-prefixed-stream/-/length-prefixed-stream-2.0.0.tgz", + "integrity": "sha512-dvjTuWTKWe0oEznQcG6a9osfiYknCs7DEFJMP88n9Y581IFhYh1sZIgAFcuDOojKB0G7ftPreKhh4D0kh/VPjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "readable-stream": "^3.1.1", + "varint": "^5.0.0" + } + }, + "node_modules/length-prefixed-stream/node_modules/varint": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", + "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==", + "dev": true, + "license": "MIT" + }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -8332,6 +8385,49 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/rpc-reflector": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/rpc-reflector/-/rpc-reflector-1.3.11.tgz", + "integrity": "sha512-TIf/RHJy11q/xmNBj0Xj2Z4GVPP1aLeaZXSRtthcMnXNuK+tv7SpZldB5Jk6RFHzA9TgxhcWvLHdHrdlEDKH0w==", + "dev": true, + "license": "ISC", + "dependencies": { + "@msgpack/msgpack": "^1.12.1", + "@types/node": "^18.16.19", + "duplexify": "^4.1.2", + "eventemitter3": "^5.0.1", + "is-stream": "^2.0.1", + "length-prefixed-stream": "^2.0.0", + "p-timeout": "^4.1.0", + "pump": "^3.0.0", + "serialize-error": "^8.1.0", + "through2": "^4.0.2", + "validate.io-array-like": "^1.0.2" + } + }, + "node_modules/rpc-reflector/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/rpc-reflector/node_modules/p-timeout": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-4.1.0.tgz", + "integrity": "sha512-+/wmHtzJuWii1sXn3HCuH/FTwGhrp4tmJTxSKJbfS+vkipci6osxXM5mY0jUiRzWKMTgUT8l7HFbeSwZAynqHw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, "node_modules/run-applescript": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.0.0.tgz", @@ -8446,6 +8542,35 @@ "version": "2.7.0", "license": "BSD-3-Clause" }, + "node_modules/serialize-error": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", + "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/serialize-error/node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/set-cookie-parser": { "version": "2.6.0", "license": "MIT" @@ -9420,6 +9545,16 @@ "node": ">=12.22" } }, + "node_modules/through2": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "readable-stream": "3" + } + }, "node_modules/timeout-refresh": { "version": "2.0.1", "license": "MIT" @@ -9907,6 +10042,32 @@ "spdx-expression-parse": "^3.0.0" } }, + "node_modules/validate.io-array-like": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/validate.io-array-like/-/validate.io-array-like-1.0.2.tgz", + "integrity": "sha512-rGLiN0cvY9OWzQcWP+RtqZR/MK9RUz3gKDTCcRLtEQ/BvlanMF5PyqtVIN+CgrIBCv/ypfme9v7r4yMJPYpbNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "const-max-uint32": "^1.0.2", + "validate.io-integer-primitive": "^1.0.0" + } + }, + "node_modules/validate.io-integer-primitive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/validate.io-integer-primitive/-/validate.io-integer-primitive-1.0.0.tgz", + "integrity": "sha512-4ARGKA4FImVWJgrgttLYsYJmDGwxlhLfDCdq09gyVgohLKKRUfD3VAo1L2vTRCLt6hDhDtFKdZiuYUTWyBggwg==", + "dev": true, + "dependencies": { + "validate.io-number-primitive": "^1.0.0" + } + }, + "node_modules/validate.io-number-primitive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/validate.io-number-primitive/-/validate.io-number-primitive-1.0.0.tgz", + "integrity": "sha512-8rlCe7N0TRTd50dwk4WNoMXNbX/4+RdtqE3TO6Bk0GJvAgbQlfL5DGr/Pl9ZLbWR6CutMjE2cu+yOoCnFWk+Qw==", + "dev": true + }, "node_modules/varint": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/varint/-/varint-6.0.0.tgz", diff --git a/package.json b/package.json index 0aa54eca..8230fbc8 100644 --- a/package.json +++ b/package.json @@ -110,6 +110,7 @@ "@bufbuild/buf": "^1.26.1", "@comapeo/cloud": "^0.1.0", "@comapeo/core2.0.1": "npm:@comapeo/core@2.0.1", + "@comapeo/ipc": "^2.1.0", "@mapeo/default-config": "5.0.0", "@mapeo/mock-data": "^2.1.1", "@sinonjs/fake-timers": "^10.0.2", @@ -165,7 +166,7 @@ "@hyperswarm/secret-stream": "^6.6.3", "@mapeo/crypto": "1.0.0-alpha.10", "@mapeo/sqlite-indexer": "1.0.0-alpha.9", - "@sinclair/typebox": "^0.29.6", + "@sinclair/typebox": "^0.33.17", "@sindresorhus/merge-streams": "^4.0.0", "b4a": "^1.6.3", "bcp-47": "^2.1.0", diff --git a/test-e2e/forked-process.js b/test-e2e/forked-process.js new file mode 100644 index 00000000..5276cd33 --- /dev/null +++ b/test-e2e/forked-process.js @@ -0,0 +1,95 @@ +import { createMapeoServer } from '@comapeo/ipc' +import { Type } from '@sinclair/typebox' +import { Value } from '@sinclair/typebox/value' +import Fastify from 'fastify' +import assert from 'node:assert/strict' +import { randomBytes } from 'node:crypto' +import * as process from 'node:process' +import { isMainThread, MessagePort, workerData } from 'node:worker_threads' +import RAM from 'random-access-memory' +import { MapeoManager } from '../src/mapeo-manager.js' + +/** + * @internal + * @typedef {ConstructorParameters[0]} MapeoManagerConstructorOptions + */ + +const mapeoManagerConstructorOptionsType = Type.Partial( + Type.Object({ + rootKey: Type.Uint8Array(), + dbFolder: Type.String(), + coreStorage: Type.String(), + projectMigrationsFolder: Type.String(), + clientMigrationsFolder: Type.String(), + defaultConfigPath: Type.String(), + customMapPath: Type.String(), + fallbackMapPath: Type.String(), + defaultOnlineStyleUrl: Type.String(), + }) +) + +/** + * @internal + * @typedef {object} ParsedWorkerData + * @prop {MessagePort} childPort + * @prop {Partial} managerConstructorOverrides + */ + +/** + * Parse data passed to this worker. If the data is invalid (which can happen + * if run with `node --test`), returns `null`. + * + * @param {unknown} workerData + * @return {null | ParsedWorkerData} + */ +function parseWorkerData(workerData) { + try { + assert( + workerData && + typeof workerData === 'object' && + 'childPort' in workerData && + workerData.childPort instanceof MessagePort && + 'managerConstructorOverrides' in workerData + ) + return { + childPort: workerData.childPort, + managerConstructorOverrides: Value.Parse( + mapeoManagerConstructorOptionsType, + workerData.managerConstructorOverrides + ), + } + } catch { + return null + } +} + +async function main() { + if (isMainThread) return + + const parsedWorkerData = parseWorkerData(workerData) + if (!parsedWorkerData) return + const { managerConstructorOverrides, childPort } = parsedWorkerData + + const manager = new MapeoManager({ + rootKey: randomBytes(16), + projectMigrationsFolder: new URL('../drizzle/project', import.meta.url) + .pathname, + clientMigrationsFolder: new URL('../drizzle/client', import.meta.url) + .pathname, + dbFolder: ':memory:', + coreStorage: () => new RAM(), + fastify: Fastify(), + ...managerConstructorOverrides, + }) + + // This `any` cast is needed because `@comapeo/ipc` expects a `MapeoManager` + // from the `@comapeo/core` install from npm, not our local development one. + createMapeoServer(/** @type {any} */ (manager), childPort) + + childPort.start() +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/test-e2e/ipc-basic.js b/test-e2e/ipc-basic.js new file mode 100644 index 00000000..2407ad1e --- /dev/null +++ b/test-e2e/ipc-basic.js @@ -0,0 +1,22 @@ +import test from 'node:test' +import assert from 'node:assert/strict' +import { createIpcManager } from './utils.js' +import { valueOf } from '@comapeo/schema' +import { generate } from '@mapeo/mock-data' + +test('basic functionality of a manager in a separate process', async (t) => { + const manager = await createIpcManager('manager', t) + await manager.setDeviceInfo({ name: 'manager', deviceType: 'mobile' }) + + const projectId = await manager.createProject() + const project = await manager.getProject(projectId) + + const { docId } = await project.observation.create( + valueOf(generate('observation')[0]) + ) + + assert( + await project.observation.getByDocId(docId), + 'can retrieve an observation we just created' + ) +}) diff --git a/test-e2e/utils.js b/test-e2e/utils.js index d07fdf8d..0ab9621d 100644 --- a/test-e2e/utils.js +++ b/test-e2e/utils.js @@ -9,6 +9,8 @@ import { createRequire } from 'node:module' import { fileURLToPath } from 'node:url' import * as v8 from 'node:v8' import { pEvent } from 'p-event' +import { createMapeoClient } from '@comapeo/ipc' +import { Worker, MessageChannel } from 'node:worker_threads' import { MapeoManager as MapeoManager_2_0_1 } from '@comapeo/core2.0.1' import { MapeoManager, roles } from '../src/index.js' @@ -267,6 +269,7 @@ export function createManager(seed, t, overrides = {}) { ...overrides, }) } + /** * @param {string} seed * @param {Partial[0]>} [overrides] @@ -293,6 +296,37 @@ export async function createOldManagerOnVersion2_0_1(seed, overrides = {}) { }) } +/** + * @param {string} seed + * @param {import('node:test').TestContext} t + * @param {Partial[0]>} [overrides] + * @returns {Promise>} + */ +export async function createIpcManager(seed, t, overrides = {}) { + const { port1: parentPort, port2: childPort } = new MessageChannel() + + const forkedProcessPath = fileURLToPath( + new URL('./forked-process.js', import.meta.url) + ) + const worker = new Worker(forkedProcessPath, { + workerData: { + managerConstructorOverrides: { rootKey: getRootKey(seed), ...overrides }, + childPort, + }, + transferList: [childPort], + }) + + t.after(() => worker.terminate()) + + // As an optimization, we can prevent the worker from keeping the test + // process alive. This isn't necessary for correctness (the call to + // `worker.terminate()` should be enough), but it can speed up tests by + // letting the process end early. + worker.unref() + + return createMapeoClient(parentPort) +} + /** * `ManagerCustodian` helps you test the creation of multiple managers accessing * the same underlying files.