diff --git a/README.md b/README.md index d8cea40c1..5744efbe9 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Perform async work synchronously in Node.js using `worker_threads` with first-cl - [Usage](#usage) - [Install](#install) - [API](#api) + - [Types](#types) - [Options](#options) - [Envs](#envs) - [TypeScript](#typescript) @@ -71,6 +72,43 @@ runAsWorker(async (...args) => { You must make sure, the `result` is serializable by [`Structured Clone Algorithm`](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm) +### Types + +````ts +export interface GlobalShim { + moduleName: string + /** + * `undefined` means side effect only + */ + globalName?: string + /** + * 1. `undefined` or empty string means `default`, for example: + * ```js + * import globalName from 'module-name' + * ``` + * + * 2. `null` means namespaced, for example: + * ```js + * import * as globalName from 'module-name' + * ``` + * + */ + named?: string | null + /** + * If not `false`, the shim will only be applied when the original `globalName` unavailable, + * for example you may only want polyfill `globalThis.fetch` when it's unavailable natively: + * ```js + * import fetch from 'node-fetch' + * + * if (!globalThis.fetch) { + * globalThis.fetch = fetch + * } + * ``` + */ + conditional?: boolean +} +```` + ### Options 1. `bufferSize` same as env `SYNCKIT_BUFFER_SIZE` @@ -78,6 +116,7 @@ You must make sure, the `result` is serializable by [`Structured Clone Algorithm 3. `execArgv` same as env `SYNCKIT_EXEC_ARGV` 4. `tsRunner` same as env `SYNCKIT_TS_RUNNER` 5. `transferList`: Please refer Node.js [`worker_threads`](https://nodejs.org/api/worker_threads.html#:~:text=Default%3A%20true.-,transferList,-%3CObject%5B%5D%3E%20If) documentation +6. `globalShims`: Similar like env `SYNCKIT_GLOBAL_SHIMS` but much more flexible which can be a `GlobalShim` `Array`, see `GlobalShim`'s [definition](#types) for more details ### Envs @@ -85,6 +124,7 @@ You must make sure, the `result` is serializable by [`Structured Clone Algorithm 2. `SYNCKIT_TIMEOUT`: `timeout` for performing the async job (no default) 3. `SYNCKIT_EXEC_ARGV`: List of node CLI options passed to the worker, split with comma `,`. (default as `[]`), see also [`node` docs](https://nodejs.org/api/worker_threads.html) 4. `SYNCKIT_TS_RUNNER`: Which TypeScript runner to be used, it could be very useful for development, could be `'ts-node' | 'esbuild-register' | 'esbuild-runner' | 'swc' | 'tsx'`, `'ts-node'` is used by default, make sure you have installed them already +5. `SYNCKIT_GLOBAL_SHIMS`: Whether to enable the default `DEFAULT_GLOBAL_SHIMS_PRESET` as `globalShims` ### TypeScript diff --git a/src/index.ts b/src/index.ts index b6fd7cb9f..081820e54 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,10 +1,12 @@ +import { type Hash, createHash } from 'node:crypto' import fs from 'node:fs' import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' import path from 'node:path' import { pathToFileURL } from 'node:url' import { MessageChannel, - TransferListItem, + type TransferListItem, Worker, parentPort, receiveMessageOnPort, @@ -14,9 +16,10 @@ import { import { findUp, isPkgAvailable, tryExtensions } from '@pkgr/utils' -import { +import type { AnyAsyncFn, AnyFn, + GlobalShim, MainToWorkerMessage, Syncify, ValueOf, @@ -46,6 +49,7 @@ const { SYNCKIT_TIMEOUT, SYNCKIT_EXEC_ARGV, SYNCKIT_TS_RUNNER, + SYNCKIT_GLOBAL_SHIMS, NODE_OPTIONS, } = process.env @@ -62,6 +66,22 @@ export const DEFAULT_EXEC_ARGV = SYNCKIT_EXEC_ARGV?.split(',') || [] export const DEFAULT_TS_RUNNER = SYNCKIT_TS_RUNNER as TsRunner | undefined +export const DEFAULT_GLOBAL_SHIMS = ['1', 'true'].includes( + SYNCKIT_GLOBAL_SHIMS!, +) + +export const DEFAULT_GLOBAL_SHIMS_PRESET: GlobalShim[] = [ + { + moduleName: 'node-fetch', + globalName: 'fetch', + }, + { + moduleName: 'node:perf_hooks', + globalName: 'performance', + named: 'performance', + }, +] + export const MTS_SUPPORTED_NODE_VERSION = 16 const syncFnCache = new Map() @@ -72,6 +92,7 @@ export interface SynckitOptions { execArgv?: string[] tsRunner?: TsRunner transferList?: TransferListItem[] + globalShims?: GlobalShim[] | boolean } // MessagePort doesn't copy the properties of Error objects. We still want @@ -278,6 +299,117 @@ const setupTsRunner = ( } } +let hash: Hash | undefined + +const md5Hash = (text: string) => + (hash ||= createHash('md5')).update(text).digest('hex') + +const encodeImportModule = ( + moduleNameOrGlobalShim: GlobalShim | string, + type: 'import' | 'require' = 'import', + // eslint-disable-next-line sonarjs/cognitive-complexity +) => { + const { moduleName, globalName, named, conditional }: GlobalShim = + typeof moduleNameOrGlobalShim === 'string' + ? { moduleName: moduleNameOrGlobalShim } + : moduleNameOrGlobalShim + const importStatement = + type === 'import' + ? `import${ + globalName + ? ' ' + + (named === null + ? '* as ' + globalName + : named?.trim() + ? `{${named}}` + : globalName) + + ' from' + : '' + } '${ + path.isAbsolute(moduleName) + ? String(pathToFileURL(moduleName)) + : moduleName + }'` + : `${ + globalName + ? 'const ' + (named?.trim() ? `{${named}}` : globalName) + '=' + : '' + }require('${moduleName + // eslint-disable-next-line unicorn/prefer-string-replace-all -- compatibility + .replace(/\\/g, '\\\\')}')` + + if (!globalName) { + return importStatement + } + + const overrideStatement = `globalThis.${globalName}=${ + named?.trim() ? named : globalName + }` + + return ( + importStatement + + (conditional === false + ? `;${overrideStatement}` + : `;if(!globalThis.${globalName})${overrideStatement}`) + ) +} + +/** + * @internal + */ +export const _generateGlobals = ( + globalShims: GlobalShim[], + type: 'import' | 'require', +) => + globalShims.reduce( + (acc, shim) => + isPkgAvailable(shim.moduleName) + ? `${acc}${acc ? ';' : ''}${encodeImportModule(shim, type)}` + : acc, + '', + ) + +const globalsCache = new Map() + +let tempDir: string + +export const generateGlobals = ( + workerPath: string, + globalShims: GlobalShim[], + type: 'import' | 'require' = 'import', +) => { + const cached = globalsCache.get(workerPath) + + if (cached) { + const [content, filepath] = cached + + if ( + (type === 'require' && !filepath) || + (type === 'import' && filepath && isFile(filepath)) + ) { + return content + } + } + + const globals = _generateGlobals(globalShims, type) + + let content = globals + let filepath: string | undefined + + if (type === 'import') { + filepath = path.resolve( + (tempDir ||= fs.realpathSync(tmpdir())), + md5Hash(workerPath) + '.mjs', + ) + content = encodeImportModule(filepath) + fs.writeFileSync(filepath, globals) + } + + globalsCache.set(workerPath, [content, filepath]) + + return content +} + // eslint-disable-next-line sonarjs/cognitive-complexity function startWorkerThread>( workerPath: string, @@ -287,6 +419,7 @@ function startWorkerThread>( execArgv = DEFAULT_EXEC_ARGV, tsRunner = DEFAULT_TS_RUNNER, transferList = [], + globalShims = DEFAULT_GLOBAL_SHIMS, }: SynckitOptions = {}, ) { const { port1: mainPort, port2: workerPort } = new MessageChannel() @@ -331,14 +464,29 @@ function startWorkerThread>( } } - const useEval = isTs && !tsUseEsm + const finalGlobalShims = + globalShims === true + ? DEFAULT_GLOBAL_SHIMS_PRESET + : Array.isArray(globalShims) + ? globalShims + : [] + + const useEval = isTs && !tsUseEsm && finalGlobalShims.length > 0 const worker = new Worker( tsUseEsm && finalTsRunner === TsRunner.TsNode - ? dataUrl(`import '${String(workerPathUrl)}'`) + ? dataUrl( + `${generateGlobals( + finalWorkerPath, + finalGlobalShims, + )};import '${String(workerPathUrl)}'`, + ) : useEval - ? // eslint-disable-next-line unicorn/prefer-string-replace-all -- compatibility - `require('${finalWorkerPath.replace(/\\/g, '\\\\')}')` + ? `${generateGlobals( + finalWorkerPath, + finalGlobalShims, + 'require', + )};${encodeImportModule(finalWorkerPath, 'require')}` : workerPathUrl, { eval: useEval, diff --git a/src/types.ts b/src/types.ts index b74288e74..806202523 100644 --- a/src/types.ts +++ b/src/types.ts @@ -40,3 +40,36 @@ export interface DataMessage { export interface WorkerToMainMessage extends DataMessage { id: number } + +export interface GlobalShim { + moduleName: string + /** + * `undefined` means side effect only + */ + globalName?: string + /** + * 1. `undefined` or empty string means `default`, for example: + * ```js + * import globalName from 'module-name' + * ``` + * + * 2. `null` means namespaced, for example: + * ```js + * import * as globalName from 'module-name' + * ``` + * + */ + named?: string | null + /** + * If not `false`, the shim will only be applied when the original `globalName` unavailable, + * for example you may only want polyfill `globalThis.fetch` when it's unavailable natively: + * ```js + * import fetch from 'node-fetch' + * + * if (!globalThis.fetch) { + * globalThis.fetch = fetch + * } + * ``` + */ + conditional?: boolean +} diff --git a/test/__snapshots__/utils.spec.ts.snap b/test/__snapshots__/utils.spec.ts.snap new file mode 100644 index 000000000..d19b70917 --- /dev/null +++ b/test/__snapshots__/utils.spec.ts.snap @@ -0,0 +1,9 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`utils generateGlobals 1`] = `"import fetch from 'node-fetch';if(!globalThis.fetch)globalThis.fetch=fetch;import {performance} from 'node:perf_hooks';if(!globalThis.performance)globalThis.performance=performance"`; + +exports[`utils generateGlobals 2`] = `"const fetch=require('node-fetch');if(!globalThis.fetch)globalThis.fetch=fetch;const {performance}=require('node:perf_hooks');if(!globalThis.performance)globalThis.performance=performance"`; + +exports[`utils generateGlobals 3`] = `"import fetch from 'node-fetch';globalThis.fetch=fetch;import {performance} from 'node:perf_hooks';if(!globalThis.performance)globalThis.performance=performance"`; + +exports[`utils generateGlobals 4`] = `"const fetch=require('node-fetch');globalThis.fetch=fetch;const {performance}=require('node:perf_hooks');if(!globalThis.performance)globalThis.performance=performance"`; diff --git a/test/utils.spec.ts b/test/utils.spec.ts index c75afba95..a6d4c7d9f 100644 --- a/test/utils.spec.ts +++ b/test/utils.spec.ts @@ -1,11 +1,82 @@ -import { fileURLToPath } from 'node:url' +import fs from 'node:fs' +import { tmpdir } from 'node:os' +import { fileURLToPath, pathToFileURL } from 'node:url' import { _dirname } from './helpers' -import { isFile } from 'synckit' +import { + DEFAULT_GLOBAL_SHIMS_PRESET, + _generateGlobals, + generateGlobals, + isFile, +} from 'synckit' -test('utils', () => { - expect(isFile(_dirname)).toBe(false) - expect(isFile('non-existed')).toBe(false) - expect(isFile(fileURLToPath(import.meta.url))).toBe(true) +describe('utils', () => { + test('isFile', () => { + expect(isFile(_dirname)).toBe(false) + expect(isFile('non-existed')).toBe(false) + expect(isFile(fileURLToPath(import.meta.url))).toBe(true) + }) + + test('generateGlobals', () => { + const _importGlobals = _generateGlobals( + DEFAULT_GLOBAL_SHIMS_PRESET, + 'import', + ) + expect(_importGlobals).toMatchSnapshot() + + const _requireGlobals = _generateGlobals( + DEFAULT_GLOBAL_SHIMS_PRESET, + 'require', + ) + expect(_requireGlobals).toMatchSnapshot() + + const tempDir = String(pathToFileURL(fs.realpathSync(tmpdir()))) + const importGlobals = generateGlobals( + 'fake.js', + DEFAULT_GLOBAL_SHIMS_PRESET, + ) + expect(importGlobals).not.toBe(_importGlobals) + expect(importGlobals).toMatch(tempDir) + expect(generateGlobals('fake.js', DEFAULT_GLOBAL_SHIMS_PRESET)).toBe( + importGlobals, + ) + + const requireGlobals = generateGlobals( + 'fake.js', + DEFAULT_GLOBAL_SHIMS_PRESET, + 'require', + ) + expect(requireGlobals).toBe(_requireGlobals) + expect(requireGlobals).not.toBe(importGlobals) + expect( + generateGlobals('fake.js', DEFAULT_GLOBAL_SHIMS_PRESET, 'require'), + ).toBe(requireGlobals) + + expect( + _generateGlobals( + [ + { + ...DEFAULT_GLOBAL_SHIMS_PRESET[0], + conditional: false, + }, + ...DEFAULT_GLOBAL_SHIMS_PRESET.slice(1), + ], + 'import', + ), + ).toMatchSnapshot() + + expect( + _generateGlobals( + [ + { + ...DEFAULT_GLOBAL_SHIMS_PRESET[0], + conditional: false, + }, + ...DEFAULT_GLOBAL_SHIMS_PRESET.slice(1), + ], + 'require', + ), + ).toMatchSnapshot() + }) })