diff --git a/.changeset/funny-meals-speak.md b/.changeset/funny-meals-speak.md new file mode 100644 index 000000000..1cb65fdd3 --- /dev/null +++ b/.changeset/funny-meals-speak.md @@ -0,0 +1,55 @@ +--- +"synckit": patch +--- + +feat: add new `globalShims` option, what means you can env `SYNCKIT_GLOBAL_SHIMS=1` to enable auto polyfilling for some modules, for example: `fetch` from `node-fetch`, `performance` from `node:perf_hooks`. + +You can also pass a custom `globalShims` option as `GlobalShim` `Array` to custom your own shims: + +````ts +export interface GlobalShim { + moduleName: string + /** + * `undefined` means side effect only + */ + globalName?: string + /** + * 1. `undefined` or empty string means `default`, for example: + * ```js + * import globalName from 'module-name' + * ``` + * + * 2. `null` means namespaced, for example: + * ```js + * import * as globalName from 'module-name' + * ``` + * + */ + named?: string | null + /** + * If not `false`, the shim will only be applied when the original `globalName` unavailable, + * for example you may only want polyfill `globalThis.fetch` when it's unavailable natively: + * ```js + * import fetch from 'node-fetch' + * + * if (!globalThis.fetch) { + * globalThis.fetch = fetch + * } + * ``` + */ + conditional?: boolean +} +```` + +You can aslo reuse the exported `DEFAULT_GLOBAL_SHIMS_PRESET` for extanding: + +```js +import { DEFAULT_GLOBAL_SHIMS_PRESET, createSyncFn } from 'synckit' + +const syncFn = createSyncFn(require.resolve('./worker'), { + globalShims: [ + ...DEFAULT_GLOBAL_SHIMS_PRESET, + // your own shim here + ] +}) +``` diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39c2dc6e7..f48b0f901 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,7 @@ jobs: node: - 16 - 18 + - 18.18 - 20 os: - macos-latest diff --git a/.github/workflows/pkg-size.yml b/.github/workflows/pkg-size.yml index 2e2b160e1..2a0b2b405 100644 --- a/.github/workflows/pkg-size.yml +++ b/.github/workflows/pkg-size.yml @@ -1,9 +1,7 @@ name: Package Size Report on: - pull_request_target: - branches: - - main + - pull_request jobs: pkg-size-report: diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 000000000..4a58985bb --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +18.18 diff --git a/README.md b/README.md index d8cea40c1..5744efbe9 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Perform async work synchronously in Node.js using `worker_threads` with first-cl - [Usage](#usage) - [Install](#install) - [API](#api) + - [Types](#types) - [Options](#options) - [Envs](#envs) - [TypeScript](#typescript) @@ -71,6 +72,43 @@ runAsWorker(async (...args) => { You must make sure, the `result` is serializable by [`Structured Clone Algorithm`](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API/Structured_clone_algorithm) +### Types + +````ts +export interface GlobalShim { + moduleName: string + /** + * `undefined` means side effect only + */ + globalName?: string + /** + * 1. `undefined` or empty string means `default`, for example: + * ```js + * import globalName from 'module-name' + * ``` + * + * 2. `null` means namespaced, for example: + * ```js + * import * as globalName from 'module-name' + * ``` + * + */ + named?: string | null + /** + * If not `false`, the shim will only be applied when the original `globalName` unavailable, + * for example you may only want polyfill `globalThis.fetch` when it's unavailable natively: + * ```js + * import fetch from 'node-fetch' + * + * if (!globalThis.fetch) { + * globalThis.fetch = fetch + * } + * ``` + */ + conditional?: boolean +} +```` + ### Options 1. `bufferSize` same as env `SYNCKIT_BUFFER_SIZE` @@ -78,6 +116,7 @@ You must make sure, the `result` is serializable by [`Structured Clone Algorithm 3. `execArgv` same as env `SYNCKIT_EXEC_ARGV` 4. `tsRunner` same as env `SYNCKIT_TS_RUNNER` 5. `transferList`: Please refer Node.js [`worker_threads`](https://nodejs.org/api/worker_threads.html#:~:text=Default%3A%20true.-,transferList,-%3CObject%5B%5D%3E%20If) documentation +6. `globalShims`: Similar like env `SYNCKIT_GLOBAL_SHIMS` but much more flexible which can be a `GlobalShim` `Array`, see `GlobalShim`'s [definition](#types) for more details ### Envs @@ -85,6 +124,7 @@ You must make sure, the `result` is serializable by [`Structured Clone Algorithm 2. `SYNCKIT_TIMEOUT`: `timeout` for performing the async job (no default) 3. `SYNCKIT_EXEC_ARGV`: List of node CLI options passed to the worker, split with comma `,`. (default as `[]`), see also [`node` docs](https://nodejs.org/api/worker_threads.html) 4. `SYNCKIT_TS_RUNNER`: Which TypeScript runner to be used, it could be very useful for development, could be `'ts-node' | 'esbuild-register' | 'esbuild-runner' | 'swc' | 'tsx'`, `'ts-node'` is used by default, make sure you have installed them already +5. `SYNCKIT_GLOBAL_SHIMS`: Whether to enable the default `DEFAULT_GLOBAL_SHIMS_PRESET` as `globalShims` ### TypeScript diff --git a/package.json b/package.json index 84008be58..49d807d54 100644 --- a/package.json +++ b/package.json @@ -90,6 +90,9 @@ "preset": "ts-jest", "testEnvironment": "node", "collectCoverage": true, + "collectCoverageFrom": [ + "src/**" + ], "extensionsToTreatAsEsm": [ ".ts" ], diff --git a/src/index.ts b/src/index.ts index b6fd7cb9f..36634c454 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,10 +1,11 @@ +import { createHash } from 'node:crypto' import fs from 'node:fs' import { createRequire } from 'node:module' import path from 'node:path' -import { pathToFileURL } from 'node:url' +import { fileURLToPath, pathToFileURL } from 'node:url' import { MessageChannel, - TransferListItem, + type TransferListItem, Worker, parentPort, receiveMessageOnPort, @@ -14,9 +15,10 @@ import { import { findUp, isPkgAvailable, tryExtensions } from '@pkgr/utils' -import { +import type { AnyAsyncFn, AnyFn, + GlobalShim, MainToWorkerMessage, Syncify, ValueOf, @@ -46,6 +48,7 @@ const { SYNCKIT_TIMEOUT, SYNCKIT_EXEC_ARGV, SYNCKIT_TS_RUNNER, + SYNCKIT_GLOBAL_SHIMS, NODE_OPTIONS, } = process.env @@ -62,6 +65,22 @@ export const DEFAULT_EXEC_ARGV = SYNCKIT_EXEC_ARGV?.split(',') || [] export const DEFAULT_TS_RUNNER = SYNCKIT_TS_RUNNER as TsRunner | undefined +export const DEFAULT_GLOBAL_SHIMS = ['1', 'true'].includes( + SYNCKIT_GLOBAL_SHIMS!, +) + +export const DEFAULT_GLOBAL_SHIMS_PRESET: GlobalShim[] = [ + { + moduleName: 'node-fetch', + globalName: 'fetch', + }, + { + moduleName: 'node:perf_hooks', + globalName: 'performance', + named: 'performance', + }, +] + export const MTS_SUPPORTED_NODE_VERSION = 16 const syncFnCache = new Map() @@ -72,6 +91,7 @@ export interface SynckitOptions { execArgv?: string[] tsRunner?: TsRunner transferList?: TransferListItem[] + globalShims?: GlobalShim[] | boolean } // MessagePort doesn't copy the properties of Error objects. We still want @@ -182,6 +202,7 @@ const setupTsRunner = ( const isTs = /\.[cm]?ts$/.test(workerPath) + let jsUseEsm = workerPath.endsWith('.mjs') let tsUseEsm = workerPath.endsWith('.mts') if (isTs) { @@ -237,6 +258,12 @@ const setupTsRunner = ( throw new Error(`Unknown ts runner: ${String(tsRunner)}`) } } + } else if (!jsUseEsm) { + const pkg = findUp(workerPath) + if (pkg) { + jsUseEsm = + (cjsRequire(pkg) as { type?: 'commonjs' | 'module' }).type === 'module' + } } /* istanbul ignore if -- https://github.com/facebook/jest/issues/5274 */ @@ -271,6 +298,7 @@ const setupTsRunner = ( return { ext, isTs, + jsUseEsm, tsRunner, tsUseEsm, workerPath, @@ -278,6 +306,117 @@ const setupTsRunner = ( } } +const md5Hash = (text: string) => createHash('md5').update(text).digest('hex') + +export const encodeImportModule = ( + moduleNameOrGlobalShim: GlobalShim | string, + type: 'import' | 'require' = 'import', + // eslint-disable-next-line sonarjs/cognitive-complexity +) => { + const { moduleName, globalName, named, conditional }: GlobalShim = + typeof moduleNameOrGlobalShim === 'string' + ? { moduleName: moduleNameOrGlobalShim } + : moduleNameOrGlobalShim + const importStatement = + type === 'import' + ? `import${ + globalName + ? ' ' + + (named === null + ? '* as ' + globalName + : named?.trim() + ? `{${named}}` + : globalName) + + ' from' + : '' + } '${ + path.isAbsolute(moduleName) + ? String(pathToFileURL(moduleName)) + : moduleName + }'` + : `${ + globalName + ? 'const ' + (named?.trim() ? `{${named}}` : globalName) + '=' + : '' + }require('${moduleName + // eslint-disable-next-line unicorn/prefer-string-replace-all -- compatibility + .replace(/\\/g, '\\\\')}')` + + if (!globalName) { + return importStatement + } + + const overrideStatement = `globalThis.${globalName}=${ + named?.trim() ? named : globalName + }` + + return ( + importStatement + + (conditional === false + ? `;${overrideStatement}` + : `;if(!globalThis.${globalName})${overrideStatement}`) + ) +} + +/** + * @internal + */ +export const _generateGlobals = ( + globalShims: GlobalShim[], + type: 'import' | 'require', +) => + globalShims.reduce( + (acc, shim) => `${acc}${acc ? ';' : ''}${encodeImportModule(shim, type)}`, + '', + ) + +const globalsCache = new Map() + +let tmpdir: string + +const _dirname = + typeof __dirname === 'undefined' + ? path.dirname(fileURLToPath(import.meta.url)) + : /* istanbul ignore next */ __dirname + +export const generateGlobals = ( + workerPath: string, + globalShims: GlobalShim[], + type: 'import' | 'require' = 'import', +) => { + const cached = globalsCache.get(workerPath) + + if (cached) { + const [content, filepath] = cached + + if ( + (type === 'require' && !filepath) || + (type === 'import' && filepath && isFile(filepath)) + ) { + return content + } + } + + const globals = _generateGlobals(globalShims, type) + + let content = globals + let filepath: string | undefined + + if (type === 'import') { + if (!tmpdir) { + tmpdir = path.resolve(findUp(_dirname), '../node_modules/.synckit') + } + fs.mkdirSync(tmpdir, { recursive: true }) + filepath = path.resolve(tmpdir, md5Hash(workerPath) + '.mjs') + content = encodeImportModule(filepath) + fs.writeFileSync(filepath, globals) + } + + globalsCache.set(workerPath, [content, filepath]) + + return content +} + // eslint-disable-next-line sonarjs/cognitive-complexity function startWorkerThread>( workerPath: string, @@ -287,6 +426,7 @@ function startWorkerThread>( execArgv = DEFAULT_EXEC_ARGV, tsRunner = DEFAULT_TS_RUNNER, transferList = [], + globalShims = DEFAULT_GLOBAL_SHIMS, }: SynckitOptions = {}, ) { const { port1: mainPort, port2: workerPort } = new MessageChannel() @@ -294,6 +434,7 @@ function startWorkerThread>( const { isTs, ext, + jsUseEsm, tsUseEsm, tsRunner: finalTsRunner, workerPath: finalWorkerPath, @@ -331,14 +472,32 @@ function startWorkerThread>( } } - const useEval = isTs && !tsUseEsm + const finalGlobalShims = ( + globalShims === true + ? DEFAULT_GLOBAL_SHIMS_PRESET + : Array.isArray(globalShims) + ? globalShims + : [] + ).filter(({ moduleName }) => isPkgAvailable(moduleName)) + + const useGlobals = finalGlobalShims.length > 0 + + const useEval = isTs ? !tsUseEsm : !jsUseEsm && useGlobals const worker = new Worker( - tsUseEsm && finalTsRunner === TsRunner.TsNode - ? dataUrl(`import '${String(workerPathUrl)}'`) + (jsUseEsm && useGlobals) || (tsUseEsm && finalTsRunner === TsRunner.TsNode) + ? dataUrl( + `${generateGlobals( + finalWorkerPath, + finalGlobalShims, + )};import '${String(workerPathUrl)}'`, + ) : useEval - ? // eslint-disable-next-line unicorn/prefer-string-replace-all -- compatibility - `require('${finalWorkerPath.replace(/\\/g, '\\\\')}')` + ? `${generateGlobals( + finalWorkerPath, + finalGlobalShims, + 'require', + )};${encodeImportModule(finalWorkerPath, 'require')}` : workerPathUrl, { eval: useEval, diff --git a/src/types.ts b/src/types.ts index b74288e74..806202523 100644 --- a/src/types.ts +++ b/src/types.ts @@ -40,3 +40,36 @@ export interface DataMessage { export interface WorkerToMainMessage extends DataMessage { id: number } + +export interface GlobalShim { + moduleName: string + /** + * `undefined` means side effect only + */ + globalName?: string + /** + * 1. `undefined` or empty string means `default`, for example: + * ```js + * import globalName from 'module-name' + * ``` + * + * 2. `null` means namespaced, for example: + * ```js + * import * as globalName from 'module-name' + * ``` + * + */ + named?: string | null + /** + * If not `false`, the shim will only be applied when the original `globalName` unavailable, + * for example you may only want polyfill `globalThis.fetch` when it's unavailable natively: + * ```js + * import fetch from 'node-fetch' + * + * if (!globalThis.fetch) { + * globalThis.fetch = fetch + * } + * ``` + */ + conditional?: boolean +} diff --git a/test/__snapshots__/utils.spec.ts.snap b/test/__snapshots__/utils.spec.ts.snap new file mode 100644 index 000000000..0b9b2796d --- /dev/null +++ b/test/__snapshots__/utils.spec.ts.snap @@ -0,0 +1,19 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`utils encodeImportModule 1`] = `"import 'module-name'"`; + +exports[`utils encodeImportModule 2`] = `"import './module-name'"`; + +exports[`utils encodeImportModule 3`] = `"import globalName from 'module-name';if(!globalThis.globalName)globalThis.globalName=globalName"`; + +exports[`utils encodeImportModule 4`] = `"import {named} from 'module-name';if(!globalThis.globalName)globalThis.globalName=named"`; + +exports[`utils encodeImportModule 5`] = `"import * as globalName from 'module-name';if(!globalThis.globalName)globalThis.globalName=globalName"`; + +exports[`utils generateGlobals 1`] = `"import fetch from 'node-fetch';if(!globalThis.fetch)globalThis.fetch=fetch;import {performance} from 'node:perf_hooks';if(!globalThis.performance)globalThis.performance=performance"`; + +exports[`utils generateGlobals 2`] = `"const fetch=require('node-fetch');if(!globalThis.fetch)globalThis.fetch=fetch;const {performance}=require('node:perf_hooks');if(!globalThis.performance)globalThis.performance=performance"`; + +exports[`utils generateGlobals 3`] = `"import fetch from 'node-fetch';globalThis.fetch=fetch;import {performance} from 'node:perf_hooks';if(!globalThis.performance)globalThis.performance=performance"`; + +exports[`utils generateGlobals 4`] = `"const fetch=require('node-fetch');globalThis.fetch=fetch;const {performance}=require('node:perf_hooks');if(!globalThis.performance)globalThis.performance=performance"`; diff --git a/test/fn.spec.ts b/test/fn.spec.ts index 70b857598..b94540cd6 100644 --- a/test/fn.spec.ts +++ b/test/fn.spec.ts @@ -6,13 +6,21 @@ import { jest } from '@jest/globals' import { _dirname, testIf, tsUseEsmSupported } from './helpers.js' import type { AsyncWorkerFn } from './types.js' -import { createSyncFn, extractProperties } from 'synckit' +import { createSyncFn } from 'synckit' + +const { SYNCKIT_TIMEOUT } = process.env beforeEach(() => { jest.resetModules() delete process.env.SYNCKIT_BUFFER_SIZE - delete process.env.SYNCKIT_TIMEOUT + delete process.env.SYNCKIT_GLOBAL_SHIMS + + if (SYNCKIT_TIMEOUT) { + process.env.SYNCKIT_TIMEOUT = SYNCKIT_TIMEOUT + } else { + delete process.env.SYNCKIT_TIMEOUT + } }) const cjsRequire = createRequire(import.meta.url) @@ -98,17 +106,29 @@ test('timeout', async () => { ) }) -test('extractProperties', () => { - expect(extractProperties()).toBeUndefined() - expect(extractProperties({})).toEqual({}) - expect(extractProperties(new Error('message'))).toEqual({}) - expect( - extractProperties( - Object.assign(new Error('message'), { - code: 'CODE', - }), - ), - ).toEqual({ - code: 'CODE', +test('globalShims env', async () => { + process.env.SYNCKIT_GLOBAL_SHIMS = '1' + + const { createSyncFn } = await import('synckit') + const syncFn = createSyncFn(workerCjsPath) + + expect(syncFn(1)).toBe(1) + expect(syncFn(2)).toBe(2) + expect(syncFn(5, 0)).toBe(5) +}) + +test('globalShims options', async () => { + const { createSyncFn } = await import('synckit') + + const syncFn = createSyncFn(workerCjsPath, { + globalShims: [ + { + moduleName: 'non-existed', + }, + ], }) + + expect(syncFn(1)).toBe(1) + expect(syncFn(2)).toBe(2) + expect(syncFn(5, 0)).toBe(5) }) diff --git a/test/helpers.ts b/test/helpers.ts index ac197def9..086922403 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -8,8 +8,8 @@ export const _dirname = path.dirname(fileURLToPath(import.meta.url)) export const nodeVersion = Number.parseFloat(process.versions.node) export const tsUseEsmSupported = - nodeVersion >= MTS_SUPPORTED_NODE_VERSION && - // ts-jest limitation - nodeVersion < 20 + // https://github.com/privatenumber/tsx/issues/354 + // eslint-disable-next-line @typescript-eslint/no-magic-numbers + nodeVersion >= MTS_SUPPORTED_NODE_VERSION && nodeVersion <= 18.18 export const testIf = (condition: boolean) => (condition ? it : it.skip) diff --git a/test/utils.spec.ts b/test/utils.spec.ts index c75afba95..9a3b56a79 100644 --- a/test/utils.spec.ts +++ b/test/utils.spec.ts @@ -1,11 +1,132 @@ -import { fileURLToPath } from 'node:url' +import path from 'node:path' +import { fileURLToPath, pathToFileURL } from 'node:url' + +import { findUp } from '@pkgr/utils' import { _dirname } from './helpers' -import { isFile } from 'synckit' +import { + DEFAULT_GLOBAL_SHIMS_PRESET, + _generateGlobals, + encodeImportModule, + extractProperties, + generateGlobals, + isFile, +} from 'synckit' + +describe('utils', () => { + test('isFile', () => { + expect(isFile(_dirname)).toBe(false) + expect(isFile('non-existed')).toBe(false) + expect(isFile(fileURLToPath(import.meta.url))).toBe(true) + }) + + test('encodeImportModule', () => { + const moduleName = 'module-name' + const onlyModuleName = encodeImportModule(moduleName) + expect(onlyModuleName).toMatchSnapshot() + expect(encodeImportModule({ moduleName })).toBe(onlyModuleName) + expect( + encodeImportModule({ moduleName: './module-name' }), + ).toMatchSnapshot() + expect( + encodeImportModule({ + moduleName, + globalName: 'globalName', + }), + ).toMatchSnapshot() + expect( + encodeImportModule({ + moduleName, + globalName: 'globalName', + named: 'named', + }), + ).toMatchSnapshot() + expect( + encodeImportModule({ + moduleName, + globalName: 'globalName', + named: null, + }), + ).toMatchSnapshot() + }) + + test('generateGlobals', () => { + const _importGlobals = _generateGlobals( + DEFAULT_GLOBAL_SHIMS_PRESET, + 'import', + ) + expect(_importGlobals).toMatchSnapshot() + + const _requireGlobals = _generateGlobals( + DEFAULT_GLOBAL_SHIMS_PRESET, + 'require', + ) + expect(_requireGlobals).toMatchSnapshot() + + const tmpdir = String( + pathToFileURL(path.resolve(findUp(_dirname), '../node_modules/.synckit')), + ) + const importGlobals = generateGlobals( + 'fake.js', + DEFAULT_GLOBAL_SHIMS_PRESET, + ) + expect(importGlobals).not.toBe(_importGlobals) + expect(importGlobals).toMatch(tmpdir) + expect(generateGlobals('fake.js', DEFAULT_GLOBAL_SHIMS_PRESET)).toBe( + importGlobals, + ) + + const requireGlobals = generateGlobals( + 'fake.js', + DEFAULT_GLOBAL_SHIMS_PRESET, + 'require', + ) + expect(requireGlobals).toBe(_requireGlobals) + expect(requireGlobals).not.toBe(importGlobals) + expect( + generateGlobals('fake.js', DEFAULT_GLOBAL_SHIMS_PRESET, 'require'), + ).toBe(requireGlobals) + + expect( + _generateGlobals( + [ + { + ...DEFAULT_GLOBAL_SHIMS_PRESET[0], + conditional: false, + }, + ...DEFAULT_GLOBAL_SHIMS_PRESET.slice(1), + ], + 'import', + ), + ).toMatchSnapshot() + + expect( + _generateGlobals( + [ + { + ...DEFAULT_GLOBAL_SHIMS_PRESET[0], + conditional: false, + }, + ...DEFAULT_GLOBAL_SHIMS_PRESET.slice(1), + ], + 'require', + ), + ).toMatchSnapshot() + }) -test('utils', () => { - expect(isFile(_dirname)).toBe(false) - expect(isFile('non-existed')).toBe(false) - expect(isFile(fileURLToPath(import.meta.url))).toBe(true) + test('extractProperties', () => { + expect(extractProperties()).toBeUndefined() + expect(extractProperties({})).toEqual({}) + expect(extractProperties(new Error('message'))).toEqual({}) + expect( + extractProperties( + Object.assign(new Error('message'), { + code: 'CODE', + }), + ), + ).toEqual({ + code: 'CODE', + }) + }) })