From 4782821ed48b7dcd64813a23806236ce20fce275 Mon Sep 17 00:00:00 2001 From: Philip <17368112+vHeemstra@users.noreply.github.com> Date: Mon, 20 Nov 2023 20:09:12 +0100 Subject: [PATCH 01/11] Feat: better cache handling - Adds option `cacheDir` - Choose the directory to use for caching - Adds option `clearCache` - Clears the cache before processing - Cache dir by default is under the package's `node_modules` folder - Relative `cacheDir` path will be relative to Vite's `` dir - Absolute `cacheDir` path will be just that --- packages/core/src/index.ts | 34 +++++++++++++++++++++----- packages/core/src/typings.d.ts | 17 +++++++++++++ packages/core/src/utils.ts | 39 +++++++++++++++++++++++++++++- packages/playground/vite.config.ts | 2 ++ 4 files changed, 85 insertions(+), 7 deletions(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index da06f29..dfa052d 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -4,8 +4,9 @@ import { mkdirSync, lstatSync, readdirSync, - unlinkSync /*, rmSync */, + unlinkSync, copyFileSync, + rmSync, } from 'node:fs' import { readFile, writeFile } from 'node:fs/promises' import { Buffer } from 'node:buffer' @@ -24,6 +25,8 @@ import { isFilterPattern, escapeRegExp, smartEnsureDirs, + getPackageDirectory, + getPackageName, } from './utils' import type { PluginOption, ResolvedConfig } from 'vite' @@ -151,6 +154,8 @@ export const parseOptions = ( ? _options.skipIfLarger : true, cache: isBoolean(_options?.cache) ? _options.cache : true, + cacheDir: isString(_options?.cacheDir) ? _options.cacheDir : undefined, + clearCache: isBoolean(_options?.clearCache) ? _options.clearCache : false, plugins, makeAvif, makeWebp, @@ -808,11 +813,25 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { : path.resolve(process.cwd(), rootDir), ) + // Note: Only cacheDir has a trailing slash. + if (isString(options.cacheDir)) { + cacheDir = + normalizePath( + path.isAbsolute(options.cacheDir) + ? options.cacheDir + : path.resolve(rootDir, options.cacheDir), + ) + '/' + } else { + const packageDir = normalizePath(getPackageDirectory()) + cacheDir = `${packageDir}/node_modules/.cache/vite-plugin-imagemin/${getPackageName( + packageDir, + )}/` + } + // sourceDir = normalizePath(path.resolve(rootDir, entry)) outDir = normalizePath(path.resolve(rootDir, config.build.outDir)) assetsDir = normalizePath(path.resolve(outDir, config.build.assetsDir)) // publicDir = normalizePath(path.resolve(rootDir, config.publicDir)) - cacheDir = `${rootDir}/node_modules/.cache/vite-plugin-imagemin/` // const emptyOutDir = config.build.emptyOutDir || pathIsWithin(rootDir, outDir) @@ -825,10 +844,15 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { logger.info('') - // Create cache + // Init cache + if (options.clearCache) { + rmSync(cacheDir.slice(0, -1), { recursive: true, force: true }) + } if (options.cache !== false) { + mkdirSync(cacheDir.slice(0, -1), { recursive: true }) + cache = (await createCache({ - // noCache: options.cache === false, + cacheDirectory: cacheDir.slice(0, -1), mode: 'content', keys: [ () => { @@ -836,8 +860,6 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { }, ], })) as CacheInterface - - mkdirSync(cacheDir.slice(0, -1), { recursive: true }) } const processDir = onlyAssets ? assetsDir : outDir diff --git a/packages/core/src/typings.d.ts b/packages/core/src/typings.d.ts index e527c29..9a18a32 100644 --- a/packages/core/src/typings.d.ts +++ b/packages/core/src/typings.d.ts @@ -86,6 +86,21 @@ export interface ConfigOptions { */ cache?: boolean + /** + * Path of the directory to use for caching. + * Either: + * - relative path to Vite's root + * - absolute path + * @default /node_modules/.cache/vite-plugin-imagemin + */ + cacheDir?: string + + /** + * Force-clear the cache. + * @default false + */ + clearCache?: boolean + /** * Only use optimized contents if smaller than original. * @default true @@ -136,6 +151,8 @@ export interface ResolvedConfigOptions { verbose: boolean skipIfLarger: boolean cache: boolean + cacheDir?: string + clearCache: boolean plugins: ResolvedPluginsConfig makeAvif: false | ResolvedMakeConfigOptions makeWebp: false | ResolvedMakeConfigOptions diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts index 58a71c2..d6cc6f7 100644 --- a/packages/core/src/utils.ts +++ b/packages/core/src/utils.ts @@ -1,4 +1,6 @@ -import { existsSync, mkdirSync } from 'node:fs' +import { existsSync, mkdirSync, readFileSync, statSync } from 'node:fs' +import { dirname, join, parse, resolve } from 'node:path' +import { cwd } from 'node:process' import { FilterPattern } from 'vite' // import { ensureDirSync } from 'fs-extra' @@ -66,3 +68,38 @@ export function smartEnsureDirs(filePaths: string[], mode = 0o0755): string[] { return dir }) } + +export function getPackageDirectory() { + let filePath = '' + let directory = resolve(cwd()) + const { root } = parse(directory) + const stopAt = resolve(directory, root) + + while (directory && directory !== stopAt && directory !== root) { + filePath = join(directory, 'package.json') + + try { + const stats = statSync(filePath, { throwIfNoEntry: false }) + if (stats?.isFile()) { + break + } + } catch { + /* ignore */ + } + + directory = dirname(directory) + } + + return filePath && dirname(filePath) +} + +export const getPackageName = (pkgPath: string) => { + const pkgFile = join(pkgPath, 'package.json') + try { + const pkg = readFileSync(pkgFile, 'utf8') + const pkgJson = JSON.parse(pkg) + return pkgJson.name + } catch { + return 'file-cache' + } +} diff --git a/packages/playground/vite.config.ts b/packages/playground/vite.config.ts index 3eddf1d..fed26c7 100644 --- a/packages/playground/vite.config.ts +++ b/packages/playground/vite.config.ts @@ -52,6 +52,8 @@ export default defineConfig({ // skipIfLargerThan: 'smallest', }, // cache: false, + // clearCache: true, + // cacheDir: 'cache', }), ], }) From 282eed45db37537c7513a08268172720e5e33084 Mon Sep 17 00:00:00 2001 From: Philip <17368112+vHeemstra@users.noreply.github.com> Date: Mon, 20 Nov 2023 20:23:37 +0100 Subject: [PATCH 02/11] Update README.md --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 94e052e..ef1b86e 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ - Can create Avif versions of supported images (jpg/png). - Skips optimized version if output is larger than original. - Skips Avif/WebP version if output is larger than original, optimized version or smallest version of an image. -- Uses cache to skip processing of unchanged files +- Uses optional cache by default to skip processing of unchanged files ## Install @@ -116,6 +116,23 @@ Default: `true` > Skip optimizing the input if it did not change since the last run. +### cacheDir + +Type: `string`
+Default: `/node_modules/.cache/vite-plugin-imagemin//` + +> Choose the directory to use for caching. +> - Relative paths will be relative to the Vite project's **root** directory. +> - Absolute paths will be use as-is. +> - Absent/non-string value will use the default location. + +### clearCache + +Type: `boolean`
+Default: `false` + +> Clears the cache folder before processing. + ### makeAvif / makeWebp Type: `object`
From b38ea058f996509250637fb74730725ccb043e78 Mon Sep 17 00:00:00 2001 From: Philip <17368112+vHeemstra@users.noreply.github.com> Date: Mon, 20 Nov 2023 20:42:21 +0100 Subject: [PATCH 03/11] Added todos --- packages/core/src/index.test.ts | 2 +- packages/core/src/utils.test.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/core/src/index.test.ts b/packages/core/src/index.test.ts index 67c59cb..649524e 100644 --- a/packages/core/src/index.test.ts +++ b/packages/core/src/index.test.ts @@ -1368,7 +1368,7 @@ describe('logErrors', () => { * dist/images/opaque-1.png.avif */ -// TODO: add tests for usage with cache +// TODO: add tests for cache usage and its options // TODO: expand after-build checks describe.skipIf(skipBuilds)('viteImagemin', () => { diff --git a/packages/core/src/utils.test.ts b/packages/core/src/utils.test.ts index 3b879b5..801fa66 100644 --- a/packages/core/src/utils.test.ts +++ b/packages/core/src/utils.test.ts @@ -213,3 +213,5 @@ describe('smartEnsureDirs', () => { }) }) }) + +// TODO: add tests for getPackageDirectory and getPackageName From f6e99694227ac3217410d90eb663ec9b9fb3262e Mon Sep 17 00:00:00 2001 From: Martijn Swart Date: Tue, 21 Nov 2023 12:11:00 +0100 Subject: [PATCH 04/11] Update: Seperated cache logic in own module --- packages/core/src/cache.ts | 127 ++++++++++++++++++++++++++++++ packages/core/src/index.ts | 154 +++++++++---------------------------- 2 files changed, 164 insertions(+), 117 deletions(-) create mode 100644 packages/core/src/cache.ts diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts new file mode 100644 index 0000000..8cce585 --- /dev/null +++ b/packages/core/src/cache.ts @@ -0,0 +1,127 @@ +import { createCache } from '@file-cache/core' +import { copyFileSync, existsSync, mkdirSync, rmSync } from 'node:fs' +import path from 'node:path' +import { normalizePath } from 'vite' + +import { + getPackageDirectory, + getPackageName, + isString, + smartEnsureDirs, +} from './utils' + +import type { CacheInterface } from '@file-cache/core/mjs/CacheInterface' +import type { ResolvedConfigOptions, StackItem } from './typings' + +let cacheEnabled = false +let cache: CacheInterface +let cacheDir = '' + +function initCacheDir(rootDir: string, _cacheDir?: string): void { + // Note: Only cacheDir has a trailing slash. + if (isString(_cacheDir)) { + cacheDir = + normalizePath( + path.isAbsolute(_cacheDir) + ? _cacheDir + : path.resolve(rootDir, _cacheDir), + ) + '/' + } else { + const packageDir = normalizePath(getPackageDirectory()) + cacheDir = `${packageDir}/node_modules/.cache/vite-plugin-imagemin/${getPackageName( + packageDir, + )}/` + } + + mkdirSync(cacheDir.slice(0, -1), { recursive: true }) +} + +export const FileCache = { + init: async (options: ResolvedConfigOptions, rootDir: string) => { + cacheEnabled = options.cache !== false + + initCacheDir(rootDir, options.cacheDir) + + if (options.clearCache) { + FileCache.clear() + } + + cache = (await createCache({ + noCache: !cacheEnabled, + cacheDirectory: cacheDir.slice(0, -1), + mode: 'content', + keys: [ + () => { + return JSON.stringify(options) + }, + ], + })) as CacheInterface + }, + + prepareDirs: (filePaths: string[]): void => { + if (cacheEnabled) { + smartEnsureDirs(filePaths.map(file => cacheDir + file)) + } + }, + + check: async ( + baseDir: string, + filePathFrom: string, + fileToStack: StackItem[] = [], + ) => { + const inputFileCache = await cache?.getAndUpdateCache( + baseDir + filePathFrom, + ) + + // Check if input file has changed or there was an error + if (inputFileCache.changed || inputFileCache.error) { + return false + } + + // Check if output files are in cache and use them if they haven't changed + const outputFilesExist = await Promise.allSettled( + fileToStack.map( + item => + new Promise((resolve, reject) => + cache + .getAndUpdateCache(cacheDir + item.toPath) + .then(outputFileCache => { + if (!outputFileCache.error && !outputFileCache.changed) { + copyFileSync(cacheDir + item.toPath, baseDir + item.toPath) + if (existsSync(baseDir + item.toPath)) { + resolve(true) + } + } + reject( + outputFileCache.error + ? `Error while checking cache [${outputFileCache.error.message}]` + : 'Could not copy cached files', + ) + }) + .catch(reject), + ), + ), + ) + + return outputFilesExist.every(p => p.status === 'fulfilled') + }, + + update: async (baseDir: string, filePathTo: string) => { + if (cacheEnabled) { + copyFileSync(baseDir + filePathTo, cacheDir + filePathTo) + await cache.getAndUpdateCache(cacheDir + filePathTo) + } + }, + + reconcile: async () => { + await cache?.reconcile() + }, + + clear: () => { + if (!cache || !cacheDir) { + return + } + + rmSync(cacheDir.slice(0, -1), { recursive: true, force: true }) + }, +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index dfa052d..0c0cb2a 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,51 +1,41 @@ -import path from 'node:path' -import { - existsSync, - mkdirSync, - lstatSync, - readdirSync, - unlinkSync, - copyFileSync, - rmSync, -} from 'node:fs' -import { readFile, writeFile } from 'node:fs/promises' -import { Buffer } from 'node:buffer' -import { performance } from 'node:perf_hooks' import chalk from 'chalk' -import { normalizePath, createFilter } from 'vite' import imagemin from 'imagemin' import isAPNG from 'is-apng' -import { createCache } from '@file-cache/core' +import { Buffer } from 'node:buffer' +import { lstatSync, readdirSync, unlinkSync } from 'node:fs' +import { readFile, writeFile } from 'node:fs/promises' +import path from 'node:path' +import { performance } from 'node:perf_hooks' +import { createFilter, normalizePath } from 'vite' + +import { FileCache } from './cache' import { - isFunction, + escapeRegExp, isBoolean, - isString, - isObject, isFilterPattern, - escapeRegExp, + isFunction, + isObject, + isString, smartEnsureDirs, - getPackageDirectory, - getPackageName, } from './utils' import type { PluginOption, ResolvedConfig } from 'vite' -import type { CacheInterface } from '@file-cache/core/mjs/CacheInterface' import type { ConfigOptions, - ResolvedConfigOptions, - ResolvedPluginsConfig, - ResolvedMakeConfigOptions, - PluginsConfig, + ErroredFile, Logger, - Stack, + PluginsConfig, ProcessFileParams, + ProcessFileReturn, + ProcessResult, + ProcessResultWhenOutput, ProcessedFile, - ErroredFile, ProcessedResults, - ProcessResultWhenOutput, - ProcessResult, - ProcessFileReturn, + ResolvedConfigOptions, + ResolvedMakeConfigOptions, + ResolvedPluginsConfig, + Stack, } from './typings' // export const pathIsWithin = (parentPath: string, childPath: string) => { @@ -204,8 +194,6 @@ export async function processFile({ precisions, bytesDivider, sizeUnit, - cacheDir = '', - cache = null, }: ProcessFileParams): ProcessFileReturn { // const start = performance.now() @@ -227,45 +215,19 @@ export async function processFile({ }) as Promise } - if (cache) { - // Check if input file hasn't changed - const inputFileCache = await cache.getAndUpdateCache(baseDir + filePathFrom) - - if (!inputFileCache.error && !inputFileCache.changed) { - // Check if output files are in cache and use them if they haven't changed - const outputFilesExist = await Promise.allSettled( - fileToStack.map( - item => - new Promise((resolve, reject) => - cache - .getAndUpdateCache(cacheDir + item.toPath) - .then(outputFileCache => { - if (!outputFileCache.error && !outputFileCache.changed) { - copyFileSync(cacheDir + item.toPath, baseDir + item.toPath) - if (existsSync(baseDir + item.toPath)) { - resolve(true) - } - } - reject( - outputFileCache.error - ? `Error while checking cache [${outputFileCache.error.message}]` - : 'Could not copy cached files', - ) - }) - .catch(reject), - ), - ), - ) + const hasValidCache = await FileCache.check( + baseDir, + filePathFrom, + fileToStack, + ) - if (outputFilesExist.every(p => p.status === 'fulfilled')) { - return Promise.reject({ - oldPath: filePathFrom, - newPath: '', - error: '', - errorType: 'cache', - }) as Promise - } - } + if (hasValidCache) { + return Promise.reject({ + oldPath: filePathFrom, + newPath: '', + error: '', + errorType: 'cache', + }) as Promise } let oldBuffer: Buffer @@ -366,10 +328,7 @@ export async function processFile({ }) .then(async () => { // Add to/update in cache - if (cache) { - copyFileSync(baseDir + filePathTo, cacheDir + filePathTo) - await cache.getAndUpdateCache(cacheDir + filePathTo) - } + await FileCache.update(baseDir, filePathTo) const duration = performance.now() - start const ratio = newSize / oldSize - 1 @@ -796,9 +755,6 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { let hadFilesToProcess = false // const mtimeCache = new Map() - let cache: CacheInterface - let cacheDir = '' - return { name: 'vite-plugin-imagemin', enforce: 'post', @@ -813,21 +769,6 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { : path.resolve(process.cwd(), rootDir), ) - // Note: Only cacheDir has a trailing slash. - if (isString(options.cacheDir)) { - cacheDir = - normalizePath( - path.isAbsolute(options.cacheDir) - ? options.cacheDir - : path.resolve(rootDir, options.cacheDir), - ) + '/' - } else { - const packageDir = normalizePath(getPackageDirectory()) - cacheDir = `${packageDir}/node_modules/.cache/vite-plugin-imagemin/${getPackageName( - packageDir, - )}/` - } - // sourceDir = normalizePath(path.resolve(rootDir, entry)) outDir = normalizePath(path.resolve(rootDir, config.build.outDir)) assetsDir = normalizePath(path.resolve(outDir, config.build.assetsDir)) @@ -845,22 +786,7 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { logger.info('') // Init cache - if (options.clearCache) { - rmSync(cacheDir.slice(0, -1), { recursive: true, force: true }) - } - if (options.cache !== false) { - mkdirSync(cacheDir.slice(0, -1), { recursive: true }) - - cache = (await createCache({ - cacheDirectory: cacheDir.slice(0, -1), - mode: 'content', - keys: [ - () => { - return JSON.stringify(options) - }, - ], - })) as CacheInterface - } + await FileCache.init(options, rootDir) const processDir = onlyAssets ? assetsDir : outDir const baseDir = `${rootDir}/` @@ -954,9 +880,7 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { // Ensure all destination (sub)directories are present smartEnsureDirs(toPaths.map(file => baseDir + file)) - if (options.cache !== false) { - smartEnsureDirs(toPaths.map(file => cacheDir + file)) - } + FileCache.prepareDirs(toPaths) // Process stack const { @@ -975,8 +899,6 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { precisions, bytesDivider, sizeUnit, - cacheDir, - cache, }), ), ) as Promise @@ -1091,9 +1013,7 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { }) // Write cache state to file for persistence - if (options.cache !== false) { - await cache.reconcile() - } + FileCache.reconcile() Object.keys(erroredFiles) .sort((a, b) => a.localeCompare(b)) // TODO: sort by (sub)folder and depth? From 2281f41889d1ec4136607f6dc71571870160d5af Mon Sep 17 00:00:00 2001 From: Martijn Swart Date: Tue, 21 Nov 2023 13:31:39 +0100 Subject: [PATCH 05/11] Update: removed all external modules --- packages/core/package.json | 3 +- packages/core/src/cache.ts | 117 ++++++++++++++++++++++++++++--------- 2 files changed, 89 insertions(+), 31 deletions(-) diff --git a/packages/core/package.json b/packages/core/package.json index cfbe326..a46557e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -61,7 +61,6 @@ "node": ">=16.0.0" }, "dependencies": { - "@file-cache/core": "^1.1.4", "chalk": "^5.2.0", "fast-glob": "^3.2.12", "fs-extra": "^11.1.1", @@ -79,4 +78,4 @@ "peerDependencies": { "vite": "^3.0.0 || ^4.0.0 || ^4.3.9" } -} +} \ No newline at end of file diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts index 8cce585..0e5799e 100644 --- a/packages/core/src/cache.ts +++ b/packages/core/src/cache.ts @@ -1,7 +1,9 @@ -import { createCache } from '@file-cache/core' -import { copyFileSync, existsSync, mkdirSync, rmSync } from 'node:fs' +import { copyFileSync, existsSync, rmSync } from 'node:fs' +import { readFile, writeFile, mkdir, copyFile } from 'node:fs/promises' + import path from 'node:path' import { normalizePath } from 'vite' +import crypto, { BinaryLike } from 'crypto' import { getPackageDirectory, @@ -10,14 +12,24 @@ import { smartEnsureDirs, } from './utils' -import type { CacheInterface } from '@file-cache/core/mjs/CacheInterface' import type { ResolvedConfigOptions, StackItem } from './typings' +type CacheContent = { + hash: string +} +type CacheMetaData = { + size: number + mtime: number +} +type CacheValue = CacheContent | CacheMetaData + let cacheEnabled = false -let cache: CacheInterface let cacheDir = '' +let cacheFile = '' +let fileCacheMap = new Map() +let entryMap = new Map() -function initCacheDir(rootDir: string, _cacheDir?: string): void { +async function initCacheDir(rootDir: string, _cacheDir?: string) { // Note: Only cacheDir has a trailing slash. if (isString(_cacheDir)) { cacheDir = @@ -33,29 +45,61 @@ function initCacheDir(rootDir: string, _cacheDir?: string): void { )}/` } - mkdirSync(cacheDir.slice(0, -1), { recursive: true }) + await mkdir(cacheDir.slice(0, -1), { recursive: true }) +} + +async function initCacheMaps() { + cacheFile = path.join(cacheDir, 'contents') + + try { + const json = JSON.parse(await readFile(cacheFile, 'utf-8')) + entryMap = new Map(Object.entries(json)) + } catch { + entryMap = new Map() + } + + fileCacheMap = new Map(entryMap) +} + +function md5(buffer: BinaryLike): string { + return crypto.createHash('md5').update(buffer).digest('hex') +} + +async function getAndUpdateCacheContent(filePath: string | URL) { + try { + const hash = md5(await readFile(filePath)) + const normalizedFilePath = filePath.toString() + const cacheValue = fileCacheMap.get(normalizedFilePath) as + | CacheContent + | undefined + if (cacheValue && cacheValue.hash === hash) { + return { + changed: false, + } + } + entryMap.set(normalizedFilePath, { hash }) + return { + changed: true, + } + } catch (error) { + return { + changed: false, + error: error as Error, + } + } } export const FileCache = { init: async (options: ResolvedConfigOptions, rootDir: string) => { cacheEnabled = options.cache !== false - initCacheDir(rootDir, options.cacheDir) + await initCacheDir(rootDir, options.cacheDir) if (options.clearCache) { FileCache.clear() } - cache = (await createCache({ - noCache: !cacheEnabled, - cacheDirectory: cacheDir.slice(0, -1), - mode: 'content', - keys: [ - () => { - return JSON.stringify(options) - }, - ], - })) as CacheInterface + await initCacheMaps() }, prepareDirs: (filePaths: string[]): void => { @@ -69,12 +113,12 @@ export const FileCache = { filePathFrom: string, fileToStack: StackItem[] = [], ) => { - const inputFileCache = await cache?.getAndUpdateCache( + const { changed, error } = await getAndUpdateCacheContent( baseDir + filePathFrom, ) // Check if input file has changed or there was an error - if (inputFileCache.changed || inputFileCache.error) { + if (changed || error) { return false } @@ -83,8 +127,7 @@ export const FileCache = { fileToStack.map( item => new Promise((resolve, reject) => - cache - .getAndUpdateCache(cacheDir + item.toPath) + getAndUpdateCacheContent(cacheDir + item.toPath) .then(outputFileCache => { if (!outputFileCache.error && !outputFileCache.changed) { copyFileSync(cacheDir + item.toPath, baseDir + item.toPath) @@ -107,21 +150,37 @@ export const FileCache = { }, update: async (baseDir: string, filePathTo: string) => { - if (cacheEnabled) { - copyFileSync(baseDir + filePathTo, cacheDir + filePathTo) - await cache.getAndUpdateCache(cacheDir + filePathTo) + if (!cacheEnabled) { + return } + + await copyFile(baseDir + filePathTo, cacheDir + filePathTo) + await getAndUpdateCacheContent(cacheDir + filePathTo) }, reconcile: async () => { - await cache?.reconcile() + if (!cacheEnabled) { + return true + } + + try { + await writeFile( + cacheFile, + JSON.stringify(Object.fromEntries(entryMap)), + 'utf-8', + ) + // reflect the changes in the cacheMap + fileCacheMap = new Map(entryMap) + return true + } catch (error) { + // console.error('Cache reconcile has failed', error) + return false + } }, clear: () => { - if (!cache || !cacheDir) { - return + if (cacheFile && cacheDir) { + rmSync(cacheDir.slice(0, -1), { recursive: true, force: true }) } - - rmSync(cacheDir.slice(0, -1), { recursive: true, force: true }) }, } From b3f934c98fbe9701738273dfd65c8d041ffb5dff Mon Sep 17 00:00:00 2001 From: Martijn Swart Date: Tue, 21 Nov 2023 13:37:36 +0100 Subject: [PATCH 06/11] Update: cleanup cache --- packages/core/src/cache.ts | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts index 0e5799e..c687708 100644 --- a/packages/core/src/cache.ts +++ b/packages/core/src/cache.ts @@ -1,9 +1,8 @@ -import { copyFileSync, existsSync, rmSync } from 'node:fs' -import { readFile, writeFile, mkdir, copyFile } from 'node:fs/promises' - +import crypto, { BinaryLike } from 'crypto' +import { copyFileSync, existsSync } from 'node:fs' +import { copyFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises' import path from 'node:path' import { normalizePath } from 'vite' -import crypto, { BinaryLike } from 'crypto' import { getPackageDirectory, @@ -14,14 +13,9 @@ import { import type { ResolvedConfigOptions, StackItem } from './typings' -type CacheContent = { +type CacheValue = { hash: string } -type CacheMetaData = { - size: number - mtime: number -} -type CacheValue = CacheContent | CacheMetaData let cacheEnabled = false let cacheDir = '' @@ -70,7 +64,7 @@ async function getAndUpdateCacheContent(filePath: string | URL) { const hash = md5(await readFile(filePath)) const normalizedFilePath = filePath.toString() const cacheValue = fileCacheMap.get(normalizedFilePath) as - | CacheContent + | CacheValue | undefined if (cacheValue && cacheValue.hash === hash) { return { @@ -95,8 +89,9 @@ export const FileCache = { await initCacheDir(rootDir, options.cacheDir) - if (options.clearCache) { - FileCache.clear() + // clear cache? + if (options.clearCache && cacheDir) { + await rm(cacheDir.slice(0, -1), { recursive: true, force: true }) } await initCacheMaps() @@ -177,10 +172,4 @@ export const FileCache = { return false } }, - - clear: () => { - if (cacheFile && cacheDir) { - rmSync(cacheDir.slice(0, -1), { recursive: true, force: true }) - } - }, } From e2015fc709726b8bbb6a51cfe2b85d524a13aa2d Mon Sep 17 00:00:00 2001 From: Philip <17368112+vHeemstra@users.noreply.github.com> Date: Wed, 22 Nov 2023 04:16:09 +0100 Subject: [PATCH 07/11] Cache handling improvements * Adds logging stats for files taken from cache (will be marked blue) * Adds cache key identifier (based on the used options) to distinguish caches. * Adds support for partial caches. Only missing and changed files will process. * Some cache logic adjustments * Some code refactoring/cleanup * Updated type definitions * Updated docs --- README.md | 58 +++++---- packages/core/src/cache.ts | 214 ++++++++++++++++++++------------ packages/core/src/index.ts | 152 +++++++++++++++-------- packages/core/src/typings.d.ts | 30 +++++ packages/core/src/utils.test.ts | 1 + 5 files changed, 304 insertions(+), 151 deletions(-) diff --git a/README.md b/README.md index ef1b86e..dc29046 100644 --- a/README.md +++ b/README.md @@ -109,30 +109,6 @@ Default: `true` > Ignore the optimized output if it is larger than the original file. -### cache - -Type: `boolean`
-Default: `true` - -> Skip optimizing the input if it did not change since the last run. - -### cacheDir - -Type: `string`
-Default: `/node_modules/.cache/vite-plugin-imagemin//` - -> Choose the directory to use for caching. -> - Relative paths will be relative to the Vite project's **root** directory. -> - Absolute paths will be use as-is. -> - Absent/non-string value will use the default location. - -### clearCache - -Type: `boolean`
-Default: `false` - -> Clears the cache folder before processing. - ### makeAvif / makeWebp Type: `object`
@@ -164,6 +140,40 @@ Default: `'optimized'` > - it is not the smallest version of the image (`'smallest'`) > - never skip (`false`) +### cache + +Type: `boolean`
+Default: `true` + +> Skip optimizing the input if it did not change since the last run. + +### cacheDir + +Type: `string`
+Default: `/node_modules/.cache/vite-plugin-imagemin//` + +> Choose the directory to use for caching. +> - Relative paths will be relative to the Vite project's **root** directory. +> - Absolute paths will be use as-is. +> - Absent/non-string value will use the default location. + +### cacheKey + +Type: `string`
+Default: `` + +> Optional string to distinguish build configs and their caches. +> +> The cache identifier is a hash of most used options including this key. +> Note: Because options like `formatFilePath` and `plugins` cannot be stringified, they will have little or no influence on the hash key and its uniqueness. + +### clearCache + +Type: `boolean`
+Default: `false` + +> Clears the cache folder before processing. + ### root Type: `string`
diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts index c687708..66e03d2 100644 --- a/packages/core/src/cache.ts +++ b/packages/core/src/cache.ts @@ -1,7 +1,7 @@ -import crypto, { BinaryLike } from 'crypto' -import { copyFileSync, existsSync } from 'node:fs' +import crypto, { BinaryLike } from 'node:crypto' +import { existsSync } from 'node:fs' import { copyFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises' -import path from 'node:path' +import { isAbsolute, resolve } from 'node:path' import { normalizePath } from 'vite' import { @@ -11,39 +11,71 @@ import { smartEnsureDirs, } from './utils' -import type { ResolvedConfigOptions, StackItem } from './typings' - -type CacheValue = { - hash: string -} +import type { CacheValue, ResolvedConfigOptions, StackItem } from './typings' let cacheEnabled = false let cacheDir = '' +let cacheKey = '' let cacheFile = '' let fileCacheMap = new Map() let entryMap = new Map() -async function initCacheDir(rootDir: string, _cacheDir?: string) { - // Note: Only cacheDir has a trailing slash. - if (isString(_cacheDir)) { - cacheDir = - normalizePath( - path.isAbsolute(_cacheDir) - ? _cacheDir - : path.resolve(rootDir, _cacheDir), - ) + '/' +function md5(buffer: BinaryLike): string { + return crypto.createHash('md5').update(buffer).digest('hex') +} + +export function createCacheKey(options: ResolvedConfigOptions) { + return md5( + Object.entries(options) + .filter( + ([k]) => + // Ignore these options, since they don't influence the files' content. + ![ + 'cache', + 'clearCache', + 'logByteDivider', + 'logger', + 'verbose', + ].includes(k), + ) + .sort(([ka], [kb]) => ka.localeCompare(kb)) + .map(([k, v], i) => `${i}_${k}_${v}`) + .join('|'), + ) +} + +async function initCacheDir(rootDir: string, options: ResolvedConfigOptions) { + cacheKey = createCacheKey(options) + + const packageDir = normalizePath(getPackageDirectory()) + const packageName = getPackageName(packageDir) + + if (isString(options.cacheDir)) { + cacheDir = normalizePath( + isAbsolute(options.cacheDir) + ? options.cacheDir + : resolve(rootDir, options.cacheDir), + ) } else { - const packageDir = normalizePath(getPackageDirectory()) - cacheDir = `${packageDir}/node_modules/.cache/vite-plugin-imagemin/${getPackageName( - packageDir, - )}/` + cacheDir = `${packageDir}/node_modules/.cache/vite-plugin-imagemin` + } + + // cacheDir = cacheDir + `/${packageName}/${cacheKey}/` + cacheDir = cacheDir + `/${packageName}/` + + if (options.clearCache) { + await rm(cacheDir.slice(0, -1), { recursive: true, force: true }) + } + + if (!cacheEnabled) { + return } await mkdir(cacheDir.slice(0, -1), { recursive: true }) } async function initCacheMaps() { - cacheFile = path.join(cacheDir, 'contents') + cacheFile = `${cacheDir}/contents-${cacheKey}.json` try { const json = JSON.parse(await readFile(cacheFile, 'utf-8')) @@ -55,102 +87,129 @@ async function initCacheMaps() { fileCacheMap = new Map(entryMap) } -function md5(buffer: BinaryLike): string { - return crypto.createHash('md5').update(buffer).digest('hex') -} - -async function getAndUpdateCacheContent(filePath: string | URL) { +async function getAndUpdateCacheContent( + filePath: string, + stats?: Omit, +): Promise<{ + changed?: boolean + value?: CacheValue + error?: Error +}> { + let hash = '' try { - const hash = md5(await readFile(filePath)) - const normalizedFilePath = filePath.toString() - const cacheValue = fileCacheMap.get(normalizedFilePath) as - | CacheValue - | undefined - if (cacheValue && cacheValue.hash === hash) { - return { - changed: false, - } - } - entryMap.set(normalizedFilePath, { hash }) + hash = md5(await readFile(filePath)) + } catch (error) { return { - changed: true, + error: error as Error, } - } catch (error) { + } + + const cacheValue = fileCacheMap.get(filePath) + if (cacheValue && cacheValue.hash === hash) { return { changed: false, - error: error as Error, + value: cacheValue, } } + + entryMap.set(filePath, { + hash, + oldSize: stats?.oldSize ?? 0, + newSize: stats?.newSize ?? 0, + }) + + return { + changed: true, + } } export const FileCache = { init: async (options: ResolvedConfigOptions, rootDir: string) => { cacheEnabled = options.cache !== false - await initCacheDir(rootDir, options.cacheDir) + await initCacheDir(rootDir, options) - // clear cache? - if (options.clearCache && cacheDir) { - await rm(cacheDir.slice(0, -1), { recursive: true, force: true }) + if (!cacheEnabled) { + return } await initCacheMaps() }, prepareDirs: (filePaths: string[]): void => { - if (cacheEnabled) { - smartEnsureDirs(filePaths.map(file => cacheDir + file)) + if (!cacheEnabled) { + return } + + smartEnsureDirs(filePaths.map(file => cacheDir + file)) }, - check: async ( + checkAndCopy: async ( baseDir: string, filePathFrom: string, fileToStack: StackItem[] = [], - ) => { - const { changed, error } = await getAndUpdateCacheContent( + ): Promise<[boolean, (string | CacheValue)[]]> => { + if (!cacheEnabled) { + return [false, []] + } + + const inputCacheStatus = await getAndUpdateCacheContent( baseDir + filePathFrom, ) // Check if input file has changed or there was an error - if (changed || error) { - return false + if (inputCacheStatus?.error || inputCacheStatus?.changed) { + return [false, []] } // Check if output files are in cache and use them if they haven't changed const outputFilesExist = await Promise.allSettled( - fileToStack.map( - item => - new Promise((resolve, reject) => - getAndUpdateCacheContent(cacheDir + item.toPath) - .then(outputFileCache => { - if (!outputFileCache.error && !outputFileCache.changed) { - copyFileSync(cacheDir + item.toPath, baseDir + item.toPath) - if (existsSync(baseDir + item.toPath)) { - resolve(true) - } - } - reject( - outputFileCache.error - ? `Error while checking cache [${outputFileCache.error.message}]` - : 'Could not copy cached files', - ) - }) - .catch(reject), - ), - ), + fileToStack.map(async item => { + const outputCacheStatus = await getAndUpdateCacheContent( + cacheDir + item.toPath, + ) + + if (outputCacheStatus?.error) { + return `Cache error [${outputCacheStatus.error.message}]` + } + + if (outputCacheStatus?.changed) { + return 'File changed' + } + + try { + await copyFile(cacheDir + item.toPath, baseDir + item.toPath) + } catch (error) { + return `Could not copy cached file [${(error as Error).message}]` + } + + if (!existsSync(baseDir + item.toPath)) { + return 'Could not use cached file' + } + + return outputCacheStatus.value as CacheValue + }), ) - return outputFilesExist.every(p => p.status === 'fulfilled') + return [ + true, + outputFilesExist.map(p => + p.status === 'fulfilled' ? p.value : p.reason, + ), + ] }, - update: async (baseDir: string, filePathTo: string) => { + update: async ( + baseDir: string, + filePathTo: string, + stats: Omit, + ) => { if (!cacheEnabled) { return } await copyFile(baseDir + filePathTo, cacheDir + filePathTo) - await getAndUpdateCacheContent(cacheDir + filePathTo) + await getAndUpdateCacheContent(cacheDir + filePathTo, stats) }, reconcile: async () => { @@ -164,11 +223,12 @@ export const FileCache = { JSON.stringify(Object.fromEntries(entryMap)), 'utf-8', ) - // reflect the changes in the cacheMap + fileCacheMap = new Map(entryMap) + return true } catch (error) { - // console.error('Cache reconcile has failed', error) + // console.error('Cache reconcile has failed', error) return false } }, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 0c0cb2a..265c34e 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -9,7 +9,6 @@ import { performance } from 'node:perf_hooks' import { createFilter, normalizePath } from 'vite' import { FileCache } from './cache' - import { escapeRegExp, isBoolean, @@ -24,12 +23,12 @@ import type { PluginOption, ResolvedConfig } from 'vite' import type { ConfigOptions, ErroredFile, + FormatProcessedFileParams, Logger, PluginsConfig, ProcessFileParams, ProcessFileReturn, ProcessResult, - ProcessResultWhenOutput, ProcessedFile, ProcessedResults, ResolvedConfigOptions, @@ -145,6 +144,7 @@ export const parseOptions = ( : true, cache: isBoolean(_options?.cache) ? _options.cache : true, cacheDir: isString(_options?.cacheDir) ? _options.cacheDir : undefined, + cacheKey: isString(_options?.cacheDir) ? _options.cacheDir : '', clearCache: isBoolean(_options?.clearCache) ? _options.clearCache : false, plugins, makeAvif, @@ -187,6 +187,43 @@ export function getAllFiles(dir: string, logger: Logger): string[] { return files } +export function formatProcessedFile({ + oldPath, + newPath, + oldSize, + newSize, + duration, + fromCache, + precisions, + bytesDivider, + sizeUnit, +}: FormatProcessedFileParams) { + const ratio = newSize / oldSize - 1 + + return { + oldPath, + newPath, + oldSize, + newSize, + ratio, + duration, + oldSizeString: `${(oldSize / bytesDivider).toFixed( + precisions.size, + )} ${sizeUnit}`, + newSizeString: `${(newSize / bytesDivider).toFixed( + precisions.size, + )} ${sizeUnit}`, + ratioString: `${ratio > 0 ? '+' : ratio === 0 ? ' ' : ''}${( + ratio * 100 + ).toFixed(precisions.ratio)} %`, + durationString: `${duration.toFixed(precisions.duration)} ms`, + fromCache, + optimizedDeleted: false as const, + avifDeleted: false as const, + webpDeleted: false as const, + } +} + export async function processFile({ filePathFrom, fileToStack = [], @@ -215,19 +252,41 @@ export async function processFile({ }) as Promise } - const hasValidCache = await FileCache.check( + const [inputCacheStatus, outputCacheStatus] = await FileCache.checkAndCopy( baseDir, filePathFrom, fileToStack, ) - if (hasValidCache) { - return Promise.reject({ - oldPath: filePathFrom, - newPath: '', - error: '', - errorType: 'cache', - }) as Promise + let cacheStack: (false | Promise)[] = [] + + if (inputCacheStatus) { + let hasFullValidCache = true + + cacheStack = outputCacheStatus.map((status, i) => { + if (isString(status)) { + hasFullValidCache = false + return false + } + + return Promise.resolve( + formatProcessedFile({ + oldPath: filePathFrom, + newPath: fileToStack[i].toPath, + oldSize: status.oldSize, + newSize: status.newSize, + duration: 0, + fromCache: true, + precisions, + bytesDivider, + sizeUnit, + }), + ) + }) + + if (hasFullValidCache) { + return Promise.allSettled(cacheStack as Promise[]) + } } let oldBuffer: Buffer @@ -283,12 +342,16 @@ export async function processFile({ } return Promise.allSettled( - fileToStack.map(item => { + fileToStack.map((item, i) => { let newBuffer: Buffer let newSize = 0 const filePathTo = item.toPath + if (cacheStack[i]) { + return cacheStack[i] as Promise + } + return imagemin .buffer(oldBuffer, { plugins: item.plugins }) .catch(e => @@ -327,29 +390,24 @@ export async function processFile({ return Promise.reject(e) }) .then(async () => { - // Add to/update in cache - await FileCache.update(baseDir, filePathTo) - - const duration = performance.now() - start - const ratio = newSize / oldSize - 1 - return Promise.resolve({ - oldPath: filePathFrom, - newPath: filePathTo, + await FileCache.update(baseDir, filePathTo, { oldSize, newSize, - ratio, - duration, - oldSizeString: `${(oldSize / bytesDivider).toFixed( - precisions.size, - )} ${sizeUnit}`, - newSizeString: `${(newSize / bytesDivider).toFixed( - precisions.size, - )} ${sizeUnit}`, - ratioString: `${ratio > 0 ? '+' : ratio === 0 ? ' ' : ''}${( - ratio * 100 - ).toFixed(precisions.ratio)} %`, - durationString: `${duration.toFixed(precisions.duration)} ms`, }) + + return Promise.resolve( + formatProcessedFile({ + oldPath: filePathFrom, + newPath: filePathTo, + oldSize, + newSize, + duration: performance.now() - start, + fromCache: false, + precisions, + bytesDivider, + sizeUnit, + }), + ) }) .catch(error => { let errorType = 'error' @@ -369,26 +427,26 @@ export async function processFile({ }) }) }), - ) as Promise + ) }) .catch(e => { let errorType = 'error' - // if (e?.message) { - if (e.message.startsWith('SKIP: ')) { - e = e.message.slice(6) + let error = (e as Error).message + + if (error.startsWith('SKIP: ')) { + error = error.slice(6) errorType = 'skip' - // } else if (e.message.startsWith('WARN: ')) { - // e = e.message.slice(6) + // } else if (error.startsWith('WARN: ')) { + // error = error.slice(6) // errorType = 'warning' } else { - e = `Error reading file [${e.message}]` + error = `Error reading file [${error}]` } - // } return Promise.reject({ oldPath: filePathFrom, newPath: '', - error: e, + error, errorType, }) as Promise }) @@ -529,7 +587,7 @@ export function logResults( // Skipped file logArray.push( // Filename - chalk.dim(basenameTo), + file.fromCache ? chalk.blue.dim(basenameTo) : chalk.dim(basenameTo), ' '.repeat( maxPathLength - bulletLength - file.newPath.length + spaceLength, ), @@ -557,7 +615,9 @@ export function logResults( logArray.push( // Filename file.ratio < 0 - ? chalk.green(basenameTo) + ? file.fromCache + ? chalk.blue(basenameTo) + : chalk.green(basenameTo) : file.ratio > 0 ? chalk.yellow(basenameTo) : basenameTo, @@ -645,9 +705,6 @@ export function logErrors( file.error, ) break - case 'cache': - logArray.push(chalk.black.bgBlue(' CACHED '), ' ', file.error) - break case 'warning': logArray.push( chalk.bgYellow(' WARNING '), @@ -689,9 +746,6 @@ export function logErrors( file.error, ) break - case 'cache': - logArray.push(chalk.black.bgBlue(' CACHED '), ' ', file.error) - break case 'warning': logArray.push( chalk.bgYellow(' WARNING '), @@ -785,7 +839,6 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { logger.info('') - // Init cache await FileCache.init(options, rootDir) const processDir = onlyAssets ? assetsDir : outDir @@ -1012,7 +1065,6 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { logResults(processedFiles[k], logger, maxLengths) }) - // Write cache state to file for persistence FileCache.reconcile() Object.keys(erroredFiles) diff --git a/packages/core/src/typings.d.ts b/packages/core/src/typings.d.ts index 9a18a32..7350115 100644 --- a/packages/core/src/typings.d.ts +++ b/packages/core/src/typings.d.ts @@ -95,6 +95,12 @@ export interface ConfigOptions { */ cacheDir?: string + /** + * Optional string to distinguish build configs and their caches. + * @default '' + */ + cacheKey?: string + /** * Force-clear the cache. * @default false @@ -153,6 +159,7 @@ export interface ResolvedConfigOptions { cache: boolean cacheDir?: string clearCache: boolean + cacheKey: string plugins: ResolvedPluginsConfig makeAvif: false | ResolvedMakeConfigOptions makeWebp: false | ResolvedMakeConfigOptions @@ -176,6 +183,22 @@ export type Stack = { [fromPath: string]: StackItem[] } +export type FormatProcessedFileParams = { + oldPath: string + newPath: string + oldSize: number + newSize: number + duration: number + fromCache: boolean + precisions: { + size: number + ratio: number + duration: number + } + bytesDivider: number + sizeUnit: string +} + export type ProcessFileParams = { baseDir?: string filePathFrom: string @@ -205,6 +228,7 @@ export type ProcessedFile = { optimizedDeleted: ResolvedMakeConfigOptions['skipIfLargerThan'] avifDeleted: ResolvedMakeConfigOptions['skipIfLargerThan'] webpDeleted: ResolvedMakeConfigOptions['skipIfLargerThan'] + fromCache: boolean } export type ErroredFile = { @@ -254,3 +278,9 @@ export type ProcessResult = | IPromiseRejectedResult export type ProcessFileReturn = Promise + +export type CacheValue = { + hash: string + oldSize: number + newSize: number +} diff --git a/packages/core/src/utils.test.ts b/packages/core/src/utils.test.ts index 801fa66..4758837 100644 --- a/packages/core/src/utils.test.ts +++ b/packages/core/src/utils.test.ts @@ -215,3 +215,4 @@ describe('smartEnsureDirs', () => { }) // TODO: add tests for getPackageDirectory and getPackageName +// TODO: create `cache.test.ts` with tests for cache functions From de25f31fe1b5e7ed2ca4c6e5ea1f00c802a2a1fd Mon Sep 17 00:00:00 2001 From: Philip <17368112+vHeemstra@users.noreply.github.com> Date: Wed, 22 Nov 2023 05:04:56 +0100 Subject: [PATCH 08/11] Better logging for cache --- packages/core/src/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 265c34e..57a34ab 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -216,7 +216,9 @@ export function formatProcessedFile({ ratioString: `${ratio > 0 ? '+' : ratio === 0 ? ' ' : ''}${( ratio * 100 ).toFixed(precisions.ratio)} %`, - durationString: `${duration.toFixed(precisions.duration)} ms`, + durationString: fromCache + ? 'Cache' + : `${duration.toFixed(precisions.duration)} ms`, fromCache, optimizedDeleted: false as const, avifDeleted: false as const, From 4cbe74d06c321d9a7b41e67660c8de1609365623 Mon Sep 17 00:00:00 2001 From: Philip <17368112+vHeemstra@users.noreply.github.com> Date: Wed, 22 Nov 2023 07:46:26 +0100 Subject: [PATCH 09/11] Update: refactor process function + test fix --- packages/core/src/index.test.ts | 18 ++- packages/core/src/index.ts | 248 ++++++++++++-------------------- 2 files changed, 103 insertions(+), 163 deletions(-) diff --git a/packages/core/src/index.test.ts b/packages/core/src/index.test.ts index 649524e..954ab42 100644 --- a/packages/core/src/index.test.ts +++ b/packages/core/src/index.test.ts @@ -642,7 +642,7 @@ describe('processFile', () => { }) }) - it('returns error object if OUTPUT PROCESS error (Error & warn)', async () => { + it('returns error object if OUTPUT PROCESS error (Error & string)', async () => { await expect( // @ts-expect-error missing properties are used after expected error processFile({ @@ -655,9 +655,9 @@ describe('processFile', () => { // optimization: 4, // strip: 'safe', // }), - () => Promise.reject(new Error('Test')), + () => Promise.reject(new Error('Test error processing file')), // () => { - // throw new Error('Test') + // throw new Error('Test error processing file') // }, ], skipIfLarger: false, @@ -668,7 +668,9 @@ describe('processFile', () => { ).resolves.toContainEqual({ status: 'rejected', reason: expect.objectContaining({ - error: expect.stringMatching(/^Error processing file/), + error: expect.stringMatching( + /^Error processing file:\s+Test error processing file/, + ), }), }) @@ -684,9 +686,9 @@ describe('processFile', () => { // optimization: 4, // strip: 'safe', // }), - () => Promise.reject('WARN: Test error processing file'), + () => Promise.reject('Test error processing file'), // () => { - // throw 'WARN: Test error processing file' + // throw 'Test error processing file' // }, ], skipIfLarger: false, @@ -697,7 +699,9 @@ describe('processFile', () => { ).resolves.toContainEqual({ status: 'rejected', reason: expect.objectContaining({ - error: 'Test error processing file', + error: expect.stringMatching( + /^Error processing file:\s*Test error processing file/, + ), }), }) }) diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 57a34ab..f1d1122 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -197,7 +197,7 @@ export function formatProcessedFile({ precisions, bytesDivider, sizeUnit, -}: FormatProcessedFileParams) { +}: FormatProcessedFileParams): ProcessedFile { const ratio = newSize / oldSize - 1 return { @@ -234,15 +234,13 @@ export async function processFile({ bytesDivider, sizeUnit, }: ProcessFileParams): ProcessFileReturn { - // const start = performance.now() - if (!filePathFrom?.length) { return Promise.reject({ oldPath: filePathFrom, newPath: '', error: 'Empty filepath', errorType: 'error', - }) as Promise + }) } if (!fileToStack?.length) { @@ -251,17 +249,24 @@ export async function processFile({ newPath: '', error: 'Empty to-stack', errorType: 'error', - }) as Promise + }) } + let cacheStack: (false | Promise)[] = [] + + // TODO: Rewrite cache check and split this: + // - get hash from cache (if present) + // - after readFile below, make current hash and compare it + // - check output cache in process pipeline as well? + // + // Reason: Since FileCache.checkAndCopy needs to read the file for checking, + // we might as well do it only once as part of the process instead of twice. const [inputCacheStatus, outputCacheStatus] = await FileCache.checkAndCopy( baseDir, filePathFrom, fileToStack, ) - let cacheStack: (false | Promise)[] = [] - if (inputCacheStatus) { let hasFullValidCache = true @@ -294,164 +299,95 @@ export async function processFile({ let oldBuffer: Buffer let oldSize = 0 - return readFile(baseDir + filePathFrom) - .then(buffer => { - const start = performance.now() - - oldBuffer = buffer - oldSize = oldBuffer.byteLength - - // TODO: handle elsewhere with - // (to-make-myself) `imagemin-apngquant` plugin - // `imagemin-apngquant` uses: - // - is-apng - // - apngdis-bin - // - (concat frames to strip) [see script at: ...?] - // - optimize using: - // - user-supplied plugin + options - // ? pngquant fallback - // - apng-asm (strip to apng) - if (filePathFrom.match(/\.a?png$/i) && isAPNG(buffer)) { - /** - * TODO: - * - apngdis the input APNG, then: - * - if output path .png: make strip + run plugins + apngasm from strip + output - * - if output path .webp: plugin each frame + run frames through webpmux-bin + single .webp output - * - * @see C:\Users\phili\Desktop\APNG\test\index.js - */ - throw new Error('SKIP: Animated PNGs not supported') - - // Copy input to tmp dir .png - - // apngdis-bin: this file to frame `.png`s - - // convert/optimize each .png to .webp - - // webpmux-bin: combine all .webp to final .webp - - // // const fileString = buffer.toString('utf8') - // const idatIdx = buffer.indexOf('IDAT') - // const actlIdx = buffer.indexOf('acTL') - // if ( - // filePathFrom.endsWith('.png') && - // idatIdx > 0 && - // actlIdx > 0 && - // idatIdx > actlIdx - // ) { - // throw new Error('SKIP: Animated PNGs not supported') - // } - } + try { + oldBuffer = await readFile(baseDir + filePathFrom) + oldSize = oldBuffer.byteLength + } catch (error) { + return Promise.reject({ + oldPath: filePathFrom, + newPath: '', + error: `Error reading file [${(error as Error).message}]`, + errorType: 'error', + }) + } - return Promise.allSettled( - fileToStack.map((item, i) => { - let newBuffer: Buffer - let newSize = 0 + if (filePathFrom.match(/\.a?png$/i) && isAPNG(oldBuffer)) { + return Promise.reject({ + oldPath: filePathFrom, + newPath: '', + error: `Animated PNGs not supported`, + errorType: 'skip', + }) + } - const filePathTo = item.toPath + const start = performance.now() - if (cacheStack[i]) { - return cacheStack[i] as Promise - } + return Promise.allSettled( + fileToStack.map(async (item, i) => { + if (cacheStack[i]) { + return cacheStack[i] as Promise + } - return imagemin - .buffer(oldBuffer, { plugins: item.plugins }) - .catch(e => - Promise.reject( - // e.message ? `Error processing file [${e.message}]` : e, - e.message ? `Error processing file:\n${e.message}` : e, - ), - ) - .then(buffer2 => { - newBuffer = buffer2 - newSize = newBuffer.byteLength - - if (false !== item.skipIfLarger && newSize > oldSize) { - // throw new Error('SKIP: Output is larger') - if (filePathTo === filePathFrom) { - // NOTE: - // If this optimized file is larger than original - // and this is not a WebP/Avif version; - // don't overwrite the original, but do continue (to have it log with the rest). - return Promise.resolve() - } - } + const filePathTo = item.toPath - // const newDirectory = path.dirname(baseDir + filePathTo) - // await ensureDir(newDirectory, 0o755) - return writeFile(baseDir + filePathTo, newBuffer) - }) - .catch(e => { - if (e?.message) { - if (e.message.startsWith('SKIP: ')) { - e = e.message - } else { - e = `Error writing file [${e.message}]` - } - } - return Promise.reject(e) - }) - .then(async () => { - await FileCache.update(baseDir, filePathTo, { - oldSize, - newSize, - }) + let newBuffer: Buffer + let newSize = 0 - return Promise.resolve( - formatProcessedFile({ - oldPath: filePathFrom, - newPath: filePathTo, - oldSize, - newSize, - duration: performance.now() - start, - fromCache: false, - precisions, - bytesDivider, - sizeUnit, - }), - ) - }) - .catch(error => { - let errorType = 'error' - if (error.startsWith('SKIP: ')) { - error = error.slice(6) - errorType = 'skip' - } else if (error.startsWith('WARN: ')) { - error = error.slice(6) - errorType = 'warning' - } + try { + newBuffer = await imagemin.buffer(oldBuffer, { plugins: item.plugins }) + newSize = newBuffer.byteLength + } catch (error) { + return Promise.reject({ + oldPath: filePathFrom, + newPath: filePathTo, + error: `Error processing file:\n${ + (error as Error)?.message ?? error + }`, + errorType: 'error', + }) + } - return Promise.reject({ - oldPath: filePathFrom, - newPath: filePathTo, - error, - errorType, - }) - }) - }), - ) - }) - .catch(e => { - let errorType = 'error' - let error = (e as Error).message - - if (error.startsWith('SKIP: ')) { - error = error.slice(6) - errorType = 'skip' - // } else if (error.startsWith('WARN: ')) { - // error = error.slice(6) - // errorType = 'warning' - } else { - error = `Error reading file [${error}]` + /** + * NOTE: Don't overwrite the original if the optimized content is larger, + * the option is set and this doesn't concern a WebP/Avif version. + */ + if ( + newSize <= oldSize || + filePathFrom !== filePathTo || + false === item.skipIfLarger + ) { + try { + await writeFile(baseDir + filePathTo, newBuffer) + } catch (error) { + return Promise.reject({ + oldPath: filePathFrom, + newPath: filePathTo, + error: `Error writing file [${(error as Error).message}]`, + errorType: 'error', + }) + } } - return Promise.reject({ - oldPath: filePathFrom, - newPath: '', - error, - errorType, - }) as Promise - }) + await FileCache.update(baseDir, filePathTo, { + oldSize, + newSize, + }) + + return Promise.resolve( + formatProcessedFile({ + oldPath: filePathFrom, + newPath: filePathTo, + oldSize, + newSize, + duration: performance.now() - start, + fromCache: false, + precisions, + bytesDivider, + sizeUnit, + }), + ) + }), + ) } export function processResults(results: ProcessResult[]): ProcessedResults { From f5968a606ee11112f4a359dc68e37c51befb2cb7 Mon Sep 17 00:00:00 2001 From: Philip <17368112+vHeemstra@users.noreply.github.com> Date: Wed, 22 Nov 2023 11:11:26 +0100 Subject: [PATCH 10/11] Update: refactor caching * Removed duplicate `readFile` * More efficient checking, restoring and updating --- packages/core/src/cache.ts | 165 ++++++++++++++++++------------------- packages/core/src/index.ts | 92 +++++++++------------ 2 files changed, 122 insertions(+), 135 deletions(-) diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts index 66e03d2..71f655c 100644 --- a/packages/core/src/cache.ts +++ b/packages/core/src/cache.ts @@ -1,6 +1,5 @@ import crypto, { BinaryLike } from 'node:crypto' -import { existsSync } from 'node:fs' -import { copyFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises' +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises' import { isAbsolute, resolve } from 'node:path' import { normalizePath } from 'vite' @@ -11,7 +10,7 @@ import { smartEnsureDirs, } from './utils' -import type { CacheValue, ResolvedConfigOptions, StackItem } from './typings' +import type { CacheValue, ResolvedConfigOptions } from './typings' let cacheEnabled = false let cacheDir = '' @@ -87,36 +86,61 @@ async function initCacheMaps() { fileCacheMap = new Map(entryMap) } -async function getAndUpdateCacheContent( - filePath: string, - stats?: Omit, -): Promise<{ +async function checkAndUpdate({ + fileName, + directory, + stats, + buffer, + restoreTo, +}: { + fileName?: string + directory?: string + stats?: Omit + buffer?: Buffer + restoreTo?: string | false +}): Promise<{ changed?: boolean value?: CacheValue error?: Error }> { - let hash = '' - try { - hash = md5(await readFile(filePath)) - } catch (error) { - return { - error: error as Error, + if (cacheEnabled) { + const filePath = (directory ?? cacheDir) + fileName + + if (!buffer) { + try { + buffer = await readFile(filePath) + } catch (error) { + return { + error: error as Error, + } + } } - } - const cacheValue = fileCacheMap.get(filePath) - if (cacheValue && cacheValue.hash === hash) { - return { - changed: false, - value: cacheValue, + const hash = md5(buffer) + const cacheValue = fileCacheMap.get(filePath) + if (cacheValue && cacheValue.hash === hash) { + if (restoreTo) { + try { + await writeFile(restoreTo + fileName, buffer) + } catch (error) { + return { + error: error as Error, + } + } + } + + return { + changed: false, + value: cacheValue, + } } - } - entryMap.set(filePath, { - hash, - oldSize: stats?.oldSize ?? 0, - newSize: stats?.newSize ?? 0, - }) + entryMap.set(filePath, { + hash, + oldSize: stats?.oldSize ?? 1, + newSize: stats?.newSize ?? 1, + }) + } return { changed: true, @@ -144,72 +168,47 @@ export const FileCache = { smartEnsureDirs(filePaths.map(file => cacheDir + file)) }, - checkAndCopy: async ( - baseDir: string, - filePathFrom: string, - fileToStack: StackItem[] = [], - ): Promise<[boolean, (string | CacheValue)[]]> => { + checkAndUpdate: checkAndUpdate, + + update: async ({ + fileName, + directory, + stats, + buffer, + }: { + fileName?: string + directory?: string + stats?: Omit + buffer: Buffer + }) => { if (!cacheEnabled) { - return [false, []] + return false } - const inputCacheStatus = await getAndUpdateCacheContent( - baseDir + filePathFrom, - ) - - // Check if input file has changed or there was an error - if (inputCacheStatus?.error || inputCacheStatus?.changed) { - return [false, []] + if (!buffer) { + return { + error: new Error('Missing content for cache file'), + } } - // Check if output files are in cache and use them if they haven't changed - const outputFilesExist = await Promise.allSettled( - fileToStack.map(async item => { - const outputCacheStatus = await getAndUpdateCacheContent( - cacheDir + item.toPath, - ) - - if (outputCacheStatus?.error) { - return `Cache error [${outputCacheStatus.error.message}]` - } - - if (outputCacheStatus?.changed) { - return 'File changed' - } - - try { - await copyFile(cacheDir + item.toPath, baseDir + item.toPath) - } catch (error) { - return `Could not copy cached file [${(error as Error).message}]` - } - - if (!existsSync(baseDir + item.toPath)) { - return 'Could not use cached file' - } + const filePath = (directory ?? cacheDir) + fileName - return outputCacheStatus.value as CacheValue - }), - ) - - return [ - true, - outputFilesExist.map(p => - p.status === 'fulfilled' ? p.value : p.reason, - ), - ] - }, - - update: async ( - baseDir: string, - filePathTo: string, - stats: Omit, - ) => { - if (!cacheEnabled) { - return + try { + await writeFile(filePath, buffer) + } catch (error) { + return { + error: new Error( + `Could not write cache file [${(error as Error).message}]`, + ), + } } - await copyFile(baseDir + filePathTo, cacheDir + filePathTo) - await getAndUpdateCacheContent(cacheDir + filePathTo, stats) + return await checkAndUpdate({ + fileName, + directory, + buffer, + stats, + }) }, reconcile: async () => { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f1d1122..6f573e2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -252,50 +252,6 @@ export async function processFile({ }) } - let cacheStack: (false | Promise)[] = [] - - // TODO: Rewrite cache check and split this: - // - get hash from cache (if present) - // - after readFile below, make current hash and compare it - // - check output cache in process pipeline as well? - // - // Reason: Since FileCache.checkAndCopy needs to read the file for checking, - // we might as well do it only once as part of the process instead of twice. - const [inputCacheStatus, outputCacheStatus] = await FileCache.checkAndCopy( - baseDir, - filePathFrom, - fileToStack, - ) - - if (inputCacheStatus) { - let hasFullValidCache = true - - cacheStack = outputCacheStatus.map((status, i) => { - if (isString(status)) { - hasFullValidCache = false - return false - } - - return Promise.resolve( - formatProcessedFile({ - oldPath: filePathFrom, - newPath: fileToStack[i].toPath, - oldSize: status.oldSize, - newSize: status.newSize, - duration: 0, - fromCache: true, - precisions, - bytesDivider, - sizeUnit, - }), - ) - }) - - if (hasFullValidCache) { - return Promise.allSettled(cacheStack as Promise[]) - } - } - let oldBuffer: Buffer let oldSize = 0 @@ -320,16 +276,44 @@ export async function processFile({ }) } + const inputFileCacheStatus = await FileCache.checkAndUpdate({ + fileName: filePathFrom, + directory: baseDir, + buffer: oldBuffer, + restoreTo: false, + }) + const skipCache = Boolean( + inputFileCacheStatus?.error || inputFileCacheStatus?.changed, + ) + const start = performance.now() return Promise.allSettled( - fileToStack.map(async (item, i) => { - if (cacheStack[i]) { - return cacheStack[i] as Promise - } - + fileToStack.map(async item => { const filePathTo = item.toPath + if (!skipCache) { + const outputFileCacheStatus = await FileCache.checkAndUpdate({ + fileName: filePathTo, + restoreTo: baseDir, + }) + if (!outputFileCacheStatus?.error && !outputFileCacheStatus?.changed) { + return Promise.resolve( + formatProcessedFile({ + oldPath: filePathFrom, + newPath: filePathTo, + oldSize: outputFileCacheStatus?.value?.oldSize ?? 1, + newSize: outputFileCacheStatus?.value?.newSize ?? 1, + duration: 0, + fromCache: true, + precisions, + bytesDivider, + sizeUnit, + }), + ) + } + } + let newBuffer: Buffer let newSize = 0 @@ -368,9 +352,13 @@ export async function processFile({ } } - await FileCache.update(baseDir, filePathTo, { - oldSize, - newSize, + await FileCache.update({ + fileName: filePathTo, + buffer: newBuffer, + stats: { + oldSize, + newSize, + }, }) return Promise.resolve( From 43b25b2732cc3af4c21cb2d5b42b394982c9e038 Mon Sep 17 00:00:00 2001 From: Philip <17368112+vHeemstra@users.noreply.github.com> Date: Wed, 22 Nov 2023 11:20:55 +0100 Subject: [PATCH 11/11] Update cache.ts --- packages/core/src/cache.ts | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts index 71f655c..4dea75f 100644 --- a/packages/core/src/cache.ts +++ b/packages/core/src/cache.ts @@ -93,7 +93,7 @@ async function checkAndUpdate({ buffer, restoreTo, }: { - fileName?: string + fileName: string directory?: string stats?: Omit buffer?: Buffer @@ -104,6 +104,12 @@ async function checkAndUpdate({ error?: Error }> { if (cacheEnabled) { + if (!fileName) { + return { + error: new Error('Missing filename'), + } + } + const filePath = (directory ?? cacheDir) + fileName if (!buffer) { @@ -172,19 +178,25 @@ export const FileCache = { update: async ({ fileName, + buffer, directory, stats, - buffer, }: { - fileName?: string + fileName: string + buffer: Buffer directory?: string stats?: Omit - buffer: Buffer }) => { if (!cacheEnabled) { return false } + if (!fileName) { + return { + error: new Error('Missing filename'), + } + } + if (!buffer) { return { error: new Error('Missing content for cache file'),