From f6e99694227ac3217410d90eb663ec9b9fb3262e Mon Sep 17 00:00:00 2001 From: Martijn Swart Date: Tue, 21 Nov 2023 12:11:00 +0100 Subject: [PATCH 1/3] Update: Seperated cache logic in own module --- packages/core/src/cache.ts | 127 ++++++++++++++++++++++++++++++ packages/core/src/index.ts | 154 +++++++++---------------------------- 2 files changed, 164 insertions(+), 117 deletions(-) create mode 100644 packages/core/src/cache.ts diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts new file mode 100644 index 0000000..8cce585 --- /dev/null +++ b/packages/core/src/cache.ts @@ -0,0 +1,127 @@ +import { createCache } from '@file-cache/core' +import { copyFileSync, existsSync, mkdirSync, rmSync } from 'node:fs' +import path from 'node:path' +import { normalizePath } from 'vite' + +import { + getPackageDirectory, + getPackageName, + isString, + smartEnsureDirs, +} from './utils' + +import type { CacheInterface } from '@file-cache/core/mjs/CacheInterface' +import type { ResolvedConfigOptions, StackItem } from './typings' + +let cacheEnabled = false +let cache: CacheInterface +let cacheDir = '' + +function initCacheDir(rootDir: string, _cacheDir?: string): void { + // Note: Only cacheDir has a trailing slash. + if (isString(_cacheDir)) { + cacheDir = + normalizePath( + path.isAbsolute(_cacheDir) + ? _cacheDir + : path.resolve(rootDir, _cacheDir), + ) + '/' + } else { + const packageDir = normalizePath(getPackageDirectory()) + cacheDir = `${packageDir}/node_modules/.cache/vite-plugin-imagemin/${getPackageName( + packageDir, + )}/` + } + + mkdirSync(cacheDir.slice(0, -1), { recursive: true }) +} + +export const FileCache = { + init: async (options: ResolvedConfigOptions, rootDir: string) => { + cacheEnabled = options.cache !== false + + initCacheDir(rootDir, options.cacheDir) + + if (options.clearCache) { + FileCache.clear() + } + + cache = (await createCache({ + noCache: !cacheEnabled, + cacheDirectory: cacheDir.slice(0, -1), + mode: 'content', + keys: [ + () => { + return JSON.stringify(options) + }, + ], + })) as CacheInterface + }, + + prepareDirs: (filePaths: string[]): void => { + if (cacheEnabled) { + smartEnsureDirs(filePaths.map(file => cacheDir + file)) + } + }, + + check: async ( + baseDir: string, + filePathFrom: string, + fileToStack: StackItem[] = [], + ) => { + const inputFileCache = await cache?.getAndUpdateCache( + baseDir + filePathFrom, + ) + + // Check if input file has changed or there was an error + if (inputFileCache.changed || inputFileCache.error) { + return false + } + + // Check if output files are in cache and use them if they haven't changed + const outputFilesExist = await Promise.allSettled( + fileToStack.map( + item => + new Promise((resolve, reject) => + cache + .getAndUpdateCache(cacheDir + item.toPath) + .then(outputFileCache => { + if (!outputFileCache.error && !outputFileCache.changed) { + copyFileSync(cacheDir + item.toPath, baseDir + item.toPath) + if (existsSync(baseDir + item.toPath)) { + resolve(true) + } + } + reject( + outputFileCache.error + ? `Error while checking cache [${outputFileCache.error.message}]` + : 'Could not copy cached files', + ) + }) + .catch(reject), + ), + ), + ) + + return outputFilesExist.every(p => p.status === 'fulfilled') + }, + + update: async (baseDir: string, filePathTo: string) => { + if (cacheEnabled) { + copyFileSync(baseDir + filePathTo, cacheDir + filePathTo) + await cache.getAndUpdateCache(cacheDir + filePathTo) + } + }, + + reconcile: async () => { + await cache?.reconcile() + }, + + clear: () => { + if (!cache || !cacheDir) { + return + } + + rmSync(cacheDir.slice(0, -1), { recursive: true, force: true }) + }, +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index dfa052d..0c0cb2a 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,51 +1,41 @@ -import path from 'node:path' -import { - existsSync, - mkdirSync, - lstatSync, - readdirSync, - unlinkSync, - copyFileSync, - rmSync, -} from 'node:fs' -import { readFile, writeFile } from 'node:fs/promises' -import { Buffer } from 'node:buffer' -import { performance } from 'node:perf_hooks' import chalk from 'chalk' -import { normalizePath, createFilter } from 'vite' import imagemin from 'imagemin' import isAPNG from 'is-apng' -import { createCache } from '@file-cache/core' +import { Buffer } from 'node:buffer' +import { lstatSync, readdirSync, unlinkSync } from 'node:fs' +import { readFile, writeFile } from 'node:fs/promises' +import path from 'node:path' +import { performance } from 'node:perf_hooks' +import { createFilter, normalizePath } from 'vite' + +import { FileCache } from './cache' import { - isFunction, + escapeRegExp, isBoolean, - isString, - isObject, isFilterPattern, - escapeRegExp, + isFunction, + isObject, + isString, smartEnsureDirs, - getPackageDirectory, - getPackageName, } from './utils' import type { PluginOption, ResolvedConfig } from 'vite' -import type { CacheInterface } from '@file-cache/core/mjs/CacheInterface' import type { ConfigOptions, - ResolvedConfigOptions, - ResolvedPluginsConfig, - ResolvedMakeConfigOptions, - PluginsConfig, + ErroredFile, Logger, - Stack, + PluginsConfig, ProcessFileParams, + ProcessFileReturn, + ProcessResult, + ProcessResultWhenOutput, ProcessedFile, - ErroredFile, ProcessedResults, - ProcessResultWhenOutput, - ProcessResult, - ProcessFileReturn, + ResolvedConfigOptions, + ResolvedMakeConfigOptions, + ResolvedPluginsConfig, + Stack, } from './typings' // export const pathIsWithin = (parentPath: string, childPath: string) => { @@ -204,8 +194,6 @@ export async function processFile({ precisions, bytesDivider, sizeUnit, - cacheDir = '', - cache = null, }: ProcessFileParams): ProcessFileReturn { // const start = performance.now() @@ -227,45 +215,19 @@ export async function processFile({ }) as Promise } - if (cache) { - // Check if input file hasn't changed - const inputFileCache = await cache.getAndUpdateCache(baseDir + filePathFrom) - - if (!inputFileCache.error && !inputFileCache.changed) { - // Check if output files are in cache and use them if they haven't changed - const outputFilesExist = await Promise.allSettled( - fileToStack.map( - item => - new Promise((resolve, reject) => - cache - .getAndUpdateCache(cacheDir + item.toPath) - .then(outputFileCache => { - if (!outputFileCache.error && !outputFileCache.changed) { - copyFileSync(cacheDir + item.toPath, baseDir + item.toPath) - if (existsSync(baseDir + item.toPath)) { - resolve(true) - } - } - reject( - outputFileCache.error - ? `Error while checking cache [${outputFileCache.error.message}]` - : 'Could not copy cached files', - ) - }) - .catch(reject), - ), - ), - ) + const hasValidCache = await FileCache.check( + baseDir, + filePathFrom, + fileToStack, + ) - if (outputFilesExist.every(p => p.status === 'fulfilled')) { - return Promise.reject({ - oldPath: filePathFrom, - newPath: '', - error: '', - errorType: 'cache', - }) as Promise - } - } + if (hasValidCache) { + return Promise.reject({ + oldPath: filePathFrom, + newPath: '', + error: '', + errorType: 'cache', + }) as Promise } let oldBuffer: Buffer @@ -366,10 +328,7 @@ export async function processFile({ }) .then(async () => { // Add to/update in cache - if (cache) { - copyFileSync(baseDir + filePathTo, cacheDir + filePathTo) - await cache.getAndUpdateCache(cacheDir + filePathTo) - } + await FileCache.update(baseDir, filePathTo) const duration = performance.now() - start const ratio = newSize / oldSize - 1 @@ -796,9 +755,6 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { let hadFilesToProcess = false // const mtimeCache = new Map() - let cache: CacheInterface - let cacheDir = '' - return { name: 'vite-plugin-imagemin', enforce: 'post', @@ -813,21 +769,6 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { : path.resolve(process.cwd(), rootDir), ) - // Note: Only cacheDir has a trailing slash. - if (isString(options.cacheDir)) { - cacheDir = - normalizePath( - path.isAbsolute(options.cacheDir) - ? options.cacheDir - : path.resolve(rootDir, options.cacheDir), - ) + '/' - } else { - const packageDir = normalizePath(getPackageDirectory()) - cacheDir = `${packageDir}/node_modules/.cache/vite-plugin-imagemin/${getPackageName( - packageDir, - )}/` - } - // sourceDir = normalizePath(path.resolve(rootDir, entry)) outDir = normalizePath(path.resolve(rootDir, config.build.outDir)) assetsDir = normalizePath(path.resolve(outDir, config.build.assetsDir)) @@ -845,22 +786,7 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { logger.info('') // Init cache - if (options.clearCache) { - rmSync(cacheDir.slice(0, -1), { recursive: true, force: true }) - } - if (options.cache !== false) { - mkdirSync(cacheDir.slice(0, -1), { recursive: true }) - - cache = (await createCache({ - cacheDirectory: cacheDir.slice(0, -1), - mode: 'content', - keys: [ - () => { - return JSON.stringify(options) - }, - ], - })) as CacheInterface - } + await FileCache.init(options, rootDir) const processDir = onlyAssets ? assetsDir : outDir const baseDir = `${rootDir}/` @@ -954,9 +880,7 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { // Ensure all destination (sub)directories are present smartEnsureDirs(toPaths.map(file => baseDir + file)) - if (options.cache !== false) { - smartEnsureDirs(toPaths.map(file => cacheDir + file)) - } + FileCache.prepareDirs(toPaths) // Process stack const { @@ -975,8 +899,6 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { precisions, bytesDivider, sizeUnit, - cacheDir, - cache, }), ), ) as Promise @@ -1091,9 +1013,7 @@ export default function viteImagemin(_options: ConfigOptions): PluginOption { }) // Write cache state to file for persistence - if (options.cache !== false) { - await cache.reconcile() - } + FileCache.reconcile() Object.keys(erroredFiles) .sort((a, b) => a.localeCompare(b)) // TODO: sort by (sub)folder and depth? From 2281f41889d1ec4136607f6dc71571870160d5af Mon Sep 17 00:00:00 2001 From: Martijn Swart Date: Tue, 21 Nov 2023 13:31:39 +0100 Subject: [PATCH 2/3] Update: removed all external modules --- packages/core/package.json | 3 +- packages/core/src/cache.ts | 117 ++++++++++++++++++++++++++++--------- 2 files changed, 89 insertions(+), 31 deletions(-) diff --git a/packages/core/package.json b/packages/core/package.json index cfbe326..a46557e 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -61,7 +61,6 @@ "node": ">=16.0.0" }, "dependencies": { - "@file-cache/core": "^1.1.4", "chalk": "^5.2.0", "fast-glob": "^3.2.12", "fs-extra": "^11.1.1", @@ -79,4 +78,4 @@ "peerDependencies": { "vite": "^3.0.0 || ^4.0.0 || ^4.3.9" } -} +} \ No newline at end of file diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts index 8cce585..0e5799e 100644 --- a/packages/core/src/cache.ts +++ b/packages/core/src/cache.ts @@ -1,7 +1,9 @@ -import { createCache } from '@file-cache/core' -import { copyFileSync, existsSync, mkdirSync, rmSync } from 'node:fs' +import { copyFileSync, existsSync, rmSync } from 'node:fs' +import { readFile, writeFile, mkdir, copyFile } from 'node:fs/promises' + import path from 'node:path' import { normalizePath } from 'vite' +import crypto, { BinaryLike } from 'crypto' import { getPackageDirectory, @@ -10,14 +12,24 @@ import { smartEnsureDirs, } from './utils' -import type { CacheInterface } from '@file-cache/core/mjs/CacheInterface' import type { ResolvedConfigOptions, StackItem } from './typings' +type CacheContent = { + hash: string +} +type CacheMetaData = { + size: number + mtime: number +} +type CacheValue = CacheContent | CacheMetaData + let cacheEnabled = false -let cache: CacheInterface let cacheDir = '' +let cacheFile = '' +let fileCacheMap = new Map() +let entryMap = new Map() -function initCacheDir(rootDir: string, _cacheDir?: string): void { +async function initCacheDir(rootDir: string, _cacheDir?: string) { // Note: Only cacheDir has a trailing slash. if (isString(_cacheDir)) { cacheDir = @@ -33,29 +45,61 @@ function initCacheDir(rootDir: string, _cacheDir?: string): void { )}/` } - mkdirSync(cacheDir.slice(0, -1), { recursive: true }) + await mkdir(cacheDir.slice(0, -1), { recursive: true }) +} + +async function initCacheMaps() { + cacheFile = path.join(cacheDir, 'contents') + + try { + const json = JSON.parse(await readFile(cacheFile, 'utf-8')) + entryMap = new Map(Object.entries(json)) + } catch { + entryMap = new Map() + } + + fileCacheMap = new Map(entryMap) +} + +function md5(buffer: BinaryLike): string { + return crypto.createHash('md5').update(buffer).digest('hex') +} + +async function getAndUpdateCacheContent(filePath: string | URL) { + try { + const hash = md5(await readFile(filePath)) + const normalizedFilePath = filePath.toString() + const cacheValue = fileCacheMap.get(normalizedFilePath) as + | CacheContent + | undefined + if (cacheValue && cacheValue.hash === hash) { + return { + changed: false, + } + } + entryMap.set(normalizedFilePath, { hash }) + return { + changed: true, + } + } catch (error) { + return { + changed: false, + error: error as Error, + } + } } export const FileCache = { init: async (options: ResolvedConfigOptions, rootDir: string) => { cacheEnabled = options.cache !== false - initCacheDir(rootDir, options.cacheDir) + await initCacheDir(rootDir, options.cacheDir) if (options.clearCache) { FileCache.clear() } - cache = (await createCache({ - noCache: !cacheEnabled, - cacheDirectory: cacheDir.slice(0, -1), - mode: 'content', - keys: [ - () => { - return JSON.stringify(options) - }, - ], - })) as CacheInterface + await initCacheMaps() }, prepareDirs: (filePaths: string[]): void => { @@ -69,12 +113,12 @@ export const FileCache = { filePathFrom: string, fileToStack: StackItem[] = [], ) => { - const inputFileCache = await cache?.getAndUpdateCache( + const { changed, error } = await getAndUpdateCacheContent( baseDir + filePathFrom, ) // Check if input file has changed or there was an error - if (inputFileCache.changed || inputFileCache.error) { + if (changed || error) { return false } @@ -83,8 +127,7 @@ export const FileCache = { fileToStack.map( item => new Promise((resolve, reject) => - cache - .getAndUpdateCache(cacheDir + item.toPath) + getAndUpdateCacheContent(cacheDir + item.toPath) .then(outputFileCache => { if (!outputFileCache.error && !outputFileCache.changed) { copyFileSync(cacheDir + item.toPath, baseDir + item.toPath) @@ -107,21 +150,37 @@ export const FileCache = { }, update: async (baseDir: string, filePathTo: string) => { - if (cacheEnabled) { - copyFileSync(baseDir + filePathTo, cacheDir + filePathTo) - await cache.getAndUpdateCache(cacheDir + filePathTo) + if (!cacheEnabled) { + return } + + await copyFile(baseDir + filePathTo, cacheDir + filePathTo) + await getAndUpdateCacheContent(cacheDir + filePathTo) }, reconcile: async () => { - await cache?.reconcile() + if (!cacheEnabled) { + return true + } + + try { + await writeFile( + cacheFile, + JSON.stringify(Object.fromEntries(entryMap)), + 'utf-8', + ) + // reflect the changes in the cacheMap + fileCacheMap = new Map(entryMap) + return true + } catch (error) { + // console.error('Cache reconcile has failed', error) + return false + } }, clear: () => { - if (!cache || !cacheDir) { - return + if (cacheFile && cacheDir) { + rmSync(cacheDir.slice(0, -1), { recursive: true, force: true }) } - - rmSync(cacheDir.slice(0, -1), { recursive: true, force: true }) }, } From b3f934c98fbe9701738273dfd65c8d041ffb5dff Mon Sep 17 00:00:00 2001 From: Martijn Swart Date: Tue, 21 Nov 2023 13:37:36 +0100 Subject: [PATCH 3/3] Update: cleanup cache --- packages/core/src/cache.ts | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts index 0e5799e..c687708 100644 --- a/packages/core/src/cache.ts +++ b/packages/core/src/cache.ts @@ -1,9 +1,8 @@ -import { copyFileSync, existsSync, rmSync } from 'node:fs' -import { readFile, writeFile, mkdir, copyFile } from 'node:fs/promises' - +import crypto, { BinaryLike } from 'crypto' +import { copyFileSync, existsSync } from 'node:fs' +import { copyFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises' import path from 'node:path' import { normalizePath } from 'vite' -import crypto, { BinaryLike } from 'crypto' import { getPackageDirectory, @@ -14,14 +13,9 @@ import { import type { ResolvedConfigOptions, StackItem } from './typings' -type CacheContent = { +type CacheValue = { hash: string } -type CacheMetaData = { - size: number - mtime: number -} -type CacheValue = CacheContent | CacheMetaData let cacheEnabled = false let cacheDir = '' @@ -70,7 +64,7 @@ async function getAndUpdateCacheContent(filePath: string | URL) { const hash = md5(await readFile(filePath)) const normalizedFilePath = filePath.toString() const cacheValue = fileCacheMap.get(normalizedFilePath) as - | CacheContent + | CacheValue | undefined if (cacheValue && cacheValue.hash === hash) { return { @@ -95,8 +89,9 @@ export const FileCache = { await initCacheDir(rootDir, options.cacheDir) - if (options.clearCache) { - FileCache.clear() + // clear cache? + if (options.clearCache && cacheDir) { + await rm(cacheDir.slice(0, -1), { recursive: true, force: true }) } await initCacheMaps() @@ -177,10 +172,4 @@ export const FileCache = { return false } }, - - clear: () => { - if (cacheFile && cacheDir) { - rmSync(cacheDir.slice(0, -1), { recursive: true, force: true }) - } - }, }