From e7a3a4e0eb5d974dd885f2a3eb42d9679661133a Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Wed, 12 Sep 2018 22:21:35 +0200 Subject: [PATCH 1/5] WIP: use FileSystem for globbing --- .../packages/wotan/test/runner.spec.js.md | 30 ++ .../packages/wotan/test/runner.spec.js.snap | Bin 559 -> 720 bytes packages/wotan/src/glob-proxy.ts | 116 ++++++++ packages/wotan/src/runner.ts | 11 +- .../wotan/src/services/cached-file-system.ts | 34 ++- .../wotan/src/services/default/file-system.ts | 5 +- packages/wotan/test/commands.spec.ts | 1 + packages/wotan/test/configuration.spec.ts | 7 + packages/wotan/test/runner.spec.ts | 273 +++++++++++------- packages/ymir/src/index.ts | 9 +- 10 files changed, 364 insertions(+), 122 deletions(-) create mode 100644 packages/wotan/src/glob-proxy.ts diff --git a/baselines/packages/wotan/test/runner.spec.js.md b/baselines/packages/wotan/test/runner.spec.js.md index 8723f579f..5771cff09 100644 --- a/baselines/packages/wotan/test/runner.spec.js.md +++ b/baselines/packages/wotan/test/runner.spec.js.md @@ -4,6 +4,36 @@ The actual snapshot is saved in `runner.spec.js.snap`. Generated by [AVA](https://ava.li). +## glob-fs + + [ + [ + 'a.ts', + { + content: 'a;', + failures: [], + fixes: 0, + }, + ], + [ + 'foo/b.ts', + { + content: `b;␊ + `, + failures: [], + fixes: 0, + }, + ], + [ + 'regular/nested/d.ts', + { + content: 'c;', + failures: [], + fixes: 0, + }, + ], + ] + ## multi-project [ diff --git a/baselines/packages/wotan/test/runner.spec.js.snap b/baselines/packages/wotan/test/runner.spec.js.snap index 320f26c49ba11e3819d74d9fc211bf8bedbcaed3..c04ff481373940bf7cc38203c787ae64654d1f5e 100644 GIT binary patch literal 720 zcmV;>0x$hRRzVJM+Gu`T5>yLWo8lzj#}}^z{7X zo*m3YUS`aqCN-uoP6HQ!E5HG2 zj?lhyLO83GCRR3^)qdY5bbwHuP>a%Q4S!hoqmjnu%gEHCsQL!XuTg!pHKLY|2<{L# zqd-2z?d!lj11TF%u)~vk$a%rBsit1bD@bGYRpT-@q)_`}wm$XNJCy!rX89IPF z9y3xN18p9I@lEcJ_K|GEbi+XLKqdc&`N~o#6svN@usJr(j^{Au*aXX!9FZF@ScDEy zT0pkMC*wJOwI;Ys#qRU`6)xjHJvx+sn0{u5=}9lywa>&~jMLwYliy;V2p3r{pSPP2 zBUi92+h(fbOU45kC1u6!WxRqS!_op@Qp^!yEhsa`Q(UmBA1FVVV_R$`k=yg<^7Aqb zicwHsF~JwZ8wjZ?oHIKAJNq)nI_B;yEhrYA|70jmhy1J-+z zq*q|Q0XBf2X^&k=lI9fRcY8?RgPx~)so54M_awaLHj2Nx^XPlnbAJHt!1!rK2mk=Z Ceq~hv literal 559 zcmV+~0?_?IRzVGT!GeIt9Dws~g3Kc$^cr|L6! zPO2A#&xG$ykA-<2KS>q%1Ulc7V>%dUl@d-d83@ zW%_4j@c7Rw#)=(vOtpAqtLrLy-;3UlT1cODASV xKj4MhL#v2T#7!z9l{TXCs@Wa entry.name); + }, + }; +} + +const propertyDescriptor = { writable: true, configurable: true }; + +export function createGlobProxy(fs: GlobFileSystem) { + const cache = new Proxy>({}, { + getOwnPropertyDescriptor() { + return propertyDescriptor; + }, + get(_, path: string) { + try { + if (!fs.statSync(path).isDirectory()) + return 'FILE'; + } catch { + return false; + } + try { + return fs.readdirSync(path); + } catch { + return []; + } + }, + set() { + // ignore cache write + return true; + }, + }); + const statCache = new Proxy>({}, { + get(_, path: string) { + try { + return fs.statSync(path).isDirectory() ? directoryStats : otherStats; + } catch { + return false; + } + }, + set() { + // ignore cache write + return true; + }, + }); + const realpathCache = new Proxy>({}, { + getOwnPropertyDescriptor() { + return propertyDescriptor; + }, + get(_, path: string) { + try { + return fs.realpathSync(path); + } catch { + return path; + } + }, + }); + const symlinks = new Proxy>({}, { + getOwnPropertyDescriptor() { + return propertyDescriptor; + }, + get(_, path: string) { + try { + return fs.lstatSync(path).isSymbolicLink(); + } catch { + return false; + } + }, + }); + return { + cache, + realpathCache, + statCache, + symlinks, + }; +} diff --git a/packages/wotan/src/runner.ts b/packages/wotan/src/runner.ts index 7f213e708..fb1e4a302 100644 --- a/packages/wotan/src/runner.ts +++ b/packages/wotan/src/runner.ts @@ -20,6 +20,7 @@ import { ConfigurationManager } from './services/configuration-manager'; import { ProjectHost } from './project-host'; import debug = require('debug'); import resolveGlob = require('to-absolute-glob'); +import { createGlobProxy, createGlobFileSystem } from './glob-proxy'; const log = debug('wotan:runner'); @@ -116,7 +117,7 @@ export class Runner { private *lintFiles(options: LintOptions, config: Configuration | undefined): LintResult { let processor: AbstractProcessor | undefined; - for (const file of getFiles(options.files, options.exclude, this.directories.getCurrentDirectory())) { + for (const file of getFiles(options.files, options.exclude, this.directories.getCurrentDirectory(), this.fs)) { if (options.config === undefined) config = this.configManager.find(file); const effectiveConfig = config && this.configManager.reduce(config, file); @@ -399,17 +400,15 @@ function getOutFileDeclarationName(outFile: string) { return outFile.slice(0, -path.extname(outFile).length) + '.d.ts'; } -function getFiles(patterns: ReadonlyArray, exclude: ReadonlyArray, cwd: string): Iterable { +// TODO make instance method +function getFiles(patterns: ReadonlyArray, exclude: ReadonlyArray, cwd: string, fs: CachedFileSystem): Iterable { const result: string[] = []; const globOptions = { cwd, absolute: true, - cache: {}, ignore: exclude, nodir: true, - realpathCache: {}, - statCache: {}, - symlinks: {}, + ...createGlobProxy(createGlobFileSystem(fs)), }; for (const pattern of patterns) { const match = glob.sync(pattern, globOptions); diff --git a/packages/wotan/src/services/cached-file-system.ts b/packages/wotan/src/services/cached-file-system.ts index ab0ad9789..3dd283066 100644 --- a/packages/wotan/src/services/cached-file-system.ts +++ b/packages/wotan/src/services/cached-file-system.ts @@ -21,10 +21,12 @@ export class CachedFileSystem { private fileKindCache: Cache; private realpathCache: Cache; private direntCache: Cache; + private symlinkCache: Cache; constructor(private fs: FileSystem, cache: CacheFactory) { this.fileKindCache = cache.create(); this.realpathCache = cache.create(); this.direntCache = cache.create(); + this.symlinkCache = cache.create(); } public isFile(file: string): boolean { @@ -48,6 +50,27 @@ export class CachedFileSystem { } } + public isSymbolicLink(file: string): boolean { + file = this.fs.normalizePath(file); + let cached = this.symlinkCache.get(file); + if (cached !== undefined) + return cached; + try { + const stats = this.fs.lstat(file); + if (stats.isSymbolicLink()) { + cached = true; + } else { + cached = false; + this.fileKindCache.set(file, statsToKind(stats)); + } + } catch { + cached = false; + this.fileKindCache.set(file, FileKind.NonExistent); + } + this.symlinkCache.set(file, cached); + return cached; + } + public readDirectory(dir: string) { dir = this.fs.normalizePath(dir); let cachedResult = this.direntCache.get(dir); @@ -63,13 +86,15 @@ export class CachedFileSystem { result.push({kind: this.getKind(path.join(dir, entry)), name: entry}); } else { cachedResult.push(entry.name); - const filePath = path.join(dir, entry.name); + const filePath = this.fs.normalizePath(path.join(dir, entry.name)); let kind: FileKind; if (entry.isSymbolicLink()) { kind = this.getKind(filePath); + this.symlinkCache.set(filePath, true); } else { + this.symlinkCache.set(filePath, false); kind = statsToKind(entry); - this.fileKindCache.set(this.fs.normalizePath(filePath), kind); + this.fileKindCache.set(filePath, kind); } result.push({kind, name: entry.name}); } @@ -126,8 +151,11 @@ export class CachedFileSystem { private updateCache(file: string, kind: FileKind) { // this currently doesn't handle directory removal as there is no API for that - if (this.fileKindCache.get(file) === kind) + const previous = this.fileKindCache.get(file); + if (previous === kind) return; + if (previous === FileKind.NonExistent || kind === FileKind.NonExistent) + this.symlinkCache.set(file, false); this.fileKindCache.set(file, kind); if (kind === FileKind.NonExistent) this.realpathCache.delete(file); // only invalidate realpath cache on file removal diff --git a/packages/wotan/src/services/default/file-system.ts b/packages/wotan/src/services/default/file-system.ts index 08c13edfd..3788ed96a 100644 --- a/packages/wotan/src/services/default/file-system.ts +++ b/packages/wotan/src/services/default/file-system.ts @@ -1,4 +1,4 @@ -import { FileSystem, Stats, MessageHandler, Dirent } from '@fimbul/ymir'; +import { FileSystem, Stats, MessageHandler, Dirent, LStats } from '@fimbul/ymir'; import * as fs from 'fs'; import { injectable } from 'inversify'; import { unixifyPath } from '../../utils'; @@ -41,6 +41,9 @@ export class NodeFileSystem implements FileSystem { public stat(path: string): Stats { return fs.statSync(path); } + public lstat(path: string): LStats { + return fs.lstatSync(path); + } public realpath(path: string) { return fs.realpathSync(path); } diff --git a/packages/wotan/test/commands.spec.ts b/packages/wotan/test/commands.spec.ts index fc12705cd..439dec5c5 100644 --- a/packages/wotan/test/commands.spec.ts +++ b/packages/wotan/test/commands.spec.ts @@ -256,6 +256,7 @@ test('SaveCommand', async (t) => { readFile() { throw new Error(); }, readDirectory() { throw new Error(); }, stat() { throw new Error(); }, + lstat() { throw new Error(); }, createDirectory() { throw new Error(); }, writeFile(f, c) { t.is(f, path.resolve('.fimbullinter.yaml')); diff --git a/packages/wotan/test/configuration.spec.ts b/packages/wotan/test/configuration.spec.ts index 66e3db863..bf0778244 100644 --- a/packages/wotan/test/configuration.spec.ts +++ b/packages/wotan/test/configuration.spec.ts @@ -12,6 +12,7 @@ import { LoadConfigurationContext, ConfigurationError, BuiltinResolver, + LStats, } from '@fimbul/ymir'; import { Container, injectable } from 'inversify'; import { CachedFileSystem } from '../src/services/cached-file-system'; @@ -291,6 +292,9 @@ test('DefaultConfigurationProvider.find', (t) => { }; throw new Error(); } + public lstat(): LStats { + throw new Error('Method not implemented.'); + } public writeFile(): void { throw new Error('Method not implemented.'); } @@ -398,6 +402,9 @@ test('DefaultConfigurationProvider.read', (t) => { public stat(): Stats { throw new Error('Method not implemented.'); } + public lstat(): LStats { + throw new Error('Method not implemented.'); + } public writeFile(): void { throw new Error('Method not implemented.'); } diff --git a/packages/wotan/test/runner.spec.ts b/packages/wotan/test/runner.spec.ts index 307982e5c..5ef9457b3 100644 --- a/packages/wotan/test/runner.spec.ts +++ b/packages/wotan/test/runner.spec.ts @@ -8,6 +8,7 @@ import * as path from 'path'; import { NodeFileSystem } from '../src/services/default/file-system'; import { FileSystem, MessageHandler, DirectoryService, FileSummary } from '@fimbul/ymir'; import { unixifyPath } from '../src/utils'; +import * as fs from 'fs'; const directories: DirectoryService = { getCurrentDirectory() { return path.resolve('packages/wotan'); }, @@ -123,39 +124,15 @@ test('throws if no tsconfig.json can be found', (t) => { test('reports warnings while parsing tsconfig.json', (t) => { const container = new Container({defaultScope: BindingScopeEnum.Singleton}); - const files: {[name: string]: string | undefined} = { - 'invalid-config.json': '{', - 'invalid-base.json': '{"extends": "./invalid-config.json"}', - 'invalid-files.json': '{"files": []}', - 'no-match.json': '{"include": ["non-existent"], "compilerOptions": {"noLib": true}}', - }; - @injectable() - class MockFileSystem extends NodeFileSystem { - constructor(logger: MessageHandler) { - super(logger); - } - public stat(file: string) { - if (isLibraryFile(file)) - return super.stat(file); - return { - isFile() { return files[path.basename(file)] !== undefined; }, - isDirectory() { return false; }, - }; - } - public readFile(file: string) { - if (isLibraryFile(file)) - return super.readFile(file); - const basename = path.basename(file); - const content = files[basename]; - if (content !== undefined) - return content; - throw new Error('ENOENT'); - } - public readDirectory(): string[] { - throw new Error('ENOENT'); - } - } - container.bind(FileSystem).to(MockFileSystem); + container.bind(MockFiles).toConstantValue({ + entries: { + 'invalid-config.json': {content: '{'}, + 'invalid-base.json': {content: '{"extends": "./invalid-config.json"}'}, + 'invalid-files.json': {content: '{"files": []}'}, + 'no-match.json': {content: '{"include": ["non-existent"], "compilerOptions": {"noLib": true}}'}, + }, + }); + container.bind(FileSystem).to(MemoryFileSystem); let warning = ''; container.bind(MessageHandler).toConstantValue({ log() {}, @@ -217,12 +194,7 @@ test('reports warnings while parsing tsconfig.json', (t) => { test.skip('excludes symlinked typeRoots', (t) => { const container = new Container({defaultScope: BindingScopeEnum.Singleton}); container.bind(DirectoryService).toConstantValue(directories); - interface FileMeta { - content?: string; - symlink?: string; - entries?: Record; - } - const files: FileMeta = { + container.bind(MockFiles).toConstantValue({ entries: { 'tsconfig.json': {content: '{"files": ["a.ts"]}'}, 'a.ts': {content: 'foo;'}, @@ -242,73 +214,8 @@ test.skip('excludes symlinked typeRoots', (t) => { }, '.wotanrc.yaml': {content: 'rules: {trailing-newline: error}'}, }, - }; - @injectable() - class MockFileSystem extends NodeFileSystem { - constructor(private dirs: DirectoryService, logger: MessageHandler) { - super(logger); - } - public stat(file: string) { - if (isLibraryFile(file)) - return super.stat(file); - const f = this.resolvePath(file); - if (f === undefined) - throw new Error('ENOENT'); - return { - isFile() { return f.resolved.content !== undefined; }, - isDirectory() { return f.resolved.content === undefined; }, - }; - } - public readFile(file: string) { - if (isLibraryFile(file)) - return super.readFile(file); - const f = this.resolvePath(file); - if (f === undefined) - throw new Error('ENOENT'); - if (f.resolved.content === undefined) - throw new Error('EISDIR'); - return f.resolved.content; - } - public readDirectory(dir: string): string[] { - const f = this.resolvePath(dir); - if (f === undefined) - throw new Error('ENOENT'); - if (f.resolved.content !== undefined) - throw new Error('ENOTDIR'); - return Object.keys(f.resolved.entries || {}); - } - public realpath(file: string): string { - if (isLibraryFile(file)) - return super.realpath(file); - const f = this.resolvePath(file); - if (f === undefined) - throw new Error('ENOENT'); - return path.resolve(this.dirs.getCurrentDirectory(), f.realpath); - } - - private resolvePath(p: string) { - const parts = path.relative(this.normalizePath(this.dirs.getCurrentDirectory()), this.normalizePath(p)).split(/\//g); - let current: FileMeta | undefined = files; - let part = parts.shift(); - let realPath = []; - while (part !== undefined) { - if (part) { - realPath.push(part); - current = current.entries && current.entries[part]; - if (current === undefined) - return; - if (current.symlink !== undefined) { - parts.unshift(...current.symlink.split(/\//g)); - realPath = []; - current = files; - } - } - part = parts.shift(); - } - return {resolved: current, realpath: realPath.join('/')}; - } - } - container.bind(FileSystem).to(MockFileSystem); + }); + container.bind(FileSystem).to(MemoryFileSystem); container.load(createCoreModule({}), createDefaultModule()); const runner = container.get(Runner); const result = Array.from(runner.lintCollection({ @@ -324,10 +231,6 @@ test.skip('excludes symlinked typeRoots', (t) => { t.is(result[0][0], unixifyPath(path.resolve('packages/wotan/a.ts'))); }); -function isLibraryFile(name: string) { - return /[\\/]typescript[\\/]lib[\\/]lib(\.es\d+(\.\w+)*)?\.d\.ts$/.test(name); -} - test('works with absolute and relative paths', (t) => { const container = new Container(); container.bind(DirectoryService).toConstantValue(directories); @@ -380,3 +283,153 @@ test('supports linting multiple (overlapping) projects in one run', (t) => { ); t.snapshot(result, {id: 'multi-project'}); }); + +test('uses FileSystem for globbing', (t) => { + const container = new Container({defaultScope: BindingScopeEnum.Singleton}); + container.bind(MockFiles).toConstantValue({ + entries: { + 'a.ts': {content: 'a;'}, + foo: { + entries: { + 'b.ts': {content: 'b;\n'}, + }, + }, + link: { + symlink: '.symlinked', // ensures symlinks are handled correctly + }, + '.symlinked': { + entries: { + nested: { + entries: { + 'c.ts': {content: 'c;'}, + }, + }, + }, + }, + regular: { + entries: { + nested: { + entries: { + 'd.ts': {symlink: '.symlinked/nested/c.ts'}, + }, + }, + }, + }, + '.wotanrc.yaml': {content: 'rules: {}'}, + }, + }); + container.bind(FileSystem).to(MemoryFileSystem); + container.load(createCoreModule({}), createDefaultModule()); + const runner = container.get(Runner); + const result = Array.from( + runner.lintCollection({ + config: undefined, + files: ['**/*.ts'], + exclude: [], + project: [], + references: false, + fix: false, + extensions: undefined, + }), + (entry): [string, FileSummary] => [unixifyPath(path.relative('', entry[0])), entry[1]], + ); + t.snapshot(result, {id: 'glob-fs'}); +}); + +function isLibraryFile(name: string) { + return /[\\/]typescript[\\/]lib[\\/]lib(\.es\d+(\.\w+)*)?\.d\.ts$/.test(name); +} + +abstract class MockFiles { + public content?: string; + public symlink?: string; + public entries?: Record; +} +@injectable() +class MemoryFileSystem implements FileSystem { + constructor(private files: MockFiles, private dirs: DirectoryService) {} + + public normalizePath(file: string) { + return NodeFileSystem.normalizePath(file); + } + public stat(file: string) { + if (isLibraryFile(file)) + return fs.statSync(file); + const f = this.resolvePath(file); + if (f === undefined) + throw new Error('ENOENT'); + return { + isFile() { return f.resolved.content !== undefined; }, + isDirectory() { return f.resolved.content === undefined; }, + }; + } + public lstat(file: string) { + const f = this.resolvePath(file); + if (f === undefined) + throw new Error(); + return { + isFile() { return !f.symlink && f.resolved.content !== undefined; }, + isDirectory() { return !f.symlink && f.resolved.content === undefined; }, + isSymbolicLink() { return f.symlink; }, + }; + } + public readFile(file: string) { + if (isLibraryFile(file)) + return fs.readFileSync(file, 'utf8'); + const f = this.resolvePath(file); + if (f === undefined) + throw new Error('ENOENT'); + if (f.resolved.content === undefined) + throw new Error('EISDIR'); + return f.resolved.content; + } + public readDirectory(dir: string): string[] { + const f = this.resolvePath(dir); + if (f === undefined) + throw new Error('ENOENT'); + if (f.resolved.content !== undefined) + throw new Error('ENOTDIR'); + return Object.keys(f.resolved.entries || {}); + } + public realpath(file: string): string { + const f = this.resolvePath(file); + if (f === undefined) + throw new Error('ENOENT'); + return path.resolve(this.dirs.getCurrentDirectory(), f.realpath); + } + public deleteFile() { + throw new Error('not implemented'); + } + public createDirectory() { + throw new Error('not implemented'); + } + public writeFile() { + throw new Error('not implemented'); + } + + private resolvePath(p: string) { + const parts = path.relative(this.normalizePath(this.dirs.getCurrentDirectory()), this.normalizePath(p)).split(/\//g); + let current: MockFiles | undefined = this.files; + let part = parts.shift(); + let realPath = []; + let symlinkedDepth = -1; + while (part !== undefined) { + --symlinkedDepth; + if (part) { + realPath.push(part); + current = current.entries && current.entries[part]; + if (current === undefined) + return; + if (current.symlink !== undefined) { + const newParts = current.symlink.split(/\//g); + parts.unshift(...newParts); + symlinkedDepth = Math.max(symlinkedDepth, 0) + newParts.length; + realPath = []; + current = this.files; + } + } + part = parts.shift(); + } + return {resolved: current, realpath: realPath.join('/'), symlink: symlinkedDepth >= 0}; + } +} diff --git a/packages/ymir/src/index.ts b/packages/ymir/src/index.ts index 6f450988b..0ae93b39d 100644 --- a/packages/ymir/src/index.ts +++ b/packages/ymir/src/index.ts @@ -384,6 +384,8 @@ export interface FileSystem { readDirectory(dir: string): Array; /** Gets the status of a file or directory. */ stat(path: string): Stats; + /** Gets the status of a file or directory, not resolving symlinks. */ + lstat(path: string): LStats; /** Gets the realpath of a given file or directory. */ realpath?(path: string): string; /** Writes content to the file, overwriting the existing content. Creates the file if necessary. */ @@ -400,11 +402,14 @@ export interface Stats { isFile(): boolean; } -export interface Dirent extends Stats { - name: string; +export interface LStats extends Stats { isSymbolicLink(): boolean; } +export interface Dirent extends LStats { + name: string; +} + export interface RuleLoaderHost { loadCoreRule(name: string): RuleConstructor | undefined; loadCustomRule(name: string, directory: string): RuleConstructor | undefined; From 6e29486f9916b6a8a961f16b8109e490559ef9c1 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Wed, 12 Sep 2018 22:22:29 +0200 Subject: [PATCH 2/5] update docs --- docs/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.md b/docs/api.md index 76122d169..42e1af648 100644 --- a/docs/api.md +++ b/docs/api.md @@ -23,7 +23,7 @@ The default implementations (targeting the Node.js runtime environment) are prov * `ConfigurationProvider` (`DefaultConfigurationProvider`) is responsible to find, resolve and load configuration files. * `DeprecationHandler` (`DefaultDeprecationHandler`) is notified everytime a deprecated rule, formatter of processor is used. This service can choose to inform the user or just swallow the event. * `DirectoryService` (`NodeDirectoryService`) provides the current directory. None of the builtin services cache the current directory. Therefore you can change it dynamically if you need to. -* `FileSystem` (`NodeFileSystem`) is responsible for the low level file system access. By providing this service, you can use an in-memory file system for example. Every file system access (except for the globbing) goes through this service. +* `FileSystem` (`NodeFileSystem`) is responsible for the low level file system access. By providing this service, you can use an in-memory file system for example. Every file system access goes through this service. * `FormatterLoaderHost` (`NodeFormatterLoader`) is used to resolve and require a formatter. * `FailureFilterFactory` (`LineSwitchFilterFactory`) creates a `FailureFilter` for a given SourceFile to determine if a failure is disabled. The default implementation parses `// wotan-disable` comments to filter failures by rulename. Your custom implementation can choose to filter by different criteria, e.g. matching the failure message. * `LineSwitchParser` (`DefaultLineSwitchParser`) is used by `LineSwitchFilterFactory` to parse the line and rulename based disable comments from the source code. A custom implementation could use a different comment format, for example `// ! package/*` and return the appropriate switch positions. From 25adb16719ba96f5d50f56844353a4f9e6ed3bc3 Mon Sep 17 00:00:00 2001 From: Klaus Meinhardt Date: Thu, 13 Sep 2018 14:19:23 +0200 Subject: [PATCH 3/5] refactor GlobFileSystem --- packages/wotan/src/glob-proxy.ts | 72 +++++++++++++------------------- packages/wotan/src/runner.ts | 1 - 2 files changed, 28 insertions(+), 45 deletions(-) diff --git a/packages/wotan/src/glob-proxy.ts b/packages/wotan/src/glob-proxy.ts index 6f5fb1166..633d04fc3 100644 --- a/packages/wotan/src/glob-proxy.ts +++ b/packages/wotan/src/glob-proxy.ts @@ -1,45 +1,43 @@ import { CachedFileSystem, FileKind } from './services/cached-file-system'; +/** A minimal abstraction of FileSystem operations needed to provide a cache proxy for 'glob'. None of the methods are expected to throw. */ export interface GlobFileSystem { - realpathSync(path: string): string; - statSync(path: string): { isDirectory(): boolean; }; - lstatSync(path: string): { isSymbolicLink(): boolean; }; - readdirSync(dir: string): string[]; + /** Returns `true` if the specified `path` is a directory, `undefined` if it doesn't exist and `false` otherwise. */ + isDirectory(path: string): boolean | undefined; + /** Returns `true` if the specified `path` is a symlink, `false` in all other cases. */ + isSymbolicLink(path: string): boolean; + /** Get the entries of a directory as string array. */ + readDirectory(dir: string): string[]; + /** Get the realpath of a given `path` by resolving all symlinks in the path. */ + realpath(path: string): string; } -const symlinkStats = { - isDirectory() { return false; }, - isSymbolicLink() { return true; }, -}; const directoryStats = { isDirectory() { return true; }, - isSymbolicLink() { return false; }, }; const otherStats = { isDirectory() { return false; }, - isSymbolicLink() { return false; }, }; export function createGlobFileSystem(fs: CachedFileSystem): GlobFileSystem { return { - realpathSync(path) { + realpath(path) { return fs.realpath === undefined ? path : fs.realpath(path); }, - statSync(path) { - const kind = fs.getKind(path); - switch (kind) { + isDirectory(path) { + switch (fs.getKind(path)) { case FileKind.Directory: - return directoryStats; + return true; case FileKind.NonExistent: - throw new Error('ENOENT'); + return; default: - return otherStats; + return false; } }, - lstatSync(path) { - return fs.isSymbolicLink(path) ? symlinkStats : otherStats; + isSymbolicLink(path) { + return fs.isSymbolicLink(path); }, - readdirSync(dir: string) { + readDirectory(dir: string) { return fs.readDirectory(dir).map((entry) => entry.name); }, }; @@ -53,16 +51,13 @@ export function createGlobProxy(fs: GlobFileSystem) { return propertyDescriptor; }, get(_, path: string) { - try { - if (!fs.statSync(path).isDirectory()) + switch (fs.isDirectory(path)) { + case true: + return fs.readDirectory(path); + case false: return 'FILE'; - } catch { - return false; - } - try { - return fs.readdirSync(path); - } catch { - return []; + default: + return false; } }, set() { @@ -72,11 +67,8 @@ export function createGlobProxy(fs: GlobFileSystem) { }); const statCache = new Proxy>({}, { get(_, path: string) { - try { - return fs.statSync(path).isDirectory() ? directoryStats : otherStats; - } catch { - return false; - } + return fs.isDirectory(path) ? directoryStats : otherStats; + }, set() { // ignore cache write @@ -88,11 +80,7 @@ export function createGlobProxy(fs: GlobFileSystem) { return propertyDescriptor; }, get(_, path: string) { - try { - return fs.realpathSync(path); - } catch { - return path; - } + return fs.realpath(path); }, }); const symlinks = new Proxy>({}, { @@ -100,11 +88,7 @@ export function createGlobProxy(fs: GlobFileSystem) { return propertyDescriptor; }, get(_, path: string) { - try { - return fs.lstatSync(path).isSymbolicLink(); - } catch { - return false; - } + return fs.isSymbolicLink(path); }, }); return { diff --git a/packages/wotan/src/runner.ts b/packages/wotan/src/runner.ts index fb1e4a302..c61e49860 100644 --- a/packages/wotan/src/runner.ts +++ b/packages/wotan/src/runner.ts @@ -405,7 +405,6 @@ function getFiles(patterns: ReadonlyArray, exclude: ReadonlyArray Date: Mon, 17 Sep 2018 16:02:20 +0200 Subject: [PATCH 4/5] move glob proxy to separate package --- packages/wotan/package.json | 1 + packages/wotan/src/glob-proxy.ts | 80 ++------------------------------ packages/wotan/src/runner.ts | 4 +- yarn.lock | 21 +++++++++ 4 files changed, 27 insertions(+), 79 deletions(-) diff --git a/packages/wotan/package.json b/packages/wotan/package.json index 4d05b8102..f79333fd6 100644 --- a/packages/wotan/package.json +++ b/packages/wotan/package.json @@ -53,6 +53,7 @@ "debug": "^4.0.0", "diff": "^3.4.0", "glob": "^7.1.2", + "glob-interceptor": "^0.0.1", "import-local": "^2.0.0", "inversify": "^4.10.0", "is-negated-glob": "^1.0.0", diff --git a/packages/wotan/src/glob-proxy.ts b/packages/wotan/src/glob-proxy.ts index 633d04fc3..fb1970e51 100644 --- a/packages/wotan/src/glob-proxy.ts +++ b/packages/wotan/src/glob-proxy.ts @@ -1,26 +1,8 @@ import { CachedFileSystem, FileKind } from './services/cached-file-system'; +import { createGlobInterceptor } from 'glob-interceptor'; -/** A minimal abstraction of FileSystem operations needed to provide a cache proxy for 'glob'. None of the methods are expected to throw. */ -export interface GlobFileSystem { - /** Returns `true` if the specified `path` is a directory, `undefined` if it doesn't exist and `false` otherwise. */ - isDirectory(path: string): boolean | undefined; - /** Returns `true` if the specified `path` is a symlink, `false` in all other cases. */ - isSymbolicLink(path: string): boolean; - /** Get the entries of a directory as string array. */ - readDirectory(dir: string): string[]; - /** Get the realpath of a given `path` by resolving all symlinks in the path. */ - realpath(path: string): string; -} - -const directoryStats = { - isDirectory() { return true; }, -}; -const otherStats = { - isDirectory() { return false; }, -}; - -export function createGlobFileSystem(fs: CachedFileSystem): GlobFileSystem { - return { +export function createGlobProxy(fs: CachedFileSystem) { + return createGlobInterceptor({ realpath(path) { return fs.realpath === undefined ? path : fs.realpath(path); }, @@ -40,61 +22,5 @@ export function createGlobFileSystem(fs: CachedFileSystem): GlobFileSystem { readDirectory(dir: string) { return fs.readDirectory(dir).map((entry) => entry.name); }, - }; -} - -const propertyDescriptor = { writable: true, configurable: true }; - -export function createGlobProxy(fs: GlobFileSystem) { - const cache = new Proxy>({}, { - getOwnPropertyDescriptor() { - return propertyDescriptor; - }, - get(_, path: string) { - switch (fs.isDirectory(path)) { - case true: - return fs.readDirectory(path); - case false: - return 'FILE'; - default: - return false; - } - }, - set() { - // ignore cache write - return true; - }, - }); - const statCache = new Proxy>({}, { - get(_, path: string) { - return fs.isDirectory(path) ? directoryStats : otherStats; - - }, - set() { - // ignore cache write - return true; - }, - }); - const realpathCache = new Proxy>({}, { - getOwnPropertyDescriptor() { - return propertyDescriptor; - }, - get(_, path: string) { - return fs.realpath(path); - }, - }); - const symlinks = new Proxy>({}, { - getOwnPropertyDescriptor() { - return propertyDescriptor; - }, - get(_, path: string) { - return fs.isSymbolicLink(path); - }, }); - return { - cache, - realpathCache, - statCache, - symlinks, - }; } diff --git a/packages/wotan/src/runner.ts b/packages/wotan/src/runner.ts index c61e49860..2c92ab216 100644 --- a/packages/wotan/src/runner.ts +++ b/packages/wotan/src/runner.ts @@ -20,7 +20,7 @@ import { ConfigurationManager } from './services/configuration-manager'; import { ProjectHost } from './project-host'; import debug = require('debug'); import resolveGlob = require('to-absolute-glob'); -import { createGlobProxy, createGlobFileSystem } from './glob-proxy'; +import { createGlobProxy } from './glob-proxy'; const log = debug('wotan:runner'); @@ -407,7 +407,7 @@ function getFiles(patterns: ReadonlyArray, exclude: ReadonlyArray Date: Mon, 17 Sep 2018 21:54:53 +0200 Subject: [PATCH 5/5] update node typings, remove useless assertion --- packages/wotan/package.json | 2 +- packages/wotan/src/services/default/file-system.ts | 2 +- yarn.lock | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/wotan/package.json b/packages/wotan/package.json index f79333fd6..6894c83a5 100644 --- a/packages/wotan/package.json +++ b/packages/wotan/package.json @@ -37,7 +37,7 @@ "@types/json5": "0.0.30", "@types/minimatch": "^3.0.1", "@types/mkdirp": "^0.5.2", - "@types/node": "^9.3.0", + "@types/node": "^10.10.1", "@types/resolve": "^0.0.8", "@types/rimraf": "^2.0.2", "@types/semver": "^5.4.0", diff --git a/packages/wotan/src/services/default/file-system.ts b/packages/wotan/src/services/default/file-system.ts index 3788ed96a..8be812932 100644 --- a/packages/wotan/src/services/default/file-system.ts +++ b/packages/wotan/src/services/default/file-system.ts @@ -36,7 +36,7 @@ export class NodeFileSystem implements FileSystem { return buf.toString('utf8'); // default to UTF8 without BOM } public readDirectory(dir: string): Array { - return fs.readdirSync(dir, {withFileTypes: true}); + return fs.readdirSync(dir, {withFileTypes: true}); } public stat(path: string): Stats { return fs.statSync(path); diff --git a/yarn.lock b/yarn.lock index 93c682b59..a38c6e51a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -227,9 +227,9 @@ version "10.9.4" resolved "https://registry.yarnpkg.com/@types/node/-/node-10.9.4.tgz#0f4cb2dc7c1de6096055357f70179043c33e9897" -"@types/node@^9.3.0": - version "9.6.31" - resolved "https://registry.yarnpkg.com/@types/node/-/node-9.6.31.tgz#4d1722987f8d808b4c194dceb8c213bd92f028e5" +"@types/node@^10.10.1": + version "10.10.1" + resolved "https://registry.yarnpkg.com/@types/node/-/node-10.10.1.tgz#d5c96ca246a418404914d180b7fdd625ad18eca6" "@types/parse5-sax-parser@^5.0.1": version "5.0.1"