WIP: use FileSystem for globbing

baselines/packages/wotan/test/runner.spec.js.md

@@ -4,6 +4,36 @@ The actual snapshot is saved in `runner.spec.js.snap`.
 
 Generated by [AVA](https://ava.li).
 
+## glob-fs
+
+    [
+      [
+        'a.ts',
+        {
+          content: 'a;',
+          failures: [],
+          fixes: 0,
+        },
+      ],
+      [
+        'foo/b.ts',
+        {
+          content: `b;␊
+          `,
+          failures: [],
+          fixes: 0,
+        },
+      ],
+      [
+        'regular/nested/d.ts',
+        {
+          content: 'c;',
+          failures: [],
+          fixes: 0,
+        },
+      ],
+    ]
+
 ## multi-project
 
     [

docs/api.md

@@ -23,7 +23,7 @@ The default implementations (targeting the Node.js runtime environment) are prov
 * `ConfigurationProvider` (`DefaultConfigurationProvider`) is responsible to find, resolve and load configuration files.
 * `DeprecationHandler` (`DefaultDeprecationHandler`) is notified everytime a deprecated rule, formatter of processor is used. This service can choose to inform the user or just swallow the event.
 * `DirectoryService` (`NodeDirectoryService`) provides the current directory. None of the builtin services cache the current directory. Therefore you can change it dynamically if you need to.
-* `FileSystem` (`NodeFileSystem`) is responsible for the low level file system access. By providing this service, you can use an in-memory file system for example. Every file system access (except for the globbing) goes through this service.
+* `FileSystem` (`NodeFileSystem`) is responsible for the low level file system access. By providing this service, you can use an in-memory file system for example. Every file system access goes through this service.
 * `FormatterLoaderHost` (`NodeFormatterLoader`) is used to resolve and require a formatter.
 * `FailureFilterFactory` (`LineSwitchFilterFactory`) creates a `FailureFilter` for a given SourceFile to determine if a failure is disabled. The default implementation parses `// wotan-disable` comments to filter failures by rulename. Your custom implementation can choose to filter by different criteria, e.g. matching the failure message.
   * `LineSwitchParser` (`DefaultLineSwitchParser`) is used by `LineSwitchFilterFactory` to parse the line and rulename based disable comments from the source code. A custom implementation could use a different comment format, for example `// ! package/*` and return the appropriate switch positions.

packages/wotan/package.json

@@ -37,7 +37,7 @@
     "@types/json5": "0.0.30",
     "@types/minimatch": "^3.0.1",
     "@types/mkdirp": "^0.5.2",
-    "@types/node": "^9.3.0",
+    "@types/node": "^10.10.1",
     "@types/resolve": "^0.0.8",
     "@types/rimraf": "^2.0.2",
     "@types/semver": "^5.4.0",
@@ -53,6 +53,7 @@
     "debug": "^4.0.0",
     "diff": "^3.4.0",
     "glob": "^7.1.2",
+    "glob-interceptor": "^0.0.1",
     "import-local": "^2.0.0",
     "inversify": "^4.10.0",
     "is-negated-glob": "^1.0.0",

packages/wotan/src/glob-proxy.ts

@@ -0,0 +1,26 @@
+import { CachedFileSystem, FileKind } from './services/cached-file-system';
+import { createGlobInterceptor } from 'glob-interceptor';
+
+export function createGlobProxy(fs: CachedFileSystem) {
+    return createGlobInterceptor({
+        realpath(path) {
+            return fs.realpath === undefined ? path : fs.realpath(path);
+        },
+        isDirectory(path) {
+            switch (fs.getKind(path)) {
+                case FileKind.Directory:
+                    return true;
+                case FileKind.NonExistent:
+                    return;
+                default:
+                    return false;
+            }
+        },
+        isSymbolicLink(path) {
+            return fs.isSymbolicLink(path);
+        },
+        readDirectory(dir: string) {
+            return fs.readDirectory(dir).map((entry) => entry.name);
+        },
+    });
+}

packages/wotan/src/runner.ts

@@ -20,6 +20,7 @@ import { ConfigurationManager } from './services/configuration-manager';
 import { ProjectHost } from './project-host';
 import debug = require('debug');
 import resolveGlob = require('to-absolute-glob');
+import { createGlobProxy } from './glob-proxy';
 
 const log = debug('wotan:runner');
 
@@ -116,7 +117,7 @@ export class Runner {
 
     private *lintFiles(options: LintOptions, config: Configuration | undefined): LintResult {
         let processor: AbstractProcessor | undefined;
-        for (const file of getFiles(options.files, options.exclude, this.directories.getCurrentDirectory())) {
+        for (const file of getFiles(options.files, options.exclude, this.directories.getCurrentDirectory(), this.fs)) {
             if (options.config === undefined)
                 config = this.configManager.find(file);
             const effectiveConfig = config && this.configManager.reduce(config, file);
@@ -399,17 +400,14 @@ function getOutFileDeclarationName(outFile: string) {
     return outFile.slice(0, -path.extname(outFile).length) + '.d.ts';
 }
 
-function getFiles(patterns: ReadonlyArray<string>, exclude: ReadonlyArray<string>, cwd: string): Iterable<string> {
+// TODO make instance method
+function getFiles(patterns: ReadonlyArray<string>, exclude: ReadonlyArray<string>, cwd: string, fs: CachedFileSystem): Iterable<string> {
     const result: string[] = [];
     const globOptions = {
         cwd,
-        absolute: true,
-        cache: {},
         ignore: exclude,
         nodir: true,
-        realpathCache: {},
-        statCache: {},
-        symlinks: {},
+        ...createGlobProxy(fs),
     };
     for (const pattern of patterns) {
         const match = glob.sync(pattern, globOptions);

packages/wotan/src/services/cached-file-system.ts

@@ -21,10 +21,12 @@ export class CachedFileSystem {
     private fileKindCache: Cache<string, FileKind>;
     private realpathCache: Cache<string, string>;
     private direntCache: Cache<string, string[]>;
+    private symlinkCache: Cache<string, boolean>;
     constructor(private fs: FileSystem, cache: CacheFactory) {
         this.fileKindCache = cache.create();
         this.realpathCache = cache.create();
         this.direntCache = cache.create();
+        this.symlinkCache = cache.create();
     }
 
     public isFile(file: string): boolean {
@@ -48,6 +50,27 @@ export class CachedFileSystem {
         }
     }
 
+    public isSymbolicLink(file: string): boolean {
+        file = this.fs.normalizePath(file);
+        let cached = this.symlinkCache.get(file);
+        if (cached !== undefined)
+            return cached;
+        try {
+            const stats = this.fs.lstat(file);
+            if (stats.isSymbolicLink()) {
+                cached = true;
+            } else {
+                cached = false;
+                this.fileKindCache.set(file, statsToKind(stats));
+            }
+        } catch {
+            cached = false;
+            this.fileKindCache.set(file, FileKind.NonExistent);
+        }
+        this.symlinkCache.set(file, cached);
+        return cached;
+    }
+
     public readDirectory(dir: string) {
         dir = this.fs.normalizePath(dir);
         let cachedResult = this.direntCache.get(dir);
@@ -63,13 +86,15 @@ export class CachedFileSystem {
                 result.push({kind: this.getKind(path.join(dir, entry)), name: entry});
             } else {
                 cachedResult.push(entry.name);
-                const filePath = path.join(dir, entry.name);
+                const filePath = this.fs.normalizePath(path.join(dir, entry.name));
                 let kind: FileKind;
                 if (entry.isSymbolicLink()) {
                     kind = this.getKind(filePath);
+                    this.symlinkCache.set(filePath, true);
                 } else {
+                    this.symlinkCache.set(filePath, false);
                     kind = statsToKind(entry);
-                    this.fileKindCache.set(this.fs.normalizePath(filePath), kind);
+                    this.fileKindCache.set(filePath, kind);
                 }
                 result.push({kind, name: entry.name});
             }
@@ -126,8 +151,11 @@ export class CachedFileSystem {
 
     private updateCache(file: string, kind: FileKind) {
         // this currently doesn't handle directory removal as there is no API for that
-        if (this.fileKindCache.get(file) === kind)
+        const previous = this.fileKindCache.get(file);
+        if (previous === kind)
             return;
+        if (previous === FileKind.NonExistent || kind === FileKind.NonExistent)
+            this.symlinkCache.set(file, false);
         this.fileKindCache.set(file, kind);
         if (kind === FileKind.NonExistent)
             this.realpathCache.delete(file); // only invalidate realpath cache on file removal

packages/wotan/src/services/default/file-system.ts

@@ -1,4 +1,4 @@
-import { FileSystem, Stats, MessageHandler, Dirent } from '@fimbul/ymir';
+import { FileSystem, Stats, MessageHandler, Dirent, LStats } from '@fimbul/ymir';
 import * as fs from 'fs';
 import { injectable } from 'inversify';
 import { unixifyPath } from '../../utils';
@@ -36,11 +36,14 @@ export class NodeFileSystem implements FileSystem {
         return buf.toString('utf8'); // default to UTF8 without BOM
     }
     public readDirectory(dir: string): Array<string | Dirent> {
-        return fs.readdirSync(dir, <any>{withFileTypes: true});
+        return fs.readdirSync(dir, {withFileTypes: true});
     }
     public stat(path: string): Stats {
         return fs.statSync(path);
     }
+    public lstat(path: string): LStats {
+        return fs.lstatSync(path);
+    }
     public realpath(path: string) {
         return fs.realpathSync(path);
     }

packages/wotan/test/commands.spec.ts

@@ -256,6 +256,7 @@ test('SaveCommand', async (t) => {
             readFile() { throw new Error(); },
             readDirectory() { throw new Error(); },
             stat() { throw new Error(); },
+            lstat() { throw new Error(); },
             createDirectory() { throw new Error(); },
             writeFile(f, c) {
                 t.is(f, path.resolve('.fimbullinter.yaml'));

packages/wotan/test/configuration.spec.ts

@@ -12,6 +12,7 @@ import {
     LoadConfigurationContext,
     ConfigurationError,
     BuiltinResolver,
+    LStats,
 } from '@fimbul/ymir';
 import { Container, injectable } from 'inversify';
 import { CachedFileSystem } from '../src/services/cached-file-system';
@@ -291,6 +292,9 @@ test('DefaultConfigurationProvider.find', (t) => {
                 };
             throw new Error();
         }
+        public lstat(): LStats {
+            throw new Error('Method not implemented.');
+        }
         public writeFile(): void {
             throw new Error('Method not implemented.');
         }
@@ -398,6 +402,9 @@ test('DefaultConfigurationProvider.read', (t) => {
         public stat(): Stats {
             throw new Error('Method not implemented.');
         }
+        public lstat(): LStats {
+            throw new Error('Method not implemented.');
+        }
         public writeFile(): void {
             throw new Error('Method not implemented.');
         }