Merge pull request #766 from BitGo/DX-389-node-modules-generator

feat(openapi-generator): support types imported from `node_modules`

.prettierignore

@@ -1,3 +1,4 @@
 coverage/
 dist/
 flake.lock
+packages/openapi-generator/test/

package.json

@@ -14,7 +14,7 @@
   },
   "homepage": "https://github.com/BitGo/api-ts#readme",
   "workspaces": [
-    "packages/**"
+    "packages/*"
   ],
   "packageManager": "[email protected]",
   "scripts": {

packages/openapi-generator/package.json

@@ -17,7 +17,8 @@
     "clean": "rm -rf -- dist",
     "format": "prettier --check .",
     "format:fix": "prettier --write .",
-    "test": "c8 --all --src src node --require @swc-node/register --test test/*.test.ts"
+    "test": "c8 --all --src src node --require @swc-node/register --test test/*.test.ts",
+    "test:target": "c8 --all --src src node --require @swc-node/register"
   },
   "dependencies": {
     "@swc/core": "1.5.7",

packages/openapi-generator/src/cli.ts

@@ -159,7 +159,7 @@ const app = command({
     });
     let schema: Schema | undefined;
     while (((schema = queue.pop()), schema !== undefined)) {
-      const refs = getRefs(schema);
+      const refs = getRefs(schema, project.right.getTypes());
       for (const ref of refs) {
         if (components[ref.name] !== undefined) {
           continue;
@@ -169,6 +169,7 @@ const app = command({
           console.error(`Could not find '${ref.name}' from '${ref.location}'`);
           process.exit(1);
         }
+
         const initE = findSymbolInitializer(project.right, sourceFile, ref.name);
         if (E.isLeft(initE)) {
           console.error(
@@ -177,6 +178,7 @@ const app = command({
           process.exit(1);
         }
         const [newSourceFile, init] = initE.right;
+
         const codecE = parseCodecInitializer(project.right, newSourceFile, init);
         if (E.isLeft(codecE)) {
           console.error(

packages/openapi-generator/src/codec.ts

@@ -96,12 +96,6 @@ function codecIdentifier(
       }
       const name = id.property.value;
 
-      if (!objectImportSym.from.startsWith('.')) {
-        return E.left(
-          `Unimplemented named member reference '${objectImportSym.localName}.${name}' from '${objectImportSym.from}'`,
-        );
-      }
-
       const newInitE = findSymbolInitializer(project, source, [
         objectImportSym.localName,
         name,
@@ -360,9 +354,17 @@ export function parseCodecInitializer(
       if (schema.type !== 'ref') {
         return E.right(schema);
       } else {
-        const refSource = project.get(schema.location);
+        let refSource = project.get(schema.location);
+
         if (refSource === undefined) {
-          return E.left(`Cannot find '${schema.name}' from '${schema.location}'`);
+          // schema.location might be a package name -> need to resolve the path from the project types
+          const path = project.getTypes()[schema.name];
+          if (path === undefined)
+            return E.left(`Cannot find module '${schema.location}' in the project`);
+          refSource = project.get(path);
+          if (refSource === undefined) {
+            return E.left(`Cannot find '${schema.name}' from '${schema.location}'`);
+          }
         }
         const initE = findSymbolInitializer(project, refSource, schema.name);
         if (E.isLeft(initE)) {

packages/openapi-generator/src/project.ts

@@ -13,10 +13,12 @@ export class Project {
   private readonly knownImports: Record<string, Record<string, KnownCodec>>;
 
   private files: Record<string, SourceFile>;
+  private types: Record<string, string>;
 
   constructor(files: Record<string, SourceFile> = {}, knownImports = KNOWN_IMPORTS) {
     this.files = files;
     this.knownImports = knownImports;
+    this.types = {};
   }
 
   add(path: string, sourceFile: SourceFile): void {
@@ -42,19 +44,33 @@ export class Project {
       const src = await this.readFile(path);
       const sourceFile = await parseSource(path, src);
 
+      if (sourceFile === undefined) continue;
+
+      // map types to their file path
+      for (const exp of sourceFile.symbols.exports) {
+        this.types[exp.exportedName] = path;
+      }
+
       this.add(path, sourceFile);
 
       for (const sym of Object.values(sourceFile.symbols.imports)) {
         if (!sym.from.startsWith('.')) {
-          continue;
-        }
-
-        const filePath = p.dirname(path);
-        const absImportPathE = this.resolve(filePath, sym.from);
-        if (E.isLeft(absImportPathE)) {
-          return absImportPathE;
-        } else if (!this.has(absImportPathE.right)) {
-          queue.push(absImportPathE.right);
+          // If we are not resolving a relative path, we need to resolve the entry point
+          const baseDir = p.dirname(sourceFile.path);
+          let entryPoint = this.resolveEntryPoint(baseDir, sym.from);
+          if (E.isLeft(entryPoint)) {
+            continue;
+          } else if (!this.has(entryPoint.right)) {
+            queue.push(entryPoint.right);
+          }
+        } else {
+          const filePath = p.dirname(path);
+          const absImportPathE = this.resolve(filePath, sym.from);
+          if (E.isLeft(absImportPathE)) {
+            return absImportPathE;
+          } else if (!this.has(absImportPathE.right)) {
+            queue.push(absImportPathE.right);
+          }
         }
       }
       for (const starExport of sourceFile.symbols.exportStarFiles) {
@@ -75,24 +91,60 @@ export class Project {
     return await readFile(filename, 'utf8');
   }
 
+  resolveEntryPoint(basedir: string, library: string): E.Either<string, string> {
+    try {
+      const packageJson = resolve.sync(`${library}/package.json`, {
+        basedir,
+        extensions: ['.json'],
+      });
+      const packageInfo = JSON.parse(fs.readFileSync(packageJson, 'utf8'));
+
+      let typesEntryPoint = '';
+
+      if (packageInfo['types']) {
+        typesEntryPoint = packageInfo['types'];
+      }
+
+      if (packageInfo['typings']) {
+        typesEntryPoint = packageInfo['typings'];
+      }
+
+      if (!typesEntryPoint) {
+        return E.left(`Could not find types entry point for ${library}`);
+      }
+
+      const entryPoint = resolve.sync(`${library}/${typesEntryPoint}`, {
+        basedir,
+        extensions: ['.ts', '.js'],
+      });
+      return E.right(entryPoint);
+    } catch (err) {
+      return E.left(`Could not resolve entry point for ${library}: ${err}`);
+    }
+  }
+
   resolve(basedir: string, path: string): E.Either<string, string> {
     try {
       const result = resolve.sync(path, {
         basedir,
         extensions: ['.ts', '.js'],
       });
       return E.right(result);
-    } catch (e: any) {
-      if (typeof e === 'object' && e.hasOwnProperty('message')) {
+    } catch (e: unknown) {
+      if (e instanceof Error && e.message) {
         return E.left(e.message);
-      } else {
-        return E.left(JSON.stringify(e));
       }
+
+      return E.left(JSON.stringify(e));
     }
   }
 
   resolveKnownImport(path: string, name: string): KnownCodec | undefined {
     const baseKey = path.startsWith('.') ? '.' : path;
     return this.knownImports[baseKey]?.[name];
   }
+
+  getTypes() {
+    return this.types;
+  }
 }

packages/openapi-generator/src/ref.ts

@@ -1,24 +1,34 @@
 import type { Schema, Reference } from './ir';
+import fs from 'fs';
 
-export function getRefs(schema: Schema): Reference[] {
+export function getRefs(schema: Schema, typeMap: Record<string, string>): Reference[] {
   if (schema.type === 'ref') {
+    if (!fs.existsSync(schema.location)) {
+      // The location is a node module - we need to populate the location here
+      const newPath = typeMap[schema.name];
+      if (!newPath) {
+        return [];
+      }
+
+      return [{ ...schema, location: newPath }];
+    }
     return [schema];
   } else if (schema.type === 'array') {
-    return getRefs(schema.items);
+    return getRefs(schema.items, typeMap);
   } else if (
     schema.type === 'intersection' ||
     schema.type === 'union' ||
     schema.type === 'tuple'
   ) {
     return schema.schemas.reduce<Reference[]>((acc, member) => {
-      return [...acc, ...getRefs(member)];
+      return [...acc, ...getRefs(member, typeMap)];
     }, []);
   } else if (schema.type === 'object') {
     return Object.values(schema.properties).reduce<Reference[]>((acc, member) => {
-      return [...acc, ...getRefs(member)];
+      return [...acc, ...getRefs(member, typeMap)];
     }, []);
   } else if (schema.type === 'record') {
-    return getRefs(schema.codomain);
+    return getRefs(schema.codomain, typeMap);
   } else {
     return [];
   }

packages/openapi-generator/src/resolveInit.ts

@@ -13,7 +13,13 @@ function resolveImportPath(
   sourceFile: SourceFile,
   path: string,
 ): E.Either<string, SourceFile> {
-  const importPathE = project.resolve(dirname(sourceFile.path), path);
+  let importPathE;
+  if (path.startsWith('.')) {
+    importPathE = project.resolve(dirname(sourceFile.path), path);
+  } else {
+    importPathE = project.resolveEntryPoint(dirname(sourceFile.path), path);
+  }
+
   if (E.isLeft(importPathE)) {
     return importPathE;
   }

packages/openapi-generator/src/sourceFile.ts

@@ -14,7 +14,10 @@ export type SourceFile = {
 // increasing counter for this, so we also need to track it globally here
 let lastSpanEnd = -1;
 
-export async function parseSource(path: string, src: string): Promise<SourceFile> {
+export async function parseSource(
+  path: string,
+  src: string,
+): Promise<SourceFile | undefined> {
   try {
     const module = await swc.parse(src, {
       syntax: 'typescript',
@@ -34,8 +37,8 @@ export async function parseSource(path: string, src: string): Promise<SourceFile
       symbols,
       span: module.span,
     };
-  } catch (e) {
-    console.error(e);
-    process.exit(1);
+  } catch (e: unknown) {
+    console.error(`Error parsing source file: ${path}`, e);
+    return undefined;
   }
 }

packages/openapi-generator/test/apiSpec.test.ts

@@ -5,6 +5,7 @@ import type { NestedDirectoryJSON } from 'memfs';
 
 import { TestProject } from './testProject';
 import { parseApiSpec, parseApiSpecComment, type Route } from '../src';
+import { MOCK_NODE_MODULES_DIR } from './externalModules';
 
 async function testCase(
   description: string,
@@ -14,7 +15,7 @@ async function testCase(
   expectedErrors: string[] = [],
 ) {
   test(description, async () => {
-    const project = new TestProject(files);
+    const project = new TestProject({ ...files, ...MOCK_NODE_MODULES_DIR });
 
     await project.parseEntryPoint(entryPoint);
     const sourceFile = project.get(entryPoint);
@@ -288,5 +289,5 @@ const MISSING_REFERENCE = {
 };
 
 testCase('missing reference', MISSING_REFERENCE, '/index.ts', {}, [
-  "Cannot find 'Foo' from 'foo'",
+  "Cannot find module 'foo' in the project",
 ]);