Dependency fix on typescript CLI (#1180)

* Fixing Tests

* Passing builds without skipping

* output files ignored

* Dependencies fix (compatibility and dependencies issues)

* package json clean up

* refactory moving common code to wrapper

* Http Support Tests

* Url config type tests

* clean up and simplify

* clean up and simplify

* Support for ProjectDirectory option

* clean up

* Unit tests for DataURL

* clean up text description

.gitignore

@@ -23,3 +23,6 @@ docs.json
 tests-integration/reference-model/Dockerfile
 
 .scala-build/
+*Debug.log*
+tests-integration/cli2-qa-test/test-data/.DS_Store
+tests-integration/cli2-qa-test/test-data/business-terms/.DS_Store

cli2/cli.ts

@@ -46,7 +46,8 @@ async function make(
   const dependencyConfig = DependencyConfig.parse({
     dependencies: morphirJson.dependencies,
     localDependencies: morphirJson.localDependencies,
-    includes: includes
+    includes: includes, 
+    projectDir: projectDir
   })
 
   //load List Of Dependency IR

cli2/dependencies.test.ts

@@ -0,0 +1,121 @@
+
+import * as dep from './dependencies';
+import { z, ZodError } from "zod";
+import * as path from 'path'
+import { decode, labelToName } from "whatwg-encoding";
+
+describe('the dependencies module', () => {
+
+    test("should declare loadAllDependencies", async () => {
+        expect(dep.loadAllDependencies).toBeDefined
+    })
+
+    test("should declare configuration types", async () => {
+        expect(dep.DataUrl).toBeDefined
+        expect(dep.FileUrl).toBeDefined
+        expect(dep.LocalFile).toBeDefined
+    })
+
+    describe("The LocalFile configuration type", () => {
+        test("should fail if can't find the file.", () => {
+            try {
+                dep.LocalFile.parse({
+                    projectDir: __dirname,
+                    sanitized: "./shouldFail.ir"
+                });
+            } catch (error) {
+                expect(error instanceof ZodError).toBeTruthy;
+                let issues = (error as ZodError).issues;
+                let expectedIssue = {
+                    "code": "custom",
+                    "message": "File not found ./shouldFail.ir",
+                };
+                expect(issues[0].message).toBe(expectedIssue.message);
+                expect(issues[0].code).toBe(expectedIssue.code);
+
+            }
+        });
+        test("should support local file.", () => {
+            let fileName = 'dependencies.ts'
+            let expectedFile = path.join(__dirname, fileName);
+
+            let expectedUrl = new URL(`file://${expectedFile}`);
+
+            let { success: urlSuccess, data: urlData } = dep.LocalFile.safeParse({
+                projectDir: __dirname,
+                sanitized: `./${fileName}`
+            });
+            expect({ success: urlSuccess, data: urlData }).toStrictEqual({ success: true, data: expectedUrl });
+        });
+        test("should support local files on an above folder", () => {
+            let fileName = 'gulpfile.js'
+            let expectedFile = path.join(__dirname, '..', fileName);
+
+            let expectedUrl = new URL(`file://${expectedFile}`);
+            let { success: urlSuccess, data: urlData } = dep.LocalFile.safeParse({
+                projectDir: __dirname, 
+                sanitized: `../${fileName}`
+            });
+            expect({ success: urlSuccess, data: urlData }).toStrictEqual({ success: true, data: expectedUrl });
+        });
+        test("should support local files on a sibling folder", () => {
+            let fileName = 'morphir.js'
+            let expectedFile = path.resolve(__dirname, "..",  "cli", fileName);
+
+            let expectedUrl = new URL(`file://${expectedFile}`);
+            let { success: urlSuccess, data: urlData } = dep.LocalFile.safeParse({
+                projectDir: __dirname, 
+                sanitized: `../cli/${fileName}`
+            });
+            expect({ success: urlSuccess, data: urlData }).toStrictEqual({ success: true, data: expectedUrl });
+        });
+    });
+    describe("The Url configuration type", () => {
+        test("should support http.", () => {
+            let url = "http://www.google.com/"
+            let { success, data: fileData } = dep.Url.safeParse(url);
+            expect(success).toBeTruthy()
+            expect(fileData).toStrictEqual(new URL(url));
+        });
+        test("should support https.", () => {
+            let url = "https://www.google.com/"
+            let { success, data: fileData } = dep.Url.safeParse(url);
+            expect(success).toBeTruthy()
+            expect(fileData).toStrictEqual(new URL(url));
+        });
+        test("should support ftp.", () => {
+            let url = "ftp://www.google.com/"
+            let { success, data: fileData } = dep.Url.safeParse(url);
+            expect(success).toBeTruthy()
+            expect(fileData).toStrictEqual(new URL(url));
+        });
+        test("should NOT support S3.", () => {
+            let url = "s3://www.aws.com/mybucket"
+            let { success, data: fileData } = dep.Url.safeParse(url);
+            expect(success).toBeFalsy()
+            expect(fileData).toBeUndefined
+        });
+    });
+
+    describe("The DataURL type", ()=>{
+
+        test("should support encoded json.", () => {
+
+            let obj = {hello: "world"}
+            let content = encodeURIComponent(JSON.stringify(obj))
+            let { success, data: parseResult } = dep.DataUrl.safeParse("data:text/html,"+content);
+            expect(success).toBeTruthy()
+
+            const encodingName = labelToName(parseResult.mimeType.parameters.get("charset") || "utf-8") || "UTF-8";
+            const bodyDecoded = decode(parseResult.body, encodingName);
+
+
+             expect(bodyDecoded).toStrictEqual(JSON.stringify(obj));
+
+        });
+
+    })
+
+
+
+})

cli2/dependencies.ts

@@ -1,15 +1,17 @@
-import * as util from "util";
 import * as fs from "fs";
-import { z } from "zod";
-import { getUri } from "get-uri";
+import { ResultAsync } from "neverthrow";
+import * as path from "path";
+import * as util from "util";
 import { decode, labelToName } from "whatwg-encoding";
+import { z } from "zod";
+import { fetchUriToJson } from "./get-uri-wrapper";
 import { Readable } from "stream";
-import { ResultAsync } from "neverthrow";
+
 
 const parseDataUrl = require("data-urls");
 const fsReadFile = util.promisify(fs.readFile);
 
-const DataUrl = z.string().trim().transform((val, ctx) => {
+export const DataUrl = z.string().trim().transform((val, ctx) => {
   const parsed = parseDataUrl(val)
   if (parsed == null) {
     ctx.addIssue({
@@ -21,7 +23,7 @@ const DataUrl = z.string().trim().transform((val, ctx) => {
   return parsed;
 });
 
-const FileUrl = z.string().trim().url().transform((val, ctx) => {
+export const FileUrl = z.string().trim().url().transform((val, ctx) => {
   if (!val.startsWith("file:")) {
     ctx.addIssue({
       code: z.ZodIssueCode.custom,
@@ -32,7 +34,30 @@ const FileUrl = z.string().trim().url().transform((val, ctx) => {
   return new URL(val);
 });
 
-const Url = z.string().url().transform((url) => new URL(url));
+type LocalFileRef = {
+  projectDir: string, 
+  original: string,
+  fullPath: string,
+  url?: URL,
+
+}
+
+export const LocalFile = z.object({
+  projectDir: z.string(),
+  sanitized: z.string(),
+}).transform(val => <LocalFileRef>{projectDir:val.projectDir, original: val.sanitized, fullPath: path.resolve(val.projectDir, val.sanitized ) })
+  .transform(ref => <LocalFileRef>({ ...ref, url: new URL(`file://${ref.fullPath}`) }))
+  .refine((ref: LocalFileRef) => fs.existsSync(ref.fullPath),
+    (ref: LocalFileRef) => {
+      console.error(`File not found ${ref.original}: ${ref.fullPath}`)
+      return { message: `File not found ${ref.original}` };
+    })
+  .transform(ref => ref.url!);
+
+const SUPPORTED_PROTOCOLS = new Set(["http:", "https:", "ftp:"]);
+
+export const Url = z.string().url().transform((url) => new URL(url))
+  .refine((url) => SUPPORTED_PROTOCOLS.has(url.protocol));
 
 const PathOrUrl = z.union([FileUrl, z.string().trim().min(1)]);
 
@@ -44,13 +69,15 @@ const GithubData = z.object({
 
 const GithubConfig = z.union([GithubData, z.string()]);
 
-const DependencySettings = z.union([DataUrl, FileUrl, z.string().trim()])
+
+const DependencySettings = z.string().trim();
 const Dependencies = z.array(DependencySettings).default([]);
 
 export const DependencyConfig = z.object({
   dependencies: Dependencies,
   localDependencies: z.array(z.string()).default([]),
   includes: z.array(z.string()).default([]),
+  projectDir: z.string()
 });
 
 const IncludeProvided = z.object({
@@ -60,7 +87,7 @@ const IncludeProvided = z.object({
 
 const LocalDependencyProvided = z.object({
   eventKind: z.literal('LocalDependencyProvided'),
-  payload: z.string()
+  payload: z.string() 
 })
 
 const DependencyProvided = z.object({
@@ -108,11 +135,11 @@ export type DependencyConfig = z.infer<typeof DependencyConfig>;
 
 export async function loadAllDependencies(config: DependencyConfig) {
   const events = DependencyConfigToDependencyEvents.parse(config);
-  const results = events.map(load);
+  const results = events.map(load(config));
   const finalResults = await Promise.all(results);
   return finalResults.flatMap((result) => {
     if (result.isOk()) {
-      console.error("Successfully loaded dependency", result.value.dependency)
+      console.info("Successfully loaded dependency", result.value.dependency)
       return result.value.dependency;
     } else {
       console.error("Error loading dependency", result.error);
@@ -121,84 +148,73 @@ export async function loadAllDependencies(config: DependencyConfig) {
   });
 }
 
-function load(event: DependencyEvent) {
-  console.error("Loading event", event);
+const load = (config: DependencyConfig) => function(event: DependencyEvent) {
+
+  //TODO: Clear this up
   let source: "dependencies" | "localDependencies" | "includes";
   let payload = event.payload;
   switch (event.eventKind) {
     case 'IncludeProvided':
       source = "includes";
-      return loadDependenciesFromString(event.payload, source)
+      return loadDependenciesFromString(config)(event.payload, source)
         .map((dependency) => ({ dependency: dependency, source: source, payload: payload }));
     case 'LocalDependencyProvided':
       source = "localDependencies";
-      return loadDependenciesFromString(event.payload, source)
+      return loadDependenciesFromString(config)(event.payload, source)
         .map((dependency) => ({ dependency: dependency, source: source, payload: payload }));
     case 'DependencyProvided':
       source = "dependencies";
       if (typeof payload === "string") {
-        return loadDependenciesFromString(payload, source)
+        return loadDependenciesFromString(config)(payload, source)
           .map((dependency) => ({ dependency: dependency, source: source, payload: payload }));
       } else {
         return loadDependenciesFromURL(payload, source)
           .map((dependency) => ({ dependency: dependency, source: source, payload: payload }));
       }
   }
 }
-
-function loadDependenciesFromString(input: string, source: string) {
+const loadDependenciesFromString = (config: DependencyConfig) => function(input: string, source: string) {
   const doWork = async () => {
     let sanitized = input.trim();
     let { success, data } = DataUrl.safeParse(sanitized);
     if (success) {
-      console.error("Loading Data url", data);
+      console.info("Loading Data url", data);
       const encodingName = labelToName(data.mimeType.parameters.get("charset") || "utf-8") || "UTF-8";
       const bodyDecoded = decode(data.body, encodingName);
-      console.error("Data from data url", bodyDecoded);
+      console.info("Data from data url", bodyDecoded);
       return JSON.parse(bodyDecoded);
     }
     let { success: fileSuccess, data: fileData } = FileUrl.safeParse(sanitized);
     if (fileSuccess && fileData !== undefined) {
-      console.error("Loading file url", fileData);
-      const data = await getUri(fileData);
-      const buffer = await toBuffer(data);
-      const jsonString = buffer.toString();
-      return JSON.parse(jsonString);
+      console.info("Loading file url", fileData);
+      return fetchUriToJson(fileData)
+
     }
     let { success: urlSuccess, data: urlData } = Url.safeParse(sanitized);
     if (urlSuccess && urlData !== undefined) {
-      console.error("Loading url", urlData);
-      if (urlData.protocol.startsWith("http") || urlData.protocol.startsWith("ftp")) {
-        console.error("Loading http or ftp url", urlData);
-        const data = await getUri(urlData);
-        const buffer = await toBuffer(data);
-        const jsonString = buffer.toString();
-        return JSON.parse(jsonString);
-      }
+      console.info("Loading url", urlData);
+      return fetchUriToJson(urlData);
     }
+    let { success: localFileSuccess, data: localUrlData } = LocalFile.safeParse({projectDir : config.projectDir, sanitized});
+    if (localFileSuccess && localUrlData !== undefined) {
+
+      console.info("Loading local file url", localUrlData);
+      return fetchUriToJson(localUrlData);
+
+    }
+
     throw new DependencyError("Invalid dependency string", input);
   }
   return ResultAsync.fromPromise(doWork(), (err) => new DependencyError("Error loading dependency", source, input, err));
 }
 
 function loadDependenciesFromURL(url: URL | Url, source: string) {
   const doWork = async () => {
-    const data = await getUri(url);
-    const buffer = await toBuffer(data);
-    const jsonString = buffer.toString();
-    return JSON.parse(jsonString);
+    return fetchUriToJson(url);
   }
   return ResultAsync.fromPromise(doWork(), (err) => new DependencyError("Error loading dependency", source, url, err));
 }
 
-async function toBuffer(stream: Readable): Promise<Buffer> {
-  const chunks: Buffer[] = [];
-  for await (const chunk of stream) {
-    chunks.push(chunk);
-  }
-  return Buffer.concat(chunks);
-}
-
 class DependencyError extends Error {
   constructor(message: string, source?: string, dependency?: string | FileUrl | DataUrl | URL, cause?: Error | unknown) {
     super(message);
@@ -237,4 +253,13 @@ class LocalDependencyNotFound extends Error {
   pathOrUrl?: PathOrUrl;
   source?: string;
 
-}
+}
+
+
+async function toBuffer(stream: Readable): Promise<Buffer> {
+  const chunks: Buffer[] = [];
+  for await (const chunk of stream) {
+    chunks.push(chunk);
+  }
+  return Buffer.concat(chunks);
+}

cli2/get-uri-wrapper.test.ts

@@ -0,0 +1,15 @@
+import * as getUriWrapper from './get-uri-wrapper';
+import * as path from 'path'
+
+describe('the get-uri-wrapper module', () => {
+
+    test("should declare fetchUriToJson", async () => {
+        expect(getUriWrapper.fetchUriToJson).toBeDefined
+    });
+    test("that fetches a document and converts it to JSON", async () => {
+        let file = path.resolve("./tsconfig.json")
+        let tsconfigJson = await getUriWrapper.fetchUriToJson(`file://${file}`);
+        expect(tsconfigJson).toHaveProperty("compilerOptions");
+
+    });
+});