fix: remove redundant template engine variable and fixed llamaparse f…

…lag (#22)
run-llama · Mar 27, 2024 · 459824d · 459824d
1 parent b08cad0
commit 459824d
Show file tree

Hide file tree

Showing 12 changed files with 134 additions and 164 deletions.
diff --git a/create-app.ts b/create-app.ts
@@ -26,7 +26,6 @@ export type InstallAppArgs = Omit<
 export async function createApp({
   template,
   framework,
-  engine,
   ui,
   appPath,
   packageManager,
@@ -76,7 +75,6 @@ export async function createApp({
     root,
     template,
     framework,
-    engine,
     ui,
     packageManager,
     isOnline,

diff --git a/e2e/basic.spec.ts b/e2e/basic.spec.ts
@@ -4,7 +4,6 @@ import { ChildProcess } from "child_process";
 import fs from "fs";
 import path from "path";
 import type {
-  TemplateEngine,
   TemplateFramework,
   TemplatePostInstallAction,
   TemplateType,
@@ -18,7 +17,7 @@ const templateFrameworks: TemplateFramework[] = [
   "express",
   "fastapi",
 ];
-const templateEngines: TemplateEngine[] = ["simple", "context"];
+const dataSources: string[] = ["--no-files", "--example-file"];
 const templateUIs: TemplateUI[] = ["shadcn", "html"];
 const templatePostInstallActions: TemplatePostInstallAction[] = [
   "none",
@@ -27,12 +26,12 @@ const templatePostInstallActions: TemplatePostInstallAction[] = [
 
 for (const templateType of templateTypes) {
   for (const templateFramework of templateFrameworks) {
-    for (const templateEngine of templateEngines) {
+    for (const dataSource of dataSources) {
       for (const templateUI of templateUIs) {
         for (const templatePostInstallAction of templatePostInstallActions) {
           const appType: AppType =
             templateFramework === "nextjs" ? "" : "--frontend";
-          test.describe(`try create-llama ${templateType} ${templateFramework} ${templateEngine} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
+          test.describe(`try create-llama ${templateType} ${templateFramework} ${dataSource} ${templateUI} ${appType} ${templatePostInstallAction}`, async () => {
             let port: number;
             let externalPort: number;
             let cwd: string;
@@ -49,7 +48,7 @@ for (const templateType of templateTypes) {
                 cwd,
                 templateType,
                 templateFramework,
-                templateEngine,
+                dataSource,
                 templateUI,
                 vectorDb,
                 appType,

diff --git a/e2e/utils.ts b/e2e/utils.ts
@@ -4,7 +4,6 @@ import { mkdir } from "node:fs/promises";
 import * as path from "path";
 import waitPort from "wait-port";
 import {
-  TemplateEngine,
   TemplateFramework,
   TemplatePostInstallAction,
   TemplateType,
@@ -67,18 +66,21 @@ export async function runCreateLlama(
   cwd: string,
   templateType: TemplateType,
   templateFramework: TemplateFramework,
-  templateEngine: TemplateEngine,
+  dataSource: string,
   templateUI: TemplateUI,
   vectorDb: TemplateVectorDB,
   appType: AppType,
   port: number,
   externalPort: number,
   postInstallAction: TemplatePostInstallAction,
 ): Promise<CreateLlamaResult> {
+  if (!process.env.OPENAI_API_KEY) {
+    throw new Error("Setting OPENAI_API_KEY is mandatory to run tests");
+  }
   const name = [
     templateType,
     templateFramework,
-    templateEngine,
+    dataSource,
     templateUI,
     appType,
   ].join("-");
@@ -89,8 +91,7 @@ export async function runCreateLlama(
     templateType,
     "--framework",
     templateFramework,
-    "--engine",
-    templateEngine,
+    dataSource,
     "--ui",
     templateUI,
     "--vector-db",
@@ -100,7 +101,7 @@ export async function runCreateLlama(
     "--embedding-model",
     EMBEDDING_MODEL,
     "--open-ai-key",
-    process.env.OPENAI_API_KEY || "testKey",
+    process.env.OPENAI_API_KEY,
     appType,
     "--eslint",
     "--use-pnpm",

diff --git a/helpers/datasources.ts b/helpers/datasources.ts
@@ -0,0 +1,27 @@
+import { TemplateDataSource } from "./types";
+
+// Example file has an empty config
+export const EXAMPLE_FILE: TemplateDataSource = {
+  type: "file",
+  config: {},
+};
+
+export function getDataSources(
+  files?: string,
+  exampleFile?: boolean,
+): TemplateDataSource[] | undefined {
+  let dataSources: TemplateDataSource[] | undefined = undefined;
+  if (files) {
+    // If user specified files option, then the program should use context engine
+    dataSources = files.split(",").map((filePath) => ({
+      type: "file",
+      config: {
+        path: filePath,
+      },
+    }));
+  }
+  if (exampleFile) {
+    dataSources = [...(dataSources ? dataSources : []), EXAMPLE_FILE];
+  }
+  return dataSources;
+}
diff --git a/helpers/index.ts b/helpers/index.ts
@@ -137,7 +137,7 @@ export const installTemplate = async (
       port: props.externalPort,
     });
 
-    if (props.engine === "context") {
+    if (props.dataSources.length > 0) {
       console.log("\nGenerating context data...\n");
       await copyContextData(props.root, props.dataSources);
       if (

diff --git a/helpers/python.ts b/helpers/python.ts
@@ -177,7 +177,6 @@ export const installPythonTemplate = async ({
   root,
   template,
   framework,
-  engine,
   vectorDb,
   dataSources,
   tools,
@@ -188,7 +187,6 @@ export const installPythonTemplate = async ({
   | "root"
   | "framework"
   | "template"
-  | "engine"
   | "vectorDb"
   | "dataSources"
   | "tools"
@@ -217,7 +215,7 @@ export const installPythonTemplate = async ({
     },
   });
 
-  if (engine === "context") {
+  if (dataSources.length > 0) {
     const enginePath = path.join(root, "app", "engine");
     const compPath = path.join(templatesDir, "components");
 
@@ -257,46 +255,44 @@ export const installPythonTemplate = async ({
       });
     }
 
-    if (dataSources.length > 0) {
-      const loaderConfigs: Record<string, any> = {};
-      const loaderPath = path.join(enginePath, "loaders");
+    const loaderConfigs: Record<string, any> = {};
+    const loaderPath = path.join(enginePath, "loaders");
 
-      // Copy loaders to enginePath
-      await copy("**", loaderPath, {
-        parents: true,
-        cwd: path.join(compPath, "loaders", "python"),
-      });
+    // Copy loaders to enginePath
+    await copy("**", loaderPath, {
+      parents: true,
+      cwd: path.join(compPath, "loaders", "python"),
+    });
 
-      // Generate loaders config
-      // Web loader config
-      if (dataSources.some((ds) => ds.type === "web")) {
-        const webLoaderConfig = dataSources
-          .filter((ds) => ds.type === "web")
-          .map((ds) => {
-            const dsConfig = ds.config as WebSourceConfig;
-            return {
-              base_url: dsConfig.baseUrl,
-              prefix: dsConfig.prefix,
-              depth: dsConfig.depth,
-            };
-          });
-        loaderConfigs["web"] = webLoaderConfig;
-      }
-      // File loader config
-      if (dataSources.some((ds) => ds.type === "file")) {
-        loaderConfigs["file"] = {
-          use_llama_parse: useLlamaParse,
-        };
-      }
-      // Write loaders config
-      if (Object.keys(loaderConfigs).length > 0) {
-        const loaderConfigPath = path.join(root, "config/loaders.json");
-        await fs.mkdir(path.join(root, "config"), { recursive: true });
-        await fs.writeFile(
-          loaderConfigPath,
-          JSON.stringify(loaderConfigs, null, 2),
-        );
-      }
+    // Generate loaders config
+    // Web loader config
+    if (dataSources.some((ds) => ds.type === "web")) {
+      const webLoaderConfig = dataSources
+        .filter((ds) => ds.type === "web")
+        .map((ds) => {
+          const dsConfig = ds.config as WebSourceConfig;
+          return {
+            base_url: dsConfig.baseUrl,
+            prefix: dsConfig.prefix,
+            depth: dsConfig.depth,
+          };
+        });
+      loaderConfigs["web"] = webLoaderConfig;
+    }
+    // File loader config
+    if (dataSources.some((ds) => ds.type === "file")) {
+      loaderConfigs["file"] = {
+        use_llama_parse: useLlamaParse,
+      };
+    }
+    // Write loaders config
+    if (Object.keys(loaderConfigs).length > 0) {
+      const loaderConfigPath = path.join(root, "config/loaders.json");
+      await fs.mkdir(path.join(root, "config"), { recursive: true });
+      await fs.writeFile(
+        loaderConfigPath,
+        JSON.stringify(loaderConfigs, null, 2),
+      );
     }
   }
 

diff --git a/helpers/types.ts b/helpers/types.ts
@@ -3,7 +3,6 @@ import { Tool } from "./tools";
 
 export type TemplateType = "streaming" | "community" | "llamapack";
 export type TemplateFramework = "nextjs" | "express" | "fastapi";
-export type TemplateEngine = "simple" | "context";
 export type TemplateUI = "html" | "shadcn";
 export type TemplateVectorDB = "none" | "mongo" | "pg" | "pinecone" | "milvus";
 export type TemplatePostInstallAction =
@@ -43,7 +42,6 @@ export interface InstallTemplateArgs {
   isOnline: boolean;
   template: TemplateType;
   framework: TemplateFramework;
-  engine: TemplateEngine;
   ui: TemplateUI;
   dataSources: TemplateDataSource[];
   eslint: boolean;

diff --git a/helpers/typescript.ts b/helpers/typescript.ts
@@ -56,7 +56,6 @@ export const installTSTemplate = async ({
   isOnline,
   template,
   framework,
-  engine,
   ui,
   eslint,
   customApiPath,
@@ -142,50 +141,41 @@ export const installTSTemplate = async ({
   /**
    * Copy the selected chat engine files to the target directory and reference it.
    */
-  let relativeEngineDestPath;
   const compPath = path.join(templatesDir, "components");
-  if (engine && (framework === "express" || framework === "nextjs")) {
-    console.log("\nUsing chat engine:", engine, "\n");
+  const relativeEngineDestPath =
+    framework === "nextjs"
+      ? path.join("app", "api", "chat")
+      : path.join("src", "controllers");
+  const enginePath = path.join(root, relativeEngineDestPath, "engine");
 
-    let vectorDBFolder: string = engine;
-
-    if (engine !== "simple" && vectorDb) {
+  if (dataSources.length === 0) {
+    // use simple hat engine if user neither select tools nor a data source
+    console.log("\nUsing simple chat engine\n");
+  } else {
+    if (vectorDb) {
+      // copy vector db component
       console.log("\nUsing vector DB:", vectorDb, "\n");
-      vectorDBFolder = vectorDb;
-    }
-
-    relativeEngineDestPath =
-      framework === "nextjs"
-        ? path.join("app", "api", "chat")
-        : path.join("src", "controllers");
-
-    const enginePath = path.join(root, relativeEngineDestPath, "engine");
-
-    // copy vector db component
-    const vectorDBPath = path.join(
-      compPath,
-      "vectordbs",
-      "typescript",
-      vectorDBFolder,
-    );
-    await copy("**", enginePath, {
-      parents: true,
-      cwd: vectorDBPath,
-    });
-
-    // copy loader component
-    const dataSourceType = dataSources[0]?.type;
-    if (dataSourceType) {
-      let loaderFolder: string;
-      loaderFolder = useLlamaParse ? "llama_parse" : dataSourceType;
+      const vectorDBPath = path.join(
+        compPath,
+        "vectordbs",
+        "typescript",
+        vectorDb,
+      );
       await copy("**", enginePath, {
         parents: true,
-        cwd: path.join(compPath, "loaders", "typescript", loaderFolder),
+        cwd: vectorDBPath,
       });
     }
-
-    // copy tools component
+    // copy loader component (TS only supports llama_parse and file for now)
+    let loaderFolder: string;
+    loaderFolder = useLlamaParse ? "llama_parse" : "file";
+    await copy("**", enginePath, {
+      parents: true,
+      cwd: path.join(compPath, "loaders", "typescript", loaderFolder),
+    });
     if (tools?.length) {
+      // use agent chat engine if user selects tools
+      console.log("\nUsing agent chat engine\n");
       await copy("**", enginePath, {
         parents: true,
         cwd: path.join(compPath, "engines", "typescript", "agent"),
@@ -201,7 +191,9 @@ export const installTSTemplate = async ({
         configFilePath,
         JSON.stringify(configContent, null, 2),
       );
-    } else if (engine !== "simple") {
+    } else {
+      // use context chat engine if user does not select tools
+      console.log("\nUsing context chat engine\n");
       await copy("**", enginePath, {
         parents: true,
         cwd: path.join(compPath, "engines", "typescript", "chat"),
@@ -248,7 +240,7 @@ export const installTSTemplate = async ({
     // modify the dev script to use the custom api path
   }
 
-  if (engine === "context" && relativeEngineDestPath) {
+  if (dataSources.length > 0 && relativeEngineDestPath) {
     // add generate script if using context engine
     packageJson.scripts = {
       ...packageJson.scripts,