fixed

lunary-ai · May 16, 2024 · 2b73a81 · 2b73a81
1 parent 4ec76d5
commit 2b73a81
Show file tree

Hide file tree

Showing 5 changed files with 13 additions and 9 deletions.
diff --git a/packages/backend/src/api/v1/datasets/utils.ts b/packages/backend/src/api/v1/datasets/utils.ts
@@ -33,9 +33,7 @@ export async function getDatasetBySlug(slug: string, projectId: string) {
       p.messages as prompt_messages,
       pv.id as variation_id,
       pv.variables,
-      pv.context,
-      pv.ideal_output,
-      pv.context
+      pv.ideal_output
     from
       dataset d 
       left join dataset_prompt p on d.id = p.dataset_id

diff --git a/packages/backend/src/checks/index.ts b/packages/backend/src/checks/index.ts
@@ -165,7 +165,10 @@ export const CHECK_RUNNERS: CheckRunner[] = [
 
       const passed = type === "contains" ? has : !has
 
-      const match = has ? run[field].match(re)[0] : ""
+      const runField =
+        typeof run[field] === "string" ? run[field] : JSON.stringify(run[field])
+
+      const match = has ? runField.match(re)[0] : ""
 
       return {
         passed,
@@ -276,7 +279,7 @@ export const CHECK_RUNNERS: CheckRunner[] = [
   {
     id: "length",
     sql: ({ field, operator, length }) =>
-      sql`length(${sql(field + "_text")} ${postgresOperators(operator)} ${length}`,
+      sql`length(${sql(field + "_text")}) ${postgresOperators(operator)} ${length}`,
   },
   {
     id: "date",
@@ -456,11 +459,14 @@ export const CHECK_RUNNERS: CheckRunner[] = [
 
       const output = lastMsg(run["output"])
 
-      if (!run.context) throw new Error("No context to compare to")
+      if (!run.idealOutput)
+        throw new Error(
+          "You need to set an ideal output for each prompt in the dataset in order to use the Rouge Evaluator",
+        )
 
       const scorer = rouge[rougeType]
 
-      const rougeScore = scorer(output, run.context) * 100
+      const rougeScore = scorer(output, run.idealOutput) * 100
 
       const passed = rougeScore >= parseInt(percent)
 

diff --git a/packages/backend/src/checks/runChecks.ts b/packages/backend/src/checks/runChecks.ts
@@ -32,7 +32,6 @@ async function sqlEval(sqlFragment: any, run: any): Promise<boolean> {
 
   // those are eval-specific and differ from the run object
   delete run.idealOutput
-  delete run.context
   delete run.radarId
   delete run.runId
   delete run.results

diff --git a/packages/backend/src/utils/db.ts b/packages/backend/src/utils/db.ts
@@ -15,7 +15,7 @@ const sql = postgres(process.env.DATABASE_URL!, {
     application_name: `backend-${isProduction ? "production" : "development"}-${new Date().getTime()}`,
   },
   // debug: process.env.DEBUG ? debugFn : () => {},
-  // onnotice: process.env.DEUG ? console.log : () => {}, // TODO: replace `() => {}` by false when porsager/postgres PR is merged
+  // onnotice: process.env.DEUG ? console.log : () => {},
 })
 
 function debugFn(

diff --git a/packages/db/0013.sql b/packages/db/0013.sql
@@ -0,0 +1 @@
+alter table dataset_prompt_variation drop column context;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		alter table dataset_prompt_variation drop column context;